Merge pull request #4107: Merge branch 'master' into jstorm-runner at commit 727253e
diff --git a/.gitattributes b/.gitattributes
index cce74a2..13a48e4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -8,7 +8,9 @@
.gitattributes text
.gitignore text
LICENSE text
+Dockerfile text
*.avsc text
+*.go text
*.html text
*.java text
*.md text
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 868edd1..bd361b7 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,12 +1,10 @@
-Be sure to do all of the following to help us incorporate your contribution
-quickly and easily:
+Follow this checklist to help us incorporate your contribution quickly and easily:
- - [ ] Make sure the PR title is formatted like:
- `[BEAM-<Jira issue #>] Description of pull request`
- - [ ] Make sure tests pass via `mvn clean verify`.
- - [ ] Replace `<Jira issue #>` in the title with the actual Jira issue
- number, if there is one.
- - [ ] If this contribution is large, please file an Apache
- [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
+ - [ ] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes.
+ - [ ] Each commit in the pull request should have a meaningful subject line and body.
+ - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue.
+ - [ ] Write a pull request description that is detailed enough to understand what the pull request does, how, and why.
+ - [ ] Run `mvn clean verify` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically.
+ - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
---
diff --git a/.gitignore b/.gitignore
index 1ecb993..8d2a6b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,8 +3,9 @@
# This is typically in files named 'src.xml' throughout this repository.
# Ignore files generated by the Maven build process.
-target/
bin/
+dependency-reduced-pom.xml
+target/
# Ignore generated archetypes
sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/
@@ -14,6 +15,7 @@
*.py[cod]
*.egg-info/
.eggs/
+nose-*.egg/
.tox/
build/
dist/
@@ -25,6 +27,7 @@
sdks/python/LICENSE
sdks/python/NOTICE
sdks/python/README.md
+sdks/python/apache_beam/portability/api/*pb2*.*
# Ignore IntelliJ files.
.idea/
@@ -41,9 +44,8 @@
.apt_generated/
.settings/
-# The build process generates the dependency-reduced POM, but it shouldn't be
-# committed.
-dependency-reduced-pom.xml
+# Ignore Visual Studio Code files.
+.vscode/
# Hotspot VM leaves this log in a non-target directory when java crashes
hs_err_pid*.log
diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy
new file mode 100644
index 0000000..131c798
--- /dev/null
+++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy
@@ -0,0 +1,129 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import hudson.model.Result
+
+int NO_BUILD = -1
+
+// These are args for the GitHub Pull Request Builder (ghprb) Plugin. Providing these arguments is
+// necessary due to a bug in the ghprb plugin where environment variables are not correctly passed
+// to jobs downstream of a Pipeline job.
+// Tracked by https://github.com/jenkinsci/ghprb-plugin/issues/572.
+List<Object> ghprbArgs = [
+ string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"),
+ string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"),
+ string(name: 'ghprbPullId', value: "${ghprbPullId}")
+]
+
+// This argument is the commit at which to build.
+List<Object> commitArg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")]
+
+int javaBuildNum = NO_BUILD
+
+final String JAVA_BUILD_TYPE = "java"
+final String PYTHON_BUILD_TYPE = "python"
+final String ALL_BUILD_TYPE = "all"
+
+def buildTypes = [
+ JAVA_BUILD_TYPE,
+ PYTHON_BUILD_TYPE,
+ ALL_BUILD_TYPE,
+]
+
+String currentBuildType = ALL_BUILD_TYPE
+String commentLower = ghprbCommentBody.toLowerCase()
+
+// Currently if there is nothing selected (e.g. the comment is just "retest this please") we select "all" by default.
+// In the future we should provide some mechanism, either via commenting or the suite failure message, to enforce
+// selection of one of the build types.
+if (!commentLower.isEmpty()) {
+ commentSplit = commentLower.split(' ')
+ buildType = commentSplit[commentSplit.length-1]
+ if (buildTypes.contains(buildType)) {
+ currentBuildType = buildType
+ }
+}
+
+// This (and the below) define "Stages" of a pipeline. These stages run serially, and inside can
+// have "parallel" blocks which execute several work steps concurrently. This work is limited to
+// simple operations -- more complicated operations need to be performed on an actual node. In this
+// case we are using the pipeline to trigger downstream builds.
+stage('Build') {
+ parallel (
+ java: {
+ if (currentBuildType == JAVA_BUILD_TYPE || currentBuildType == ALL_BUILD_TYPE) {
+ def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs
+ if (javaBuild.getResult() == Result.SUCCESS.toString()) {
+ javaBuildNum = javaBuild.getNumber()
+ }
+ } else {
+ echo 'Skipping Java due to comment selecting non-Java execution: ' + ghprbCommentBody
+ }
+ },
+ python_unit: { // Python doesn't have a build phase, so we include this here.
+ if (currentBuildType == PYTHON_BUILD_TYPE || currentBuildType == ALL_BUILD_TYPE) {
+ try {
+ build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs
+ } catch (Exception e) {
+ echo 'Python build failed: ' + e.toString()
+ }
+ } else {
+ echo 'Skipping Python due to comment selecting non-Python execution: ' + ghprbCommentBody
+ }
+ }
+ )
+}
+
+// This argument is provided to downstream jobs so they know from which build to pull artifacts.
+javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")]
+javaUnitPassed = false
+
+stage('Unit Test / Code Health') {
+ parallel (
+ java_unit: {
+ if(javaBuildNum != NO_BUILD) {
+ def javaTest = build job: 'beam_Java_UnitTest', parameters: javaBuildArg + ghprbArgs
+ if(javaTest.getResult() == Result.SUCCESS.toString()) {
+ javaUnitPassed = true
+ }
+ }
+ },
+ java_codehealth: {
+ if(javaBuildNum != NO_BUILD) {
+ try {
+ build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs
+ } catch (Exception e) {
+ echo 'Java CodeHealth Build Failed: ' + e.toString()
+ }
+ }
+ }
+ )
+}
+
+stage('Integration Test') {
+ parallel (
+ // Not gated on codehealth because codehealth shouldn't affect whether tests provide useful
+ // signal.
+ java_integration: {
+ if(javaUnitPassed) {
+ build job: 'beam_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs
+ }
+ }
+ )
+}
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index f47ab28..2930d74 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -22,18 +22,41 @@
// http://groovy-lang.org/style-guide.html
class common_job_properties {
+ static String checkoutDir = 'src'
+
+ static void setSCM(def context, String repositoryName) {
+ context.scm {
+ git {
+ remote {
+ // Double quotes here mean ${repositoryName} is interpolated.
+ github("apache/${repositoryName}")
+ // Single quotes here mean that ${ghprbPullId} is not interpolated and instead passed
+ // through to Jenkins where it refers to the environment variable.
+ refspec('+refs/heads/*:refs/remotes/origin/* ' +
+ '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*')
+ }
+ branch('${sha1}')
+ extensions {
+ cleanAfterCheckout()
+ relativeTargetDirectory(checkoutDir)
+ }
+ }
+ }
+ }
+
// Sets common top-level job properties for website repository jobs.
- static void setTopLevelWebsiteJobProperties(context) {
+ static void setTopLevelWebsiteJobProperties(def context,
+ String branch = 'asf-site') {
setTopLevelJobProperties(
context,
'beam-site',
- 'asf-site',
+ branch,
'beam',
30)
}
// Sets common top-level job properties for main repository jobs.
- static void setTopLevelMainJobProperties(context,
+ static void setTopLevelMainJobProperties(def context,
String branch = 'master',
int timeout = 100,
String jenkinsExecutorLabel = 'beam') {
@@ -47,7 +70,7 @@
// Sets common top-level job properties. Accessed through one of the above
// methods to protect jobs from internal details of param defaults.
- private static void setTopLevelJobProperties(context,
+ private static void setTopLevelJobProperties(def context,
String repositoryName,
String defaultBranch,
String jenkinsExecutorLabel,
@@ -70,19 +93,7 @@
}
// Source code management.
- context.scm {
- git {
- remote {
- url('https://github.com/apache/' + repositoryName + '.git')
- refspec('+refs/heads/*:refs/remotes/origin/* ' +
- '+refs/pull/*:refs/remotes/origin/pr/*')
- }
- branch('${sha1}')
- extensions {
- cleanAfterCheckout()
- }
- }
- }
+ setSCM(context, repositoryName)
context.parameters {
// This is a recommended setup if you want to run the job manually. The
@@ -114,8 +125,9 @@
// below to insulate callers from internal parameter defaults.
private static void setPullRequestBuildTrigger(context,
String commitStatusContext,
- String successComment = '--none--',
- String prTriggerPhrase = '') {
+ String prTriggerPhrase = '',
+ boolean onlyTriggerPhraseToggle = true,
+ String successComment = '--none--') {
context.triggers {
githubPullRequest {
admins(['asfbot'])
@@ -130,6 +142,8 @@
// required to start it.
if (prTriggerPhrase) {
triggerPhrase(prTriggerPhrase)
+ }
+ if (onlyTriggerPhraseToggle) {
onlyTriggerPhrase()
}
@@ -140,41 +154,19 @@
delegate.context("Jenkins: " + commitStatusContext)
}
- /*
- This section is disabled, because of jenkinsci/ghprb-plugin#417 issue.
- For the time being, an equivalent configure section below is added.
-
// Comment messages after build completes.
buildStatus {
completedStatus('SUCCESS', successComment)
completedStatus('FAILURE', '--none--')
completedStatus('ERROR', '--none--')
}
- */
}
}
}
-
- // Comment messages after build completes.
- context.configure {
- def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message(successComment)
- result('SUCCESS')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('ERROR')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('FAILURE')
- }
- }
}
// Sets common config for Maven jobs.
- static void setMavenConfig(context, mavenInstallation='Maven 3.3.3') {
+ static void setMavenConfig(context, String mavenInstallation='Maven 3.3.3') {
context.mavenInstallation(mavenInstallation)
context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
@@ -182,21 +174,24 @@
// tiered compilation to make the JVM startup times faster during the tests.
context.mavenOpts('-XX:+TieredCompilation')
context.mavenOpts('-XX:TieredStopAtLevel=1')
- context.rootPOM('pom.xml')
+ context.rootPOM(checkoutDir + '/pom.xml')
// Use a repository local to the workspace for better isolation of jobs.
context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE)
// Disable archiving the built artifacts by default, as this is slow and flaky.
// We can usually recreate them easily, and we can also opt-in individual jobs
// to artifact archiving.
- context.archivingDisabled(true)
+ if (context.metaClass.respondsTo(context, 'archivingDisabled', boolean)) {
+ context.archivingDisabled(true)
+ }
}
// Sets common config for PreCommit jobs.
static void setPreCommit(context,
String commitStatusName,
+ String prTriggerPhrase = '',
String successComment = '--none--') {
// Set pull request build trigger.
- setPullRequestBuildTrigger(context, commitStatusName, successComment)
+ setPullRequestBuildTrigger(context, commitStatusName, prTriggerPhrase, false, successComment)
}
// Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase
@@ -208,8 +203,9 @@
setPullRequestBuildTrigger(
context,
commitStatusName,
- '--none--',
- prTriggerPhrase)
+ prTriggerPhrase,
+ true,
+ '--none--')
}
// Sets common config for PostCommit jobs.
@@ -233,10 +229,19 @@
}
}
+ static def mapToArgString(LinkedHashMap<String, String> inputArgs) {
+ List argList = []
+ inputArgs.each({
+ // FYI: Replacement only works with double quotes.
+ key, value -> argList.add("--$key=$value")
+ })
+ return argList.join(' ')
+ }
+
// Configures the argument list for performance tests, adding the standard
// performance test job arguments.
private static def genPerformanceArgs(def argMap) {
- def standard_args = [
+ LinkedHashMap<String, String> standardArgs = [
project: 'apache-beam-testing',
dpb_log_level: 'INFO',
maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn',
@@ -245,13 +250,8 @@
official: 'true'
]
// Note: in case of key collision, keys present in ArgMap win.
- def joined_args = standard_args.plus(argMap)
- def argList = []
- joined_args.each({
- // FYI: Replacement only works with double quotes.
- key, value -> argList.add("--$key=$value")
- })
- return argList.join(' ')
+ LinkedHashMap<String, String> joinedArgs = standardArgs.plus(argMap)
+ return mapToArgString(joinedArgs)
}
// Adds the standard performance test job steps.
@@ -262,10 +262,114 @@
shell('rm -rf PerfKitBenchmarker')
// Clone appropriate perfkit branch
shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git')
- // Install job requirements.
+ // Install Perfkit benchmark requirements.
shell('pip install --user -r PerfKitBenchmarker/requirements.txt')
+ // Install job requirements for Python SDK.
+ shell('pip install --user -e sdks/python/[gcp,test]')
// Launch performance test.
shell("python PerfKitBenchmarker/pkb.py $pkbArgs")
}
}
+
+ /**
+ * Sets properties for all jobs which are run by a pipeline top-level (maven) job.
+ * @param context The delegate from the top level of a MavenJob.
+ * @param jobTimeout How long (in minutes) to wait for the job to finish.
+ * @param descriptor A short string identifying the job, e.g. "Java Unit Test".
+ */
+ static def setPipelineJobProperties(def context, int jobTimeout, String descriptor) {
+ context.parameters {
+ stringParam(
+ 'ghprbGhRepository',
+ 'N/A',
+ 'Repository name for use by ghprb plugin.')
+ stringParam(
+ 'ghprbActualCommit',
+ 'N/A',
+ 'Commit ID for use by ghprb plugin.')
+ stringParam(
+ 'ghprbPullId',
+ 'N/A',
+ 'PR # for use by ghprb plugin.')
+
+ }
+
+ // Set JDK version.
+ context.jdk('JDK 1.8 (latest)')
+
+ // Restrict this project to run only on Jenkins executors as specified
+ context.label('beam')
+
+ // Execute concurrent builds if necessary.
+ context.concurrentBuild()
+
+ context.wrappers {
+ timeout {
+ absolute(jobTimeout)
+ abortBuild()
+ }
+ credentialsBinding {
+ string("COVERALLS_REPO_TOKEN", "beam-coveralls-token")
+ }
+ downstreamCommitStatus {
+ delegate.context("Jenkins: ${descriptor}")
+ triggeredStatus("${descriptor} Pending")
+ startedStatus("Running ${descriptor}")
+ statusUrl()
+ completedStatus('SUCCESS', "${descriptor} Passed")
+ completedStatus('FAILURE', "${descriptor} Failed")
+ completedStatus('ERROR', "Error Executing ${descriptor}")
+ }
+ // Set SPARK_LOCAL_IP for spark tests.
+ environmentVariables {
+ env('SPARK_LOCAL_IP', '127.0.0.1')
+ }
+ }
+
+ // Set Maven parameters.
+ setMavenConfig(context)
+ }
+
+ /**
+ * Sets job properties common to pipeline jobs which are responsible for being the root of a
+ * build tree. Downstream jobs should pull artifacts from these jobs.
+ * @param context The delegate from the top level of a MavenJob.
+ */
+ static def setPipelineBuildJobProperties(def context) {
+ context.properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ context.parameters {
+ stringParam(
+ 'sha1',
+ 'master',
+ 'Commit id or refname (e.g. origin/pr/9/head) you want to build.')
+ }
+
+ // Source code management.
+ setSCM(context, 'beam')
+ }
+
+ /**
+ * Sets common job parameters for jobs which consume artifacts built for them by an upstream job.
+ * @param context The delegate from the top level of a MavenJob.
+ * @param jobName The job from which to copy artifacts.
+ */
+ static def setPipelineDownstreamJobProperties(def context, String jobName) {
+ context.parameters {
+ stringParam(
+ 'buildNum',
+ 'N/A',
+ "Build number of ${jobName} to copy from.")
+ }
+
+ context.preBuildSteps {
+ copyArtifacts(jobName) {
+ buildSelector {
+ buildNumber('${buildNum}')
+ }
+ }
+ }
+ }
}
diff --git a/.test-infra/jenkins/job_00_seed.groovy b/.test-infra/jenkins/job_00_seed.groovy
new file mode 100644
index 0000000..9fcd9d6
--- /dev/null
+++ b/.test-infra/jenkins/job_00_seed.groovy
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Defines the seed job, which creates or updates all other Jenkins projects.
+job('beam_SeedJob') {
+ description('Automatically configures all Apache Beam Jenkins projects based' +
+ ' on Jenkins DSL groovy files checked into the code repository.')
+
+ properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ // Restrict to only run on Jenkins executors labeled 'beam'
+ label('beam')
+
+ logRotator {
+ daysToKeep(14)
+ }
+
+ scm {
+ git {
+ remote {
+ github('apache/beam')
+
+ // ${ghprbPullId} is not interpolated by groovy, but passed through to Jenkins where it
+ // refers to the environment variable
+ refspec(['+refs/heads/*:refs/remotes/origin/*',
+ '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*']
+ .join(' '))
+
+ // The variable ${sha1} is not interpolated by groovy, but a parameter of the Jenkins job
+ branch('${sha1}')
+
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+ }
+
+ parameters {
+ // Setup for running this job from a pull request
+ stringParam(
+ 'sha1',
+ 'master',
+ 'Commit id or refname (eg: origin/pr/4001/head) you want to build against.')
+ }
+
+ wrappers {
+ timeout {
+ absolute(60)
+ abortBuild()
+ }
+ }
+
+ triggers {
+ // Run once per day
+ cron('0 */6 * * *')
+
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+
+ // Also run when manually kicked on a pull request
+ triggerPhrase('Run Seed Job')
+ onlyTriggerPhrase()
+
+ extensions {
+ commitStatus {
+ context("Jenkins: Seed Job")
+ }
+
+ buildStatus {
+ completedStatus('SUCCESS', '--none--')
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ }
+ }
+ }
+
+ // If anything goes wrong, mail the main dev list, because it is a big deal
+ publishers {
+ mailer('dev@beam.apache.org', false, true)
+ }
+
+ steps {
+ dsl {
+ // A list or a glob of other groovy files to process.
+ external('.test-infra/jenkins/job_*.groovy')
+
+ // If a job is removed from the script, disable it (rather than deleting).
+ removeAction('DISABLE')
+ }
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy
new file mode 100644
index 0000000..87aa98d
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_Build.groovy
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which builds artifacts for downstream jobs to consume.
+mavenJob('beam_Java_Build') {
+ description('Builds Beam Java SDK and archives artifacts. Meant to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Build")
+ // Set standard properties for a pipeline job which needs to pull from GitHub instead of an
+ // upstream job.
+ common_job_properties.setPipelineBuildJobProperties(delegate)
+
+ configure { project ->
+ // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually.
+ project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' {
+ 'string' "beam_*"
+ }
+ // The Build Discarder also doesn't support the job DSL in the right way so we have to configure it manually.
+ // -1 indicates that a property is "infinite".
+ project / 'properties' / 'jenkins.model.BuildDiscarderProperty' / 'strategy'(class:'hudson.tasks.LogRotator') {
+ 'daysToKeep'(-1)
+ 'numToKeep'(-1)
+ 'artifactDaysToKeep'(1)
+ 'artifactNumToKeep'(-1)
+ }
+ }
+
+ // Construct Maven goals for this job.
+ args = [
+ '-B',
+ '-e',
+ 'clean',
+ 'install',
+ "-pl '!sdks/python,!sdks/java/javadoc'",
+ '-DskipTests',
+ '-Dcheckstyle.skip',
+ ]
+ goals(args.join(' '))
+
+ // This job publishes artifacts so that downstream jobs can use them.
+ publishers {
+ archiveArtifacts {
+ pattern('.repository/org/apache/beam/**/*')
+ pattern('.test-infra/**/*')
+ pattern('.github/**/*')
+ pattern('examples/**/*')
+ pattern('runners/**/*')
+ pattern('sdks/**/*')
+ pattern('target/**/*')
+ pattern('pom.xml')
+ exclude('examples/**/*.jar,runners/**/*.jar,sdks/**/*.jar,target/**/*.jar')
+ onlyIfSuccessful()
+ defaultExcludes()
+ }
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
new file mode 100644
index 0000000..41a4536
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the Beam code health checks.
+mavenJob('beam_Java_CodeHealth') {
+ description('Runs Java code health checks. Meant to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Code Health")
+ // This job runs downstream of the beam_Java_Build job and gets artifacts from that job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ args = [
+ '-B',
+ '-e',
+ "-pl '!sdks/python'",
+ 'checkstyle:check',
+ 'findbugs:check',
+ 'org.apache.rat:apache-rat-plugin:check',
+ ]
+ goals(args.join(' '))
+}
diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
new file mode 100644
index 0000000..56daf73
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the set of precommit integration tests.
+mavenJob('beam_Java_IntegrationTest') {
+ description('Runs Java Failsafe integration tests. Designed to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Integration Tests")
+ // Set standard properties for a job which pulls artifacts from an upstream job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ // Profiles to activate in order to ensure runners are available at test time.
+ profiles = [
+ 'jenkins-precommit',
+ 'direct-runner',
+ 'dataflow-runner',
+ 'spark-runner',
+ 'flink-runner',
+ 'apex-runner'
+ ]
+ // In the case of the precommit integration tests, we are currently only running the integration
+ // tests in the examples directory. By directly invoking failsafe with an execution name (which we
+ // do in order to avoid building artifacts again) we are required to enumerate each execution we
+ // want to run, something which is feasible in this case.
+ examples_integration_executions = [
+ 'apex-runner-integration-tests',
+ 'dataflow-runner-integration-tests',
+ 'dataflow-runner-integration-tests-streaming',
+ 'direct-runner-integration-tests',
+ 'flink-runner-integration-tests',
+ 'spark-runner-integration-tests',
+ ]
+ // Arguments to provide Maven.
+ args = [
+ '-B',
+ '-e',
+ "-P${profiles.join(',')}",
+ "-pl examples/java",
+ ]
+ // This adds executions for each of the failsafe invocations listed above to the list of goals.
+ examples_integration_executions.each({
+ value -> args.add("failsafe:integration-test@${value}")
+ })
+ goals(args.join(' '))
+}
diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
new file mode 100644
index 0000000..e558eea
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the current set of standard unit tests.
+mavenJob('beam_Java_UnitTest') {
+ description('Runs Java Surefire unit tests. Designed to be run by a pipeline job.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Unit Tests")
+ // Set standard properties for a job which pulls artifacts from an upstream job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ // Construct Maven goals for this job.
+ args = [
+ '-B',
+ '-e',
+ 'surefire:test@default-test',
+ "-pl '!sdks/python'",
+ '-DrepoToken=$COVERALLS_REPO_TOKEN',
+ '-DpullRequest=$ghprbPullId',
+ ]
+ goals(args.join(' '))
+}
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
new file mode 100644
index 0000000..6a71bda
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Beam Python performance tests on PerfKit Benchmarker.
+job('beam_PerformanceTests_Python'){
+ // Set default Beam job properties.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Run job in postcommit every 6 hours, don't trigger every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 */6 * * *',
+ false,
+ 'commits@beam.apache.org')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Python SDK Performance Test',
+ 'Run Python Performance Test')
+
+ def pipelineArgs = [
+ project: 'apache-beam-testing',
+ staging_location: 'gs://temp-storage-for-end-to-end-tests/staging-it',
+ temp_location: 'gs://temp-storage-for-end-to-end-tests/temp-it',
+ output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
+ ]
+ def pipelineArgList = []
+ pipelineArgs.each({
+ key, value -> pipelineArgList.add("--$key=$value")
+ })
+ def pipelineArgsJoined = pipelineArgList.join(',')
+
+ def argMap = [
+ beam_sdk : 'python',
+ benchmarks: 'beam_integration_benchmark',
+ beam_it_args: pipelineArgsJoined
+ ]
+
+ common_job_properties.buildPerformanceTest(delegate, argMap)
+}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_JDKVersionsTest.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_JDKVersionsTest.groovy
new file mode 100644
index 0000000..df0a2c7
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_JDKVersionsTest.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Java postcommit tests cross multiple JDK versions.
+matrixJob('beam_PostCommit_Java_JDK_Versions_Test') {
+ description('Runs postcommit tests on the Java SDK in multiple Jdk versions.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set JDK versions.
+ axes {
+ label('label', 'beam')
+ jdk('JDK 1.7 (latest)',
+ 'OpenJDK 7 (on Ubuntu only)',
+ 'OpenJDK 8 (on Ubuntu only)')
+ }
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 */6 * * *',
+ false,
+ '', // TODO: Remove last two args once test is stable again.
+ false)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Java JDK Version Test',
+ 'Run Java JDK Version Test')
+
+ // Maven build for this job.
+ steps {
+ maven {
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Maven build project.
+ // Skip beam-sdks-python since this test is only apply to Java.
+ // TODO[BEAM-2322,BEAM-2323,BEAM-2324]: Re-enable beam-runners-apex once the build is passed.
+ goals('-B -e -P dataflow-runner clean install -pl \'!org.apache.beam:beam-sdks-python,!org.apache.beam:beam-runners-apex\' -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=TestDataflowRunner" ]\'')
+ }
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
index 2f05c38..0dda772 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
@@ -29,7 +29,7 @@
concurrentBuild()
// Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 240)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
@@ -44,5 +44,22 @@
'Run Java PostCommit')
// Maven goals for this job.
- goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=TestDataflowRunner" ]\'')
+ goals([
+ 'clean',
+ 'install',
+ '--projects sdks/java/core,runners/direct-java,sdks/java/fn-execution',
+ ' --also-make',
+ '--also-make-dependents',
+ '--batch-mode',
+ '--errors',
+ '--fail-at-end',
+ '-P release,dataflow-runner',
+ '-DrepoToken=$COVERALLS_REPO_TOKEN',
+ '-D skipITs=false',
+ '''-D integrationTestPipelineOptions=\'[ \
+ "--project=apache-beam-testing", \
+ "--tempRoot=gs://temp-storage-for-end-to-end-tests", \
+ "--runner=TestDataflowRunner" \
+ ]\' '''
+ ].join(' '))
}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy
index f781b4e..f1ba704 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy
@@ -32,7 +32,8 @@
common_job_properties.setMavenConfig(delegate, 'Maven 3.3.3 (Windows)')
// Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '0 */6 * * *', false)
+ // TODO(BEAM-1042, BEAM-1045, BEAM-2269, BEAM-2299) Turn notifications back on once fixed.
+ common_job_properties.setPostCommit(delegate, '0 */6 * * *', false, '', false)
// Allows triggering this build against pull requests.
common_job_properties.enablePhraseTriggeringFromPullRequest(
@@ -41,5 +42,5 @@
'Run Java Windows PostCommit')
// Maven goals for this job.
- goals('-B -e -Prelease,direct-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+ goals('-B -e -Prelease,direct-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install')
}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index 1348a19..e1cbafe 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@ -45,5 +45,5 @@
'Run Gearpump ValidatesRunner')
// Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
+ goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'')
}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
new file mode 100644
index 0000000..06bbfb7
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of Python ValidatesRunner tests against the
+// Dataflow runner.
+job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
+ description('Runs Python ValidatesRunner suite on the Dataflow runner.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Google Cloud Dataflow Runner Python ValidatesRunner Tests',
+ 'Run Python Dataflow ValidatesRunner')
+
+ // Allow the test to only run on particular nodes
+ // TODO(BEAM-1817): Remove once the tests can run on all nodes
+ parameters {
+ nodeParam('TEST_HOST') {
+ description('select test host as either beam1, 2 or 3')
+ defaultNodes(['beam3'])
+ allowedNodes(['beam1', 'beam2', 'beam3'])
+ trigger('multiSelectionDisallowed')
+ eligibility('IgnoreOfflineNodeEligibility')
+ }
+ }
+
+ // Execute shell command to test Python SDK.
+ steps {
+ shell('bash sdks/python/run_validatesrunner.sh')
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Go_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Go_MavenInstall.groovy
new file mode 100644
index 0000000..c616edc
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Go_MavenInstall.groovy
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Go precommit which runs a maven install, and the current set
+// of precommit tests.
+mavenJob('beam_PreCommit_Go_MavenInstall') {
+ description('Runs an install of the current GitHub Pull Request.')
+
+ previousNames('beam_PreCommit_MavenVerify')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'master',
+ 150)
+
+ // Set Maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PreCommit job.
+ common_job_properties.setPreCommit(delegate, 'mvn clean install -pl sdks/go -am -amd', 'Run Go PreCommit')
+
+ // Maven goals for this job: The Go SDK, its dependencies, and things that depend on it.
+ goals('''\
+ --batch-mode \
+ --errors \
+ --activate-profiles release,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \
+ --projects sdks/go \
+ --also-make \
+ --also-make-dependents \
+ -D pullRequest=$ghprbPullId \
+ help:effective-settings \
+ clean \
+ install
+ ''')
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index bc130ec..0775e2f 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -32,14 +32,25 @@
common_job_properties.setTopLevelMainJobProperties(
delegate,
'master',
- 120)
+ 240)
// Set Maven parameters.
common_job_properties.setMavenConfig(delegate)
// Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, 'Maven clean install')
+ common_job_properties.setPreCommit(delegate, 'mvn clean install -pl sdks/java/core,runners/direct-java,sdks/java/fn-execution -am -amd', 'Run Java PreCommit')
- // Maven goals for this job.
- goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+ // Maven goals for this job: The Java SDK, its dependencies, and things that depend on it.
+ goals([
+ '--batch-mode',
+ '--errors',
+ '--activate-profiles release,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner',
+ '--projects sdks/java/core,runners/direct-java,sdks/java/fn-execution',
+ '--also-make',
+ '--also-make-dependents',
+ '-D pullRequest=$ghprbPullId',
+ 'help:effective-settings',
+ 'clean',
+ 'install'
+ ].join(' '))
}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
new file mode 100644
index 0000000..dadc10c
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
@@ -0,0 +1,84 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job owns the overall execution of the precommit pipeline. The actual pipeline code is in
+// Precommit_Pipeline.groovy.
+pipelineJob('beam_PreCommit_Pipeline') {
+ description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.')
+
+ properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ parameters {
+ // Allow building at a specific commit.
+ stringParam(
+ 'commit',
+ 'master',
+ 'Commit id or refname (e.g. origin/pr/9/head) you want to build.')
+ }
+
+ wrappers {
+ // Set a timeout appropriate for the precommit tests.
+ timeout {
+ absolute(120)
+ abortBuild()
+ }
+ }
+
+ // Restrict this project to run only on Jenkins executors as specified
+ label('beam')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ triggers {
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+ // Remove once Pipeline Build is default.
+ triggerPhrase('^Run PreCommit Pipeline (((Python|Java))|All)$')
+ onlyTriggerPhrase()
+ displayBuildErrorsOnDownstreamBuilds()
+ extensions {
+ commitStatus {
+ context("Jenkins: PreCommit Pipeline")
+ }
+ buildStatus {
+ completedStatus('SUCCESS', '--none--')
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ }
+ }
+ }
+
+ definition {
+ cpsScm {
+ // Source code management.
+ common_job_properties.setSCM(delegate, 'beam')
+ scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy')
+ }
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_MavenInstall.groovy
new file mode 100644
index 0000000..f0429e4
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Python_MavenInstall.groovy
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Python precommit which runs a maven install, and the current set
+// of precommit tests.
+mavenJob('beam_PreCommit_Python_MavenInstall') {
+ description('Runs an install of the current GitHub Pull Request.')
+
+ previousNames('beam_PreCommit_MavenVerify')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'master',
+ 150)
+
+ // Set Maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PreCommit job.
+ common_job_properties.setPreCommit(delegate, 'mvn clean install -pl sdks/python -am -amd', 'Run Python PreCommit')
+
+ // Maven modules for this job: The Python SDK, its dependencies, and things that depend on it,
+ // excluding the container.
+ goals([
+ '--batch-mode',
+ '--errors',
+ '--activate-profiles release',
+ '--projects sdks/python,!sdks/python/container',
+ '--also-make',
+ '--also-make-dependents',
+ '-D pullRequest=$ghprbPullId',
+ 'help:effective-settings',
+ 'clean',
+ 'install',
+ ].join(' '))
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
new file mode 100644
index 0000000..f386d85
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Merge') {
+ description('Runs website tests for mergebot.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelWebsiteJobProperties(delegate, 'mergebot')
+
+ triggers {
+ githubPush()
+ }
+
+ steps {
+ // Run the following shell script as a build step.
+ shell '''
+ # Install RVM per instructions at https://rvm.io/rvm/install.
+ RVM_GPG_KEY=409B6B1796C275462A1703113804BB82D39DC0E3
+ gpg --keyserver hkp://keys.gnupg.net --recv-keys $RVM_GPG_KEY
+
+ \\curl -sSL https://get.rvm.io | bash
+ source /home/jenkins/.rvm/scripts/rvm
+
+ # Install Ruby.
+ RUBY_VERSION_NUM=2.3.0
+ rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+ # Install Bundler gem
+ PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+ GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+ gem install bundler --user-install
+
+ # Enter the git clone for remaining commands
+ cd src
+
+ # Install all needed gems.
+ bundle install --path ~/.gem/
+
+ # Build the new site and test it.
+ rm -fr ./content/
+ bundle exec rake test
+ '''.stripIndent().trim()
+ }
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
index 7c64f11..0b4d738 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
@@ -56,6 +56,9 @@
GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
gem install bundler --user-install
+ # Enter the git clone for remaining commands
+ cd src
+
# Install all needed gems.
bundle install --path ~/.gem/
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
index 421b58a..9b0aa74 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
@@ -54,6 +54,9 @@
GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
gem install bundler --user-install
+ # Enter the git clone for remaining commands
+ cd src
+
# Install all needed gems.
bundle install --path ~/.gem/
diff --git a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
new file mode 100644
index 0000000..89701d4
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Python Jenkins job which runs a maven install, and the current set of precommit
+// tests.
+mavenJob('beam_Python_UnitTest') {
+ description('Runs Python unit tests on a specific commit. Designed to be run by a pipeline job.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 35, "Python Unit Tests")
+ // Set standard properties for a pipeline job which needs to pull from GitHub instead of an
+ // upstream job.
+ common_job_properties.setPipelineBuildJobProperties(delegate)
+
+ // Construct Maven goals for this job.
+ args = [
+ '-B',
+ '-e',
+ 'clean install',
+ '-pl sdks/python',
+ ]
+ goals(args.join(' '))
+}
diff --git a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
index 7284acd..2e1f40d 100644
--- a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -27,8 +27,12 @@
// Execute concurrent builds if necessary.
concurrentBuild()
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
+ // Set common parameters. Huge timeout because we really do need to
+ // run all the ITs and release the artifacts.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'master',
+ 240)
// Set maven paramaters.
common_job_properties.setMavenConfig(delegate)
@@ -41,5 +45,17 @@
'dev@beam.apache.org')
// Maven goals for this job.
- goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=TestDataflowRunner" ]\'')
+ goals('''\
+ clean deploy \
+ --batch-mode \
+ --errors \
+ --fail-at-end \
+ -P release,dataflow-runner \
+ -D skipITs=false \
+ -D integrationTestPipelineOptions=\'[ \
+ "--project=apache-beam-testing", \
+ "--tempRoot=gs://temp-storage-for-end-to-end-tests", \
+ "--runner=TestDataflowRunner" \
+ ]\'\
+ ''')
}
diff --git a/.test-infra/jenkins/job_seed.groovy b/.test-infra/jenkins/job_seed.groovy
deleted file mode 100644
index 2d1b07c..0000000
--- a/.test-infra/jenkins/job_seed.groovy
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob') {
- description('Automatically configures all Apache Beam Jenkins projects based' +
- ' on Jenkins DSL groovy files checked into the code repository.')
-
- previousNames('beam_SeedJob_Main')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 6 * * *',
- false,
- 'dev@beam.apache.org')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Seed Job',
- 'Run Seed Job')
-
- steps {
- dsl {
- // A list or a glob of other groovy files to process.
- external('.test-infra/jenkins/job_*.groovy')
-
- // If a job is removed from the script, disable it (rather than deleting).
- removeAction('DISABLE')
- }
- }
-}
diff --git a/.test-infra/jenkins/job_seed_standalone.groovy b/.test-infra/jenkins/job_seed_standalone.groovy
new file mode 100644
index 0000000..beaecd9
--- /dev/null
+++ b/.test-infra/jenkins/job_seed_standalone.groovy
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Defines the seed job, which creates or updates all other Jenkins projects.
+job('beam_SeedJob_Standalone') {
+ description('Automatically configures all Apache Beam Jenkins projects based' +
+ ' on Jenkins DSL groovy files checked into the code repository.')
+
+ properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ // Restrict to only run on Jenkins executors labeled 'beam'
+ label('beam')
+
+ logRotator {
+ daysToKeep(14)
+ }
+
+ scm {
+ git {
+ remote {
+ github('apache/beam')
+
+ // ${ghprbPullId} is not interpolated by groovy, but passed through to Jenkins where it
+ // refers to the environment variable
+ refspec(['+refs/heads/*:refs/remotes/origin/*',
+ '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*']
+ .join(' '))
+
+ // The variable ${sha1} is not interpolated by groovy, but a parameter of the Jenkins job
+ branch('${sha1}')
+
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+ }
+
+ parameters {
+ // Setup for running this job from a pull request
+ stringParam(
+ 'sha1',
+ 'master',
+ 'Commit id or refname (eg: origin/pr/4001/head) you want to build against.')
+ }
+
+ wrappers {
+ timeout {
+ absolute(60)
+ abortBuild()
+ }
+ }
+
+ triggers {
+ // Run once per day
+ cron('0 */5 * * *')
+
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+
+ // Also run when manually kicked on a pull request
+ triggerPhrase('Run Standalone Seed Job')
+ onlyTriggerPhrase()
+
+ extensions {
+ commitStatus {
+ context("Jenkins: Standalone Seed Job")
+ }
+
+ buildStatus {
+ completedStatus('SUCCESS', '--none--')
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ }
+ }
+ }
+
+ // If anything goes wrong, mail the main dev list, because it is a big deal
+ publishers {
+ mailer('dev@beam.apache.org', false, true)
+ }
+
+ steps {
+ dsl {
+ // A list or a glob of other groovy files to process.
+ external('.test-infra/jenkins/job_*.groovy')
+
+ // If a job is removed from the script, disable it (rather than deleting).
+ removeAction('DISABLE')
+ }
+ }
+}
diff --git a/.test-infra/kubernetes/cassandra/LargeITCluster/setup.sh b/.test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
new file mode 100644
index 0000000..7bc0809
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Create Cassandra services and statefulset.
+kubectl create -f cassandra-svc-statefulset.yaml
diff --git a/.test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh b/.test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh
deleted file mode 100644
index 7341209..0000000
--- a/.test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Create Cassandra services and statefulset.
-kubectl create -f cassandra-service-for-local-dev.yaml
-kubectl create -f cassandra-svc-statefulset.yaml
diff --git a/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh b/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
index 367b604..3d040a6 100644
--- a/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
+++ b/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
@@ -20,6 +20,5 @@
# Delete Cassandra services and statefulset.
kubectl delete -f cassandra-svc-statefulset.yaml
-kubectl delete -f cassandra-service-for-local-dev.yaml
# Delete the persistent storage media for the PersistentVolumes
kubectl delete pvc -l app=cassandra
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/setup.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
new file mode 100644
index 0000000..fad6df0
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Create Cassandra services and Replication controller.
+kubectl create -f cassandra-svc-rc.yaml
+
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
deleted file mode 100644
index 9377a9c..0000000
--- a/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Create Cassandra services and Replication controller.
-kubectl create -f cassandra-service-for-local-dev.yaml
-kubectl create -f cassandra-svc-rc.yaml
-
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
index f4ad0be..f538a75 100644
--- a/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
@@ -19,4 +19,3 @@
# Delete Cassandra services and Replication controller.
kubectl delete -f cassandra-svc-rc.yaml
-kubectl delete -f cassandra-service-for-local-dev.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
new file mode 100644
index 0000000..9fbb6c3
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#!/bin/sh
+set -e
+
+# Create Elasticsearch services and deployments.
+kubectl create -f es-services-deployments.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
deleted file mode 100644
index 93022c7..0000000
--- a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#!/bin/sh
-set -e
-
-# Create Elasticsearch services and deployments.
-kubectl create -f elasticsearch-service-for-local-dev.yaml
-kubectl create -f es-services-deployments.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
index bdc9ab9..18568a3 100644
--- a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
@@ -18,4 +18,3 @@
# Delete elasticsearch services and deployments.
kubectl delete -f es-services-deployments.yaml
-kubectl delete -f elasticsearch-service-for-local-dev.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
new file mode 100644
index 0000000..e8cf275
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#!/bin/sh
+set -e
+
+# Create Elasticsearch services and deployments.
+kubectl create -f elasticsearch-svc-rc.yaml
+
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
deleted file mode 100644
index 2d6522e..0000000
--- a/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#!/bin/sh
-set -e
-
-# Create Elasticsearch services and deployments.
-kubectl create -f elasticsearch-service-for-local-dev.yaml
-kubectl create -f elasticsearch-svc-rc.yaml
-
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
index 61c079f..079141d 100644
--- a/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
@@ -18,4 +18,3 @@
# Delete elasticsearch services and deployments.
kubectl delete -f elasticsearch-svc-rc.yaml
-kubectl delete -f elasticsearch-service-for-local-dev.yaml
diff --git a/.test-infra/kubernetes/postgres/pkb-config-local.yml b/.test-infra/kubernetes/postgres/pkb-config-local.yml
new file mode 100644
index 0000000..1bac0c4
--- /dev/null
+++ b/.test-infra/kubernetes/postgres/pkb-config-local.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is a pkb benchmark configuration file, used when running the IO ITs
+# that use this data store. It allows users to run tests when they are on a
+# separate network from the kubernetes cluster by reading the postgres IP
+# address from the LoadBalancer service.
+#
+# This file defines pipeline options to pass to beam, as well as how to derive
+# the values for those pipeline options from kubernetes (where appropriate.)
+
+static_pipeline_options:
+ - postgresUsername: postgres
+ - postgresPassword: uuinkks
+ - postgresDatabaseName: postgres
+ - postgresSsl: false
+dynamic_pipeline_options:
+ - name: postgresServerName
+ type: LoadBalancerIp
+ serviceName: postgres-for-dev
diff --git a/.test-infra/kubernetes/postgres/pkb-config.yml b/.test-infra/kubernetes/postgres/pkb-config.yml
new file mode 100644
index 0000000..b943b17
--- /dev/null
+++ b/.test-infra/kubernetes/postgres/pkb-config.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is a pkb benchmark configuration file, used when running the IO ITs
+# that use this data store.
+#
+# This file defines pipeline options to pass to beam, as well as how to derive
+# the values for those pipeline options from kubernetes (where appropriate.)
+
+static_pipeline_options:
+ - postgresUsername: postgres
+ - postgresPassword: uuinkks
+ - postgresDatabaseName: postgres
+ - postgresSsl: false
+dynamic_pipeline_options:
+ - name: postgresServerName
+ type: NodePortIp
+ podLabel: name=postgres
diff --git a/README.md b/README.md
index 52c056f..8190baf 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@
## Getting Started
-Please refer to the [Quickstart](http://beam.apache.org/get-started/quickstart/) available on our website.
+Please refer to the Quickstart[[Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py)] available on our website.
If you'd like to build and install the whole project from the source distribution, you may need some additional tools installed
in your system. In a Debian-based distribution:
@@ -102,4 +102,4 @@
* [Apache Beam](http://beam.apache.org)
* [Overview](http://beam.apache.org/use/beam-overview/)
-* [Quickstart](http://beam.apache.org/use/quickstart/)
+* Quickstart: [Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py)
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 701e4fe..e47e9a1 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-examples-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -34,10 +34,6 @@
<packaging>jar</packaging>
- <properties>
- <spark.version>1.6.2</spark.version>
- </properties>
-
<profiles>
<!--
@@ -66,6 +62,12 @@
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-apex</artifactId>
<scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!--
Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from
@@ -95,6 +97,12 @@
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-flink_2.10</artifactId>
<scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
</profile>
@@ -116,13 +124,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
@@ -359,32 +365,7 @@
</profiles>
<build>
- <pluginManagement>
- <plugins>
- <!-- BEAM-933 -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
-
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamUseDummyRunner />
- <beamTestPipelineOptions>
- </beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </plugin>
-
<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
@@ -510,6 +491,12 @@
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<!-- Hamcrest and JUnit are required dependencies of PAssert,
which is used in the main code of DebuggingWordCount example. -->
@@ -524,7 +511,6 @@
</dependency>
<!-- Test dependencies -->
-
<!--
For testing the example itself, use the direct runner. This is separate from
the use of ValidatesRunner tests for testing a particular runner.
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
index 20b48e4..5c039cd 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
@@ -98,7 +98,6 @@
* 2-hour period.
*/
static class AddTimestampFn extends DoFn<String, String> {
- private static final Duration RAND_RANGE = Duration.standardHours(1);
private final Instant minTimestamp;
private final Instant maxTimestamp;
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index bfa7eb3..2d568ce 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -21,6 +21,7 @@
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Distribution;
import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
@@ -88,9 +89,12 @@
*/
static class ExtractWordsFn extends DoFn<String, String> {
private final Counter emptyLines = Metrics.counter(ExtractWordsFn.class, "emptyLines");
+ private final Distribution lineLenDist = Metrics.distribution(
+ ExtractWordsFn.class, "lineLenDistro");
@ProcessElement
public void processElement(ProcessContext c) {
+ lineLenDist.update(c.element().length());
if (c.element().trim().isEmpty()) {
emptyLines.inc();
}
diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
index 5e6df9c..abd14b7 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
@@ -17,17 +17,20 @@
*/
package org.apache.beam.examples.common;
-import static com.google.common.base.Verify.verifyNotNull;
+import static com.google.common.base.MoreObjects.firstNonNull;
import javax.annotation.Nullable;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
import org.joda.time.format.DateTimeFormatter;
@@ -53,22 +56,12 @@
@Override
public PDone expand(PCollection<String> input) {
- // filenamePrefix may contain a directory and a filename component. Pull out only the filename
- // component from that path for the PerWindowFiles.
- String prefix = "";
ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
- if (!resource.isDirectory()) {
- prefix = verifyNotNull(
- resource.getFilename(),
- "A non-directory resource should have a non-null filename: %s",
- resource);
- }
-
-
- TextIO.Write write = TextIO.write()
- .to(resource.getCurrentDirectory())
- .withFilenamePolicy(new PerWindowFiles(prefix))
- .withWindowedWrites();
+ TextIO.Write write =
+ TextIO.write()
+ .to(new PerWindowFiles(resource))
+ .withTempDirectory(resource.getCurrentDirectory())
+ .withWindowedWrites();
if (numShards != null) {
write = write.withNumShards(numShards);
}
@@ -83,31 +76,41 @@
*/
public static class PerWindowFiles extends FilenamePolicy {
- private final String prefix;
+ private final ResourceId baseFilename;
- public PerWindowFiles(String prefix) {
- this.prefix = prefix;
+ public PerWindowFiles(ResourceId baseFilename) {
+ this.baseFilename = baseFilename;
}
public String filenamePrefixForWindow(IntervalWindow window) {
+ String prefix =
+ baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), "");
return String.format("%s-%s-%s",
prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end()));
}
@Override
- public ResourceId windowedFilename(
- ResourceId outputDirectory, WindowedContext context, String extension) {
- IntervalWindow window = (IntervalWindow) context.getWindow();
- String filename = String.format(
- "%s-%s-of-%s%s",
- filenamePrefixForWindow(window), context.getShardNumber(), context.getNumShards(),
- extension);
- return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ public ResourceId windowedFilename(int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ IntervalWindow intervalWindow = (IntervalWindow) window;
+ String filename =
+ String.format(
+ "%s-%s-of-%s%s",
+ filenamePrefixForWindow(intervalWindow),
+ shardNumber,
+ numShards,
+ outputFileHints.getSuggestedFilenameSuffix());
+ return baseFilename
+ .getCurrentDirectory()
+ .resolve(filename, StandardResolveOptions.RESOLVE_FILE);
}
@Override
public ResourceId unwindowedFilename(
- ResourceId outputDirectory, Context context, String extension) {
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
throw new UnsupportedOperationException("Unsupported.");
}
}
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
index 7552b94..cfc413c 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.examples.complete;
+import com.google.common.base.Optional;
import java.io.File;
import java.io.IOException;
import java.net.URI;
@@ -24,7 +25,6 @@
import java.util.HashSet;
import java.util.Set;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringDelegateCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -121,7 +121,7 @@
Set<URI> uris = new HashSet<>();
if (absoluteUri.getScheme().equals("file")) {
File directory = new File(absoluteUri);
- for (String entry : directory.list()) {
+ for (String entry : Optional.fromNullable(directory.list()).or(new String[] {})) {
File path = new File(directory, entry);
uris.add(path.toURI());
}
@@ -154,11 +154,6 @@
}
@Override
- public Coder<?> getDefaultOutputCoder() {
- return KvCoder.of(StringDelegateCoder.of(URI.class), StringUtf8Coder.of());
- }
-
- @Override
public PCollection<KV<URI, String>> expand(PBegin input) {
Pipeline pipeline = input.getPipeline();
@@ -178,9 +173,11 @@
uriString = uri.toString();
}
- PCollection<KV<URI, String>> oneUriToLines = pipeline
- .apply("TextIO.Read(" + uriString + ")", TextIO.read().from(uriString))
- .apply("WithKeys(" + uriString + ")", WithKeys.<URI, String>of(uri));
+ PCollection<KV<URI, String>> oneUriToLines =
+ pipeline
+ .apply("TextIO.Read(" + uriString + ")", TextIO.read().from(uriString))
+ .apply("WithKeys(" + uriString + ")", WithKeys.<URI, String>of(uri))
+ .setCoder(KvCoder.of(StringDelegateCoder.of(URI.class), StringUtf8Coder.of()));
urisToLines = urisToLines.and(oneUriToLines);
}
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
index 478e2dc..3691e53 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
@@ -162,17 +162,18 @@
public PCollection<String> expand(PCollection<TableRow> input) {
return input
.apply(ParDo.of(new ExtractUserAndTimestamp()))
-
- .apply("SampleUsers", ParDo.of(
- new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) {
- if (Math.abs(c.element().hashCode()) <= Integer.MAX_VALUE * samplingThreshold) {
- c.output(c.element());
- }
- }
- }))
-
+ .apply(
+ "SampleUsers",
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (Math.abs((long) c.element().hashCode())
+ <= Integer.MAX_VALUE * samplingThreshold) {
+ c.output(c.element());
+ }
+ }
+ }))
.apply(new ComputeSessions())
.apply("SessionsToStrings", ParDo.of(new SessionsToStringsDoFn()))
.apply(new TopPerMonth())
@@ -191,7 +192,6 @@
@Default.String(EXPORTED_WIKI_TABLE)
String getInput();
void setInput(String value);
-
@Description("File to output results to")
@Validation.Required
String getOutput();
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
index c9ba18c..fb16eb4 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
@@ -29,6 +29,8 @@
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+
import org.apache.avro.reflect.Nullable;
import org.apache.beam.examples.common.ExampleBigQueryTableOptions;
import org.apache.beam.examples.common.ExampleOptions;
@@ -112,6 +114,23 @@
public int compareTo(StationSpeed other) {
return Long.compare(this.timestamp, other.timestamp);
}
+
+ @Override
+ public boolean equals(Object object) {
+ if (object == null) {
+ return false;
+ }
+ if (object.getClass() != getClass()) {
+ return false;
+ }
+ StationSpeed otherStationSpeed = (StationSpeed) object;
+ return Objects.equals(this.timestamp, otherStationSpeed.timestamp);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.timestamp.hashCode();
+ }
}
/**
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
index 07a3edd..df9ff5a 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java
@@ -156,7 +156,7 @@
fields.add(new TableFieldSchema().setName("tornado_count").setType("INTEGER"));
TableSchema schema = new TableSchema().setFields(fields);
- p.apply(BigQueryIO.read().from(options.getInput()))
+ p.apply(BigQueryIO.readTableRows().from(options.getInput()))
.apply(new CountTornadoes())
.apply(BigQueryIO.writeTableRows()
.to(options.getOutput())
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
index 693f0c4..1e91aec 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java
@@ -195,7 +195,7 @@
fields.add(new TableFieldSchema().setName("all_plays").setType("STRING"));
TableSchema schema = new TableSchema().setFields(fields);
- p.apply(BigQueryIO.read().from(options.getInput()))
+ p.apply(BigQueryIO.readTableRows().from(options.getInput()))
.apply(new PlaysForWord())
.apply(BigQueryIO.writeTableRows()
.to(options.getOutput())
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
index fed9db7..a4fe425 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java
@@ -237,7 +237,7 @@
TableSchema schema = buildWeatherSchemaProjection();
- p.apply(BigQueryIO.read().from(options.getInput()))
+ p.apply(BigQueryIO.readTableRows().from(options.getInput()))
.apply(ParDo.of(new ProjectionFn()))
.apply(new BelowGlobalMean(options.getMonthFilter()))
.apply(BigQueryIO.writeTableRows()
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java
index d1fffb4..ae8c59c 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java
@@ -166,8 +166,10 @@
Pipeline p = Pipeline.create(options);
// the following two 'applys' create multiple inputs to our pipeline, one for each
// of our two input sources.
- PCollection<TableRow> eventsTable = p.apply(BigQueryIO.read().from(GDELT_EVENTS_TABLE));
- PCollection<TableRow> countryCodes = p.apply(BigQueryIO.read().from(COUNTRY_CODES));
+ PCollection<TableRow> eventsTable = p.apply(
+ BigQueryIO.readTableRows().from(GDELT_EVENTS_TABLE));
+ PCollection<TableRow> countryCodes = p.apply(
+ BigQueryIO.readTableRows().from(COUNTRY_CODES));
PCollection<String> formattedResults = joinEvents(eventsTable, countryCodes);
formattedResults.apply(TextIO.write().to(options.getOutput()));
p.run().waitUntilFinish();
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java
index 295b3f4..992580e 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java
@@ -149,7 +149,7 @@
fields.add(new TableFieldSchema().setName("max_mean_temp").setType("FLOAT"));
TableSchema schema = new TableSchema().setFields(fields);
- p.apply(BigQueryIO.read().from(options.getInput()))
+ p.apply(BigQueryIO.readTableRows().from(options.getInput()))
.apply(new MaxMeanTemp())
.apply(BigQueryIO.writeTableRows()
.to(options.getOutput())
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
index e7596aa..651c242 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
@@ -23,6 +23,7 @@
import com.google.api.services.bigquery.model.TableSchema;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.beam.examples.common.ExampleBigQueryTableOptions;
import org.apache.beam.examples.common.ExampleOptions;
@@ -476,9 +477,10 @@
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
Instant timestamp = Instant.now();
- if (Math.random() < THRESHOLD){
+ Random random = new Random();
+ if (random.nextDouble() < THRESHOLD){
int range = MAX_DELAY - MIN_DELAY;
- int delayInMinutes = (int) (Math.random() * range) + MIN_DELAY;
+ int delayInMinutes = random.nextInt(range) + MIN_DELAY;
long delayInMillis = TimeUnit.MINUTES.toMillis(delayInMinutes);
timestamp = new Instant(timestamp.getMillis() - delayInMillis);
}
diff --git a/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java b/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java
index 054277a..be48a99 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java
@@ -35,6 +35,13 @@
public class DebuggingWordCountTest {
@Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+ private String getFilePath(String filePath) {
+ if (filePath.contains(":")) {
+ return filePath.replace("\\", "/").split(":")[1];
+ }
+ return filePath;
+ }
+
@Test
public void testDebuggingWordCount() throws Exception {
File inputFile = tmpFolder.newFile();
@@ -45,8 +52,8 @@
StandardCharsets.UTF_8);
WordCountOptions options =
TestPipeline.testingPipelineOptions().as(WordCountOptions.class);
- options.setInputFile(inputFile.getAbsolutePath());
- options.setOutput(outputFile.getAbsolutePath());
+ options.setInputFile(getFilePath(inputFile.getAbsolutePath()));
+ options.setOutput(getFilePath(outputFile.getAbsolutePath()));
DebuggingWordCount.main(TestPipeline.convertToArgs(options));
}
}
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index eb7e4c4..bec7952 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -32,6 +32,7 @@
import org.apache.beam.examples.common.ExampleUtils;
import org.apache.beam.examples.common.WriteOneFilePerWindow.PerWindowFiles;
import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -149,7 +150,8 @@
String outputPrefix = options.getOutput();
- PerWindowFiles filenamePolicy = new PerWindowFiles(outputPrefix);
+ PerWindowFiles filenamePolicy =
+ new PerWindowFiles(FileBasedSink.convertToFileResourceIfPossible(outputPrefix));
List<ShardedFile> expectedOutputFiles = Lists.newArrayListWithCapacity(6);
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 56295a4..7651845 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-examples-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -35,10 +35,6 @@
<packaging>jar</packaging>
- <properties>
- <spark.version>1.6.2</spark.version>
- </properties>
-
<profiles>
<!--
The direct runner is available by default.
@@ -66,6 +62,12 @@
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-apex</artifactId>
<scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!--
Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from
@@ -95,6 +97,12 @@
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-flink_2.10</artifactId>
<scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
</profile>
@@ -116,13 +124,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
@@ -145,21 +151,21 @@
</dependency>
</dependencies>
</profile>
+
+ <!-- Include the Apache Gearpump (incubating) runner with -P gearpump-runner -->
+ <profile>
+ <id>gearpump-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<build>
- <pluginManagement>
- <plugins>
- <!-- BEAM-934 -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
@@ -172,17 +178,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamTestPipelineOptions>
- </beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </plugin>
-
<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
index b9a3ff2..d9667ad 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
@@ -167,7 +167,7 @@
return startTimeInMillis;
}
long getEndTimeInMillis() {
- return startTimeInMillis + (expirationPeriod * 60 * 1000);
+ return startTimeInMillis + (expirationPeriod * 60L * 1000L);
}
String getRandomUser() {
int userNum = random.nextInt(numMembers);
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java
index 8cba6c2..1667f3a 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java
@@ -93,7 +93,7 @@
Topic topic = client.projects().topics()
.create(fullTopicName, new Topic())
.execute();
- System.out.printf("Topic %s was created.\n", topic.getName());
+ System.out.printf("Topic %s was created.%n", topic.getName());
}
}
}
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java
index e6c8ddb..6b7c928 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java
@@ -18,7 +18,6 @@
package org.apache.beam.examples.complete.game.utils;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Verify.verifyNotNull;
import java.io.Serializable;
import java.util.ArrayList;
@@ -28,6 +27,7 @@
import java.util.stream.Collectors;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -36,6 +36,7 @@
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
import org.joda.time.DateTimeZone;
@@ -111,21 +112,12 @@
checkArgument(
input.getWindowingStrategy().getWindowFn().windowCoder() == IntervalWindow.getCoder());
- // filenamePrefix may contain a directory and a filename component. Pull out only the filename
- // component from that path for the PerWindowFiles.
- String prefix = "";
ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
- if (!resource.isDirectory()) {
- prefix = verifyNotNull(
- resource.getFilename(),
- "A non-directory resource should have a non-null filename: %s",
- resource);
- }
return input.apply(
TextIO.write()
- .to(resource.getCurrentDirectory())
- .withFilenamePolicy(new PerWindowFiles(prefix))
+ .to(new PerWindowFiles(resource))
+ .withTempDirectory(resource.getCurrentDirectory())
.withWindowedWrites()
.withNumShards(3));
}
@@ -139,31 +131,38 @@
*/
protected static class PerWindowFiles extends FilenamePolicy {
- private final String prefix;
+ private final ResourceId prefix;
- public PerWindowFiles(String prefix) {
+ public PerWindowFiles(ResourceId prefix) {
this.prefix = prefix;
}
public String filenamePrefixForWindow(IntervalWindow window) {
- return String.format("%s-%s-%s",
- prefix, formatter.print(window.start()), formatter.print(window.end()));
+ String filePrefix = prefix.isDirectory() ? "" : prefix.getFilename();
+ return String.format(
+ "%s-%s-%s", filePrefix, formatter.print(window.start()), formatter.print(window.end()));
}
@Override
- public ResourceId windowedFilename(
- ResourceId outputDirectory, WindowedContext context, String extension) {
- IntervalWindow window = (IntervalWindow) context.getWindow();
- String filename = String.format(
- "%s-%s-of-%s%s",
- filenamePrefixForWindow(window), context.getShardNumber(), context.getNumShards(),
- extension);
- return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ public ResourceId windowedFilename(int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ IntervalWindow intervalWindow = (IntervalWindow) window;
+ String filename =
+ String.format(
+ "%s-%s-of-%s%s",
+ filenamePrefixForWindow(intervalWindow),
+ shardNumber,
+ numShards,
+ outputFileHints.getSuggestedFilenameSuffix());
+ return prefix.getCurrentDirectory().resolve(filename, StandardResolveOptions.RESOLVE_FILE);
}
@Override
public ResourceId unwindowedFilename(
- ResourceId outputDirectory, Context context, String extension) {
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
throw new UnsupportedOperationException("Unsupported.");
}
}
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java b/examples/java8/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java
new file mode 100644
index 0000000..f17171e
--- /dev/null
+++ b/examples/java8/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.examples;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Code snippets used in webdocs.
+ */
+public class Snippets {
+
+ /* Helper function to format results in coGroupByKeyTuple */
+ public static String formatCoGbkResults(
+ String name, Iterable<String> emails, Iterable<String> phones) {
+
+ List<String> emailsList = new ArrayList<>();
+ for (String elem : emails) {
+ emailsList.add("'" + elem + "'");
+ }
+ Collections.<String>sort(emailsList);
+ String emailsStr = "[" + String.join(", ", emailsList) + "]";
+
+ List<String> phonesList = new ArrayList<>();
+ for (String elem : phones) {
+ phonesList.add("'" + elem + "'");
+ }
+ Collections.<String>sort(phonesList);
+ String phonesStr = "[" + String.join(", ", phonesList) + "]";
+
+ return name + "; " + emailsStr + "; " + phonesStr;
+ }
+
+ public static PCollection<String> coGroupByKeyTuple(
+ TupleTag<String> emailsTag,
+ TupleTag<String> phonesTag,
+ PCollection<KV<String, String>> emails,
+ PCollection<KV<String, String>> phones) {
+
+ // [START CoGroupByKeyTuple]
+ PCollection<KV<String, CoGbkResult>> results =
+ KeyedPCollectionTuple
+ .of(emailsTag, emails)
+ .and(phonesTag, phones)
+ .apply(CoGroupByKey.<String>create());
+
+ PCollection<String> contactLines = results.apply(ParDo.of(
+ new DoFn<KV<String, CoGbkResult>, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ KV<String, CoGbkResult> e = c.element();
+ String name = e.getKey();
+ Iterable<String> emailsIter = e.getValue().getAll(emailsTag);
+ Iterable<String> phonesIter = e.getValue().getAll(phonesTag);
+ String formattedResult = Snippets.formatCoGbkResults(name, emailsIter, phonesIter);
+ c.output(formattedResult);
+ }
+ }
+ ));
+ // [END CoGroupByKeyTuple]
+ return contactLines;
+ }
+}
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java
index 745c210..611e2b3 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java
@@ -276,6 +276,8 @@
.addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)),
event(TestUser.BLUE_TWO, 3, Duration.ZERO),
event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3)))
+ // Move the watermark to the end of the window to output on time
+ .advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION))
// Move the watermark past the end of the allowed lateness plus the end of the window
.advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS)
.plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1)))
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java b/examples/java8/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java
new file mode 100644
index 0000000..3ca6c9a
--- /dev/null
+++ b/examples/java8/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.examples;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+
+/**
+ * Tests for Snippets.
+ */
+@RunWith(JUnit4.class)
+public class SnippetsTest implements Serializable {
+
+ @Rule
+ public transient TestPipeline p = TestPipeline.create();
+
+ /* Tests CoGroupByKeyTuple */
+ @Test
+ public void testCoGroupByKeyTuple() throws IOException {
+ // [START CoGroupByKeyTupleInputs]
+ final List<KV<String, String>> emailsList = Arrays.asList(
+ KV.of("amy", "amy@example.com"),
+ KV.of("carl", "carl@example.com"),
+ KV.of("julia", "julia@example.com"),
+ KV.of("carl", "carl@email.com"));
+
+ final List<KV<String, String>> phonesList = Arrays.asList(
+ KV.of("amy", "111-222-3333"),
+ KV.of("james", "222-333-4444"),
+ KV.of("amy", "333-444-5555"),
+ KV.of("carl", "444-555-6666"));
+
+ PCollection<KV<String, String>> emails = p.apply("CreateEmails", Create.of(emailsList));
+ PCollection<KV<String, String>> phones = p.apply("CreatePhones", Create.of(phonesList));
+ // [END CoGroupByKeyTupleInputs]
+
+ // [START CoGroupByKeyTupleOutputs]
+ final TupleTag<String> emailsTag = new TupleTag();
+ final TupleTag<String> phonesTag = new TupleTag();
+
+ final List<KV<String, CoGbkResult>> expectedResults = Arrays.asList(
+ KV.of("amy", CoGbkResult
+ .of(emailsTag, Arrays.asList("amy@example.com"))
+ .and(phonesTag, Arrays.asList("111-222-3333", "333-444-5555"))),
+ KV.of("carl", CoGbkResult
+ .of(emailsTag, Arrays.asList("carl@email.com", "carl@example.com"))
+ .and(phonesTag, Arrays.asList("444-555-6666"))),
+ KV.of("james", CoGbkResult
+ .of(emailsTag, Arrays.asList())
+ .and(phonesTag, Arrays.asList("222-333-4444"))),
+ KV.of("julia", CoGbkResult
+ .of(emailsTag, Arrays.asList("julia@example.com"))
+ .and(phonesTag, Arrays.asList())));
+ // [END CoGroupByKeyTupleOutputs]
+
+ PCollection<String> actualFormattedResults =
+ Snippets.coGroupByKeyTuple(emailsTag, phonesTag, emails, phones);
+
+ // [START CoGroupByKeyTupleFormattedOutputs]
+ final List<String> formattedResults = Arrays.asList(
+ "amy; ['amy@example.com']; ['111-222-3333', '333-444-5555']",
+ "carl; ['carl@email.com', 'carl@example.com']; ['444-555-6666']",
+ "james; []; ['222-333-4444']",
+ "julia; ['julia@example.com']; []");
+ // [END CoGroupByKeyTupleFormattedOutputs]
+
+ // Make sure that both 'expectedResults' and 'actualFormattedResults' match with the
+ // 'formattedResults'. 'expectedResults' will have to be formatted before comparing
+ List<String> expectedFormattedResultsList = new ArrayList<String>(expectedResults.size());
+ for (KV<String, CoGbkResult> e : expectedResults) {
+ String name = e.getKey();
+ Iterable<String> emailsIter = e.getValue().getAll(emailsTag);
+ Iterable<String> phonesIter = e.getValue().getAll(phonesTag);
+ String formattedResult = Snippets.formatCoGbkResults(name, emailsIter, phonesIter);
+ expectedFormattedResultsList.add(formattedResult);
+ }
+ PCollection<String> expectedFormattedResultsPColl =
+ p.apply(Create.of(expectedFormattedResultsList));
+ PAssert.that(expectedFormattedResultsPColl).containsInAnyOrder(formattedResults);
+ PAssert.that(actualFormattedResults).containsInAnyOrder(formattedResults);
+
+ p.run();
+ }
+}
diff --git a/examples/pom.xml b/examples/pom.xml
index a7e61dd..9eea99a 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/model/fn-execution/pom.xml b/model/fn-execution/pom.xml
new file mode 100644
index 0000000..b5b5fdf
--- /dev/null
+++ b/model/fn-execution/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-fn-execution</artifactId>
+ <name>Apache Beam :: Model :: Fn Execution</name>
+ <description>Portable definitions for execution user-defined functions</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/model/fn-execution/src/main/proto/beam_fn_api.proto b/model/fn-execution/src/main/proto/beam_fn_api.proto
new file mode 100644
index 0000000..132d366
--- /dev/null
+++ b/model/fn-execution/src/main/proto/beam_fn_api.proto
@@ -0,0 +1,729 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Fn API and boostrapping.
+ *
+ * TODO: Usage of plural names in lists looks awkward in Java
+ * e.g. getOutputsMap, addCodersBuilder
+ *
+ * TODO: gRPC / proto field names conflict with generated code
+ * e.g. "class" in java, "output" in python
+ */
+
+syntax = "proto3";
+
+/* TODO: Consider consolidating common components in another package
+ * and lanaguage namespaces for re-use with Runner Api.
+ */
+
+package org.apache.beam.model.fn_execution.v1;
+
+option go_package = "fnexecution_v1";
+option java_package = "org.apache.beam.model.fnexecution.v1";
+option java_outer_classname = "BeamFnApi";
+
+import "beam_runner_api.proto";
+import "endpoints.proto";
+import "google/protobuf/timestamp.proto";
+
+/*
+ * Constructs that define the pipeline shape.
+ *
+ * These are mostly unstable due to the missing pieces to be shared with
+ * the Runner Api like windowing strategy, display data, .... There are still
+ * some modelling questions related to whether a side input is modelled
+ * as another field on a PrimitiveTransform or as part of inputs and we
+ * still are missing things like the CompositeTransform.
+ */
+
+// A representation of an input or output definition on a primitive transform.
+// Stable
+message Target {
+ // A repeated list of target definitions.
+ message List {
+ repeated Target target = 1;
+ }
+
+ // (Required) The id of the PrimitiveTransform which is the target.
+ string primitive_transform_reference = 1;
+
+ // (Required) The local name of an input or output defined on the primitive
+ // transform.
+ string name = 2;
+}
+
+// A descriptor for connecting to a remote port using the Beam Fn Data API.
+// Allows for communication between two environments (for example between the
+// runner and the SDK).
+// Stable
+message RemoteGrpcPort {
+ // (Required) An API descriptor which describes where to
+ // connect to including any authentication that is required.
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor api_service_descriptor = 1;
+}
+
+/*
+ * Control Plane API
+ *
+ * Progress reporting and splitting still need further vetting. Also, this may change
+ * with the addition of new types of instructions/responses related to metrics.
+ */
+
+// An API that describes the work that a SDK harness is meant to do.
+// Stable
+service BeamFnControl {
+ // Instructions sent by the runner to the SDK requesting different types
+ // of work.
+ rpc Control(
+ // A stream of responses to instructions the SDK was asked to be performed.
+ stream InstructionResponse
+ ) returns (
+ // A stream of instructions requested of the SDK to be performed.
+ stream InstructionRequest
+ ) {}
+}
+
+// A request sent by a runner which the SDK is asked to fulfill.
+// For any unsupported request type, an error should be returned with a
+// matching instruction id.
+// Stable
+message InstructionRequest {
+ // (Required) An unique identifier provided by the runner which represents
+ // this requests execution. The InstructionResponse MUST have the matching id.
+ string instruction_id = 1;
+
+ // (Required) A request that the SDK Harness needs to interpret.
+ oneof request {
+ RegisterRequest register = 1000;
+ ProcessBundleRequest process_bundle = 1001;
+ ProcessBundleProgressRequest process_bundle_progress = 1002;
+ ProcessBundleSplitRequest process_bundle_split = 1003;
+ }
+}
+
+// The response for an associated request the SDK had been asked to fulfill.
+// Stable
+message InstructionResponse {
+ // (Required) A reference provided by the runner which represents a requests
+ // execution. The InstructionResponse MUST have the matching id when
+ // responding to the runner.
+ string instruction_id = 1;
+
+ // If this is specified, then this instruction has failed.
+ // A human readable string representing the reason as to why processing has
+ // failed.
+ string error = 2;
+
+ // If the instruction did not fail, it is required to return an equivalent
+ // response type depending on the request this matches.
+ oneof response {
+ RegisterResponse register = 1000;
+ ProcessBundleResponse process_bundle = 1001;
+ ProcessBundleProgressResponse process_bundle_progress = 1002;
+ ProcessBundleSplitResponse process_bundle_split = 1003;
+ }
+}
+
+// A list of objects which can be referred to by the runner in
+// future requests.
+// Stable
+message RegisterRequest {
+ // (Optional) The set of descriptors used to process bundles.
+ repeated ProcessBundleDescriptor process_bundle_descriptor = 1;
+}
+
+// Stable
+message RegisterResponse {
+}
+
+// Definitions that should be used to construct the bundle processing graph.
+message ProcessBundleDescriptor {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ string id = 1;
+
+ // (Required) A map from pipeline-scoped id to PTransform.
+ map<string, org.apache.beam.model.pipeline.v1.PTransform> transforms = 2;
+
+ // (Required) A map from pipeline-scoped id to PCollection.
+ map<string, org.apache.beam.model.pipeline.v1.PCollection> pcollections = 3;
+
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ map<string, org.apache.beam.model.pipeline.v1.WindowingStrategy> windowing_strategies = 4;
+
+ // (Required) A map from pipeline-scoped id to Coder.
+ map<string, org.apache.beam.model.pipeline.v1.Coder> coders = 5;
+
+ // (Required) A map from pipeline-scoped id to Environment.
+ map<string, org.apache.beam.model.pipeline.v1.Environment> environments = 6;
+
+ // A descriptor describing the end point to use for State API
+ // calls. Required if the Runner intends to send remote references over the
+ // data plane or if any of the transforms rely on user state or side inputs.
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor state_api_service_descriptor = 7;
+}
+
+// A request to process a given bundle.
+// Stable
+message ProcessBundleRequest {
+ // (Required) A reference to the process bundle descriptor that must be
+ // instantiated and executed by the SDK harness.
+ string process_bundle_descriptor_reference = 1;
+
+ // (Optional) A list of cache tokens that can be used by an SDK to reuse
+ // cached data returned by the State API across multiple bundles.
+ repeated bytes cache_tokens = 2;
+}
+
+// Stable
+message ProcessBundleResponse {
+ // (Optional) If metrics reporting is supported by the SDK, this represents
+ // the final metrics to record for this bundle.
+ Metrics metrics = 1;
+}
+
+// A request to report progress information for a given bundle.
+// This is an optional request to be handled and is used to support advanced
+// SDK features such as SplittableDoFn, user level metrics etc.
+message ProcessBundleProgressRequest {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ string instruction_reference = 1;
+}
+
+message Metrics {
+ // PTransform level metrics.
+ // These metrics are split into processed and active element groups for
+ // progress reporting purposes. This allows a Runner to see what is measured,
+ // what is estimated and what can be extrapolated to be able to accurately
+ // estimate the backlog of remaining work.
+ message PTransform {
+ // Metrics that are measured for processed and active element groups.
+ message Measured {
+ // (Optional) Map from local input name to number of elements processed
+ // from this input.
+ // If unset, assumed to be the sum of the outputs of all producers to
+ // this transform (for ProcessedElements) and 0 (for ActiveElements).
+ map<string, int64> input_element_counts = 1;
+
+ // (Required) Map from local output name to number of elements produced
+ // for this output.
+ map<string, int64> output_element_counts = 2;
+
+ // (Optional) The total time spent so far in processing the elements in
+ // this group, in seconds.
+ double total_time_spent = 3;
+
+ // TODO: Add other element group level metrics.
+ }
+
+ // Metrics for fully processed elements.
+ message ProcessedElements {
+ // (Required)
+ Measured measured = 1;
+ }
+
+ // Metrics for active elements.
+ // An element is considered active if the SDK has started but not finished
+ // processing it yet.
+ message ActiveElements {
+ // (Required)
+ Measured measured = 1;
+
+ // Estimated metrics.
+
+ // (Optional) Sum of estimated fraction of known work remaining for all
+ // active elements, as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ // TODO: Handle the case when known work is infinite.
+ double fraction_remaining = 2;
+
+ // (Optional) Map from local output name to sum of estimated number
+ // of elements remaining for this output from all active elements,
+ // as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ map<string, int64> output_elements_remaining = 3;
+ }
+
+ // (Required): Metrics for processed elements.
+ ProcessedElements processed_elements = 1;
+ // (Required): Metrics for active elements.
+ ActiveElements active_elements = 2;
+
+ // (Optional): Map from local output name to its watermark.
+ // The watermarks reported are tentative, to get a better sense of progress
+ // while processing a bundle but before it is committed. At bundle commit
+ // time, a Runner needs to also take into account the timers set to compute
+ // the actual watermarks.
+ map<string, int64> watermarks = 3;
+
+ // TODO: Define other transform level system metrics.
+ }
+
+ // User defined metrics
+ message User {
+ // TODO: Define it.
+ }
+
+ map<string, PTransform> ptransforms = 1;
+ map<string, User> user = 2;
+}
+
+message ProcessBundleProgressResponse {
+ // (Required)
+ Metrics metrics = 1;
+}
+
+message ProcessBundleSplitRequest {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ string instruction_reference = 1;
+
+ // (Required) The fraction of work (when compared to the known amount of work)
+ // the process bundle request should try to split at.
+ double fraction = 2;
+}
+
+// urn:org.apache.beam:restriction:element-count:1.0
+message ElementCountRestriction {
+ // A restriction representing the number of elements that should be processed.
+ // Effectively the range [0, count]
+ int64 count = 1;
+}
+
+// urn:org.apache.beam:restriction:element-count-skip:1.0
+message ElementCountSkipRestriction {
+ // A restriction representing the number of elements that should be skipped.
+ // Effectively the range (count, infinity]
+ int64 count = 1;
+}
+
+// Each primitive transform that is splittable is defined by a restriction
+// it is currently processing. During splitting, that currently active
+// restriction (R_initial) is split into 2 components:
+// * a restriction (R_done) representing all elements that will be fully
+// processed
+// * a restriction (R_todo) representing all elements that will not be fully
+// processed
+//
+// where:
+// R_initial = R_done ⋃ R_todo
+message PrimitiveTransformSplit {
+ // (Required) A reference to a primitive transform with the given id that
+ // is part of the active process bundle request with the given instruction
+ // id.
+ string primitive_transform_reference = 1;
+
+ // (Required) A function specification describing the restriction
+ // that has been completed by the primitive transform.
+ //
+ // For example, a remote GRPC source will have a specific urn and data
+ // block containing an ElementCountRestriction.
+ org.apache.beam.model.pipeline.v1.FunctionSpec completed_restriction = 2;
+
+ // (Required) A function specification describing the restriction
+ // representing the remainder of work for the primitive transform.
+ //
+ // FOr example, a remote GRPC source will have a specific urn and data
+ // block contain an ElemntCountSkipRestriction.
+ org.apache.beam.model.pipeline.v1.FunctionSpec remaining_restriction = 3;
+}
+
+message ProcessBundleSplitResponse {
+ // (Optional) A set of split responses for a currently active work item.
+ //
+ // If primitive transform B is a descendant of primitive transform A and both
+ // A and B report a split. Then B's restriction is reported as an element
+ // restriction pair and thus the fully reported restriction is:
+ // R = A_done
+ // ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ A_todo
+ // If there is a decendant of B named C, then C would similarly report a
+ // set of element pair restrictions.
+ //
+ // This restriction is processed and completed by the currently active process
+ // bundle request:
+ // A_done ⋃ (A_boundary ⋂ B_done)
+ // and these restrictions will be processed by future process bundle requests:
+ // A_boundary â‹‚ B_todo (passed to SDF B directly)
+ // A_todo (passed to SDF A directly)
+
+ // If primitive transform B and C are siblings and descendants of A and A, B,
+ // and C report a split. Then B and C's restrictions are relative to A's.
+ // R = A_done
+ // ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ C_todo)
+ // ⋃ A_todo
+ // If there is no descendant of B or C also reporting a split, than
+ // B_boundary = ∅ and C_boundary = ∅
+ //
+ // This restriction is processed and completed by the currently active process
+ // bundle request:
+ // A_done ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ C_done)
+ // and these restrictions will be processed by future process bundle requests:
+ // A_boundary â‹‚ B_todo (passed to SDF B directly)
+ // A_boundary â‹‚ C_todo (passed to SDF C directly)
+ // A_todo (passed to SDF A directly)
+ //
+ // Note that descendants splits should only be reported if it is inexpensive
+ // to compute the boundary restriction intersected with descendants splits.
+ // Also note, that the boundary restriction may represent a set of elements
+ // produced by a parent primitive transform which can not be split at each
+ // element or that there are intermediate unsplittable primitive transforms
+ // between an ancestor splittable function and a descendant splittable
+ // function which may have more than one output per element. Finally note
+ // that the descendant splits should only be reported if the split
+ // information is relatively compact.
+ repeated PrimitiveTransformSplit splits = 1;
+}
+
+/*
+ * Data Plane API
+ */
+
+// Messages used to represent logical byte streams.
+// Stable
+message Elements {
+ // Represents multiple encoded elements in nested context for a given named
+ // instruction and target.
+ message Data {
+ // (Required) A reference to an active instruction request with the given
+ // instruction id.
+ string instruction_reference = 1;
+
+ // (Required) A definition representing a consumer or producer of this data.
+ // If received by a harness, this represents the consumer within that
+ // harness that should consume these bytes. If sent by a harness, this
+ // represents the producer of these bytes.
+ //
+ // Note that a single element may span multiple Data messages.
+ //
+ // Note that a sending/receiving pair should share the same target
+ // identifier.
+ Target target = 2;
+
+ // (Optional) Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ //
+ // An empty data block represents the end of stream for the given
+ // instruction and target.
+ bytes data = 3;
+ }
+
+ // (Required) A list containing parts of logical byte streams.
+ repeated Data data = 1;
+}
+
+// Stable
+service BeamFnData {
+ // Used to send data between harnesses.
+ rpc Data(
+ // A stream of data representing input.
+ stream Elements
+ ) returns (
+ // A stream of data representing output.
+ stream Elements
+ ) {}
+}
+
+/*
+ * State API
+ */
+
+message StateRequest {
+ // (Required) An unique identifier provided by the SDK which represents this
+ // requests execution. The StateResponse corresponding with this request
+ // will have the matching id.
+ string id = 1;
+
+ // (Required) The associated instruction id of the work that is currently
+ // being processed. This allows for the runner to associate any modifications
+ // to state to be committed with the appropriate work execution.
+ string instruction_reference = 2;
+
+ // (Required) The state key this request is for.
+ StateKey state_key = 3;
+
+ // (Required) The action to take on this request.
+ oneof request {
+ // A request to get state.
+ StateGetRequest get = 1000;
+
+ // A request to append to state.
+ StateAppendRequest append = 1001;
+
+ // A request to clear state.
+ StateClearRequest clear = 1002;
+ }
+}
+
+message StateResponse {
+ // (Required) A reference provided by the SDK which represents a requests
+ // execution. The StateResponse must have the matching id when responding
+ // to the SDK.
+ string id = 1;
+
+ // (Optional) If this is specified, then the state request has failed.
+ // A human readable string representing the reason as to why the request
+ // failed.
+ string error = 2;
+
+ // (Optional) If this is specified, then the result of this state request
+ // can be cached using the supplied token.
+ bytes cache_token = 3;
+
+ // A corresponding response matching the request will be populated.
+ oneof response {
+ // A response to getting state.
+ StateGetResponse get = 1000;
+
+ // A response to appending to state.
+ StateAppendResponse append = 1001;
+
+ // A response to clearing state.
+ StateClearResponse clear = 1002;
+ }
+}
+
+service BeamFnState {
+ // Used to get/append/clear state stored by the runner on behalf of the SDK.
+ rpc State(
+ // A stream of state instructions requested of the runner.
+ stream StateRequest
+ ) returns (
+ // A stream of responses to state instructions the runner was asked to be
+ // performed.
+ stream StateResponse
+ ) {}
+}
+
+message StateKey {
+ message Runner {
+ // (Required) Opaque information supplied by the runner. Used to support
+ // remote references.
+ bytes key = 1;
+ }
+
+ message MultimapSideInput {
+ // (Required) The id of the PTransform containing a side input.
+ string ptransform_id = 1;
+ // (Required) The id of the side input.
+ string side_input_id = 2;
+ // (Required) The window (after mapping the currently executing elements
+ // window into the side input windows domain) encoded in a nested context.
+ bytes window = 3;
+ // (Required) The key encoded in a nested context.
+ bytes key = 4;
+ }
+
+ message BagUserState {
+ // (Required) The id of the PTransform containing user state.
+ string ptransform_id = 1;
+ // (Required) The id of the user state.
+ string user_state_id = 2;
+ // (Required) The window encoded in a nested context.
+ bytes window = 3;
+ // (Required) The key of the currently executing element encoded in a
+ // nested context.
+ bytes key = 4;
+ }
+
+ // (Required) One of the following state keys must be set.
+ oneof type {
+ Runner runner = 1;
+ MultimapSideInput multimap_side_input = 2;
+ BagUserState bag_user_state = 3;
+ // TODO: represent a state key for user map state
+ }
+}
+
+// A request to get state.
+message StateGetRequest {
+ // (Optional) If specified, signals to the runner that the response
+ // should resume from the following continuation token.
+ //
+ // If unspecified, signals to the runner that the response should start
+ // from the beginning of the logical continuable stream.
+ bytes continuation_token = 1;
+}
+
+// A response to get state representing a logical byte stream which can be
+// continued using the state API.
+message StateGetResponse {
+ // (Optional) If specified, represents a token which can be used with the
+ // state API to get the next chunk of this logical byte stream. The end of
+ // the logical byte stream is signalled by this field being unset.
+ bytes continuation_token = 1;
+
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ bytes data = 2;
+}
+
+// A request to append state.
+message StateAppendRequest {
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // multiple append requests are concatenated together.
+ bytes data = 1;
+}
+
+// A response to append state.
+message StateAppendResponse {
+}
+
+// A request to clear state.
+message StateClearRequest {
+}
+
+// A response to clear state.
+message StateClearResponse {
+}
+
+/*
+ * Logging API
+ *
+ * This is very stable. There can be some changes to how we define a LogEntry,
+ * to increase/decrease the severity types, the way we format an exception/stack
+ * trace, or the log site.
+ */
+
+// A log entry
+message LogEntry {
+ // A list of log entries, enables buffering and batching of multiple
+ // log messages using the logging API.
+ message List {
+ // (Required) One or or more log messages.
+ repeated LogEntry log_entries = 1;
+ }
+
+ // The severity of the event described in a log entry, expressed as one of the
+ // severity levels listed below. For your reference, the levels are
+ // assigned the listed numeric values. The effect of using numeric values
+ // other than those listed is undefined.
+ //
+ // If you are writing log entries, you should map other severity encodings to
+ // one of these standard levels. For example, you might map all of
+ // Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`.
+ //
+ // This list is intentionally not comprehensive; the intent is to provide a
+ // common set of "good enough" severity levels so that logging front ends
+ // can provide filtering and searching across log types. Users of the API are
+ // free not to use all severity levels in their log messages.
+ message Severity {
+ enum Enum {
+ UNSPECIFIED = 0;
+ // Trace level information, also the default log level unless
+ // another severity is specified.
+ TRACE = 1;
+ // Debugging information.
+ DEBUG = 2;
+ // Normal events.
+ INFO = 3;
+ // Normal but significant events, such as start up, shut down, or
+ // configuration.
+ NOTICE = 4;
+ // Warning events might cause problems.
+ WARN = 5;
+ // Error events are likely to cause problems.
+ ERROR = 6;
+ // Critical events cause severe problems or brief outages and may
+ // indicate that a person must take action.
+ CRITICAL = 7;
+ }
+ }
+
+ // (Required) The severity of the log statement.
+ Severity.Enum severity = 1;
+
+ // (Required) The time at which this log statement occurred.
+ google.protobuf.Timestamp timestamp = 2;
+
+ // (Required) A human readable message.
+ string message = 3;
+
+ // (Optional) An optional trace of the functions involved. For example, in
+ // Java this can include multiple causes and multiple suppressed exceptions.
+ string trace = 4;
+
+ // (Optional) A reference to the instruction this log statement is associated
+ // with.
+ string instruction_reference = 5;
+
+ // (Optional) A reference to the primitive transform this log statement is
+ // associated with.
+ string primitive_transform_reference = 6;
+
+ // (Optional) Human-readable name of the function or method being invoked,
+ // with optional context such as the class or package name. The format can
+ // vary by language. For example:
+ // qual.if.ied.Class.method (Java)
+ // dir/package.func (Go)
+ // module.function (Python)
+ // file.cc:382 (C++)
+ string log_location = 7;
+
+ // (Optional) The name of the thread this log statement is associated with.
+ string thread = 8;
+}
+
+message LogControl {
+}
+
+// Stable
+service BeamFnLogging {
+ // Allows for the SDK to emit log entries which the runner can
+ // associate with the active job.
+ rpc Logging(
+ // A stream of log entries batched into lists emitted by the SDK harness.
+ stream LogEntry.List
+ ) returns (
+ // A stream of log control messages used to configure the SDK.
+ stream LogControl
+ ) {}
+}
+
+/*
+ * Environment types
+ */
+// A Docker container configuration for launching the SDK harness to execute
+// user specified functions.
+message DockerContainer {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ string id = 1;
+
+ // (Required) The Docker container URI
+ // For example "dataflow.gcr.io/v1beta3/java-batch:1.5.1"
+ string uri = 2;
+
+ // (Optional) Docker registry specification.
+ // If unspecified, the uri is expected to be able to be fetched without
+ // requiring additional configuration by a runner.
+ string registry_reference = 3;
+}
+
diff --git a/model/fn-execution/src/main/proto/beam_provision_api.proto b/model/fn-execution/src/main/proto/beam_provision_api.proto
new file mode 100644
index 0000000..086af10
--- /dev/null
+++ b/model/fn-execution/src/main/proto/beam_provision_api.proto
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Provision API, for communicating with a runner
+ * for job and environment provisioning information over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.fn_execution.v1;
+
+option go_package = "fnexecution_v1";
+option java_package = "org.apache.beam.model.fnexecution.v1";
+option java_outer_classname = "ProvisionApi";
+
+import "google/protobuf/struct.proto";
+
+// A service to provide runtime provisioning information to the SDK harness
+// worker instances -- such as pipeline options, resource constraints and
+// other job metadata -- needed by an SDK harness instance to initialize.
+service ProvisionService {
+ // Get provision information for the SDK harness worker instance.
+ rpc GetProvisionInfo(GetProvisionInfoRequest) returns (GetProvisionInfoResponse);
+}
+
+// A request to get the provision info of a SDK harness worker instance.
+message GetProvisionInfoRequest { }
+
+// A response containing the provision info of a SDK harness worker instance.
+message GetProvisionInfoResponse {
+ ProvisionInfo info = 1;
+}
+
+// Runtime provisioning information for a SDK harness worker instance,
+// such as pipeline options, resource constraints and other job metadata
+message ProvisionInfo {
+ // (required) The job ID.
+ string job_id = 1;
+ // (required) The job name.
+ string job_name = 2;
+
+ // (required) Pipeline options. For non-template jobs, the options are
+ // identical to what is passed to job submission.
+ google.protobuf.Struct pipeline_options = 3;
+
+ // (optional) Resource limits that the SDK harness worker should respect.
+ // Runners may -- but are not required to -- enforce any limits provided.
+ Resources resource_limits = 4;
+}
+
+// Resources specify limits for local resources, such memory and cpu. It
+// is used to inform SDK harnesses of their allocated footprint.
+message Resources {
+ // Memory limits.
+ message Memory {
+ // (optional) Hard limit in bytes. A zero value means unspecified.
+ uint64 size = 1;
+
+ // TOOD(herohde) 10/20/2017: consider soft limits, shm usage?
+ }
+ // (optional) Memory usage limits. SDKs can use this value to configure
+ // internal buffer sizes and language specific sizes.
+ Memory memory = 1;
+
+ // CPU limits.
+ message Cpu {
+ // (optional) Shares of a cpu to use. Fractional values, such as "0.2"
+ // or "2.5", are fine. Any value <= 0 means unspecified.
+ float shares = 1;
+
+ // TODO(herohde) 10/20/2017: consider cpuset?
+ }
+ // (optional) CPU usage limits.
+ Cpu cpu = 2;
+
+ // Disk limits.
+ message Disk {
+ // (optional) Hard limit in bytes. A zero value means unspecified.
+ uint64 size = 1;
+ }
+ // (optional) Disk size limits for the semi-persistent location.
+ Disk semi_persistent_disk = 3;
+}
diff --git a/sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml b/model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml
similarity index 100%
rename from sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
rename to model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml
diff --git a/model/job-management/pom.xml b/model/job-management/pom.xml
new file mode 100644
index 0000000..580188c
--- /dev/null
+++ b/model/job-management/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-job-management</artifactId>
+ <name>Apache Beam :: Model :: Job Management</name>
+ <description>Portable definitions for submitting pipelines.</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/model/job-management/src/main/proto/beam_artifact_api.proto b/model/job-management/src/main/proto/beam_artifact_api.proto
new file mode 100644
index 0000000..387e63f
--- /dev/null
+++ b/model/job-management/src/main/proto/beam_artifact_api.proto
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Artifact API, for communicating with a runner
+ * for artifact staging and retrieval over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.job_management.v1;
+
+option go_package = "jobmanagement_v1";
+option java_package = "org.apache.beam.model.jobmanagement.v1";
+option java_outer_classname = "ArtifactApi";
+
+// A service to stage artifacts for use in a Job.
+//
+// RPCs made to an ArtifactStagingService endpoint should include some form of identification for
+// the job as a header.
+service ArtifactStagingService {
+ // Stage an artifact to be available during job execution. The first request must contain the
+ // name of the artifact. All future requests must contain sequential chunks of the content of
+ // the artifact.
+ rpc PutArtifact(stream PutArtifactRequest) returns (PutArtifactResponse);
+
+ // Commit the manifest for a Job. All artifacts must have been successfully uploaded
+ // before this call is made.
+ //
+ // Throws error INVALID_ARGUMENT if not all of the members of the manifest are present
+ rpc CommitManifest(CommitManifestRequest) returns (CommitManifestResponse);
+}
+
+// A service to retrieve artifacts for use in a Job.
+service ArtifactRetrievalService {
+ // Get the manifest for the job
+ rpc GetManifest(GetManifestRequest) returns (GetManifestResponse);
+
+ // Get an artifact staged for the job. The requested artifact must be within the manifest
+ rpc GetArtifact(GetArtifactRequest) returns (stream ArtifactChunk);
+}
+
+// An artifact identifier and associated metadata.
+message ArtifactMetadata {
+ // (Required) The name of the artifact.
+ string name = 1;
+
+ // (Optional) The Unix-like permissions of the artifact
+ uint32 permissions = 2;
+
+ // (Optional) The base64-encoded md5 checksum of the artifact. Used, among other things, by
+ // harness boot code to validate the integrity of the artifact.
+ string md5 = 3;
+}
+
+// A collection of artifacts.
+message Manifest {
+ repeated ArtifactMetadata artifact = 1;
+}
+
+// A manifest with location information.
+message ProxyManifest {
+ Manifest manifest = 1;
+ message Location {
+ string name = 1;
+ string uri = 2;
+ }
+ repeated Location location = 2;
+}
+
+// A request to get the manifest of a Job.
+message GetManifestRequest {}
+
+// A response containing a job manifest.
+message GetManifestResponse {
+ Manifest manifest = 1;
+}
+
+// A request to get an artifact. The artifact must be present in the manifest for the job.
+message GetArtifactRequest {
+ // (Required) The name of the artifact to retrieve.
+ string name = 1;
+}
+
+// Part of an artifact.
+message ArtifactChunk {
+ bytes data = 1;
+}
+
+// A request to stage an artifact.
+message PutArtifactRequest {
+ // (Required)
+ oneof content {
+ // The Artifact metadata. The first message in a PutArtifact call must contain the name
+ // of the artifact.
+ ArtifactMetadata metadata = 1;
+
+ // A chunk of the artifact. All messages after the first in a PutArtifact call must contain a
+ // chunk.
+ ArtifactChunk data = 2;
+ }
+}
+
+message PutArtifactResponse {
+}
+
+// A request to commit the manifest for a Job. All artifacts must have been successfully uploaded
+// before this call is made.
+message CommitManifestRequest {
+ // (Required) The manifest to commit.
+ Manifest manifest = 1;
+}
+
+// The result of committing a manifest.
+message CommitManifestResponse {
+ // (Required) An opaque token representing the entirety of the staged artifacts.
+ string staging_token = 1;
+}
+
diff --git a/model/job-management/src/main/proto/beam_job_api.proto b/model/job-management/src/main/proto/beam_job_api.proto
new file mode 100644
index 0000000..a045ad3
--- /dev/null
+++ b/model/job-management/src/main/proto/beam_job_api.proto
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Job API, api for communicating with a runner
+ * for job submission over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.job_management.v1;
+
+option go_package = "jobmanagement_v1";
+option java_package = "org.apache.beam.model.jobmanagement.v1";
+option java_outer_classname = "JobApi";
+
+import "beam_runner_api.proto";
+import "endpoints.proto";
+import "google/protobuf/struct.proto";
+
+
+// Job Service for running RunnerAPI pipelines
+service JobService {
+ // Prepare a job for execution. The job will not be executed until a call is made to run with the
+ // returned preparationId.
+ rpc Prepare (PrepareJobRequest) returns (PrepareJobResponse);
+
+ // Submit the job for execution
+ rpc Run (RunJobRequest) returns (RunJobResponse);
+
+ // Get the current state of the job
+ rpc GetState (GetJobStateRequest) returns (GetJobStateResponse);
+
+ // Cancel the job
+ rpc Cancel (CancelJobRequest) returns (CancelJobResponse);
+
+ // Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
+ rpc GetStateStream (GetJobStateRequest) returns (stream GetJobStateResponse);
+
+ // Subscribe to a stream of state changes and messages from the job
+ rpc GetMessageStream (JobMessagesRequest) returns (stream JobMessagesResponse);
+}
+
+
+// Prepare is a synchronous request that returns a preparationId back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error ALREADY_EXISTS if the jobName is reused. Runners are permitted to deduplicate based on the name of the job.
+// Throws error UNKNOWN for all other issues
+message PrepareJobRequest {
+ org.apache.beam.model.pipeline.v1.Pipeline pipeline = 1; // (required)
+ google.protobuf.Struct pipeline_options = 2; // (required)
+ string job_name = 3; // (required)
+}
+
+message PrepareJobResponse {
+ // (required) The ID used to associate calls made while preparing the job. preparationId is used
+ // to run the job, as well as in other pre-execution APIs such as Artifact staging.
+ string preparation_id = 1;
+
+ // An endpoint which exposes the Beam Artifact Staging API. Artifacts used by the job should be
+ // staged to this endpoint, and will be available during job execution.
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor artifact_staging_endpoint = 2;
+}
+
+
+// Run is a synchronous request that returns a jobId back.
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the preparation ID does not exist
+// Throws error UNKNOWN for all other issues
+message RunJobRequest {
+ // (required) The ID provided by an earlier call to prepare. Runs the job. All prerequisite tasks
+ // must have been completed.
+ string preparation_id = 1;
+ // (optional) If any artifacts have been staged for this job, contains the staging_token returned
+ // from the CommitManifestResponse.
+ string staging_token = 2;
+}
+
+
+message RunJobResponse {
+ string job_id = 1; // (required) The ID for the executing job
+}
+
+
+// Cancel is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+message CancelJobRequest {
+ string job_id = 1; // (required)
+
+}
+
+// Valid responses include any terminal state or CANCELLING
+message CancelJobResponse {
+ JobState.Enum state = 1; // (required)
+}
+
+
+// GetState is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+message GetJobStateRequest {
+ string job_id = 1; // (required)
+
+}
+
+message GetJobStateResponse {
+ JobState.Enum state = 1; // (required)
+}
+
+
+// GetJobMessages is a streaming api for streaming job messages from the service
+// One request will connect you to the job and you'll get a stream of job state
+// and job messages back; one is used for logging and the other for detecting
+// the job ended.
+message JobMessagesRequest {
+ string job_id = 1; // (required)
+
+}
+
+message JobMessage {
+ string message_id = 1;
+ string time = 2;
+ MessageImportance importance = 3;
+ string message_text = 4;
+
+ enum MessageImportance {
+ MESSAGE_IMPORTANCE_UNSPECIFIED = 0;
+ JOB_MESSAGE_DEBUG = 1;
+ JOB_MESSAGE_DETAILED = 2;
+ JOB_MESSAGE_BASIC = 3;
+ JOB_MESSAGE_WARNING = 4;
+ JOB_MESSAGE_ERROR = 5;
+ }
+}
+
+message JobMessagesResponse {
+ oneof response {
+ JobMessage message_response = 1;
+ GetJobStateResponse state_response = 2;
+ }
+}
+
+// Enumeration of all JobStates
+message JobState {
+ enum Enum {
+ UNSPECIFIED = 0;
+ STOPPED = 1;
+ RUNNING = 2;
+ DONE = 3;
+ FAILED = 4;
+ CANCELLED = 5;
+ UPDATED = 6;
+ DRAINING = 7;
+ DRAINED = 8;
+ STARTING = 9;
+ CANCELLING = 10;
+ }
+}
diff --git a/model/pipeline/pom.xml b/model/pipeline/pom.xml
new file mode 100644
index 0000000..21d97a2
--- /dev/null
+++ b/model/pipeline/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-pipeline</artifactId>
+ <name>Apache Beam :: Model :: Pipeline</name>
+ <description>Portable definitions for building pipelines</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/model/pipeline/src/main/proto/beam_runner_api.proto b/model/pipeline/src/main/proto/beam_runner_api.proto
new file mode 100644
index 0000000..b45be09
--- /dev/null
+++ b/model/pipeline/src/main/proto/beam_runner_api.proto
@@ -0,0 +1,843 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Runner API, which is the runner-independent,
+ * SDK-independent definition of the Beam model.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.pipeline.v1;
+
+option go_package = "pipeline_v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
+option java_outer_classname = "RunnerApi";
+
+import "google/protobuf/any.proto";
+
+// A set of mappings from id to message. This is included as an optional field
+// on any proto message that may contain references needing resolution.
+message Components {
+ // (Required) A map from pipeline-scoped id to PTransform.
+ map<string, PTransform> transforms = 1;
+
+ // (Required) A map from pipeline-scoped id to PCollection.
+ map<string, PCollection> pcollections = 2;
+
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ map<string, WindowingStrategy> windowing_strategies = 3;
+
+ // (Required) A map from pipeline-scoped id to Coder.
+ map<string, Coder> coders = 4;
+
+ // (Required) A map from pipeline-scoped id to Environment.
+ map<string, Environment> environments = 5;
+}
+
+// A disjoint union of all the things that may contain references
+// that require Components to resolve.
+message MessageWithComponents {
+
+ // (Optional) The by-reference components of the root message,
+ // enabling a standalone message.
+ //
+ // If this is absent, it is expected that there are no
+ // references.
+ Components components = 1;
+
+ // (Required) The root message that may contain pointers
+ // that should be resolved by looking inside components.
+ oneof root {
+ Coder coder = 2;
+ CombinePayload combine_payload = 3;
+ SdkFunctionSpec sdk_function_spec = 4;
+ ParDoPayload par_do_payload = 6;
+ PTransform ptransform = 7;
+ PCollection pcollection = 8;
+ ReadPayload read_payload = 9;
+ SideInput side_input = 11;
+ WindowIntoPayload window_into_payload = 12;
+ WindowingStrategy windowing_strategy = 13;
+ FunctionSpec function_spec = 14;
+ }
+}
+
+// A Pipeline is a hierarchical graph of PTransforms, linked
+// by PCollections.
+//
+// This is represented by a number of by-reference maps to nodes,
+// PCollections, SDK environments, UDF, etc., for
+// supporting compact reuse and arbitrary graph structure.
+//
+// All of the keys in the maps here are arbitrary strings that are only
+// required to be internally consistent within this proto message.
+message Pipeline {
+
+ // (Required) The coders, UDFs, graph nodes, etc, that make up
+ // this pipeline.
+ Components components = 1;
+
+ // (Required) The ids of all PTransforms that are not contained within another PTransform.
+ // These must be in shallow topological order, so that traversing them recursively
+ // in this order yields a recursively topological traversal.
+ repeated string root_transform_ids = 2;
+
+ // (Optional) Static display data for the pipeline. If there is none,
+ // it may be omitted.
+ DisplayData display_data = 3;
+}
+
+// An applied PTransform! This does not contain the graph data, but only the
+// fields specific to a graph node that is a Runner API transform
+// between PCollections.
+message PTransform {
+
+ // (Required) A unique name for the application node.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here and be unique, even if it is
+ // autogenerated.
+ string unique_name = 5;
+
+ // (Optional) A URN and payload that, together, fully defined the semantics
+ // of this transform.
+ //
+ // If absent, this must be an "anonymous" composite transform.
+ //
+ // For primitive transform in the Runner API, this is required, and the
+ // payloads are well-defined messages. When the URN indicates ParDo it
+ // is a ParDoPayload, and so on.
+ //
+ // TODO: document the standardized URNs and payloads
+ // TODO: separate standardized payloads into a separate proto file
+ //
+ // For some special composite transforms, the payload is also officially
+ // defined:
+ //
+ // - when the URN is "urn:beam:transforms:combine" it is a CombinePayload
+ //
+ FunctionSpec spec = 1;
+
+ // (Optional) if this node is a composite, a list of the ids of
+ // transforms that it contains.
+ repeated string subtransforms = 2;
+
+ // (Required) A map from local names of inputs (unique only with this map, and
+ // likely embedded in the transform payload and serialized user code) to
+ // PCollection ids.
+ //
+ // The payload for this transform may clarify the relationship of these
+ // inputs. For example:
+ //
+ // - for a Flatten transform they are merged
+ // - for a ParDo transform, some may be side inputs
+ //
+ // All inputs are recorded here so that the topological ordering of
+ // the graph is consistent whether or not the payload is understood.
+ //
+ map<string, string> inputs = 3;
+
+ // (Required) A map from local names of outputs (unique only within this map,
+ // and likely embedded in the transform payload and serialized user code)
+ // to PCollection ids.
+ //
+ // The URN or payload for this transform node may clarify the type and
+ // relationship of these outputs. For example:
+ //
+ // - for a ParDo transform, these are tags on PCollections, which will be
+ // embedded in the DoFn.
+ //
+ map<string, string> outputs = 4;
+
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData display_data = 6;
+}
+
+// A PCollection!
+message PCollection {
+
+ // (Required) A unique name for the PCollection.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here, even if it is autogenerated.
+ string unique_name = 1;
+
+ // (Required) The id of the Coder for this PCollection.
+ string coder_id = 2;
+
+ // (Required) Whether this PCollection is bounded or unbounded
+ IsBounded.Enum is_bounded = 3;
+
+ // (Required) The id of the windowing strategy for this PCollection.
+ string windowing_strategy_id = 4;
+
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData display_data = 5;
+}
+
+// The payload for the primitive ParDo transform.
+message ParDoPayload {
+
+ // (Required) The SdkFunctionSpec of the DoFn.
+ SdkFunctionSpec do_fn = 1;
+
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ repeated Parameter parameters = 2;
+
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ map<string, SideInput> side_inputs = 3;
+
+ // (Optional) A mapping of local state names to state specifications.
+ map<string, StateSpec> state_specs = 4;
+
+ // (Optional) A mapping of local timer names to timer specifications.
+ map<string, TimerSpec> timer_specs = 5;
+
+ // Whether the DoFn is splittable
+ bool splittable = 6;
+}
+
+// Parameters that a UDF might require.
+//
+// The details of how a runner sends these parameters to the SDK harness
+// are the subject of the Fn API.
+//
+// The details of how an SDK harness delivers them to the UDF is entirely
+// up to the SDK. (for some SDKs there may be parameters that are not
+// represented here if the runner doesn't need to do anything)
+//
+// Here, the parameters are simply indicators to the runner that they
+// need to run the function a particular way.
+//
+// TODO: the evolution of the Fn API will influence what needs explicit
+// representation here
+message Parameter {
+ Type.Enum type = 1;
+
+ message Type {
+ enum Enum {
+ UNSPECIFIED = 0;
+ WINDOW = 1;
+ PIPELINE_OPTIONS = 2;
+ RESTRICTION_TRACKER = 3;
+ }
+ }
+}
+
+message StateSpec {
+ oneof spec {
+ ValueStateSpec value_spec = 1;
+ BagStateSpec bag_spec = 2;
+ CombiningStateSpec combining_spec = 3;
+ MapStateSpec map_spec = 4;
+ SetStateSpec set_spec = 5;
+ }
+}
+
+message ValueStateSpec {
+ string coder_id = 1;
+}
+
+message BagStateSpec {
+ string element_coder_id = 1;
+}
+
+message CombiningStateSpec {
+ string accumulator_coder_id = 1;
+ SdkFunctionSpec combine_fn = 2;
+}
+
+message MapStateSpec {
+ string key_coder_id = 1;
+ string value_coder_id = 2;
+}
+
+message SetStateSpec {
+ string element_coder_id = 1;
+}
+
+message TimerSpec {
+ TimeDomain.Enum time_domain = 1;
+}
+
+message IsBounded {
+ enum Enum {
+ UNSPECIFIED = 0;
+ UNBOUNDED = 1;
+ BOUNDED = 2;
+ }
+}
+
+// The payload for the primitive Read transform.
+message ReadPayload {
+
+ // (Required) The SdkFunctionSpec of the source for this Read.
+ SdkFunctionSpec source = 1;
+
+ // (Required) Whether the source is bounded or unbounded
+ IsBounded.Enum is_bounded = 2;
+
+ // TODO: full audit of fields required by runners as opposed to SDK harness
+}
+
+// The payload for the WindowInto transform.
+message WindowIntoPayload {
+
+ // (Required) The SdkFunctionSpec of the WindowFn.
+ SdkFunctionSpec window_fn = 1;
+}
+
+// The payload for the special-but-not-primitive Combine transform.
+message CombinePayload {
+
+ // (Required) The SdkFunctionSpec of the CombineFn.
+ SdkFunctionSpec combine_fn = 1;
+
+ // (Required) A reference to the Coder to use for accumulators of the CombineFn
+ string accumulator_coder_id = 2;
+
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ repeated Parameter parameters = 3;
+
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ map<string, SideInput> side_inputs = 4;
+}
+
+// The payload for the test-only primitive TestStream
+message TestStreamPayload {
+
+ // (Required) the coder for elements in the TestStream events
+ string coder_id = 1;
+
+ repeated Event events = 2;
+
+ message Event {
+ oneof event {
+ AdvanceWatermark watermark_event = 1;
+ AdvanceProcessingTime processing_time_event = 2;
+ AddElements element_event = 3;
+ }
+
+ message AdvanceWatermark {
+ int64 new_watermark = 1;
+ }
+
+ message AdvanceProcessingTime {
+ int64 advance_duration = 1;
+ }
+
+ message AddElements {
+ repeated TimestampedElement elements = 1;
+ }
+ }
+
+ message TimestampedElement {
+ bytes encoded_element = 1;
+ int64 timestamp = 2;
+ }
+}
+// The payload for the special-but-not-primitive WriteFiles transform.
+message WriteFilesPayload {
+
+ // (Required) The SdkFunctionSpec of the FileBasedSink.
+ SdkFunctionSpec sink = 1;
+
+ // (Required) The format function.
+ SdkFunctionSpec format_function = 2;
+
+ bool windowed_writes = 3;
+
+ bool runner_determined_sharding = 4;
+
+ map<string, SideInput> side_inputs = 5;
+}
+
+// A coder, the binary format for serialization and deserialization of data in
+// a pipeline.
+message Coder {
+
+ // (Required) A specification for the coder, as a URN plus parameters. This
+ // may be a cross-language agreed-upon format, or it may be a "custom coder"
+ // that can only be used by a particular SDK. It does not include component
+ // coders, as it is beneficial for these to be comprehensible to a runner
+ // regardless of whether the binary format is agree-upon.
+ SdkFunctionSpec spec = 1;
+
+ // (Optional) If this coder is parametric, such as ListCoder(VarIntCoder),
+ // this is a list of the components. In order for encodings to be identical,
+ // the SdkFunctionSpec and all components must be identical, recursively.
+ repeated string component_coder_ids = 2;
+}
+
+// A windowing strategy describes the window function, triggering, allowed
+// lateness, and accumulation mode for a PCollection.
+//
+// TODO: consider inlining field on PCollection
+message WindowingStrategy {
+
+ // (Required) The SdkFunctionSpec of the UDF that assigns windows,
+ // merges windows, and shifts timestamps before they are
+ // combined according to the OutputTime.
+ SdkFunctionSpec window_fn = 1;
+
+ // (Required) Whether or not the window fn is merging.
+ //
+ // This knowledge is required for many optimizations.
+ MergeStatus.Enum merge_status = 2;
+
+ // (Required) The coder for the windows of this PCollection.
+ string window_coder_id = 3;
+
+ // (Required) The trigger to use when grouping this PCollection.
+ Trigger trigger = 4;
+
+ // (Required) The accumulation mode indicates whether new panes are a full
+ // replacement for prior panes or whether they are deltas to be combined
+ // with other panes (the combine should correspond to whatever the upstream
+ // grouping transform is).
+ AccumulationMode.Enum accumulation_mode = 5;
+
+ // (Required) The OutputTime specifies, for a grouping transform, how to
+ // compute the aggregate timestamp. The window_fn will first possibly shift
+ // it later, then the OutputTime takes the max, min, or ignores it and takes
+ // the end of window.
+ //
+ // This is actually only for input to grouping transforms, but since they
+ // may be introduced in runner-specific ways, it is carried along with the
+ // windowing strategy.
+ OutputTime.Enum output_time = 6;
+
+ // (Required) Indicate when output should be omitted upon window expiration.
+ ClosingBehavior.Enum closing_behavior = 7;
+
+ // (Required) The duration, in milliseconds, beyond the end of a window at
+ // which the window becomes droppable.
+ int64 allowed_lateness = 8;
+
+ // (Required) Indicate whether empty on-time panes should be omitted.
+ OnTimeBehavior.Enum OnTimeBehavior = 9;
+
+ // (Required) Whether or not the window fn assigns inputs to exactly one window
+ //
+ // This knowledge is required for some optimizations
+ bool assigns_to_one_window = 10;
+}
+
+// Whether or not a PCollection's WindowFn is non-merging, merging, or
+// merging-but-already-merged, in which case a subsequent GroupByKey is almost
+// always going to do something the user does not want
+message MergeStatus {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The WindowFn does not require merging.
+ // Examples: global window, FixedWindows, SlidingWindows
+ NON_MERGING = 1;
+
+ // The WindowFn is merging and the PCollection has not had merging
+ // performed.
+ // Example: Sessions prior to a GroupByKey
+ NEEDS_MERGE = 2;
+
+ // The WindowFn is merging and the PCollection has had merging occur
+ // already.
+ // Example: Sessions after a GroupByKey
+ ALREADY_MERGED = 3;
+ }
+}
+
+// Whether or not subsequent outputs of aggregations should be entire
+// replacement values or just the aggregation of inputs received since
+// the prior output.
+message AccumulationMode {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The aggregation is discarded when it is output
+ DISCARDING = 1;
+
+ // The aggregation is accumulated across outputs
+ ACCUMULATING = 2;
+ }
+}
+
+// Controls whether or not an aggregating transform should output data
+// when a window expires.
+message ClosingBehavior {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Emit output when a window expires, whether or not there has been
+ // any new data since the last output.
+ EMIT_ALWAYS = 1;
+
+ // Only emit output when new data has arrives since the last output
+ EMIT_IF_NONEMPTY = 2;
+ }
+}
+
+// Controls whether or not an aggregating transform should output data
+// when an on-time pane is empty.
+message OnTimeBehavior {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Always fire the on-time pane. Even if there is no new data since
+ // the previous firing, an element will be produced.
+ FIRE_ALWAYS = 1;
+
+ // Only fire the on-time pane if there is new data since the previous firing.
+ FIRE_IF_NONEMPTY = 2;
+ }
+}
+
+// When a number of windowed, timestamped inputs are aggregated, the timestamp
+// for the resulting output.
+message OutputTime {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The output has the timestamp of the end of the window.
+ END_OF_WINDOW = 1;
+
+ // The output has the latest timestamp of the input elements since
+ // the last output.
+ LATEST_IN_PANE = 2;
+
+ // The output has the earliest timestamp of the input elements since
+ // the last output.
+ EARLIEST_IN_PANE = 3;
+ }
+}
+
+// The different time domains in the Beam model.
+message TimeDomain {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Event time is time from the perspective of the data
+ EVENT_TIME = 1;
+
+ // Processing time is time from the perspective of the
+ // execution of your pipeline
+ PROCESSING_TIME = 2;
+
+ // Synchronized processing time is the minimum of the
+ // processing time of all pending elements.
+ //
+ // The "processing time" of an element refers to
+ // the local processing time at which it was emitted
+ SYNCHRONIZED_PROCESSING_TIME = 3;
+ }
+}
+
+// A small DSL for expressing when to emit new aggregations
+// from a GroupByKey or CombinePerKey
+//
+// A trigger is described in terms of when it is _ready_ to permit output.
+message Trigger {
+
+ // Ready when all subtriggers are ready.
+ message AfterAll {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Ready when any subtrigger is ready.
+ message AfterAny {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Starting with the first subtrigger, ready when the _current_ subtrigger
+ // is ready. After output, advances the current trigger by one.
+ message AfterEach {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Ready after the input watermark is past the end of the window.
+ //
+ // May have implicitly-repeated subtriggers for early and late firings.
+ // When the end of the window is reached, the trigger transitions between
+ // the subtriggers.
+ message AfterEndOfWindow {
+
+ // (Optional) A trigger governing output prior to the end of the window.
+ Trigger early_firings = 1;
+
+ // (Optional) A trigger governing output after the end of the window.
+ Trigger late_firings = 2;
+ }
+
+ // After input arrives, ready when the specified delay has passed.
+ message AfterProcessingTime {
+
+ // (Required) The transforms to apply to an arriving element's timestamp,
+ // in order
+ repeated TimestampTransform timestamp_transforms = 1;
+ }
+
+ // Ready whenever upstream processing time has all caught up with
+ // the arrival time of an input element
+ message AfterSynchronizedProcessingTime {
+ }
+
+ // The default trigger. Equivalent to Repeat { AfterEndOfWindow } but
+ // specially denoted to indicate the user did not alter the triggering.
+ message Default {
+ }
+
+ // Ready whenever the requisite number of input elements have arrived
+ message ElementCount {
+ int32 element_count = 1;
+ }
+
+ // Never ready. There will only be an ON_TIME output and a final
+ // output at window expiration.
+ message Never {
+ }
+
+ // Always ready. This can also be expressed as ElementCount(1) but
+ // is more explicit.
+ message Always {
+ }
+
+ // Ready whenever either of its subtriggers are ready, but finishes output
+ // when the finally subtrigger fires.
+ message OrFinally {
+
+ // (Required) Trigger governing main output; may fire repeatedly.
+ Trigger main = 1;
+
+ // (Required) Trigger governing termination of output.
+ Trigger finally = 2;
+ }
+
+ // Ready whenever the subtrigger is ready; resets state when the subtrigger
+ // completes.
+ message Repeat {
+ // (Require) Trigger that is run repeatedly.
+ Trigger subtrigger = 1;
+ }
+
+ // The full disjoint union of possible triggers.
+ oneof trigger {
+ AfterAll after_all = 1;
+ AfterAny after_any = 2;
+ AfterEach after_each = 3;
+ AfterEndOfWindow after_end_of_window = 4;
+ AfterProcessingTime after_processing_time = 5;
+ AfterSynchronizedProcessingTime after_synchronized_processing_time = 6;
+ Always always = 12;
+ Default default = 7;
+ ElementCount element_count = 8;
+ Never never = 9;
+ OrFinally or_finally = 10;
+ Repeat repeat = 11;
+ }
+}
+
+// A specification for a transformation on a timestamp.
+//
+// Primarily used by AfterProcessingTime triggers to transform
+// the arrival time of input to a target time for firing.
+message TimestampTransform {
+ oneof timestamp_transform {
+ Delay delay = 1;
+ AlignTo align_to = 2;
+ }
+
+ message Delay {
+ // (Required) The delay, in milliseconds.
+ int64 delay_millis = 1;
+ }
+
+ message AlignTo {
+ // (Required) A duration to which delays should be quantized
+ // in milliseconds.
+ int64 period = 3;
+
+ // (Required) An offset from 0 for the quantization specified by
+ // alignment_size, in milliseconds
+ int64 offset = 4;
+ }
+}
+
+// A specification for how to "side input" a PCollection.
+message SideInput {
+ // (Required) URN of the access pattern required by the `view_fn` to present
+ // the desired SDK-specific interface to a UDF.
+ //
+ // This access pattern defines the SDK harness <-> Runner Harness RPC
+ // interface for accessing a side input.
+ //
+ // The only access pattern intended for Beam, because of its superior
+ // performance possibilities, is "urn:beam:sideinput:multimap" (or some such
+ // URN)
+ FunctionSpec access_pattern = 1;
+
+ // (Required) The SdkFunctionSpec of the UDF that adapts a particular
+ // access_pattern to a user-facing view type.
+ //
+ // For example, View.asSingleton() may include a `view_fn` that adapts a
+ // specially-designed multimap to a single value per window.
+ SdkFunctionSpec view_fn = 2;
+
+ // (Required) The SdkFunctionSpec of the UDF that maps a main input window
+ // to a side input window.
+ //
+ // For example, when the main input is in fixed windows of one hour, this
+ // can specify that the side input should be accessed according to the day
+ // in which that hour falls.
+ SdkFunctionSpec window_mapping_fn = 3;
+}
+
+// An environment for executing UDFs. Generally an SDK container URL, but
+// there can be many for a single SDK, for example to provide dependency
+// isolation.
+message Environment {
+
+ // (Required) The URL of a container
+ //
+ // TODO: reconcile with Fn API's DockerContainer structure by
+ // adding adequate metadata to know how to interpret the container
+ string url = 1;
+}
+
+// A specification of a user defined function.
+//
+message SdkFunctionSpec {
+
+ // (Required) A full specification of this function.
+ FunctionSpec spec = 1;
+
+ // (Required) Reference to an execution environment capable of
+ // invoking this function.
+ string environment_id = 2;
+}
+
+// A URN along with a parameter object whose schema is determined by the
+// URN.
+//
+// This structure is reused in two distinct, but compatible, ways:
+//
+// 1. This can be a specification of the function over PCollections
+// that a PTransform computes.
+// 2. This can be a specification of a user-defined function, possibly
+// SDK-specific. (external to this message must be adequate context
+// to indicate the environment in which the UDF can be understood).
+//
+// Though not explicit in this proto, there are two possibilities
+// for the relationship of a runner to this specification that
+// one should bear in mind:
+//
+// 1. The runner understands the URN. For example, it might be
+// a well-known URN like "urn:beam:transform:Top" or
+// "urn:beam:windowfn:FixedWindows" with
+// an agreed-upon payload (e.g. a number or duration,
+// respectively).
+// 2. The runner does not understand the URN. It might be an
+// SDK specific URN such as "urn:beam:dofn:javasdk:1.0"
+// that indicates to the SDK what the payload is,
+// such as a serialized Java DoFn from a particular
+// version of the Beam Java SDK. The payload will often
+// then be an opaque message such as bytes in a
+// language-specific serialization format.
+message FunctionSpec {
+
+ // (Required) A URN that describes the accompanying payload.
+ // For any URN that is not recognized (by whomever is inspecting
+ // it) the parameter payload should be treated as opaque and
+ // passed as-is.
+ string urn = 1;
+
+ // (Optional) The data specifying any parameters to the URN. If
+ // the URN does not require any arguments, this may be omitted.
+ bytes payload = 3;
+}
+
+// TODO: transfer javadoc here
+message DisplayData {
+
+ // (Required) The list of display data.
+ repeated Item items = 1;
+
+ // A complete identifier for a DisplayData.Item
+ message Identifier {
+
+ // (Required) The transform originating this display data.
+ string transform_id = 1;
+
+ // (Optional) The URN indicating the type of the originating transform,
+ // if there is one.
+ string transform_urn = 2;
+
+ string key = 3;
+ }
+
+ // A single item of display data.
+ message Item {
+ // (Required)
+ Identifier id = 1;
+
+ // (Required)
+ Type.Enum type = 2;
+
+ // (Required)
+ google.protobuf.Any value = 3;
+
+ // (Optional)
+ google.protobuf.Any short_value = 4;
+
+ // (Optional)
+ string label = 5;
+
+ // (Optional)
+ string link_url = 6;
+ }
+
+ message Type {
+ enum Enum {
+ UNSPECIFIED = 0;
+ STRING = 1;
+ INTEGER = 2;
+ FLOAT = 3;
+ BOOLEAN = 4;
+ TIMESTAMP = 5;
+ DURATION = 6;
+ JAVA_CLASS = 7;
+ }
+ }
+}
diff --git a/model/pipeline/src/main/proto/endpoints.proto b/model/pipeline/src/main/proto/endpoints.proto
new file mode 100644
index 0000000..d807140
--- /dev/null
+++ b/model/pipeline/src/main/proto/endpoints.proto
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing endpoints containing a service.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.pipeline.v1;
+
+option go_package = "pipeline_v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
+option java_outer_classname = "Endpoints";
+
+message ApiServiceDescriptor {
+ // (Required) The URL to connect to.
+ string url = 2;
+
+ // (Optional) The method for authentication. If unspecified, access to the
+ // url is already being performed in a trusted context (e.g. localhost,
+ // private network).
+ oneof authentication {
+ OAuth2ClientCredentialsGrant oauth2_client_credentials_grant = 3;
+ }
+}
+
+message OAuth2ClientCredentialsGrant {
+ // (Required) The URL to submit a "client_credentials" grant type request for
+ // an OAuth access token which will be used as a bearer token for requests.
+ string url = 1;
+}
diff --git a/model/pipeline/src/main/proto/standard_window_fns.proto b/model/pipeline/src/main/proto/standard_window_fns.proto
new file mode 100644
index 0000000..db26d91
--- /dev/null
+++ b/model/pipeline/src/main/proto/standard_window_fns.proto
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Runner API, which is the runner-independent,
+ * SDK-independent definition of the Beam model.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.model.pipeline.v1;
+
+option go_package = "pipeline_v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
+option java_outer_classname = "StandardWindowFns";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+// beam:windowfn:global_windows:v0.1
+// empty payload
+
+// beam:windowfn:fixed_windows:v0.1
+message FixedWindowsPayload {
+ google.protobuf.Duration size = 1;
+ google.protobuf.Timestamp offset = 2;
+}
+
+// beam:windowfn:sliding_windows:v0.1
+message SlidingWindowsPayload {
+ google.protobuf.Duration size = 1;
+ google.protobuf.Timestamp offset = 2;
+ google.protobuf.Duration period = 3;
+}
+
+// beam:windowfn:session_windows:v0.1
+message SessionsPayload {
+ google.protobuf.Duration gap_size = 1;
+}
diff --git a/model/pom.xml b/model/pom.xml
new file mode 100644
index 0000000..a7ffd3d
--- /dev/null
+++ b/model/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-parent</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: Model</name>
+
+ <modules>
+ <module>pipeline</module>
+ <module>job-management</module>
+ <module>fn-execution</module>
+ </modules>
+</project>
diff --git a/pom.xml b/pom.xml
index a978f58..adfef71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,7 +34,7 @@
<url>http://beam.apache.org/</url>
<inceptionYear>2016</inceptionYear>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<licenses>
<license>
@@ -101,53 +101,93 @@
<beamSurefireArgline />
<!-- If updating dependencies, please update any relevant javadoc offlineLinks -->
- <apache.commons.lang.version>3.5</apache.commons.lang.version>
- <apache.commons.compress.version>1.9</apache.commons.compress.version>
+ <apache.commons.compress.version>1.14</apache.commons.compress.version>
+ <apache.commons.lang.version>3.6</apache.commons.lang.version>
+ <apache.commons.text.version>1.1</apache.commons.text.version>
<apex.kryo.version>2.24.0</apex.kryo.version>
- <avro.version>1.8.1</avro.version>
- <bigquery.version>v2-rev295-1.22.0</bigquery.version>
- <bigtable.version>0.9.6.2</bigtable.version>
+ <api-common.version>1.0.0-rc2</api-common.version>
+ <args4j.version>2.33</args4j.version>
+ <avro.version>1.8.2</avro.version>
+ <bigquery.version>v2-rev355-1.22.0</bigquery.version>
+ <bigtable.version>1.0.0-pre3</bigtable.version>
<cloudresourcemanager.version>v1-rev6-1.22.0</cloudresourcemanager.version>
- <pubsubgrpc.version>0.1.0</pubsubgrpc.version>
+ <pubsubgrpc.version>0.1.18</pubsubgrpc.version>
<clouddebugger.version>v2-rev8-1.22.0</clouddebugger.version>
- <dataflow.version>v1b3-rev196-1.22.0</dataflow.version>
+ <dataflow.version>v1b3-rev218-1.22.0</dataflow.version>
<dataflow.proto.version>0.5.160222</dataflow.proto.version>
<datastore.client.version>1.4.0</datastore.client.version>
<datastore.proto.version>1.3.0</datastore.proto.version>
+ <google-api-common.version>1.0.0-rc2</google-api-common.version>
<google-auto-service.version>1.0-rc2</google-auto-service.version>
- <google-auto-value.version>1.4.1</google-auto-value.version>
- <google-auth.version>0.6.1</google-auth.version>
+ <google-auto-value.version>1.5.1</google-auto-value.version>
+ <google-auth.version>0.7.1</google-auth.version>
<google-clients.version>1.22.0</google-clients.version>
<google-cloud-bigdataoss.version>1.4.5</google-cloud-bigdataoss.version>
+ <google-cloud-core.version>1.0.2</google-cloud-core.version>
<google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version>
<guava.version>20.0</guava.version>
<grpc.version>1.2.0</grpc.version>
- <grpc-google-common-protos.version>0.1.0</grpc-google-common-protos.version>
+ <grpc-google-common-protos.version>0.1.9</grpc-google-common-protos.version>
+ <!--
+ This is the version of Hadoop used to compile the module that depend on Hadoop.
+ This dependency is defined with a provided scope.
+ Users must supply their own Hadoop version at runtime.
+ -->
+ <hadoop.version>2.7.3</hadoop.version>
<hamcrest.version>1.3</hamcrest.version>
- <jackson.version>2.8.8</jackson.version>
+ <jackson.version>2.8.9</jackson.version>
<findbugs.version>3.0.1</findbugs.version>
+ <findbugs.annotations.version>1.3.9-1</findbugs.annotations.version>
<joda.version>2.4</joda.version>
<junit.version>4.12</junit.version>
<mockito.version>1.9.5</mockito.version>
<netty.version>4.1.8.Final</netty.version>
<netty.tcnative.version>1.1.33.Fork26</netty.tcnative.version>
- <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
<protobuf.version>3.2.0</protobuf.version>
<pubsub.version>v1-rev10-1.22.0</pubsub.version>
- <slf4j.version>1.7.14</slf4j.version>
- <spark.version>1.6.2</spark.version>
+ <slf4j.version>1.7.25</slf4j.version>
+ <spanner.version>0.20.0-beta</spanner.version>
+ <spark.version>1.6.3</spark.version>
+ <spring.version>4.3.5.RELEASE</spring.version>
<stax2.version>3.1.4</stax2.version>
<storage.version>v1-rev71-1.22.0</storage.version>
<woodstox.version>4.4.1</woodstox.version>
<spring.version>4.3.5.RELEASE</spring.version>
+ <snappy-java.version>1.1.4</snappy-java.version>
+ <kafka.clients.version>0.11.0.1</kafka.clients.version>
+ <commons.csv.version>1.4</commons.csv.version>
+
+ <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
+ <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
<groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
<surefire-plugin.version>2.20</surefire-plugin.version>
<failsafe-plugin.version>2.20</failsafe-plugin.version>
+ <maven-compiler-plugin.version>3.6.2</maven-compiler-plugin.version>
+ <maven-dependency-plugin.version>3.0.1</maven-dependency-plugin.version>
+ <maven-enforcer-plugin.version>3.0.0-M1</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>1.6.0</maven-exec-plugin.version>
+ <maven-jar-plugin.version>3.0.2</maven-jar-plugin.version>
+ <maven-javadoc-plugin.version>3.0.0-M1</maven-javadoc-plugin.version>
+ <maven-license-plugin.version>1.13</maven-license-plugin.version>
<maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
-
+ <maven-shade-plugin.version>3.0.0</maven-shade-plugin.version>
+ <reproducible-build-maven-plugin.version>0.3</reproducible-build-maven-plugin.version>
+
<compiler.error.flag>-Werror</compiler.error.flag>
<compiler.default.pkginfo.flag>-Xpkginfo:always</compiler.default.pkginfo.flag>
<compiler.default.exclude>nothing</compiler.default.exclude>
+ <gax-grpc.version>0.20.0</gax-grpc.version>
+
+ <!-- standard binary for kubectl -->
+ <kubectl>kubectl</kubectl>
+ <!-- the standard location for kubernete's config file -->
+ <kubeconfig>${user.home}/.kube/config</kubeconfig>
+
+ <!-- For container builds, override to push containers to any registry -->
+ <docker-repository-root>${user.name}-docker-apache.bintray.io/beam</docker-repository-root>
+
+ <!-- Default skipping -->
+ <rat.skip>true</rat.skip>
</properties>
<packaging>pom</packaging>
@@ -156,6 +196,7 @@
<!-- sdks/java/build-tools has project-wide configuration. To make these available
in all modules, link it directly to the parent pom.xml. -->
<module>sdks/java/build-tools</module>
+ <module>model</module>
<module>sdks</module>
<module>runners</module>
<module>examples</module>
@@ -164,8 +205,8 @@
</modules>
<profiles>
- <!-- A global profile defined for all modules for release-level verification.
- Optional processes such as building source and javadoc should be limited
+ <!-- A global profile defined for all modules for release-level verification.
+ Optional processes such as building source and javadoc should be limited
to this profile. -->
<profile>
<id>release</id>
@@ -210,15 +251,11 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
+ <configuration>
+ <skip>false</skip>
+ </configuration>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
@@ -242,6 +279,57 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>io.github.zlika</groupId>
+ <artifactId>reproducible-build-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>java8-enable-like-dependencies</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <!-- Override Beam parent to allow Java8 dependencies -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>${maven-enforcer-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <enforceBytecodeVersion>
+ <maxJdkVersion>1.8</maxJdkVersion>
+ <excludes>
+ <!--
+ Supplied by the user JDK and compiled with matching
+ version. Is not shaded, so safe to ignore.
+ -->
+ <exclude>jdk.tools:jdk.tools</exclude>
+ </excludes>
+ </enforceBytecodeVersion>
+ <requireJavaVersion>
+ <version>[1.7,)</version>
+ </requireJavaVersion>
+ <requireMavenVersion>
+ <!-- Keep aligned with preqrequisite section below. -->
+ <version>[3.2,)</version>
+ </requireMavenVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
@@ -325,31 +413,64 @@
</pluginManagement>
</build>
</profile>
+
+ <profile>
+ <id>build-containers</id>
+ <build>
+ <!-- TODO(BEAM-2878): enable container build for releases -->
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <configuration>
+ <noCache>true</noCache>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
+
</profiles>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-job-management</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>${project.version}</version>
</dependency>
@@ -381,12 +502,31 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-sketching</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-sorter</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-harness</artifactId>
<version>${project.version}</version>
</dependency>
@@ -406,12 +546,46 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-amqp</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-cassandra</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-elasticsearch</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-5</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
<version>${project.version}</version>
</dependency>
@@ -443,6 +617,12 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-hcatalog</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-jdbc</artifactId>
<version>${project.version}</version>
</dependency>
@@ -477,12 +657,18 @@
<version>${project.version}</version>
</dependency>
- <dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-solr</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
- <version>${project.version}</version>
+ <version>${project.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-core-construction-java</artifactId>
@@ -497,6 +683,25 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-java-fn-execution</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-reference-job-orchestrator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-runners-direct-java</artifactId>
<version>${project.version}</version>
</dependency>
@@ -527,6 +732,12 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-examples-java</artifactId>
<version>${project.version}</version>
</dependency>
@@ -550,6 +761,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ <version>${apache.commons.text.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-all</artifactId>
<version>${grpc.version}</version>
@@ -604,6 +821,18 @@
</dependency>
<dependency>
+ <groupId>com.google.api</groupId>
+ <artifactId>api-common</artifactId>
+ <version>${google-api-common.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api</groupId>
+ <artifactId>gax-grpc</artifactId>
+ <version>${gax-grpc.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>${google-clients.version}</version>
@@ -720,13 +949,13 @@
<artifactId>google-auth-library-credentials</artifactId>
<version>${google-auth.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-oauth2-http</artifactId>
<version>${google-auth.version}</version>
<exclusions>
- <!-- Exclude an old version of guava that is being pulled in by a transitive
+ <!-- Exclude an old version of guava that is being pulled in by a transitive
dependency of google-api-client -->
<exclusion>
<groupId>com.google.guava</groupId>
@@ -773,7 +1002,7 @@
<dependency>
<groupId>com.google.api.grpc</groupId>
- <artifactId>grpc-google-pubsub-v1</artifactId>
+ <artifactId>grpc-google-cloud-pubsub-v1</artifactId>
<version>${pubsubgrpc.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled in by a transitive
@@ -792,11 +1021,29 @@
</dependency>
<dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-cloud-pubsub-v1</artifactId>
+ <version>${pubsubgrpc.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
+ <version>${grpc-google-common-protos.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ <version>${grpc-google-common-protos.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-storage</artifactId>
<version>${storage.version}</version>
<exclusions>
- <!-- Exclude an old version of guava that is being pulled in by a transitive
+ <!-- Exclude an old version of guava that is being pulled in by a transitive
dependency of google-api-client -->
<exclusion>
<groupId>com.google.guava</groupId>
@@ -806,6 +1053,11 @@
</dependency>
<dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-core-grpc</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-protos</artifactId>
<version>${bigtable.version}</version>
@@ -855,12 +1107,30 @@
</dependency>
<dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <version>${findbugs.annotations.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcsio</artifactId>
<version>${google-cloud-bigdataoss.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-core</artifactId>
+ <version>${google-cloud-core.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-spanner</artifactId>
+ <version>${spanner.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util</artifactId>
<version>${google-cloud-bigdataoss.version}</version>
@@ -871,7 +1141,7 @@
<artifactId>google-api-services-dataflow</artifactId>
<version>${dataflow.version}</version>
<exclusions>
- <!-- Exclude an old version of guava that is being pulled in by a transitive
+ <!-- Exclude an old version of guava that is being pulled in by a transitive
dependency of google-api-client -->
<exclusion>
<groupId>com.google.guava</groupId>
@@ -885,7 +1155,7 @@
<artifactId>google-api-services-clouddebugger</artifactId>
<version>${clouddebugger.version}</version>
<exclusions>
- <!-- Exclude an old version of guava that is being pulled in by a transitive
+ <!-- Exclude an old version of guava that is being pulled in by a transitive
dependency of google-api-client -->
<exclusion>
<groupId>com.google.guava</groupId>
@@ -901,6 +1171,12 @@
</dependency>
<dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>grpc-google-common-protos</artifactId>
<version>${grpc-google-common-protos.version}</version>
@@ -976,6 +1252,12 @@
</dependency>
<dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>${args4j.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
@@ -986,13 +1268,67 @@
<artifactId>byte-buddy</artifactId>
<version>1.6.8</version>
</dependency>
-
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-expression</artifactId>
<version>${spring.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy-java.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-common_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.clients.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>${commons.csv.version}</version>
+ </dependency>
+
<!-- Testing -->
<dependency>
@@ -1062,6 +1398,27 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -1087,7 +1444,7 @@
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -1131,7 +1488,7 @@
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.6.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
@@ -1186,7 +1543,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
- <version>3.0.2</version>
+ <version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<id>default-jar</id>
@@ -1207,6 +1564,19 @@
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>${maven-license-plugin.version}</version>
+ <configuration>
+ <licenseMerges>
+ <licenseMerge>The Apache Software License, version 2.0|Apache License, Version 2.0|Apache 2.0|Apache License 2.0|Apache|Apache-2.0|Apache License Version 2.0|Apache License Version 2|Apache Software License - Version 2.0|Apache 2.0 License|the Apache License, ASL Version 2.0|Apache v2|The Apache License, Version 2.0|http://www.apache.org/licenses/LICENSE-2.0.txt|ASL, version 2</licenseMerge>
+ <licenseMerge>MIT License|MIT|MIT License|The MIT License</licenseMerge>
+ <licenseMerge>CDDL 1.0|Common Development and Distribution License (CDDL) v1.0|COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0</licenseMerge>
+ </licenseMerges>
+ </configuration>
+ </plugin>
+
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.0.1</version>
@@ -1221,52 +1591,115 @@
here, we leave things simple here. -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
+ <version>${maven-javadoc-plugin.version}</version>
<configuration>
+ <!-- exclude this auto-generated packages from javadoc -->
+ <excludePackageNames>org.apache.beam.sdk.extensions.sql.impl.parser.impl</excludePackageNames>
<additionalparam>${beam.javadoc_opts}</additionalparam>
<windowtitle>Apache Beam SDK for Java, version ${project.version} API</windowtitle>
<doctitle>Apache Beam SDK for Java, version ${project.version}</doctitle>
<use>false</use>
<quiet>true</quiet>
+ <notimestamp>true</notimestamp>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
- <version>0.12</version>
+ <version>${apache-rat-plugin.version}</version>
+ <!-- Apache RAT checks all files in the project, only run once. -->
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
<configuration>
<reportFile>${project.build.directory}/${project.build.finalName}.rat</reportFile>
<excludeSubProjects>false</excludeSubProjects>
<consoleOutput>true</consoleOutput>
<useDefaultExcludes>true</useDefaultExcludes>
+
+ <!--
+ Keep excludes in sync with .gitignore, with consistent
+ order and sections for easy cross-checking.
+
+ Patterns are relative to $PWD, not the RAT ${basedir},
+ so each _must_ be prefixed with `**` or `${project.basedir}`.
+ -->
<excludes>
- <!-- Keep exclude sync with .gitignore -->
+ <!-- .gitignore: Ignore files generated by the Maven build process -->
<exclude>**/target/**/*</exclude>
+ <exclude>**/bin/**/*</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
- <exclude>**/hs_err_pid*.log</exclude>
- <exclude>.github/**/*</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>**/.idea/**/*</exclude>
+
+ <!-- .gitignore: Ignore files generated by the Python build process -->
+ <exclude>**/*.pyc</exclude>
+ <exclude>**/*.pyo</exclude>
+ <exclude>**/*.pyd</exclude>
<exclude>**/*.egg-info/**/*</exclude>
+ <exclude>**/.eggs/**/*</exclude>
+ <exclude>**/nose-*.egg/**/*</exclude>
+ <exclude>**/.tox/**/*</exclude>
+ <exclude>**/build/**/*</exclude>
+ <exclude>**/dist/**/*</exclude>
+ <exclude>**/distribute-*/**/*</exclude>
+ <exclude>**/env/**/*</exclude>
+ <exclude>sdks/python/**/*.c</exclude>
+ <exclude>sdks/python/**/*.so</exclude>
+ <exclude>sdks/python/LICENSE</exclude>
+ <exclude>sdks/python/NOTICE</exclude>
+ <exclude>sdks/python/README.md</exclude>
+ <exclude>sdks/python/apache_beam/portability/api/*pb2*.*</exclude>
+
+ <!-- .gitignore: Ignore IntelliJ files. -->
+ <exclude>**/idea/**/*</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/*.ipr</exclude>
+ <exclude>**/*.iws</exclude>
+
+ <!-- .gitignore: Ignore Eclipse files. -->
+ <exclude>**/.classpath</exclude>
+ <exclude>**/.project</exclude>
+ <exclude>**/.factorypath</exclude>
+ <exclude>**/.checkstyle</exclude>
+ <exclude>**/.fbExcludeFilterFile</exclude>
+ <exclude>**/.apt_generated/**/*</exclude>
+ <exclude>**/.settings/**/*</exclude>
+
+ <!-- .gitignore: Ignore Visual Studio Code files. -->
+ <exclude>**/.vscode/*/**</exclude>
+
+ <!-- .gitignore: Hotspot VM leaves this log in a non-target directory when java crashes -->
+ <exclude>**/hs_err_pid*.log</exclude>
+
+ <!-- .gitignore: Ignore files that end with '~', since they
+ are most likely auto-save files produced by a text editor. -->
+ <exclude>**/*~</exclude>
+
+ <!-- .gitignore: Ignore MacOSX files. -->
+ <exclude>**/.DS_Store/**/*</exclude>
+
+ <!-- Ignore files we track but do not distribute -->
+ <exclude>.github/**/*</exclude>
+
<exclude>**/package-list</exclude>
<exclude>**/user.avsc</exclude>
<exclude>**/test/resources/**/*.txt</exclude>
<exclude>**/test/**/.placeholder</exclude>
- <exclude>.repository/**/*</exclude>
- <exclude>**/nose-*.egg/**/*</exclude>
- <exclude>**/.eggs/**/*</exclude>
- <exclude>**/.tox/**/*</exclude>
<!-- Default eclipse excludes neglect subprojects -->
- <exclude>**/.checkstyle</exclude>
- <exclude>**/.classpath</exclude>
- <exclude>**/.factorypath</exclude>
- <exclude>**/.project</exclude>
- <exclude>**/.settings/**/*</exclude>
<!-- Proto/grpc generated wrappers -->
- <exclude>**/sdks/python/apache_beam/runners/api/*.py</exclude>
+ <exclude>**/apache_beam/portability/api/*_pb2*.py</exclude>
+ <exclude>**/go/pkg/beam/model/**/*.pb.go</exclude>
+
+ <!-- VCF test files -->
+ <exclude>**/apache_beam/testing/data/vcf/*</exclude>
</excludes>
</configuration>
</plugin>
@@ -1358,7 +1791,7 @@
</configuration>
</plugin>
- <!-- This plugin's configuration tells the m2e plugin how to import this
+ <!-- This plugin's configuration tells the m2e plugin how to import this
Maven project into the Eclipse environment. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
@@ -1418,7 +1851,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<id>bundle-and-repackage</id>
@@ -1590,6 +2023,36 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>io.github.zlika</groupId>
+ <artifactId>reproducible-build-maven-plugin</artifactId>
+ <version>${reproducible-build-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>strip-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <version>2.1.6</version>
+ <extensions>true</extensions>
+ <configuration>
+ <goVersion>1.9</goVersion>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <version>1.3.5</version>
+ <!-- no executions by default. Use build-containers profile -->
+ </plugin>
</plugins>
</pluginManagement>
@@ -1597,7 +2060,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven-dependency-plugin.version}</version>
<executions>
<execution>
<goals><goal>analyze-only</goal></goals>
@@ -1605,6 +2068,11 @@
<!-- Ignore runtime-only dependencies in analysis -->
<ignoreNonCompile>true</ignoreNonCompile>
<failOnWarning>true</failOnWarning>
+
+ <!-- ignore jsr305 for both "used but undeclared" and "declared but unused" -->
+ <ignoredDependencies>
+ <ignoredDependency>com.google.code.findbugs:jsr305</ignoredDependency>
+ </ignoredDependencies>
</configuration>
</execution>
</executions>
@@ -1612,7 +2080,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
- <version>1.4.1</version>
+ <version>${maven-enforcer-plugin.version}</version>
<executions>
<execution>
<id>enforce</id>
@@ -1629,6 +2097,7 @@
version. Is not shaded, so safe to ignore.
-->
<exclude>jdk.tools:jdk.tools</exclude>
+ <exclude>com.google.auto.value:auto-value</exclude>
</excludes>
</enforceBytecodeVersion>
<requireJavaVersion>
@@ -1695,7 +2164,7 @@
</goals>
<configuration>
<outputDirectory>${basedir}/sdks/python</outputDirectory>
- <resources>
+ <resources>
<resource>
<directory>${basedir}</directory>
<includes>
@@ -1704,8 +2173,8 @@
<include>README.md</include>
</includes>
</resource>
- </resources>
- </configuration>
+ </resources>
+ </configuration>
</execution>
</executions>
</plugin>
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index f1a8a62..f70e67e 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -63,23 +63,27 @@
<version>${apex.malhar.version}</version>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.apex</groupId>
<artifactId>apex-engine</artifactId>
<version>${apex.core.version}</version>
<scope>runtime</scope>
+ <exclusions>
+ <!-- Fix build on JDK-9 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Beam -->
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<exclusions>
<exclusion>
@@ -175,7 +179,14 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -207,7 +218,6 @@
<groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
<excludedGroups>
org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
- org.apache.beam.sdk.testing.UsesStatefulParDo,
org.apache.beam.sdk.testing.UsesTimersInParDo,
org.apache.beam.sdk.testing.UsesSplittableParDo,
org.apache.beam.sdk.testing.UsesAttemptedMetrics,
@@ -244,12 +254,12 @@
<configuration>
<ignoredUsedUndeclaredDependencies>
<ignoredUsedUndeclaredDependency>org.apache.apex:apex-api:jar:${apex.core.version}</ignoredUsedUndeclaredDependency>
- <ignoredUsedUndeclaredDependency>org.apache.commons:commons-lang3::3.1</ignoredUsedUndeclaredDependency>
+ <ignoredUsedUndeclaredDependency>org.apache.commons:commons-lang3::${apache.commons.lang.version}</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>commons-io:commons-io:jar:2.4</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>com.esotericsoftware.kryo:kryo::${apex.kryo.version}</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>com.datatorrent:netlet::1.3.0</ignoredUsedUndeclaredDependency>
- <ignoredUsedUndeclaredDependency>org.slf4j:slf4j-api:jar:1.7.14</ignoredUsedUndeclaredDependency>
- <ignoredUsedUndeclaredDependency>org.apache.hadoop:hadoop-common:jar:2.6.0</ignoredUsedUndeclaredDependency>
+ <ignoredUsedUndeclaredDependency>org.slf4j:slf4j-api:jar:${slf4j.version}</ignoredUsedUndeclaredDependency>
+ <ignoredUsedUndeclaredDependency>org.apache.hadoop:hadoop-common:jar:${hadoop.version}</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>joda-time:joda-time:jar:2.4</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>com.google.guava:guava:jar:20.0</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
index 92f6e8f..8db7c7a 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
@@ -25,7 +25,7 @@
/**
* Options that configure the Apex pipeline.
*/
-public interface ApexPipelineOptions extends PipelineOptions, java.io.Serializable {
+public interface ApexPipelineOptions extends PipelineOptions {
@Description("set unique application name for Apex runner")
void setApplicationName(String name);
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index 2fd0b22..57d2593 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -39,12 +39,13 @@
import org.apache.apex.api.Launcher.AppHandle;
import org.apache.apex.api.Launcher.LaunchMode;
import org.apache.beam.runners.apex.translation.ApexPipelineTranslator;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.construction.PTransformMatchers;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.PrimitiveCreate;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.runners.core.construction.SplittableParDo;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.coders.Coder;
@@ -56,15 +57,12 @@
import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.View.AsIterable;
-import org.apache.beam.sdk.transforms.View.AsSingleton;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
@@ -110,20 +108,20 @@
new PrimitiveCreate.Factory()))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
- new StreamingViewAsSingleton.Factory()))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class),
new StreamingViewAsIterable.Factory()))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new StreamingCombineGloballyAsSingletonView.Factory()))
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class),
+ new StreamingWrapSingletonInList.Factory()))
.add(
PTransformOverride.of(
PTransformMatchers.splittableParDoMulti(),
new SplittableParDoOverrideFactory<>()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+ new SplittableParDoViaKeyedWorkItems.OverrideFactory<>()))
.build();
}
@@ -214,7 +212,7 @@
* @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
*/
public static class CreateApexPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private static final long serialVersionUID = 1L;
private PCollectionView<ViewT> view;
@@ -228,7 +226,12 @@
}
@Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
return view;
}
}
@@ -240,134 +243,61 @@
}
}
- private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
- extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
+ private static class StreamingWrapSingletonInList<T>
+ extends PTransform<PCollection<T>, PCollection<T>> {
private static final long serialVersionUID = 1L;
- Combine.GloballyAsSingletonView<InputT, OutputT> transform;
+ CreatePCollectionView<T, T> transform;
/**
* Builds an instance of this class from the overridden transform.
*/
- private StreamingCombineGloballyAsSingletonView(
- Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
+ private StreamingWrapSingletonInList(
+ CreatePCollectionView<T, T> transform) {
this.transform = transform;
}
@Override
- public PCollectionView<OutputT> expand(PCollection<InputT> input) {
- PCollection<OutputT> combined = input
- .apply(Combine.globally(transform.getCombineFn())
- .withoutDefaults().withFanout(transform.getFanout()));
-
- PCollectionView<OutputT> view = PCollectionViews.singletonView(combined,
- combined.getWindowingStrategy(), transform.getInsertDefault(),
- transform.getInsertDefault() ? transform.getCombineFn().defaultValue() : null,
- combined.getCoder());
- return combined.apply(ParDo.of(new WrapAsList<OutputT>()))
- .apply(CreateApexPCollectionView.<OutputT, OutputT> of(view));
+ public PCollection<T> expand(PCollection<T> input) {
+ input
+ .apply(ParDo.of(new WrapAsList<T>()))
+ .apply(CreateApexPCollectionView.<List<T>, T>of(transform.getView()));
+ return input;
}
@Override
protected String getKindString() {
- return "StreamingCombineGloballyAsSingletonView";
- }
-
- static class Factory<InputT, OutputT>
- extends SingleInputOutputOverrideFactory<
- PCollection<InputT>, PCollectionView<OutputT>,
- Combine.GloballyAsSingletonView<InputT, OutputT>> {
- @Override
- public PTransformReplacement<PCollection<InputT>, PCollectionView<OutputT>>
- getReplacementTransform(
- AppliedPTransform<
- PCollection<InputT>, PCollectionView<OutputT>,
- GloballyAsSingletonView<InputT, OutputT>>
- transform) {
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- new StreamingCombineGloballyAsSingletonView<>(transform.getTransform()));
- }
- }
- }
-
- private static class StreamingViewAsSingleton<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
- private static final long serialVersionUID = 1L;
-
- private View.AsSingleton<T> transform;
-
- public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<T> expand(PCollection<T> input) {
- Combine.Globally<T, T> combine = Combine
- .globally(new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
- if (!transform.hasDefaultValue()) {
- combine = combine.withoutDefaults();
- }
- return input.apply(combine.asSingletonView());
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsSingleton";
- }
-
- private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
- private boolean hasDefaultValue;
- private T defaultValue;
-
- SingletonCombine(boolean hasDefaultValue, T defaultValue) {
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- }
-
- @Override
- public T apply(T left, T right) {
- throw new IllegalArgumentException("PCollection with more than one element "
- + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
- + "combine the PCollection into a single value");
- }
-
- @Override
- public T identity() {
- if (hasDefaultValue) {
- return defaultValue;
- } else {
- throw new IllegalArgumentException("Empty PCollection accessed as a singleton view. "
- + "Consider setting withDefault to provide a default value");
- }
- }
+ return "StreamingWrapSingletonInList";
}
static class Factory<T>
extends SingleInputOutputOverrideFactory<
- PCollection<T>, PCollectionView<T>, View.AsSingleton<T>> {
+ PCollection<T>, PCollection<T>,
+ CreatePCollectionView<T, T>> {
@Override
- public PTransformReplacement<PCollection<T>, PCollectionView<T>> getReplacementTransform(
- AppliedPTransform<PCollection<T>, PCollectionView<T>, AsSingleton<T>> transform) {
+ public PTransformReplacement<PCollection<T>, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PCollection<T>, PCollection<T>, CreatePCollectionView<T, T>>
+ transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new StreamingViewAsSingleton<>(transform.getTransform()));
+ new StreamingWrapSingletonInList<>(transform.getTransform()));
}
}
}
private static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
+ extends PTransform<PCollection<T>, PCollection<T>> {
private static final long serialVersionUID = 1L;
+ private final PCollectionView<Iterable<T>> view;
- private StreamingViewAsIterable() {}
+ private StreamingViewAsIterable(PCollectionView<Iterable<T>> view) {
+ this.view = view;
+ }
@Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(input, input.getWindowingStrategy(), input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateApexPCollectionView.<T, Iterable<T>> of(view));
+ public PCollection<T> expand(PCollection<T> input) {
+ return ((PCollection<T>)
+ input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults()))
+ .apply(CreateApexPCollectionView.<T, Iterable<T>>of(view));
}
@Override
@@ -377,15 +307,17 @@
static class Factory<T>
extends SingleInputOutputOverrideFactory<
- PCollection<T>, PCollectionView<Iterable<T>>, View.AsIterable<T>> {
+ PCollection<T>, PCollection<T>, CreatePCollectionView<T, Iterable<T>>> {
@Override
- public PTransformReplacement<PCollection<T>, PCollectionView<Iterable<T>>>
+ public PTransformReplacement<PCollection<T>, PCollection<T>>
getReplacementTransform(
- AppliedPTransform<PCollection<T>, PCollectionView<Iterable<T>>, AsIterable<T>>
+ AppliedPTransform<
+ PCollection<T>, PCollection<T>,
+ CreatePCollectionView<T, Iterable<T>>>
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new StreamingViewAsIterable<T>());
+ new StreamingViewAsIterable<T>(transform.getTransform().getView()));
}
}
}
@@ -447,8 +379,9 @@
public PTransformReplacement<PCollection<InputT>, PCollectionTuple> getReplacementTransform(
AppliedPTransform<PCollection<InputT>, PCollectionTuple, MultiOutput<InputT, OutputT>>
transform) {
- return PTransformReplacement.of(PTransformReplacements.getSingletonMainInput(transform),
- new SplittableParDo<>(transform.getTransform()));
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform),
+ SplittableParDo.forAppliedParDo(transform));
}
@Override
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
index cc24ddd..6ed61cf 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
@@ -19,6 +19,7 @@
import com.datatorrent.api.DAG;
import java.io.IOException;
+import javax.annotation.Nullable;
import org.apache.apex.api.Launcher.AppHandle;
import org.apache.apex.api.Launcher.ShutdownMode;
import org.apache.beam.sdk.Pipeline;
@@ -52,7 +53,8 @@
}
@Override
- public State waitUntilFinish(Duration duration) {
+ @Nullable
+ public State waitUntilFinish(@Nullable Duration duration) {
long timeout = (duration == null || duration.getMillis() < 1) ? Long.MAX_VALUE
: System.currentTimeMillis() + duration.getMillis();
try {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
index b3a6d1c..02f53ec 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
@@ -25,7 +25,8 @@
import org.apache.beam.runners.apex.ApexRunner.CreateApexPCollectionView;
import org.apache.beam.runners.apex.translation.operators.ApexProcessFnOperator;
import org.apache.beam.runners.apex.translation.operators.ApexReadUnboundedInputOperator;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems;
import org.apache.beam.runners.core.construction.PrimitiveCreate;
import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter;
import org.apache.beam.sdk.Pipeline;
@@ -38,7 +39,6 @@
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +48,7 @@
* into Apex logical plan {@link DAG}.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
-public class ApexPipelineTranslator implements Pipeline.PipelineVisitor {
+public class ApexPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
private static final Logger LOG = LoggerFactory.getLogger(ApexPipelineTranslator.class);
/**
@@ -63,9 +63,9 @@
static {
// register TransformTranslators
registerTransformTranslator(ParDo.MultiOutput.class, new ParDoTranslator<>());
- registerTransformTranslator(SplittableParDo.ProcessElements.class,
+ registerTransformTranslator(SplittableParDoViaKeyedWorkItems.ProcessElements.class,
new ParDoTranslator.SplittableProcessElementsTranslator());
- registerTransformTranslator(SplittableParDo.GBKIntoKeyedWorkItems.class,
+ registerTransformTranslator(GBKIntoKeyedWorkItems.class,
new GBKIntoKeyedWorkItemsTranslator());
registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
@@ -109,7 +109,7 @@
throw new UnsupportedOperationException(
"no translator registered for " + transform);
}
- translationContext.setCurrentTransform(node);
+ translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
translator.translate(transform, translationContext);
}
@@ -153,7 +153,6 @@
unboundedSource, true, context.getPipelineOptions());
context.addOperator(operator, operator.output);
}
-
}
private static class CreateApexPCollectionViewTranslator<ElemT, ViewT>
@@ -161,11 +160,10 @@
private static final long serialVersionUID = 1L;
@Override
- public void translate(CreateApexPCollectionView<ElemT, ViewT> transform,
- TranslationContext context) {
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.addView(view);
- LOG.debug("view {}", view.getName());
+ public void translate(
+ CreateApexPCollectionView<ElemT, ViewT> transform, TranslationContext context) {
+ context.addView(transform.getView());
+ LOG.debug("view {}", transform.getView().getName());
}
}
@@ -176,18 +174,17 @@
@Override
public void translate(
CreatePCollectionView<ElemT, ViewT> transform, TranslationContext context) {
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.addView(view);
- LOG.debug("view {}", view.getName());
+ context.addView(transform.getView());
+ LOG.debug("view {}", transform.getView().getName());
}
}
private static class GBKIntoKeyedWorkItemsTranslator<K, InputT>
- implements TransformTranslator<SplittableParDo.GBKIntoKeyedWorkItems<K, InputT>> {
+ implements TransformTranslator<GBKIntoKeyedWorkItems<K, InputT>> {
@Override
public void translate(
- SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform, TranslationContext context) {
+ GBKIntoKeyedWorkItems<K, InputT> transform, TranslationContext context) {
// https://issues.apache.org/jira/browse/BEAM-1850
ApexProcessFnOperator<KV<K, InputT>> operator = ApexProcessFnOperator.toKeyedWorkItems(
context.getPipelineOptions());
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslator.java
index 440b801..189cb65 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslator.java
@@ -110,8 +110,12 @@
}
if (collections.size() > 2) {
- PCollection<T> intermediateCollection = intermediateCollection(collection,
- collection.getCoder());
+ PCollection<T> intermediateCollection =
+ PCollection.createPrimitiveOutputInternal(
+ collection.getPipeline(),
+ collection.getWindowingStrategy(),
+ collection.isBounded(),
+ collection.getCoder());
context.addOperator(operator, operator.out, intermediateCollection);
remainingCollections.add(intermediateCollection);
} else {
@@ -135,11 +139,4 @@
}
}
- static <T> PCollection<T> intermediateCollection(PCollection<T> input, Coder<T> outputCoder) {
- PCollection<T> output = PCollection.createPrimitiveOutputInternal(input.getPipeline(),
- input.getWindowingStrategy(), input.isBounded());
- output.setCoder(outputCoder);
- return output;
- }
-
}
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
index 9133cb6..dd4bd67 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
@@ -30,15 +30,12 @@
import java.util.Map.Entry;
import org.apache.beam.runners.apex.ApexRunner;
import org.apache.beam.runners.apex.translation.operators.ApexParDoOperator;
-import org.apache.beam.runners.core.SplittableParDo;
-import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
@@ -64,15 +61,6 @@
String.format(
"%s does not support splittable DoFn: %s", ApexRunner.class.getSimpleName(), doFn));
}
- if (signature.stateDeclarations().size() > 0) {
- throw new UnsupportedOperationException(
- String.format(
- "Found %s annotations on %s, but %s cannot yet be used with state in the %s.",
- DoFn.StateId.class.getSimpleName(),
- doFn.getClass().getName(),
- DoFn.class.getSimpleName(),
- ApexRunner.class.getSimpleName()));
- }
if (signature.timerDeclarations().size() > 0) {
throw new UnsupportedOperationException(
@@ -87,10 +75,6 @@
Map<TupleTag<?>, PValue> outputs = context.getOutputs();
PCollection<InputT> input = context.getInput();
List<PCollectionView<?>> sideInputs = transform.getSideInputs();
- Coder<InputT> inputCoder = input.getCoder();
- WindowedValueCoder<InputT> wvInputCoder =
- FullWindowedValueCoder.of(
- inputCoder, input.getWindowingStrategy().getWindowFn().windowCoder());
ApexParDoOperator<InputT, OutputT> operator = new ApexParDoOperator<>(
context.getPipelineOptions(),
@@ -99,7 +83,7 @@
transform.getAdditionalOutputTags().getAll(),
input.getWindowingStrategy(),
sideInputs,
- wvInputCoder,
+ input.getCoder(),
context.getStateBackend());
Map<PCollection<?>, OutputPort<?>> ports = Maps.newHashMapWithExpectedSize(outputs.size());
@@ -132,23 +116,18 @@
}
}
- static class SplittableProcessElementsTranslator<InputT, OutputT,
- RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
- implements TransformTranslator<SplittableParDo.ProcessElements<InputT, OutputT,
- RestrictionT, TrackerT>> {
+ static class SplittableProcessElementsTranslator<
+ InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+ implements TransformTranslator<ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
@Override
public void translate(
- SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
+ ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
TranslationContext context) {
Map<TupleTag<?>, PValue> outputs = context.getOutputs();
PCollection<InputT> input = context.getInput();
List<PCollectionView<?>> sideInputs = transform.getSideInputs();
- Coder<InputT> inputCoder = input.getCoder();
- WindowedValueCoder<InputT> wvInputCoder =
- FullWindowedValueCoder.of(
- inputCoder, input.getWindowingStrategy().getWindowFn().windowCoder());
@SuppressWarnings({ "rawtypes", "unchecked" })
DoFn<InputT, OutputT> doFn = (DoFn) transform.newProcessFn(transform.getFn());
@@ -159,7 +138,7 @@
transform.getAdditionalOutputTags().getAll(),
input.getWindowingStrategy(),
sideInputs,
- wvInputCoder,
+ input.getCoder(),
context.getStateBackend());
Map<PCollection<?>, OutputPort<?>> ports = Maps.newHashMapWithExpectedSize(outputs.size());
@@ -242,8 +221,11 @@
}
PCollection<Object> resultCollection =
- FlattenPCollectionTranslator.intermediateCollection(
- firstSideInput, firstSideInput.getCoder());
+ PCollection.createPrimitiveOutputInternal(
+ firstSideInput.getPipeline(),
+ firstSideInput.getWindowingStrategy(),
+ firstSideInput.isBounded(),
+ firstSideInput.getCoder());
FlattenPCollectionTranslator.flattenCollections(
sourceCollections, unionTags, resultCollection, context);
return resultCollection;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index a5e3028..94d13e1 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,9 +34,9 @@
import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -77,8 +77,8 @@
this.pipelineOptions = pipelineOptions;
}
- public void setCurrentTransform(TransformHierarchy.Node treeNode) {
- this.currentTransform = treeNode.toAppliedPTransform();
+ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
+ this.currentTransform = transform;
}
public ApexPipelineOptions getPipelineOptions() {
@@ -94,7 +94,8 @@
}
public <InputT extends PValue> InputT getInput() {
- return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
+ return (InputT)
+ Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
}
public Map<TupleTag<?>, PValue> getOutputs() {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 1d48e20..5c0d72f 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -33,7 +33,6 @@
import org.apache.beam.runners.apex.ApexPipelineOptions;
import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
-import org.apache.beam.runners.apex.translation.utils.SerializablePipelineOptions;
import org.apache.beam.runners.core.NullSideInputReader;
import org.apache.beam.runners.core.OutputWindowedValue;
import org.apache.beam.runners.core.ReduceFnRunner;
@@ -41,7 +40,8 @@
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.TimerInternals.TimerData;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.sdk.coders.Coder;
@@ -149,7 +149,9 @@
@Override
public void setup(OperatorContext context) {
- this.traceTuples = ApexStreamTuple.Logging.isDebugEnabled(serializedOptions.get(), this);
+ this.traceTuples =
+ ApexStreamTuple.Logging.isDebugEnabled(
+ serializedOptions.get().as(ApexPipelineOptions.class), this);
}
@Override
@@ -163,7 +165,7 @@
windowingStrategy,
ExecutableTriggerStateMachine.create(
TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger()))),
+ TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
stateInternalsFactory.stateInternalsForKey(key),
timerInternals,
new OutputWindowedValue<KV<K, Iterable<V>>>() {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
index 7fee0d5..a66bb5b 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
@@ -40,7 +40,6 @@
import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
import org.apache.beam.runners.apex.translation.utils.NoOpStepContext;
-import org.apache.beam.runners.apex.translation.utils.SerializablePipelineOptions;
import org.apache.beam.runners.apex.translation.utils.StateInternalsProxy;
import org.apache.beam.runners.apex.translation.utils.ValueAndCoderKryoSerializable;
import org.apache.beam.runners.core.DoFnRunner;
@@ -55,7 +54,7 @@
import org.apache.beam.runners.core.SideInputHandler;
import org.apache.beam.runners.core.SideInputReader;
import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateInternalsFactory;
import org.apache.beam.runners.core.StateNamespace;
@@ -64,6 +63,7 @@
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.TimerInternals.TimerData;
import org.apache.beam.runners.core.TimerInternalsFactory;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.ListCoder;
@@ -73,11 +73,14 @@
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
@@ -133,7 +136,7 @@
List<TupleTag<?>> additionalOutputTags,
WindowingStrategy<?, ?> windowingStrategy,
List<PCollectionView<?>> sideInputs,
- Coder<WindowedValue<InputT>> inputCoder,
+ Coder<InputT> linputCoder,
ApexStateBackend stateBackend
) {
this.pipelineOptions = new SerializablePipelineOptions(pipelineOptions);
@@ -151,22 +154,33 @@
throw new UnsupportedOperationException(msg);
}
- Coder<List<WindowedValue<InputT>>> listCoder = ListCoder.of(inputCoder);
+ WindowedValueCoder<InputT> wvCoder =
+ FullWindowedValueCoder.of(
+ linputCoder, this.windowingStrategy.getWindowFn().windowCoder());
+ Coder<List<WindowedValue<InputT>>> listCoder = ListCoder.of(wvCoder);
this.pushedBack = new ValueAndCoderKryoSerializable<>(new ArrayList<WindowedValue<InputT>>(),
listCoder);
- this.inputCoder = inputCoder;
+ this.inputCoder = wvCoder;
TimerInternals.TimerDataCoder timerCoder =
TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
this.currentKeyTimerInternals = new ApexTimerInternals<>(timerCoder);
- if (doFn instanceof SplittableParDo.ProcessFn) {
+ if (doFn instanceof ProcessFn) {
// we know that it is keyed on String
Coder<?> keyCoder = StringUtf8Coder.of();
this.currentKeyStateInternals = new StateInternalsProxy<>(
stateBackend.newStateInternalsFactory(keyCoder));
+ } else {
+ DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
+ if (signature.usesState()) {
+ checkArgument(linputCoder instanceof KvCoder, "keyed input required for stateful DoFn");
+ @SuppressWarnings("rawtypes")
+ Coder<?> keyCoder = ((KvCoder) linputCoder).getKeyCoder();
+ this.currentKeyStateInternals = new StateInternalsProxy<>(
+ stateBackend.newStateInternalsFactory(keyCoder));
+ }
}
-
}
@SuppressWarnings("unused") // for Kryo
@@ -359,10 +373,7 @@
}
}
if (sideInputs.isEmpty()) {
- if (traceTuples) {
- LOG.debug("\nemitting watermark {}\n", mark);
- }
- output.emit(mark);
+ outputWatermark(mark);
return;
}
@@ -370,16 +381,28 @@
Math.min(pushedBackWatermark.get(), currentInputWatermark);
if (potentialOutputWatermark > currentOutputWatermark) {
currentOutputWatermark = potentialOutputWatermark;
- if (traceTuples) {
- LOG.debug("\nemitting watermark {}\n", currentOutputWatermark);
+ outputWatermark(ApexStreamTuple.WatermarkTuple.of(currentOutputWatermark));
+ }
+ }
+
+ private void outputWatermark(ApexStreamTuple.WatermarkTuple<?> mark) {
+ if (traceTuples) {
+ LOG.debug("\nemitting {}\n", mark);
+ }
+ output.emit(mark);
+ if (!additionalOutputPortMapping.isEmpty()) {
+ for (DefaultOutputPort<ApexStreamTuple<?>> additionalOutput :
+ additionalOutputPortMapping.values()) {
+ additionalOutput.emit(mark);
}
- output.emit(ApexStreamTuple.WatermarkTuple.of(currentOutputWatermark));
}
}
@Override
public void setup(OperatorContext context) {
- this.traceTuples = ApexStreamTuple.Logging.isDebugEnabled(pipelineOptions.get(), this);
+ this.traceTuples =
+ ApexStreamTuple.Logging.isDebugEnabled(
+ pipelineOptions.get().as(ApexPipelineOptions.class), this);
SideInputReader sideInputReader = NullSideInputReader.of(sideInputs);
if (!sideInputs.isEmpty()) {
sideInputHandler = new SideInputHandler(sideInputs, sideInputStateInternals);
@@ -445,15 +468,15 @@
pushbackDoFnRunner =
SimplePushbackSideInputDoFnRunner.create(doFnRunner, sideInputs, sideInputHandler);
- if (doFn instanceof SplittableParDo.ProcessFn) {
+ if (doFn instanceof ProcessFn) {
@SuppressWarnings("unchecked")
StateInternalsFactory<String> stateInternalsFactory =
(StateInternalsFactory<String>) this.currentKeyStateInternals.getFactory();
@SuppressWarnings({ "rawtypes", "unchecked" })
- SplittableParDo.ProcessFn<InputT, OutputT, Object, RestrictionTracker<Object>>
- splittableDoFn = (SplittableParDo.ProcessFn) doFn;
+ ProcessFn<InputT, OutputT, Object, RestrictionTracker<Object>>
+ splittableDoFn = (ProcessFn) doFn;
splittableDoFn.setStateInternalsFactory(stateInternalsFactory);
TimerInternalsFactory<String> timerInternalsFactory = new TimerInternalsFactory<String>() {
@Override
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
index 1549560..21fb9d2 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
@@ -30,8 +30,8 @@
import org.apache.beam.runners.apex.ApexPipelineOptions;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple.DataTuple;
-import org.apache.beam.runners.apex.translation.utils.SerializablePipelineOptions;
import org.apache.beam.runners.apex.translation.utils.ValuesSource;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -119,7 +119,9 @@
@Override
public void setup(OperatorContext context) {
- this.traceTuples = ApexStreamTuple.Logging.isDebugEnabled(pipelineOptions.get(), this);
+ this.traceTuples =
+ ApexStreamTuple.Logging.isDebugEnabled(
+ pipelineOptions.get().as(ApexPipelineOptions.class), this);
try {
reader = source.createReader(this.pipelineOptions.get(), null);
available = reader.start();
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
index 18ea8e4..978a793 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
@@ -37,7 +37,6 @@
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTag.StateBinder;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.coders.ListCoder;
@@ -141,7 +140,6 @@
namespace,
address,
accumCoder,
- key,
combineFn
);
}
@@ -184,7 +182,7 @@
// TODO: reuse input
Input input = new Input(buf);
try {
- return coder.decode(input, Context.OUTER);
+ return coder.decode(input);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -195,7 +193,7 @@
public void writeValue(T input) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
try {
- coder.encode(input, output, Context.OUTER);
+ coder.encode(input, output);
stateTable.put(namespace.stringKey(), address.getId(), output.toByteArray());
} catch (IOException e) {
throw new RuntimeException(e);
@@ -306,15 +304,13 @@
private final class ApexCombiningState<K, InputT, AccumT, OutputT>
extends AbstractState<AccumT>
implements CombiningState<InputT, AccumT, OutputT> {
- private final K key;
private final CombineFn<InputT, AccumT, OutputT> combineFn;
private ApexCombiningState(StateNamespace namespace,
StateTag<CombiningState<InputT, AccumT, OutputT>> address,
Coder<AccumT> coder,
- K key, CombineFn<InputT, AccumT, OutputT> combineFn) {
+ CombineFn<InputT, AccumT, OutputT> combineFn) {
super(namespace, address, coder);
- this.key = key;
this.combineFn = combineFn;
}
@@ -330,8 +326,7 @@
@Override
public void add(InputT input) {
- AccumT accum = getAccum();
- combineFn.addInput(accum, input);
+ AccumT accum = combineFn.addInput(getAccum(), input);
writeValue(accum);
}
@@ -431,7 +426,7 @@
/**
* Serializable state for internals (namespace to state tag to coded value).
*/
- private Map<Slice, Table<String, String, byte[]>> perKeyState = new HashMap<>();
+ private Map<Slice, HashBasedTable<String, String, byte[]>> perKeyState = new HashMap<>();
private final Coder<K> keyCoder;
private ApexStateInternalsFactory(Coder<K> keyCoder) {
@@ -451,7 +446,7 @@
} catch (CoderException e) {
throw new RuntimeException(e);
}
- Table<String, String, byte[]> stateTable = perKeyState.get(keyBytes);
+ HashBasedTable<String, String, byte[]> stateTable = perKeyState.get(keyBytes);
if (stateTable == null) {
stateTable = HashBasedTable.create();
perKeyState.put(keyBytes, stateTable);
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
index 721eecd..b49e4da 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
@@ -17,49 +17,18 @@
*/
package org.apache.beam.runners.apex.translation.utils;
-import java.io.IOException;
import java.io.Serializable;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
/**
- * Serializable {@link ExecutionContext.StepContext} that does nothing.
+ * Serializable {@link StepContext} that does nothing.
*/
-public class NoOpStepContext implements ExecutionContext.StepContext, Serializable {
+public class NoOpStepContext implements StepContext, Serializable {
private static final long serialVersionUID = 1L;
@Override
- public String getStepName() {
- return null;
- }
-
- @Override
- public String getTransformName() {
- return null;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {
- }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder, W window, Coder<W> windowCoder) throws
- IOException {
-
- }
-
- @Override
public StateInternals stateInternals() {
return null;
}
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
deleted file mode 100644
index 46b04fc..0000000
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.apex.translation.utils;
-
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.beam.runners.apex.ApexPipelineOptions;
-import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.common.ReflectHelpers;
-
-/**
- * A wrapper to enable serialization of {@link PipelineOptions}.
- */
-public class SerializablePipelineOptions implements Externalizable {
-
- /* Used to ensure we initialize file systems exactly once, because it's a slow operation. */
- private static final AtomicBoolean FILE_SYSTEMS_INTIIALIZED = new AtomicBoolean(false);
-
- private transient ApexPipelineOptions pipelineOptions;
-
- public SerializablePipelineOptions(ApexPipelineOptions pipelineOptions) {
- this.pipelineOptions = pipelineOptions;
- }
-
- public SerializablePipelineOptions() {
- }
-
- public ApexPipelineOptions get() {
- return this.pipelineOptions;
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeUTF(createMapper().writeValueAsString(pipelineOptions));
- }
-
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- String s = in.readUTF();
- this.pipelineOptions = createMapper().readValue(s, PipelineOptions.class)
- .as(ApexPipelineOptions.class);
-
- if (FILE_SYSTEMS_INTIIALIZED.compareAndSet(false, true)) {
- FileSystems.setDefaultPipelineOptions(pipelineOptions);
- }
- }
-
- /**
- * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
- * for user specified configuration injection into the ObjectMapper. This supports user custom
- * types on {@link PipelineOptions}.
- */
- private static ObjectMapper createMapper() {
- return new ObjectMapper().registerModules(
- ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
- }
-}
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
index 41f027f..193da74 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
@@ -77,11 +77,7 @@
}
@Override
- public void validate() {
- }
-
- @Override
- public Coder<T> getDefaultOutputCoder() {
+ public Coder<T> getOutputCoder() {
return iterableCoder.getElemCoder();
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
index 68ec2ea..adaf67b 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
@@ -43,12 +43,15 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
/**
* Test for dependency resolution for pipeline execution on YARN.
*/
public class ApexYarnLauncherTest {
+ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
@Test
public void testGetYarnDeployDependencies() throws Exception {
@@ -119,10 +122,9 @@
@Test
public void testCreateJar() throws Exception {
- File baseDir = new File("./target/testCreateJar");
- File srcDir = new File(baseDir, "src");
+ File baseDir = tmpFolder.newFolder("target", "testCreateJar");
+ File srcDir = tmpFolder.newFolder("target", "testCreateJar", "src");
String file1 = "file1";
- FileUtils.forceMkdir(srcDir);
FileUtils.write(new File(srcDir, file1), "file1");
File jarFile = new File(baseDir, "test.jar");
@@ -134,6 +136,5 @@
Assert.assertTrue("manifest", Files.isRegularFile(zipfs.getPath(JarFile.MANIFEST_NAME)));
Assert.assertTrue("file1", Files.isRegularFile(zipfs.getPath(file1)));
}
-
}
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
index c590a2e..7949129 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
@@ -55,11 +55,7 @@
}
@Override
- public void validate() {
- }
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
index e76096e..ba75746 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
@@ -123,11 +123,15 @@
options.setInputFile(new File(inputFile).getAbsolutePath());
String outputFilePrefix = "target/wordcountresult.txt";
options.setOutput(outputFilePrefix);
- WordCountTest.main(TestPipeline.convertToArgs(options));
File outFile1 = new File(outputFilePrefix + "-00000-of-00002");
File outFile2 = new File(outputFilePrefix + "-00001-of-00002");
- Assert.assertTrue(outFile1.exists() && outFile2.exists());
+ Assert.assertTrue(!outFile1.exists() || outFile1.delete());
+ Assert.assertTrue(!outFile2.exists() || outFile2.delete());
+
+ WordCountTest.main(TestPipeline.convertToArgs(options));
+
+ Assert.assertTrue("result files exist", outFile1.exists() && outFile2.exists());
HashSet<String> results = new HashSet<>();
results.addAll(FileUtils.readLines(outFile1));
results.addAll(FileUtils.readLines(outFile2));
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
index 206b430..63a218b 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
@@ -59,9 +59,9 @@
WindowingStrategy<?, ?> ws = WindowingStrategy.of(FixedWindows.of(
Duration.standardSeconds(10)));
- PCollection<KV<String, Integer>> input = PCollection.createPrimitiveOutputInternal(pipeline,
- ws, IsBounded.BOUNDED);
- input.setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
+ PCollection<KV<String, Integer>> input =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline, ws, IsBounded.BOUNDED, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
ApexGroupByKeyOperator<String, Integer> operator = new ApexGroupByKeyOperator<>(options,
input, new ApexStateInternals.ApexStateBackend()
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
index 929778a..1ad9622 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
@@ -53,7 +53,6 @@
@Test
public void test() throws Exception {
ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
- options.setApplicationName("FlattenPCollection");
options.setRunner(ApexRunner.class);
Pipeline p = Pipeline.create(options);
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/GroupByKeyTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/GroupByKeyTranslatorTest.java
index 9c61b47..516ae79 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/GroupByKeyTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/GroupByKeyTranslatorTest.java
@@ -149,11 +149,7 @@
}
@Override
- public void validate() {
- }
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
index 736b0e7..73382e3 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
@@ -42,7 +42,6 @@
import org.apache.beam.runners.apex.translation.utils.ApexStateInternals;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
@@ -202,7 +201,6 @@
.as(ApexPipelineOptions.class);
options.setRunner(TestApexRunner.class);
Pipeline pipeline = Pipeline.create(options);
- Coder<WindowedValue<Integer>> coder = WindowedValue.getValueOnlyCoder(VarIntCoder.of());
PCollectionView<Integer> singletonView = pipeline.apply(Create.of(1))
.apply(Sum.integersGlobally().asSingletonView());
@@ -215,7 +213,7 @@
TupleTagList.empty().getAll(),
WindowingStrategy.globalDefault(),
Collections.<PCollectionView<?>>singletonList(singletonView),
- coder,
+ VarIntCoder.of(),
new ApexStateInternals.ApexStateBackend());
operator.setup(null);
operator.beginWindow(0);
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
index a7e64af4..87aa8c2 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
@@ -18,350 +18,109 @@
package org.apache.beam.runners.apex.translation.utils;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import com.datatorrent.lib.util.KryoCloneUtils;
-import java.util.Arrays;
-import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
-import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateInternalsFactory;
-import org.apache.beam.runners.core.StateMerging;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateNamespaceForTest;
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTags;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.state.BagState;
-import org.apache.beam.sdk.state.CombiningState;
-import org.apache.beam.sdk.state.GroupingState;
-import org.apache.beam.sdk.state.ReadableState;
import org.apache.beam.sdk.state.ValueState;
-import org.apache.beam.sdk.state.WatermarkHoldState;
-import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.hamcrest.Matchers;
-import org.joda.time.Instant;
-import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.Suite;
/**
* Tests for {@link ApexStateInternals}. This is based on the tests for
- * {@code InMemoryStateInternals}.
+ * {@code StateInternalsTest}.
*/
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ ApexStateInternalsTest.StandardStateInternalsTests.class,
+ ApexStateInternalsTest.OtherTests.class
+})
public class ApexStateInternalsTest {
- private static final BoundedWindow WINDOW_1 = new IntervalWindow(new Instant(0), new Instant(10));
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
- private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
- private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
- StateTags.value("stringValue", StringUtf8Coder.of());
- private static final StateTag<CombiningState<Integer, int[], Integer>>
- SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
- "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
- private static final StateTag<WatermarkHoldState>
- WATERMARK_EARLIEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_LATEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_EOW_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW);
-
- private ApexStateInternals<String> underTest;
-
- @Before
- public void initStateInternals() {
- underTest = new ApexStateInternals.ApexStateBackend()
+ private static StateInternals newStateInternals() {
+ return new ApexStateInternals.ApexStateBackend()
.newStateInternalsFactory(StringUtf8Coder.of())
- .stateInternalsForKey((String) null);
+ .stateInternalsForKey("dummyKey");
}
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ /**
+ * A standard StateInternals test. Ignore set and map tests.
+ */
+ @RunWith(JUnit4.class)
+ public static class StandardStateInternalsTests extends StateInternalsTest {
+ @Override
+ protected StateInternals createStateInternals() {
+ return newStateInternals();
+ }
- assertEquals(value, underTest.state(NAMESPACE_1, STRING_BAG_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, STRING_BAG_ADDR)));
+ @Override
+ @Ignore
+ public void testSet() {}
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello"));
+ @Override
+ @Ignore
+ public void testSetIsEmpty() {}
- value.add("world");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello", "world"));
+ @Override
+ @Ignore
+ public void testMergeSetIntoSource() {}
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertEquals(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), value);
+ @Override
+ @Ignore
+ public void testMergeSetIntoNewNamespace() {}
+ @Override
+ @Ignore
+ public void testMap() {}
+
+ @Override
+ @Ignore
+ public void testSetReadable() {}
+
+ @Override
+ @Ignore
+ public void testMapReadable() {}
}
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ /**
+ * A specific test of ApexStateInternalsTest.
+ */
+ @RunWith(JUnit4.class)
+ public static class OtherTests {
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
+ private static final StateNamespace NAMESPACE = new StateNamespaceForTest("ns");
+ private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
+ StateTags.value("stringValue", StringUtf8Coder.of());
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
+ @Test
+ public void testSerialization() throws Exception {
+ ApexStateInternals.ApexStateInternalsFactory<String> sif =
+ new ApexStateInternals.ApexStateBackend().
+ newStateInternalsFactory(StringUtf8Coder.of());
+ ApexStateInternals<String> keyAndState = sif.stateInternalsForKey("dummy");
+
+ ValueState<String> value = keyAndState.state(NAMESPACE, STRING_VALUE_ADDR);
+ assertEquals(keyAndState.state(NAMESPACE, STRING_VALUE_ADDR), value);
+ value.write("hello");
+
+ ApexStateInternals.ApexStateInternalsFactory<String> cloned;
+ assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(sif));
+ ApexStateInternals<String> clonedKeyAndState = cloned.stateInternalsForKey("dummy");
+
+ ValueState<String> clonedValue = clonedKeyAndState.state(NAMESPACE, STRING_VALUE_ADDR);
+ assertThat(clonedValue.read(), Matchers.equalTo("hello"));
+ assertEquals(clonedKeyAndState.state(NAMESPACE, STRING_VALUE_ADDR), value);
+ }
}
-
- @Test
- public void testMergeBagIntoSource() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
-
- // Reading the merged bag gets both the contents
- assertThat(bag1.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeBagIntoNewNamespace() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
- BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
-
- // Reading the merged bag gets both the contents
- assertThat(bag3.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag1.read(), Matchers.emptyIterable());
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testCombiningValue() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR)));
-
- assertThat(value.read(), Matchers.equalTo(0));
- value.add(2);
- assertThat(value.read(), Matchers.equalTo(2));
-
- value.add(3);
- assertThat(value.read(), Matchers.equalTo(5));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(0));
- assertEquals(underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR), value);
- }
-
- @Test
- public void testCombiningIsEmpty() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(5);
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeCombiningValueIntoSource() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- assertThat(value1.read(), Matchers.equalTo(11));
- assertThat(value2.read(), Matchers.equalTo(10));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1);
-
- assertThat(value1.read(), Matchers.equalTo(21));
- assertThat(value2.read(), Matchers.equalTo(0));
- }
-
- @Test
- public void testMergeCombiningValueIntoNewNamespace() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value3 =
- underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3);
-
- // Merging clears the old values and updates the result value.
- assertThat(value1.read(), Matchers.equalTo(0));
- assertThat(value2.read(), Matchers.equalTo(0));
- assertThat(value3.read(), Matchers.equalTo(21));
- }
-
- @Test
- public void testWatermarkEarliestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), Matchers.equalTo(new Instant(1000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), value);
- }
-
- @Test
- public void testWatermarkLatestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), Matchers.equalTo(new Instant(3000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), Matchers.equalTo(new Instant(3000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), value);
- }
-
- @Test
- public void testWatermarkEndOfWindowState() throws Exception {
- WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), value);
- }
-
- @Test
- public void testWatermarkStateIsEmpty() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(new Instant(1000));
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeEarliestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the merged value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value1, WINDOW_1);
-
- assertThat(value1.read(), Matchers.equalTo(new Instant(2000)));
- assertThat(value2.read(), Matchers.equalTo(null));
- }
-
- @Test
- public void testMergeLatestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value3 =
- underTest.state(NAMESPACE_3, WATERMARK_LATEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value3, WINDOW_1);
-
- // Merging clears the old values and updates the result value.
- assertThat(value3.read(), Matchers.equalTo(new Instant(5000)));
- assertThat(value1.read(), Matchers.equalTo(null));
- assertThat(value2.read(), Matchers.equalTo(null));
- }
-
- @Test
- public void testSerialization() throws Exception {
- ApexStateInternalsFactory<String> sif = new ApexStateBackend().
- newStateInternalsFactory(StringUtf8Coder.of());
- ApexStateInternals<String> keyAndState = sif.stateInternalsForKey("dummy");
-
- ValueState<String> value = keyAndState.state(NAMESPACE_1, STRING_VALUE_ADDR);
- assertEquals(keyAndState.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
- value.write("hello");
-
- ApexStateInternalsFactory<String> cloned;
- assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(sif));
- ApexStateInternals<String> clonedKeyAndState = cloned.stateInternalsForKey("dummy");
-
- ValueState<String> clonedValue = clonedKeyAndState.state(NAMESPACE_1, STRING_VALUE_ADDR);
- assertThat(clonedValue.read(), Matchers.equalTo("hello"));
- assertEquals(clonedKeyAndState.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
- }
-
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
index 288aade..4769829 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
@@ -63,11 +63,7 @@
}
@Override
- public void validate() {
- }
-
- @Override
- public Coder<T> getDefaultOutputCoder() {
+ public Coder<T> getOutputCoder() {
return coder;
}
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
deleted file mode 100644
index 118ff99..0000000
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.apex.translation.utils;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import com.datatorrent.common.util.FSStorageAgent;
-import com.esotericsoftware.kryo.serializers.FieldSerializer.Bind;
-import com.esotericsoftware.kryo.serializers.JavaSerializer;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.auto.service.AutoService;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import org.apache.beam.runners.apex.ApexPipelineOptions;
-import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.Description;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.Test;
-
-/**
- * Tests the serialization of PipelineOptions.
- */
-public class PipelineOptionsTest {
-
- /**
- * Interface for testing.
- */
- public interface MyOptions extends ApexPipelineOptions {
- @Description("Bla bla bla")
- @Default.String("Hello")
- String getTestOption();
- void setTestOption(String value);
- }
-
- private static class OptionsWrapper {
- private OptionsWrapper() {
- this(null); // required for Kryo
- }
- private OptionsWrapper(ApexPipelineOptions options) {
- this.options = new SerializablePipelineOptions(options);
- }
- @Bind(JavaSerializer.class)
- private final SerializablePipelineOptions options;
- }
-
- @Test
- public void testSerialization() {
- OptionsWrapper wrapper = new OptionsWrapper(
- PipelineOptionsFactory.fromArgs("--testOption=nothing").as(MyOptions.class));
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- FSStorageAgent.store(bos, wrapper);
-
- ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
- OptionsWrapper wrapperCopy = (OptionsWrapper) FSStorageAgent.retrieve(bis);
- assertNotNull(wrapperCopy.options);
- assertEquals("nothing", wrapperCopy.options.get().as(MyOptions.class).getTestOption());
- }
-
- @Test
- public void testSerializationWithUserCustomType() {
- OptionsWrapper wrapper = new OptionsWrapper(
- PipelineOptionsFactory.fromArgs("--jacksonIncompatible=\"testValue\"")
- .as(JacksonIncompatibleOptions.class));
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- FSStorageAgent.store(bos, wrapper);
-
- ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
- OptionsWrapper wrapperCopy = (OptionsWrapper) FSStorageAgent.retrieve(bis);
- assertNotNull(wrapperCopy.options);
- assertEquals("testValue",
- wrapperCopy.options.get().as(JacksonIncompatibleOptions.class)
- .getJacksonIncompatible().value);
- }
-
- /** PipelineOptions used to test auto registration of Jackson modules. */
- public interface JacksonIncompatibleOptions extends ApexPipelineOptions {
- JacksonIncompatible getJacksonIncompatible();
- void setJacksonIncompatible(JacksonIncompatible value);
- }
-
- /** A Jackson {@link Module} to test auto-registration of modules. */
- @AutoService(Module.class)
- public static class RegisteredTestModule extends SimpleModule {
- public RegisteredTestModule() {
- super("RegisteredTestModule");
- setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
- }
- }
-
- /** A class which Jackson does not know how to serialize/deserialize. */
- public static class JacksonIncompatible {
- private final String value;
- public JacksonIncompatible(String value) {
- this.value = value;
- }
- }
-
- /** A Jackson mixin used to add annotations to other classes. */
- @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
- @JsonSerialize(using = JacksonIncompatibleSerializer.class)
- public static final class JacksonIncompatibleMixin {}
-
- /** A Jackson deserializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleDeserializer extends
- JsonDeserializer<JacksonIncompatible> {
-
- @Override
- public JacksonIncompatible deserialize(JsonParser jsonParser,
- DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- return new JacksonIncompatible(jsonParser.readValueAs(String.class));
- }
- }
-
- /** A Jackson serializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
-
- @Override
- public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
- jsonGenerator.writeString(jacksonIncompatible.value);
- }
- }
-}
diff --git a/runners/core-construction-java/pom.xml b/runners/core-construction-java/pom.xml
index abf0b65..9f71959 100644
--- a/runners/core-construction-java/pom.xml
+++ b/runners/core-construction-java/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>beam-runners-parent</artifactId>
<groupId>org.apache.beam</groupId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -56,7 +56,12 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-job-management</artifactId>
</dependency>
<dependency>
@@ -65,11 +70,31 @@
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
@@ -90,11 +115,27 @@
</dependency>
<dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
@@ -114,6 +155,5 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
new file mode 100644
index 0000000..095b549
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.io.BaseEncoding;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.protobuf.ByteString;
+import io.grpc.Channel;
+import io.grpc.stub.StreamObserver;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.MessageDigest;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactChunk;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.CommitManifestRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceBlockingStub;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceStub;
+
+/** A client to stage files on an {@link ArtifactStagingServiceGrpc ArtifactService}. */
+public class ArtifactServiceStager {
+ // 2 MB per file-request
+ private static final int DEFAULT_BUFFER_SIZE = 2 * 1024 * 1024;
+
+ public static ArtifactServiceStager overChannel(Channel channel) {
+ return overChannel(channel, DEFAULT_BUFFER_SIZE);
+ }
+
+ /**
+ * Create a new ArtifactServiceStager with the specified buffer size. Useful for testing
+ * multi-part uploads.
+ *
+ * @param bufferSize the maximum size of the artifact chunk, in bytes.
+ */
+ static ArtifactServiceStager overChannel(Channel channel, int bufferSize) {
+ return new ArtifactServiceStager(channel, bufferSize);
+ }
+
+ private final int bufferSize;
+ private final ArtifactStagingServiceStub stub;
+ private final ArtifactStagingServiceBlockingStub blockingStub;
+ private final ListeningExecutorService executorService =
+ MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
+
+ private ArtifactServiceStager(Channel channel, int bufferSize) {
+ this.stub = ArtifactStagingServiceGrpc.newStub(channel);
+ this.blockingStub = ArtifactStagingServiceGrpc.newBlockingStub(channel);
+ this.bufferSize = bufferSize;
+ }
+
+ public void stage(Iterable<File> files) throws IOException, InterruptedException {
+ final Map<File, ListenableFuture<ArtifactMetadata>> futures = new HashMap<>();
+ for (File file : files) {
+ futures.put(file, executorService.submit(new StagingCallable(file)));
+ }
+ ListenableFuture<StagingResult> stagingResult =
+ Futures.whenAllComplete(futures.values()).call(new ExtractStagingResultsCallable(futures));
+ stageManifest(stagingResult);
+ }
+
+ private void stageManifest(ListenableFuture<StagingResult> stagingFuture)
+ throws InterruptedException {
+ try {
+ StagingResult stagingResult = stagingFuture.get();
+ if (stagingResult.isSuccess()) {
+ Manifest manifest =
+ Manifest.newBuilder().addAllArtifact(stagingResult.getMetadata()).build();
+ blockingStub.commitManifest(
+ CommitManifestRequest.newBuilder().setManifest(manifest).build());
+ } else {
+ RuntimeException failure =
+ new RuntimeException(
+ String.format(
+ "Failed to stage %s files: %s",
+ stagingResult.getFailures().size(), stagingResult.getFailures().keySet()));
+ for (Throwable t : stagingResult.getFailures().values()) {
+ failure.addSuppressed(t);
+ }
+ throw failure;
+ }
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private class StagingCallable implements Callable<ArtifactMetadata> {
+ private final File file;
+
+ private StagingCallable(File file) {
+ this.file = file;
+ }
+
+ @Override
+ public ArtifactMetadata call() throws Exception {
+ // TODO: Add Retries
+ PutArtifactResponseObserver responseObserver = new PutArtifactResponseObserver();
+ StreamObserver<PutArtifactRequest> requestObserver = stub.putArtifact(responseObserver);
+ ArtifactMetadata metadata = ArtifactMetadata.newBuilder().setName(file.getName()).build();
+ requestObserver.onNext(PutArtifactRequest.newBuilder().setMetadata(metadata).build());
+
+ MessageDigest md5Digest = MessageDigest.getInstance("MD5");
+ FileChannel channel = new FileInputStream(file).getChannel();
+ ByteBuffer readBuffer = ByteBuffer.allocate(bufferSize);
+ while (!responseObserver.isTerminal() && channel.position() < channel.size()) {
+ readBuffer.clear();
+ channel.read(readBuffer);
+ readBuffer.flip();
+ md5Digest.update(readBuffer);
+ readBuffer.rewind();
+ PutArtifactRequest request =
+ PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactChunk.newBuilder().setData(ByteString.copyFrom(readBuffer)).build())
+ .build();
+ requestObserver.onNext(request);
+ }
+
+ requestObserver.onCompleted();
+ responseObserver.awaitTermination();
+ if (responseObserver.err.get() != null) {
+ throw new RuntimeException(responseObserver.err.get());
+ }
+ return metadata.toBuilder().setMd5(BaseEncoding.base64().encode(md5Digest.digest())).build();
+ }
+
+ private class PutArtifactResponseObserver implements StreamObserver<PutArtifactResponse> {
+ private final CountDownLatch completed = new CountDownLatch(1);
+ private final AtomicReference<Throwable> err = new AtomicReference<>(null);
+
+ @Override
+ public void onNext(PutArtifactResponse value) {}
+
+ @Override
+ public void onError(Throwable t) {
+ err.set(t);
+ completed.countDown();
+ throw new RuntimeException(t);
+ }
+
+ @Override
+ public void onCompleted() {
+ completed.countDown();
+ }
+
+ public boolean isTerminal() {
+ return completed.getCount() == 0;
+ }
+
+ public void awaitTermination() throws InterruptedException {
+ completed.await();
+ }
+ }
+ }
+
+ private static class ExtractStagingResultsCallable implements Callable<StagingResult> {
+ private final Map<File, ListenableFuture<ArtifactMetadata>> futures;
+
+ private ExtractStagingResultsCallable(
+ Map<File, ListenableFuture<ArtifactMetadata>> futures) {
+ this.futures = futures;
+ }
+
+ @Override
+ public StagingResult call() throws Exception {
+ Set<ArtifactMetadata> metadata = new HashSet<>();
+ Map<File, Throwable> failures = new HashMap<>();
+ for (Entry<File, ListenableFuture<ArtifactMetadata>> stagedFileResult : futures.entrySet()) {
+ try {
+ metadata.add(stagedFileResult.getValue().get());
+ } catch (ExecutionException ee) {
+ failures.put(stagedFileResult.getKey(), ee.getCause());
+ } catch (InterruptedException ie) {
+ throw new AssertionError(
+ "This should never happen. " + "All of the futures are complete by construction", ie);
+ }
+ }
+ if (failures.isEmpty()) {
+ return StagingResult.success(metadata);
+ } else {
+ return StagingResult.failure(failures);
+ }
+ }
+ }
+
+ @AutoValue
+ abstract static class StagingResult {
+ static StagingResult success(Set<ArtifactMetadata> metadata) {
+ return new AutoValue_ArtifactServiceStager_StagingResult(
+ metadata, Collections.<File, Throwable>emptyMap());
+ }
+
+ static StagingResult failure(Map<File, Throwable> failures) {
+ return new AutoValue_ArtifactServiceStager_StagingResult(
+ null, failures);
+ }
+
+ boolean isSuccess() {
+ return getMetadata() != null;
+ }
+
+ @Nullable
+ abstract Set<ArtifactMetadata> getMetadata();
+
+ abstract Map<File, Throwable> getFailures();
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
new file mode 100644
index 0000000..2b00ce4
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.LengthPrefixCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
+
+/** Converts to and from Beam Runner API representations of {@link Coder Coders}. */
+public class CoderTranslation {
+ // This URN says that the coder is just a UDF blob this SDK understands
+ // TODO: standardize such things
+ public static final String JAVA_SERIALIZED_CODER_URN = "urn:beam:coders:javasdk:0.1";
+
+ // The URNs for coders which are shared across languages
+ @VisibleForTesting
+ static final BiMap<Class<? extends StructuredCoder>, String> KNOWN_CODER_URNS =
+ ImmutableBiMap.<Class<? extends StructuredCoder>, String>builder()
+ .put(ByteArrayCoder.class, "urn:beam:coders:bytes:0.1")
+ .put(KvCoder.class, "urn:beam:coders:kv:0.1")
+ .put(VarLongCoder.class, "urn:beam:coders:varint:0.1")
+ .put(IntervalWindowCoder.class, "urn:beam:coders:interval_window:0.1")
+ .put(IterableCoder.class, "urn:beam:coders:stream:0.1")
+ .put(LengthPrefixCoder.class, "urn:beam:coders:length_prefix:0.1")
+ .put(GlobalWindow.Coder.class, "urn:beam:coders:global_window:0.1")
+ .put(FullWindowedValueCoder.class, "urn:beam:coders:windowed_value:0.1")
+ .build();
+
+ @VisibleForTesting
+ static final Map<Class<? extends StructuredCoder>, CoderTranslator<? extends StructuredCoder>>
+ KNOWN_TRANSLATORS =
+ ImmutableMap
+ .<Class<? extends StructuredCoder>, CoderTranslator<? extends StructuredCoder>>
+ builder()
+ .put(ByteArrayCoder.class, CoderTranslators.atomic(ByteArrayCoder.class))
+ .put(VarLongCoder.class, CoderTranslators.atomic(VarLongCoder.class))
+ .put(IntervalWindowCoder.class, CoderTranslators.atomic(IntervalWindowCoder.class))
+ .put(GlobalWindow.Coder.class, CoderTranslators.atomic(GlobalWindow.Coder.class))
+ .put(KvCoder.class, CoderTranslators.kv())
+ .put(IterableCoder.class, CoderTranslators.iterable())
+ .put(LengthPrefixCoder.class, CoderTranslators.lengthPrefix())
+ .put(FullWindowedValueCoder.class, CoderTranslators.fullWindowedValue())
+ .build();
+
+ public static RunnerApi.MessageWithComponents toProto(Coder<?> coder) throws IOException {
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.Coder coderProto = toProto(coder, components);
+ return RunnerApi.MessageWithComponents.newBuilder()
+ .setCoder(coderProto)
+ .setComponents(components.toComponents())
+ .build();
+ }
+
+ public static RunnerApi.Coder toProto(
+ Coder<?> coder, @SuppressWarnings("unused") SdkComponents components) throws IOException {
+ if (KNOWN_CODER_URNS.containsKey(coder.getClass())) {
+ return toKnownCoder(coder, components);
+ }
+ return toCustomCoder(coder);
+ }
+
+ private static RunnerApi.Coder toKnownCoder(Coder<?> coder, SdkComponents components)
+ throws IOException {
+ checkArgument(
+ coder instanceof StructuredCoder,
+ "A Known %s must implement %s, but %s of class %s does not",
+ Coder.class.getSimpleName(),
+ StructuredCoder.class.getSimpleName(),
+ coder,
+ coder.getClass().getName());
+ StructuredCoder<?> stdCoder = (StructuredCoder<?>) coder;
+ CoderTranslator translator = KNOWN_TRANSLATORS.get(stdCoder.getClass());
+ List<String> componentIds = registerComponents(coder, translator, components);
+ return RunnerApi.Coder.newBuilder()
+ .addAllComponentCoderIds(componentIds)
+ .setSpec(
+ SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder().setUrn(KNOWN_CODER_URNS.get(stdCoder.getClass()))))
+ .build();
+ }
+
+ private static <T extends Coder<?>> List<String> registerComponents(
+ T coder, CoderTranslator<T> translator, SdkComponents components) throws IOException {
+ List<String> componentIds = new ArrayList<>();
+ for (Coder<?> component : translator.getComponents(coder)) {
+ componentIds.add(components.registerCoder(component));
+ }
+ return componentIds;
+ }
+
+ private static RunnerApi.Coder toCustomCoder(Coder<?> coder) throws IOException {
+ RunnerApi.Coder.Builder coderBuilder = RunnerApi.Coder.newBuilder();
+ return coderBuilder
+ .setSpec(
+ SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(JAVA_SERIALIZED_CODER_URN)
+ .setPayload(
+ ByteString.copyFrom(SerializableUtils.serializeToByteArray(coder)))
+ .build()))
+ .build();
+ }
+
+ public static Coder<?> fromProto(
+ RunnerApi.Coder protoCoder, RehydratedComponents components)
+ throws IOException {
+ String coderSpecUrn = protoCoder.getSpec().getSpec().getUrn();
+ if (coderSpecUrn.equals(JAVA_SERIALIZED_CODER_URN)) {
+ return fromCustomCoder(protoCoder);
+ }
+ return fromKnownCoder(protoCoder, components);
+ }
+
+ private static Coder<?> fromKnownCoder(RunnerApi.Coder coder, RehydratedComponents components)
+ throws IOException {
+ String coderUrn = coder.getSpec().getSpec().getUrn();
+ List<Coder<?>> coderComponents = new LinkedList<>();
+ for (String componentId : coder.getComponentCoderIdsList()) {
+ Coder<?> innerCoder = components.getCoder(componentId);
+ coderComponents.add(innerCoder);
+ }
+ Class<? extends StructuredCoder> coderType = KNOWN_CODER_URNS.inverse().get(coderUrn);
+ CoderTranslator<?> translator = KNOWN_TRANSLATORS.get(coderType);
+ checkArgument(
+ translator != null,
+ "Unknown Coder URN %s. Known URNs: %s",
+ coderUrn,
+ KNOWN_CODER_URNS.values());
+ return translator.fromComponents(coderComponents);
+ }
+
+ private static Coder<?> fromCustomCoder(RunnerApi.Coder protoCoder) throws IOException {
+ return (Coder<?>)
+ SerializableUtils.deserializeFromByteArray(
+ protoCoder
+ .getSpec()
+ .getSpec()
+ .getPayload()
+ .toByteArray(),
+ "Custom Coder Bytes");
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java
deleted file mode 100644
index 6c2caa9..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableMap;
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.LengthPrefixCoder;
-import org.apache.beam.sdk.coders.StructuredCoder;
-import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
-import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-
-/** Converts to and from Beam Runner API representations of {@link Coder Coders}. */
-public class Coders {
- // This URN says that the coder is just a UDF blob this SDK understands
- // TODO: standardize such things
- public static final String JAVA_SERIALIZED_CODER_URN = "urn:beam:coders:javasdk:0.1";
-
- // The URNs for coders which are shared across languages
- @VisibleForTesting
- static final BiMap<Class<? extends StructuredCoder>, String> KNOWN_CODER_URNS =
- ImmutableBiMap.<Class<? extends StructuredCoder>, String>builder()
- .put(ByteArrayCoder.class, "urn:beam:coders:bytes:0.1")
- .put(KvCoder.class, "urn:beam:coders:kv:0.1")
- .put(VarLongCoder.class, "urn:beam:coders:varint:0.1")
- .put(IntervalWindowCoder.class, "urn:beam:coders:interval_window:0.1")
- .put(IterableCoder.class, "urn:beam:coders:stream:0.1")
- .put(LengthPrefixCoder.class, "urn:beam:coders:length_prefix:0.1")
- .put(GlobalWindow.Coder.class, "urn:beam:coders:global_window:0.1")
- .put(FullWindowedValueCoder.class, "urn:beam:coders:windowed_value:0.1")
- .build();
-
- @VisibleForTesting
- static final Map<Class<? extends StructuredCoder>, CoderTranslator<? extends StructuredCoder>>
- KNOWN_TRANSLATORS =
- ImmutableMap
- .<Class<? extends StructuredCoder>, CoderTranslator<? extends StructuredCoder>>
- builder()
- .put(ByteArrayCoder.class, CoderTranslators.atomic(ByteArrayCoder.class))
- .put(VarLongCoder.class, CoderTranslators.atomic(VarLongCoder.class))
- .put(IntervalWindowCoder.class, CoderTranslators.atomic(IntervalWindowCoder.class))
- .put(GlobalWindow.Coder.class, CoderTranslators.atomic(GlobalWindow.Coder.class))
- .put(KvCoder.class, CoderTranslators.kv())
- .put(IterableCoder.class, CoderTranslators.iterable())
- .put(LengthPrefixCoder.class, CoderTranslators.lengthPrefix())
- .put(FullWindowedValueCoder.class, CoderTranslators.fullWindowedValue())
- .build();
-
- public static RunnerApi.MessageWithComponents toProto(Coder<?> coder) throws IOException {
- SdkComponents components = SdkComponents.create();
- RunnerApi.Coder coderProto = toProto(coder, components);
- return RunnerApi.MessageWithComponents.newBuilder()
- .setCoder(coderProto)
- .setComponents(components.toComponents())
- .build();
- }
-
- public static RunnerApi.Coder toProto(
- Coder<?> coder, @SuppressWarnings("unused") SdkComponents components) throws IOException {
- if (KNOWN_CODER_URNS.containsKey(coder.getClass())) {
- return toKnownCoder(coder, components);
- }
- return toCustomCoder(coder);
- }
-
- private static RunnerApi.Coder toKnownCoder(Coder<?> coder, SdkComponents components)
- throws IOException {
- checkArgument(
- coder instanceof StructuredCoder,
- "A Known %s must implement %s, but %s of class %s does not",
- Coder.class.getSimpleName(),
- StructuredCoder.class.getSimpleName(),
- coder,
- coder.getClass().getName());
- StructuredCoder<?> stdCoder = (StructuredCoder<?>) coder;
- CoderTranslator translator = KNOWN_TRANSLATORS.get(stdCoder.getClass());
- List<String> componentIds = registerComponents(coder, translator, components);
- return RunnerApi.Coder.newBuilder()
- .addAllComponentCoderIds(componentIds)
- .setSpec(
- SdkFunctionSpec.newBuilder()
- .setSpec(
- FunctionSpec.newBuilder().setUrn(KNOWN_CODER_URNS.get(stdCoder.getClass()))))
- .build();
- }
-
- private static <T extends Coder<?>> List<String> registerComponents(
- T coder, CoderTranslator<T> translator, SdkComponents components) throws IOException {
- List<String> componentIds = new ArrayList<>();
- for (Coder<?> component : translator.getComponents(coder)) {
- componentIds.add(components.registerCoder(component));
- }
- return componentIds;
- }
-
- private static RunnerApi.Coder toCustomCoder(Coder<?> coder) throws IOException {
- RunnerApi.Coder.Builder coderBuilder = RunnerApi.Coder.newBuilder();
- return coderBuilder
- .setSpec(
- SdkFunctionSpec.newBuilder()
- .setSpec(
- FunctionSpec.newBuilder()
- .setUrn(JAVA_SERIALIZED_CODER_URN)
- .setParameter(
- Any.pack(
- BytesValue.newBuilder()
- .setValue(
- ByteString.copyFrom(
- SerializableUtils.serializeToByteArray(coder)))
- .build()))))
- .build();
- }
-
- public static Coder<?> fromProto(RunnerApi.Coder protoCoder, Components components)
- throws IOException {
- String coderSpecUrn = protoCoder.getSpec().getSpec().getUrn();
- if (coderSpecUrn.equals(JAVA_SERIALIZED_CODER_URN)) {
- return fromCustomCoder(protoCoder, components);
- }
- return fromKnownCoder(protoCoder, components);
- }
-
- private static Coder<?> fromKnownCoder(RunnerApi.Coder coder, Components components)
- throws IOException {
- String coderUrn = coder.getSpec().getSpec().getUrn();
- List<Coder<?>> coderComponents = new LinkedList<>();
- for (String componentId : coder.getComponentCoderIdsList()) {
- Coder<?> innerCoder = fromProto(components.getCodersOrThrow(componentId), components);
- coderComponents.add(innerCoder);
- }
- Class<? extends StructuredCoder> coderType = KNOWN_CODER_URNS.inverse().get(coderUrn);
- CoderTranslator<?> translator = KNOWN_TRANSLATORS.get(coderType);
- checkArgument(
- translator != null,
- "Unknown Coder URN %s. Known URNs: %s",
- coderUrn,
- KNOWN_CODER_URNS.values());
- return translator.fromComponents(coderComponents);
- }
-
- private static Coder<?> fromCustomCoder(
- RunnerApi.Coder protoCoder, @SuppressWarnings("unused") Components components)
- throws IOException {
- return (Coder<?>)
- SerializableUtils.deserializeFromByteArray(
- protoCoder
- .getSpec()
- .getSpec()
- .getParameter()
- .unpack(BytesValue.class)
- .getValue()
- .toByteArray(),
- "Custom Coder Bytes");
- }
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
new file mode 100644
index 0000000..ff431fc
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.COMBINE_TRANSFORM_URN;
+
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.CombinePayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.AppliedCombineFn;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/**
+ * Methods for translating between {@link Combine.PerKey} {@link PTransform PTransforms} and {@link
+ * RunnerApi.CombinePayload} protos.
+ */
+public class CombineTranslation {
+
+ public static final String JAVA_SERIALIZED_COMBINE_FN_URN = "urn:beam:combinefn:javasdk:v1";
+
+ /** A {@link TransformPayloadTranslator} for {@link Combine.PerKey}. */
+ public static class CombinePayloadTranslator
+ implements PTransformTranslation.TransformPayloadTranslator<Combine.PerKey<?, ?, ?>> {
+ public static TransformPayloadTranslator create() {
+ return new CombinePayloadTranslator();
+ }
+
+ private CombinePayloadTranslator() {}
+
+ @Override
+ public String getUrn(Combine.PerKey<?, ?, ?> transform) {
+ return COMBINE_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>> transform, SdkComponents components)
+ throws IOException {
+ return FunctionSpec.newBuilder()
+ .setUrn(COMBINE_TRANSFORM_URN)
+ .setPayload(payloadForCombine((AppliedPTransform) transform, components).toByteString())
+ .build();
+ }
+
+ @Override
+ public PTransformTranslation.RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ checkArgument(
+ protoTransform.getSpec() != null,
+ "%s received transform with null spec",
+ getClass().getSimpleName());
+ checkArgument(protoTransform.getSpec().getUrn().equals(COMBINE_TRANSFORM_URN));
+ return new RawCombine<>(
+ CombinePayload.parseFrom(protoTransform.getSpec().getPayload()), rehydratedComponents);
+ }
+
+ /** Registers {@link CombinePayloadTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(Combine.PerKey.class, new CombinePayloadTranslator());
+ }
+
+ @Override
+ public Map<String, ? extends TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.singletonMap(COMBINE_TRANSFORM_URN, new CombinePayloadTranslator());
+ }
+ }
+ }
+
+ /**
+ * These methods drive to-proto translation for both Java SDK transforms and rehydrated
+ * transforms.
+ */
+ interface CombineLike {
+ RunnerApi.SdkFunctionSpec getCombineFn();
+
+ Coder<?> getAccumulatorCoder();
+
+ Map<String, RunnerApi.SideInput> getSideInputs();
+ }
+
+ /** Produces a {@link RunnerApi.CombinePayload} from a portable {@link CombineLike}. */
+ static RunnerApi.CombinePayload payloadForCombineLike(
+ CombineLike combine, SdkComponents components) throws IOException {
+ return RunnerApi.CombinePayload.newBuilder()
+ .setAccumulatorCoderId(components.registerCoder(combine.getAccumulatorCoder()))
+ .putAllSideInputs(combine.getSideInputs())
+ .setCombineFn(combine.getCombineFn())
+ .build();
+ }
+
+ static <K, InputT, OutputT> CombinePayload payloadForCombine(
+ final AppliedPTransform<
+ PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>,
+ Combine.PerKey<K, InputT, OutputT>>
+ combine,
+ SdkComponents components)
+ throws IOException {
+
+ return payloadForCombineLike(
+ new CombineLike() {
+ @Override
+ public SdkFunctionSpec getCombineFn() {
+ return SdkFunctionSpec.newBuilder()
+ // TODO: Set Java SDK Environment
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN)
+ .setPayload(
+ ByteString.copyFrom(
+ SerializableUtils.serializeToByteArray(
+ combine.getTransform().getFn())))
+ .build())
+ .build();
+ }
+
+ @Override
+ public Coder<?> getAccumulatorCoder() {
+ GlobalCombineFn<?, ?, ?> combineFn = combine.getTransform().getFn();
+ try {
+ return extractAccumulatorCoder(combineFn, (AppliedPTransform) combine);
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Map<String, SideInput> getSideInputs() {
+ Map<String, SideInput> sideInputs = new HashMap<>();
+ for (PCollectionView<?> sideInput : combine.getTransform().getSideInputs()) {
+ sideInputs.put(
+ sideInput.getTagInternal().getId(), ParDoTranslation.toProto(sideInput));
+ }
+ return sideInputs;
+ }
+ },
+ components);
+ }
+
+ private static class RawCombine<K, InputT, AccumT, OutputT>
+ extends PTransformTranslation.RawPTransform<
+ PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>
+ implements CombineLike {
+
+ private final transient RehydratedComponents rehydratedComponents;
+ private final FunctionSpec spec;
+ private final CombinePayload payload;
+ private final Coder<AccumT> accumulatorCoder;
+
+ private RawCombine(CombinePayload payload, RehydratedComponents rehydratedComponents) {
+ this.rehydratedComponents = rehydratedComponents;
+ this.payload = payload;
+ this.spec =
+ FunctionSpec.newBuilder()
+ .setUrn(COMBINE_TRANSFORM_URN)
+ .setPayload(payload.toByteString())
+ .build();
+
+ // Eagerly extract the coder to throw a good exception here
+ try {
+ this.accumulatorCoder =
+ (Coder<AccumT>) rehydratedComponents.getCoder(payload.getAccumulatorCoderId());
+ } catch (IOException exc) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Failure extracting accumulator coder with id '%s' for %s",
+ payload.getAccumulatorCoderId(), Combine.class.getSimpleName()),
+ exc);
+ }
+ }
+
+ @Override
+ public String getUrn() {
+ return COMBINE_TRANSFORM_URN;
+ }
+
+ @Nonnull
+ @Override
+ public FunctionSpec getSpec() {
+ return spec;
+ }
+
+ @Override
+ public RunnerApi.FunctionSpec migrate(SdkComponents sdkComponents) throws IOException {
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(COMBINE_TRANSFORM_URN)
+ .setPayload(payloadForCombineLike(this, sdkComponents).toByteString())
+ .build();
+ }
+
+ @Override
+ public SdkFunctionSpec getCombineFn() {
+ return payload.getCombineFn();
+ }
+
+ @Override
+ public Coder<?> getAccumulatorCoder() {
+ return accumulatorCoder;
+ }
+
+ @Override
+ public Map<String, SideInput> getSideInputs() {
+ return payload.getSideInputsMap();
+ }
+ }
+
+ @VisibleForTesting
+ static CombinePayload toProto(
+ AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>> combine, SdkComponents sdkComponents)
+ throws IOException {
+ GlobalCombineFn<?, ?, ?> combineFn = combine.getTransform().getFn();
+ try {
+ Coder<?> accumulatorCoder = extractAccumulatorCoder(combineFn, (AppliedPTransform) combine);
+ Map<String, SideInput> sideInputs = new HashMap<>();
+ return RunnerApi.CombinePayload.newBuilder()
+ .setAccumulatorCoderId(sdkComponents.registerCoder(accumulatorCoder))
+ .putAllSideInputs(sideInputs)
+ .setCombineFn(toProto(combineFn))
+ .build();
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
+ GlobalCombineFn<InputT, AccumT, ?> combineFn,
+ AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform)
+ throws CannotProvideCoderException {
+ @SuppressWarnings("unchecked")
+ PCollection<KV<K, InputT>> mainInput =
+ (PCollection<KV<K, InputT>>)
+ Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
+ KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) mainInput.getCoder();
+ return AppliedCombineFn.withInputCoder(
+ combineFn,
+ transform.getPipeline().getCoderRegistry(),
+ inputCoder,
+ transform.getTransform().getSideInputs(),
+ ((PCollection<?>) Iterables.getOnlyElement(transform.getOutputs().values()))
+ .getWindowingStrategy())
+ .getAccumulatorCoder();
+ }
+
+ public static SdkFunctionSpec toProto(GlobalCombineFn<?, ?, ?> combineFn) {
+ return SdkFunctionSpec.newBuilder()
+ // TODO: Set Java SDK Environment URN
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn)))
+ .build())
+ .build();
+ }
+
+ public static Coder<?> getAccumulatorCoder(
+ CombinePayload payload, RehydratedComponents components) throws IOException {
+ String id = payload.getAccumulatorCoderId();
+ return components.getCoder(id);
+ }
+
+ public static Coder<?> getAccumulatorCoder(AppliedPTransform<?, ?, ?> transform)
+ throws IOException {
+ SdkComponents sdkComponents = SdkComponents.create();
+ String id = getCombinePayload(transform, sdkComponents).getAccumulatorCoderId();
+ Components components = sdkComponents.toComponents();
+ return CoderTranslation.fromProto(
+ components.getCodersOrThrow(id), RehydratedComponents.forComponents(components));
+ }
+
+ public static GlobalCombineFn<?, ?, ?> getCombineFn(CombinePayload payload) throws IOException {
+ checkArgument(payload.getCombineFn().getSpec().getUrn().equals(JAVA_SERIALIZED_COMBINE_FN_URN));
+ return (GlobalCombineFn<?, ?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ payload.getCombineFn().getSpec().getPayload().toByteArray(), "CombineFn");
+ }
+
+ public static GlobalCombineFn<?, ?, ?> getCombineFn(AppliedPTransform<?, ?, ?> transform)
+ throws IOException {
+ return getCombineFn(getCombinePayload(transform));
+ }
+
+ private static CombinePayload getCombinePayload(AppliedPTransform<?, ?, ?> transform)
+ throws IOException {
+ return getCombinePayload(transform, SdkComponents.create());
+ }
+
+ private static CombinePayload getCombinePayload(
+ AppliedPTransform<?, ?, ?> transform, SdkComponents components) throws IOException {
+ return CombinePayload.parseFrom(
+ PTransformTranslation.toProto(
+ transform, Collections.<AppliedPTransform<?, ?, ?>>emptyList(), components)
+ .getSpec()
+ .getPayload());
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
new file mode 100644
index 0000000..709cb8a
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.service.AutoService;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/**
+ * Utility methods for translating a {@link View} transforms to and from {@link RunnerApi}
+ * representations.
+ *
+ * @deprecated this should generally be done as part of {@link ParDo} translation, or moved into a
+ * dedicated runners-core-construction auxiliary class
+ */
+@Deprecated
+public class CreatePCollectionViewTranslation {
+
+ /**
+ * @deprecated Since {@link CreatePCollectionView} is not a part of the Beam model, there is no
+ * SDK-agnostic specification. Using this method means your runner is tied to Java.
+ */
+ @Deprecated
+ public static <ElemT, ViewT> PCollectionView<ViewT> getView(
+ AppliedPTransform<
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>>
+ application)
+ throws IOException {
+
+ RunnerApi.PTransform transformProto =
+ PTransformTranslation.toProto(
+ application,
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+ SdkComponents.create());
+
+ checkArgument(
+ PTransformTranslation.CREATE_VIEW_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
+ "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"",
+ PCollectionView.class.getSimpleName(),
+ application.getTransform(),
+ application.getFullName(),
+ transformProto.getSpec().getUrn());
+
+ return (PCollectionView<ViewT>)
+ SerializableUtils.deserializeFromByteArray(
+ transformProto
+ .getSpec()
+ .getPayload()
+ .toByteArray(),
+ PCollectionView.class.getSimpleName());
+ }
+
+ /**
+ * @deprecated runners should move away from translating `CreatePCollectionView` and treat this
+ * as part of the translation for a `ParDo` side input.
+ */
+ @Deprecated
+ static class CreatePCollectionViewTranslator
+ extends TransformPayloadTranslator.WithDefaultRehydration<View.CreatePCollectionView<?, ?>> {
+ @Override
+ public String getUrn(View.CreatePCollectionView<?, ?> transform) {
+ return PTransformTranslation.CREATE_VIEW_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform,
+ SdkComponents components) {
+ return FunctionSpec.newBuilder()
+ .setUrn(getUrn(transform.getTransform()))
+ .setPayload(
+ ByteString.copyFrom(
+ SerializableUtils.serializeToByteArray(transform.getTransform().getView())))
+ .build();
+ }
+ }
+
+ /**
+ * Registers {@link CreatePCollectionViewTranslator}.
+ *
+ * @deprecated runners should move away from translating `CreatePCollectionView` and treat this
+ * as part of the translation for a `ParDo` side input.
+ */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ @Deprecated
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(
+ View.CreatePCollectionView.class, new CreatePCollectionViewTranslator());
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
new file mode 100644
index 0000000..8a9394d
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.protobuf.Any;
+import com.google.protobuf.BoolValue;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+/** Utilities for going to/from DisplayData protos. */
+public class DisplayDataTranslation {
+ public static RunnerApi.DisplayData toProto(DisplayData displayData) {
+ // TODO https://issues.apache.org/jira/browse/BEAM-2645
+ return RunnerApi.DisplayData.newBuilder()
+ .addItems(
+ RunnerApi.DisplayData.Item.newBuilder()
+ .setId(RunnerApi.DisplayData.Identifier.newBuilder().setKey("stubImplementation"))
+ .setLabel("Stub implementation")
+ .setType(RunnerApi.DisplayData.Type.Enum.BOOLEAN)
+ .setValue(Any.pack(BoolValue.newBuilder().setValue(true).build())))
+ .build();
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
new file mode 100644
index 0000000..972c453
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.auto.service.AutoService;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
+
+/**
+ * Utility methods for translating a {@link Assign} to and from {@link RunnerApi} representations.
+ */
+public class FlattenTranslator
+ extends TransformPayloadTranslator.WithDefaultRehydration<Flatten.PCollections<?>> {
+
+ public static TransformPayloadTranslator create() {
+ return new FlattenTranslator();
+ }
+
+ private FlattenTranslator() {}
+
+ @Override
+ public String getUrn(Flatten.PCollections<?> transform) {
+ return PTransformTranslation.FLATTEN_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Flatten.PCollections<?>> transform, SdkComponents components) {
+ return RunnerApi.FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build();
+ }
+
+ /** Registers {@link FlattenTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(Flatten.PCollections.class, new FlattenTranslator());
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ForwardingPTransform.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ForwardingPTransform.java
index ca25ba7..ccf41f3 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ForwardingPTransform.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ForwardingPTransform.java
@@ -18,7 +18,6 @@
package org.apache.beam.runners.core.construction;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -37,7 +36,16 @@
@Override
public OutputT expand(InputT input) {
- return delegate().expand(input);
+ OutputT res = delegate().expand(input);
+ if (res instanceof PCollection) {
+ PCollection pc = (PCollection) res;
+ try {
+ pc.setCoder(delegate().getDefaultOutputCoder(input, pc));
+ } catch (CannotProvideCoderException e) {
+ // Let coder inference happen later.
+ }
+ }
+ return res;
}
@Override
@@ -51,12 +59,6 @@
}
@Override
- public <T> Coder<T> getDefaultOutputCoder(InputT input, PCollection<T> output)
- throws CannotProvideCoderException {
- return delegate().getDefaultOutputCoder(input, output);
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
builder.delegate(delegate());
}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
new file mode 100644
index 0000000..0803ad3
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.auto.service.AutoService;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+
+/**
+ * Utility methods for translating a {@link GroupByKey} to and from {@link RunnerApi}
+ * representations.
+ */
+public class GroupByKeyTranslation {
+
+ static class GroupByKeyTranslator
+ extends TransformPayloadTranslator.WithDefaultRehydration<GroupByKey<?, ?>> {
+ @Override
+ public String getUrn(GroupByKey<?, ?> transform) {
+ return PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, GroupByKey<?, ?>> transform, SdkComponents components) {
+ return FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build();
+ }
+ }
+
+ /** Registers {@link GroupByKeyTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(GroupByKey.class, new GroupByKeyTranslator());
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
new file mode 100644
index 0000000..b85efe6
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import java.io.IOException;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+
+/**
+ * Utility methods for translating {@link PCollection PCollections} to and from Runner API protos.
+ */
+public class PCollectionTranslation {
+ private PCollectionTranslation() {}
+
+ public static RunnerApi.PCollection toProto(PCollection<?> pCollection, SdkComponents components)
+ throws IOException {
+ String coderId = components.registerCoder(pCollection.getCoder());
+ String windowingStrategyId =
+ components.registerWindowingStrategy(pCollection.getWindowingStrategy());
+ // TODO: Display Data
+
+ return RunnerApi.PCollection.newBuilder()
+ .setUniqueName(pCollection.getName())
+ .setCoderId(coderId)
+ .setIsBounded(toProto(pCollection.isBounded()))
+ .setWindowingStrategyId(windowingStrategyId)
+ .build();
+ }
+
+ public static PCollection<?> fromProto(
+ RunnerApi.PCollection pCollection, Pipeline pipeline, RehydratedComponents components)
+ throws IOException {
+
+ Coder<?> coder = components.getCoder(pCollection.getCoderId());
+ return PCollection.createPrimitiveOutputInternal(
+ pipeline,
+ components.getWindowingStrategy(pCollection.getWindowingStrategyId()),
+ fromProto(pCollection.getIsBounded()),
+ (Coder) coder);
+ }
+
+ public static IsBounded isBounded(RunnerApi.PCollection pCollection) {
+ return fromProto(pCollection.getIsBounded());
+ }
+
+ static RunnerApi.IsBounded.Enum toProto(IsBounded bounded) {
+ switch (bounded) {
+ case BOUNDED:
+ return RunnerApi.IsBounded.Enum.BOUNDED;
+ case UNBOUNDED:
+ return RunnerApi.IsBounded.Enum.UNBOUNDED;
+ default:
+ throw new IllegalArgumentException(
+ String.format("Unknown %s %s", IsBounded.class.getSimpleName(), bounded));
+ }
+ }
+
+ static IsBounded fromProto(RunnerApi.IsBounded.Enum isBounded) {
+ switch (isBounded) {
+ case BOUNDED:
+ return IsBounded.BOUNDED;
+ case UNBOUNDED:
+ return IsBounded.UNBOUNDED;
+ case UNRECOGNIZED:
+ default:
+ // Whether or not this enum cannot be recognized by the proto (due to the version of the
+ // generated code we link to) or the switch hasn't been updated to handle it,
+ // the situation is the same: we don't know what this IsBounded means
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ RunnerApi.IsBounded.class.getCanonicalName(),
+ IsBounded.class.getCanonicalName(),
+ isBounded));
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
deleted file mode 100644
index 0f2fcb7..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import java.io.IOException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * Utility methods for translating {@link PCollection PCollections} to and from Runner API protos.
- */
-public class PCollections {
- private PCollections() {}
-
- public static RunnerApi.PCollection toProto(PCollection<?> pCollection, SdkComponents components)
- throws IOException {
- String coderId = components.registerCoder(pCollection.getCoder());
- String windowingStrategyId =
- components.registerWindowingStrategy(pCollection.getWindowingStrategy());
- // TODO: Display Data
-
- return RunnerApi.PCollection.newBuilder()
- .setUniqueName(pCollection.getName())
- .setCoderId(coderId)
- .setIsBounded(toProto(pCollection.isBounded()))
- .setWindowingStrategyId(windowingStrategyId)
- .build();
- }
-
- public static IsBounded isBounded(RunnerApi.PCollection pCollection) {
- return fromProto(pCollection.getIsBounded());
- }
-
- public static Coder<?> getCoder(
- RunnerApi.PCollection pCollection, RunnerApi.Components components) throws IOException {
- return Coders.fromProto(components.getCodersOrThrow(pCollection.getCoderId()), components);
- }
-
- public static WindowingStrategy<?, ?> getWindowingStrategy(
- RunnerApi.PCollection pCollection, RunnerApi.Components components)
- throws InvalidProtocolBufferException {
- return WindowingStrategies.fromProto(
- components.getWindowingStrategiesOrThrow(pCollection.getWindowingStrategyId()), components);
- }
-
- private static RunnerApi.IsBounded toProto(IsBounded bounded) {
- switch (bounded) {
- case BOUNDED:
- return RunnerApi.IsBounded.BOUNDED;
- case UNBOUNDED:
- return RunnerApi.IsBounded.UNBOUNDED;
- default:
- throw new IllegalArgumentException(
- String.format("Unknown %s %s", IsBounded.class.getSimpleName(), bounded));
- }
- }
-
- private static IsBounded fromProto(RunnerApi.IsBounded isBounded) {
- switch (isBounded) {
- case BOUNDED:
- return IsBounded.BOUNDED;
- case UNBOUNDED:
- return IsBounded.UNBOUNDED;
- case UNRECOGNIZED:
- default:
- // Whether or not this enum cannot be recognized by the proto (due to the version of the
- // generated code we link to) or the switch hasn't been updated to handle it,
- // the situation is the same: we don't know what this IsBounded means
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- RunnerApi.IsBounded.class.getCanonicalName(),
- IsBounded.class.getCanonicalName(),
- isBounded));
- }
- }
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
index bfe24a0..0d27241 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
@@ -17,12 +17,14 @@
*/
package org.apache.beam.runners.core.construction;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.WRITE_FILES_TRANSFORM_URN;
+
import com.google.common.base.MoreObjects;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformMatcher;
import org.apache.beam.sdk.transforms.DoFn;
@@ -50,6 +52,34 @@
private PTransformMatchers() {}
/**
+ * Returns a {@link PTransformMatcher} that matches a {@link PTransform} if the URN of the
+ * {@link PTransform} is equal to the URN provided ot this matcher.
+ */
+ public static PTransformMatcher urnEqualTo(String urn) {
+ return new EqualUrnPTransformMatcher(urn);
+ }
+
+ private static class EqualUrnPTransformMatcher implements PTransformMatcher {
+ private final String urn;
+
+ private EqualUrnPTransformMatcher(String urn) {
+ this.urn = urn;
+ }
+
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ return urn.equals(PTransformTranslation.urnForTransformOrNull(application.getTransform()));
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("urn", urn)
+ .toString();
+ }
+ }
+
+ /**
* Returns a {@link PTransformMatcher} that matches a {@link PTransform} if the class of the
* {@link PTransform} is equal to the {@link Class} provided ot this matcher.
*/
@@ -151,6 +181,68 @@
}
/**
+ * A {@link PTransformMatcher} that matches a {@link ParDo} by URN if it has a splittable {@link
+ * DoFn}.
+ */
+ public static PTransformMatcher splittableParDo() {
+ return new PTransformMatcher() {
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ if (PTransformTranslation.PAR_DO_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+
+ try {
+ return ParDoTranslation.isSplittable(application);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s could not be translated",
+ PTransformTranslation.PAR_DO_TRANSFORM_URN),
+ e);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper("SplittableParDoMultiMatcher").toString();
+ }
+ };
+ }
+
+ /**
+ * A {@link PTransformMatcher} that matches a {@link ParDo} transform by URN
+ * and whether it contains state or timers as specified by {@link ParDoTranslation}.
+ */
+ public static PTransformMatcher stateOrTimerParDo() {
+ return new PTransformMatcher() {
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ if (PTransformTranslation.PAR_DO_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+
+ try {
+ return ParDoTranslation.usesStateOrTimers(application);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s could not be translated",
+ PTransformTranslation.PAR_DO_TRANSFORM_URN),
+ e);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper("StateOrTimerParDoMatcher").toString();
+ }
+ };
+ }
+
+ /**
* A {@link PTransformMatcher} that matches a {@link ParDo.MultiOutput} containing a {@link DoFn}
* that uses state or timers, as specified by {@link DoFnSignature#usesState()} and
* {@link DoFnSignature#usesTimers()}.
@@ -268,9 +360,18 @@
return new PTransformMatcher() {
@Override
public boolean matches(AppliedPTransform<?, ?, ?> application) {
- if (application.getTransform() instanceof WriteFiles) {
- WriteFiles write = (WriteFiles) application.getTransform();
- return write.getSharding() == null && write.getNumShards() == null;
+ if (WRITE_FILES_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+ try {
+ return WriteFilesTranslation.isRunnerDeterminedSharding(
+ (AppliedPTransform) application);
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s failed to parse: %s",
+ WRITE_FILES_TRANSFORM_URN, application.getTransform()),
+ exc);
+ }
}
return false;
}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
index 706a956..35bad15 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkArgument;
+import com.google.common.collect.Iterables;
import java.util.Map;
import java.util.Set;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -66,4 +67,9 @@
ignoredTags);
return mainInput;
}
+
+ public static <T> PCollection<T> getSingletonMainOutput(
+ AppliedPTransform<?, PCollection<T>, ? extends PTransform<?, PCollection<T>>> transform) {
+ return ((PCollection<T>) Iterables.getOnlyElement(transform.getOutputs().values()));
+ }
}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
new file mode 100644
index 0000000..a3a5a1f
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -0,0 +1,443 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Joiner;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Utilities for converting {@link PTransform PTransforms} to and from {@link RunnerApi Runner API
+ * protocol buffers}.
+ */
+public class PTransformTranslation {
+
+ public static final String PAR_DO_TRANSFORM_URN = "urn:beam:transform:pardo:v1";
+ public static final String FLATTEN_TRANSFORM_URN = "urn:beam:transform:flatten:v1";
+ public static final String GROUP_BY_KEY_TRANSFORM_URN = "urn:beam:transform:groupbykey:v1";
+ public static final String READ_TRANSFORM_URN = "urn:beam:transform:read:v1";
+ public static final String WINDOW_TRANSFORM_URN = "urn:beam:transform:window:v1";
+ public static final String TEST_STREAM_TRANSFORM_URN = "urn:beam:transform:teststream:v1";
+
+ // Not strictly a primitive transform
+ public static final String COMBINE_TRANSFORM_URN = "urn:beam:transform:combine:v1";
+
+ public static final String RESHUFFLE_URN = "urn:beam:transform:reshuffle:v1";
+
+ // Less well-known. And where shall these live?
+ public static final String WRITE_FILES_TRANSFORM_URN = "urn:beam:transform:write_files:0.1";
+
+ /**
+ * @deprecated runners should move away from translating `CreatePCollectionView` and treat this as
+ * part of the translation for a `ParDo` side input.
+ */
+ @Deprecated
+ public static final String CREATE_VIEW_TRANSFORM_URN = "urn:beam:transform:create_view:v1";
+
+ private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
+ KNOWN_PAYLOAD_TRANSLATORS = loadTransformPayloadTranslators();
+
+ private static final Map<String, TransformPayloadTranslator> KNOWN_REHYDRATORS =
+ loadTransformRehydrators();
+
+ private static final TransformPayloadTranslator<?> DEFAULT_REHYDRATOR =
+ new RawPTransformTranslator();
+
+ private static Map<Class<? extends PTransform>, TransformPayloadTranslator>
+ loadTransformPayloadTranslators() {
+ HashMap<Class<? extends PTransform>, TransformPayloadTranslator> translators = new HashMap<>();
+
+ for (TransformPayloadTranslatorRegistrar registrar :
+ ServiceLoader.load(TransformPayloadTranslatorRegistrar.class)) {
+
+ Map<Class<? extends PTransform>, TransformPayloadTranslator> newTranslators =
+ (Map) registrar.getTransformPayloadTranslators();
+
+ Set<Class<? extends PTransform>> alreadyRegistered =
+ Sets.intersection(translators.keySet(), newTranslators.keySet());
+
+ if (!alreadyRegistered.isEmpty()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Classes already registered: %s", Joiner.on(", ").join(alreadyRegistered)));
+ }
+
+ translators.putAll(newTranslators);
+ }
+ return ImmutableMap.copyOf(translators);
+ }
+
+ private static Map<String, TransformPayloadTranslator> loadTransformRehydrators() {
+ HashMap<String, TransformPayloadTranslator> rehydrators = new HashMap<>();
+
+ for (TransformPayloadTranslatorRegistrar registrar :
+ ServiceLoader.load(TransformPayloadTranslatorRegistrar.class)) {
+
+ Map<String, ? extends TransformPayloadTranslator> newRehydrators =
+ registrar.getTransformRehydrators();
+
+ Set<String> alreadyRegistered =
+ Sets.intersection(rehydrators.keySet(), newRehydrators.keySet());
+
+ if (!alreadyRegistered.isEmpty()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "URNs already registered: %s", Joiner.on(", ").join(alreadyRegistered)));
+ }
+
+ rehydrators.putAll(newRehydrators);
+ }
+ return ImmutableMap.copyOf(rehydrators);
+ }
+
+ private PTransformTranslation() {}
+
+ /**
+ * Translates an {@link AppliedPTransform} into a runner API proto.
+ *
+ * <p>Does not register the {@code appliedPTransform} within the provided {@link SdkComponents}.
+ */
+ static RunnerApi.PTransform toProto(
+ AppliedPTransform<?, ?, ?> appliedPTransform,
+ List<AppliedPTransform<?, ?, ?>> subtransforms,
+ SdkComponents components)
+ throws IOException {
+ // TODO include DisplayData https://issues.apache.org/jira/browse/BEAM-2645
+ RunnerApi.PTransform.Builder transformBuilder = RunnerApi.PTransform.newBuilder();
+ for (Map.Entry<TupleTag<?>, PValue> taggedInput : appliedPTransform.getInputs().entrySet()) {
+ checkArgument(
+ taggedInput.getValue() instanceof PCollection,
+ "Unexpected input type %s",
+ taggedInput.getValue().getClass());
+ transformBuilder.putInputs(
+ toProto(taggedInput.getKey()),
+ components.registerPCollection((PCollection<?>) taggedInput.getValue()));
+ }
+ for (Map.Entry<TupleTag<?>, PValue> taggedOutput : appliedPTransform.getOutputs().entrySet()) {
+ // TODO: Remove gating
+ if (taggedOutput.getValue() instanceof PCollection) {
+ checkArgument(
+ taggedOutput.getValue() instanceof PCollection,
+ "Unexpected output type %s",
+ taggedOutput.getValue().getClass());
+ transformBuilder.putOutputs(
+ toProto(taggedOutput.getKey()),
+ components.registerPCollection((PCollection<?>) taggedOutput.getValue()));
+ }
+ }
+ for (AppliedPTransform<?, ?, ?> subtransform : subtransforms) {
+ transformBuilder.addSubtransforms(components.getExistingPTransformId(subtransform));
+ }
+
+ transformBuilder.setUniqueName(appliedPTransform.getFullName());
+ transformBuilder.setDisplayData(
+ DisplayDataTranslation.toProto(DisplayData.from(appliedPTransform.getTransform())));
+
+ PTransform<?, ?> transform = appliedPTransform.getTransform();
+
+ // A RawPTransform directly vends its payload. Because it will generally be
+ // a subclass, we cannot do dictionary lookup in KNOWN_PAYLOAD_TRANSLATORS.
+ if (transform instanceof RawPTransform) {
+ // The raw transform was parsed in the context of other components; this puts it in the
+ // context of our current serialization
+ FunctionSpec spec = ((RawPTransform<?, ?>) transform).migrate(components);
+
+ // A composite transform is permitted to have a null spec. There are also some pseudo-
+ // primitives not yet supported by the portability framework that have null specs
+ if (spec != null) {
+ transformBuilder.setSpec(spec);
+ }
+ } else if (KNOWN_PAYLOAD_TRANSLATORS.containsKey(transform.getClass())) {
+ transformBuilder.setSpec(
+ KNOWN_PAYLOAD_TRANSLATORS
+ .get(transform.getClass())
+ .translate(appliedPTransform, components));
+ }
+
+ return transformBuilder.build();
+ }
+
+ /**
+ * Translates a {@link RunnerApi.PTransform} to a {@link RawPTransform} specialized for the URN
+ * and spec.
+ */
+ static RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+
+ @Nullable
+ TransformPayloadTranslator<?> rehydrator =
+ KNOWN_REHYDRATORS.get(
+ protoTransform.getSpec() == null ? null : protoTransform.getSpec().getUrn());
+
+ if (rehydrator == null) {
+ return DEFAULT_REHYDRATOR.rehydrate(protoTransform, rehydratedComponents);
+ } else {
+ return rehydrator.rehydrate(protoTransform, rehydratedComponents);
+ }
+ }
+
+ /**
+ * Translates a composite {@link AppliedPTransform} into a runner API proto with no component
+ * transforms.
+ *
+ * <p>This should not be used when translating a {@link Pipeline}.
+ *
+ * <p>Does not register the {@code appliedPTransform} within the provided {@link SdkComponents}.
+ */
+ static RunnerApi.PTransform toProto(
+ AppliedPTransform<?, ?, ?> appliedPTransform, SdkComponents components) throws IOException {
+ return toProto(
+ appliedPTransform, Collections.<AppliedPTransform<?, ?, ?>>emptyList(), components);
+ }
+
+ private static String toProto(TupleTag<?> tag) {
+ return tag.getId();
+ }
+
+ /** Returns the URN for the transform if it is known, otherwise {@code null}. */
+ @Nullable
+ public static String urnForTransformOrNull(PTransform<?, ?> transform) {
+
+ // A RawPTransform directly vends its URN. Because it will generally be
+ // a subclass, we cannot do dictionary lookup in KNOWN_PAYLOAD_TRANSLATORS.
+ if (transform instanceof RawPTransform) {
+ return ((RawPTransform) transform).getUrn();
+ }
+
+ TransformPayloadTranslator translator = KNOWN_PAYLOAD_TRANSLATORS.get(transform.getClass());
+ if (translator == null) {
+ return null;
+ }
+ return translator.getUrn(transform);
+ }
+
+ /** Returns the URN for the transform if it is known, otherwise throws. */
+ public static String urnForTransform(PTransform<?, ?> transform) {
+ String urn = urnForTransformOrNull(transform);
+ if (urn == null) {
+ throw new IllegalStateException(
+ String.format("No translator known for %s", transform.getClass().getName()));
+ }
+ return urn;
+ }
+
+ /**
+ * A bi-directional translator between a Java-based {@link PTransform} and a protobuf payload for
+ * that transform.
+ *
+ * <p>When going to a protocol buffer message, the translator produces a payload corresponding to
+ * the Java representation while registering components that payload references.
+ *
+ * <p>When "rehydrating" a protocol buffer message, the translator returns a {@link RawPTransform}
+ * - because the transform may not be Java-based, it is not possible to rebuild a Java-based
+ * {@link PTransform}. The resulting {@link RawPTransform} subclass encapsulates the knowledge of
+ * which components are referenced in the payload.
+ */
+ public interface TransformPayloadTranslator<T extends PTransform<?, ?>> {
+ String getUrn(T transform);
+
+ FunctionSpec translate(AppliedPTransform<?, ?, T> application, SdkComponents components)
+ throws IOException;
+
+ RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException;
+
+ /**
+ * A {@link TransformPayloadTranslator} for transforms that contain no references to components,
+ * so they do not need a specialized rehydration.
+ */
+ abstract class WithDefaultRehydration<T extends PTransform<?, ?>>
+ implements TransformPayloadTranslator<T> {
+ @Override
+ public final RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ return UnknownRawPTransform.forSpec(protoTransform.getSpec());
+ }
+ }
+
+ /**
+ * A {@link TransformPayloadTranslator} for transforms that contain no references to components,
+ * so they do not need a specialized rehydration.
+ */
+ abstract class NotSerializable<T extends PTransform<?, ?>>
+ implements TransformPayloadTranslator<T> {
+
+ public static NotSerializable<?> forUrn(final String urn) {
+ return new NotSerializable<PTransform<?, ?>>() {
+ @Override
+ public String getUrn(PTransform<?, ?> transform) {
+ return urn;
+ }
+ };
+ }
+
+ @Override
+ public final FunctionSpec translate(
+ AppliedPTransform<?, ?, T> transform, SdkComponents components) throws IOException {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s should never be translated",
+ transform.getTransform().getClass().getCanonicalName()));
+ }
+
+ @Override
+ public final RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s.rehydrate should never be called; there is no serialized form",
+ getClass().getCanonicalName()));
+ }
+ }
+ }
+
+ /**
+ * A {@link PTransform} that indicates its URN and payload directly.
+ *
+ * <p>This is the result of rehydrating transforms from a pipeline proto. There is no {@link
+ * #expand} method since the definition of the transform may be lost. The transform is already
+ * fully expanded in the pipeline proto.
+ */
+ public abstract static class RawPTransform<InputT extends PInput, OutputT extends POutput>
+ extends PTransform<InputT, OutputT> {
+
+ /** The URN for this transform, if standardized. */
+ @Nullable
+ public String getUrn() {
+ return getSpec() == null ? null : getSpec().getUrn();
+ }
+
+ /** The payload for this transform, if any. */
+ @Nullable
+ public abstract FunctionSpec getSpec();
+
+ /**
+ * Build a new payload set in the context of the given {@link SdkComponents}, if applicable.
+ *
+ * <p>When re-serializing this transform, the ids reference in the rehydrated payload may
+ * conflict with those defined by the serialization context. In that case, the components must
+ * be re-registered and a new payload returned.
+ */
+ public FunctionSpec migrate(SdkComponents components) throws IOException {
+ return getSpec();
+ }
+
+ /**
+ * By default, throws an exception, but can be overridden.
+ *
+ * <p>It is permissible for runner-specific transforms to be both a {@link RawPTransform} that
+ * directly vends its proto representation and also to expand, for convenience of not having to
+ * register a translator.
+ */
+ @Override
+ public OutputT expand(InputT input) {
+ throw new IllegalStateException(
+ String.format(
+ "%s should never be asked to expand;"
+ + " it is the result of deserializing an already-constructed Pipeline",
+ getClass().getSimpleName()));
+ }
+ }
+
+ @AutoValue
+ abstract static class UnknownRawPTransform extends RawPTransform<PInput, POutput> {
+
+ @Override
+ public String getUrn() {
+ return getSpec() == null ? null : getSpec().getUrn();
+ }
+
+ @Nullable
+ public abstract RunnerApi.FunctionSpec getSpec();
+
+ public static UnknownRawPTransform forSpec(RunnerApi.FunctionSpec spec) {
+ return new AutoValue_PTransformTranslation_UnknownRawPTransform(spec);
+ }
+
+ @Override
+ public POutput expand(PInput input) {
+ throw new IllegalStateException(
+ String.format(
+ "%s should never be asked to expand;"
+ + " it is the result of deserializing an already-constructed Pipeline",
+ getClass().getSimpleName()));
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("urn", getUrn())
+ .add("payload", getSpec())
+ .toString();
+ }
+
+ public RunnerApi.FunctionSpec getSpecForComponents(SdkComponents components) {
+ return getSpec();
+ }
+ }
+
+ /** A translator that uses the explicit URN and payload from a {@link RawPTransform}. */
+ public static class RawPTransformTranslator
+ implements TransformPayloadTranslator<RawPTransform<?, ?>> {
+ @Override
+ public String getUrn(RawPTransform<?, ?> transform) {
+ return transform.getUrn();
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, RawPTransform<?, ?>> transform, SdkComponents components)
+ throws IOException {
+ return transform.getTransform().migrate(components);
+ }
+
+ @Override
+ public RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents) {
+ return UnknownRawPTransform.forSpec(protoTransform.getSpec());
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
deleted file mode 100644
index d25d342..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.collect.ImmutableMap;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/**
- * Utilities for converting {@link PTransform PTransforms} to and from {@link RunnerApi Runner API
- * protocol buffers}.
- */
-public class PTransforms {
- private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
- KNOWN_PAYLOAD_TRANSLATORS =
- ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder().build();
- // TODO: ParDoPayload, WindowIntoPayload, ReadPayload, CombinePayload
- // TODO: "Flatten Payload", etc?
- // TODO: Load via service loader.
- private PTransforms() {}
-
- /**
- * Translates an {@link AppliedPTransform} into a runner API proto.
- *
- * <p>Does not register the {@code appliedPTransform} within the provided {@link SdkComponents}.
- */
- static RunnerApi.PTransform toProto(
- AppliedPTransform<?, ?, ?> appliedPTransform,
- List<AppliedPTransform<?, ?, ?>> subtransforms,
- SdkComponents components)
- throws IOException {
- RunnerApi.PTransform.Builder transformBuilder = RunnerApi.PTransform.newBuilder();
- for (Map.Entry<TupleTag<?>, PValue> taggedInput : appliedPTransform.getInputs().entrySet()) {
- checkArgument(
- taggedInput.getValue() instanceof PCollection,
- "Unexpected input type %s",
- taggedInput.getValue().getClass());
- transformBuilder.putInputs(
- toProto(taggedInput.getKey()),
- components.registerPCollection((PCollection<?>) taggedInput.getValue()));
- }
- for (Map.Entry<TupleTag<?>, PValue> taggedOutput : appliedPTransform.getOutputs().entrySet()) {
- // TODO: Remove gating
- if (taggedOutput.getValue() instanceof PCollection) {
- checkArgument(
- taggedOutput.getValue() instanceof PCollection,
- "Unexpected output type %s",
- taggedOutput.getValue().getClass());
- transformBuilder.putOutputs(
- toProto(taggedOutput.getKey()),
- components.registerPCollection((PCollection<?>) taggedOutput.getValue()));
- }
- }
- for (AppliedPTransform<?, ?, ?> subtransform : subtransforms) {
- transformBuilder.addSubtransforms(components.getExistingPTransformId(subtransform));
- }
-
- transformBuilder.setUniqueName(appliedPTransform.getFullName());
- // TODO: Display Data
-
- PTransform<?, ?> transform = appliedPTransform.getTransform();
- if (KNOWN_PAYLOAD_TRANSLATORS.containsKey(transform.getClass())) {
- FunctionSpec payload =
- KNOWN_PAYLOAD_TRANSLATORS
- .get(transform.getClass())
- .translate(appliedPTransform, components);
- transformBuilder.setSpec(payload);
- }
-
- return transformBuilder.build();
- }
-
- private static String toProto(TupleTag<?> tag) {
- return tag.getId();
- }
-
- /**
- * A translator consumes a {@link PTransform} application and produces the appropriate
- * FunctionSpec for a distinguished or primitive transform within the Beam runner API.
- */
- public interface TransformPayloadTranslator<T extends PTransform<?, ?>> {
- FunctionSpec translate(AppliedPTransform<?, ?, T> transform, SdkComponents components);
- }
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
new file mode 100644
index 0000000..f88cbe5
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -0,0 +1,768 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.PAR_DO_TRANSFORM_URN;
+
+import com.google.auto.service.AutoService;
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Parameter.Type;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput.Builder;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Materializations;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerDeclaration;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/** Utilities for interacting with {@link ParDo} instances and {@link ParDoPayload} protos. */
+public class ParDoTranslation {
+ /** The URN for an unknown Java {@link DoFn}. */
+ public static final String CUSTOM_JAVA_DO_FN_URN = "urn:beam:dofn:javasdk:0.1";
+ /** The URN for an unknown Java {@link ViewFn}. */
+ public static final String CUSTOM_JAVA_VIEW_FN_URN = "urn:beam:viewfn:javasdk:0.1";
+ /** The URN for an unknown Java {@link WindowMappingFn}. */
+ public static final String CUSTOM_JAVA_WINDOW_MAPPING_FN_URN =
+ "urn:beam:windowmappingfn:javasdk:0.1";
+
+ /** A {@link TransformPayloadTranslator} for {@link ParDo}. */
+ public static class ParDoPayloadTranslator
+ implements TransformPayloadTranslator<MultiOutput<?, ?>> {
+ public static TransformPayloadTranslator create() {
+ return new ParDoPayloadTranslator();
+ }
+
+ private ParDoPayloadTranslator() {}
+
+ @Override
+ public String getUrn(ParDo.MultiOutput<?, ?> transform) {
+ return PAR_DO_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components)
+ throws IOException {
+ ParDoPayload payload = toProto(transform.getTransform(), components);
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(PAR_DO_TRANSFORM_URN)
+ .setPayload(payload.toByteString())
+ .build();
+ }
+
+ @Override
+ public PTransformTranslation.RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ return new RawParDo<>(protoTransform, rehydratedComponents);
+ }
+
+ /** Registers {@link ParDoPayloadTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(ParDo.MultiOutput.class, new ParDoPayloadTranslator());
+ }
+
+ @Override
+ public Map<String, ? extends TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.singletonMap(PAR_DO_TRANSFORM_URN, new ParDoPayloadTranslator());
+ }
+ }
+ }
+
+ public static ParDoPayload toProto(final ParDo.MultiOutput<?, ?> parDo, SdkComponents components)
+ throws IOException {
+
+ final DoFn<?, ?> doFn = parDo.getFn();
+ final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
+
+ return payloadForParDoLike(
+ new ParDoLike() {
+ @Override
+ public SdkFunctionSpec translateDoFn(SdkComponents newComponents) {
+ return toProto(parDo.getFn(), parDo.getMainOutputTag());
+ }
+
+ @Override
+ public List<RunnerApi.Parameter> translateParameters() {
+ List<RunnerApi.Parameter> parameters = new ArrayList<>();
+ for (Parameter parameter : signature.processElement().extraParameters()) {
+ Optional<RunnerApi.Parameter> protoParameter = toProto(parameter);
+ if (protoParameter.isPresent()) {
+ parameters.add(protoParameter.get());
+ }
+ }
+ return parameters;
+ }
+
+ @Override
+ public Map<String, SideInput> translateSideInputs(SdkComponents components) {
+ Map<String, SideInput> sideInputs = new HashMap<>();
+ for (PCollectionView<?> sideInput : parDo.getSideInputs()) {
+ sideInputs.put(sideInput.getTagInternal().getId(), toProto(sideInput));
+ }
+ return sideInputs;
+ }
+
+ @Override
+ public Map<String, RunnerApi.StateSpec> translateStateSpecs(SdkComponents components)
+ throws IOException {
+ Map<String, RunnerApi.StateSpec> stateSpecs = new HashMap<>();
+ for (Map.Entry<String, StateDeclaration> state :
+ signature.stateDeclarations().entrySet()) {
+ RunnerApi.StateSpec spec =
+ toProto(getStateSpecOrCrash(state.getValue(), doFn), components);
+ stateSpecs.put(state.getKey(), spec);
+ }
+ return stateSpecs;
+ }
+
+ @Override
+ public Map<String, RunnerApi.TimerSpec> translateTimerSpecs(SdkComponents newComponents) {
+ Map<String, RunnerApi.TimerSpec> timerSpecs = new HashMap<>();
+ for (Map.Entry<String, TimerDeclaration> timer :
+ signature.timerDeclarations().entrySet()) {
+ RunnerApi.TimerSpec spec = toProto(getTimerSpecOrCrash(timer.getValue(), doFn));
+ timerSpecs.put(timer.getKey(), spec);
+ }
+ return timerSpecs;
+ }
+
+ @Override
+ public boolean isSplittable() {
+ return signature.processElement().isSplittable();
+ }
+ },
+ components);
+ }
+
+ private static StateSpec<?> getStateSpecOrCrash(
+ StateDeclaration stateDeclaration, DoFn<?, ?> target) {
+ try {
+ Object fieldValue = stateDeclaration.field().get(target);
+ checkState(
+ fieldValue instanceof StateSpec,
+ "Malformed %s class %s: state declaration field %s does not have type %s.",
+ DoFn.class.getSimpleName(),
+ target.getClass().getName(),
+ stateDeclaration.field().getName(),
+ StateSpec.class);
+
+ return (StateSpec<?>) stateDeclaration.field().get(target);
+ } catch (IllegalAccessException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Malformed %s class %s: state declaration field %s is not accessible.",
+ DoFn.class.getSimpleName(),
+ target.getClass().getName(),
+ stateDeclaration.field().getName()));
+ }
+ }
+
+ private static TimerSpec getTimerSpecOrCrash(
+ TimerDeclaration timerDeclaration, DoFn<?, ?> target) {
+ try {
+ Object fieldValue = timerDeclaration.field().get(target);
+ checkState(
+ fieldValue instanceof TimerSpec,
+ "Malformed %s class %s: timer declaration field %s does not have type %s.",
+ DoFn.class.getSimpleName(),
+ target.getClass().getName(),
+ timerDeclaration.field().getName(),
+ TimerSpec.class);
+
+ return (TimerSpec) timerDeclaration.field().get(target);
+ } catch (IllegalAccessException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Malformed %s class %s: timer declaration field %s is not accessible.",
+ DoFn.class.getSimpleName(),
+ target.getClass().getName(),
+ timerDeclaration.field().getName()));
+ }
+ }
+
+ public static DoFn<?, ?> getDoFn(ParDoPayload payload) throws InvalidProtocolBufferException {
+ return doFnAndMainOutputTagFromProto(payload.getDoFn()).getDoFn();
+ }
+
+ public static DoFn<?, ?> getDoFn(AppliedPTransform<?, ?, ?> application) throws IOException {
+ PTransform<?, ?> transform = application.getTransform();
+ if (transform instanceof ParDo.MultiOutput) {
+ return ((ParDo.MultiOutput<?, ?>) transform).getFn();
+ }
+
+ return getDoFn(getParDoPayload(application));
+ }
+
+ public static TupleTag<?> getMainOutputTag(ParDoPayload payload)
+ throws InvalidProtocolBufferException {
+ return doFnAndMainOutputTagFromProto(payload.getDoFn()).getMainOutputTag();
+ }
+
+ public static TupleTag<?> getMainOutputTag(AppliedPTransform<?, ?, ?> application)
+ throws IOException {
+ PTransform<?, ?> transform = application.getTransform();
+ if (transform instanceof ParDo.MultiOutput) {
+ return ((ParDo.MultiOutput<?, ?>) transform).getMainOutputTag();
+ }
+
+ return getMainOutputTag(getParDoPayload(application));
+ }
+
+ public static TupleTagList getAdditionalOutputTags(AppliedPTransform<?, ?, ?> application)
+ throws IOException {
+ PTransform<?, ?> transform = application.getTransform();
+ if (transform instanceof ParDo.MultiOutput) {
+ return ((ParDo.MultiOutput<?, ?>) transform).getAdditionalOutputTags();
+ }
+
+ RunnerApi.PTransform protoTransform =
+ PTransformTranslation.toProto(application, SdkComponents.create());
+
+ ParDoPayload payload = ParDoPayload.parseFrom(protoTransform.getSpec().getPayload());
+ TupleTag<?> mainOutputTag = getMainOutputTag(payload);
+ Set<String> outputTags =
+ Sets.difference(
+ protoTransform.getOutputsMap().keySet(), Collections.singleton(mainOutputTag.getId()));
+
+ ArrayList<TupleTag<?>> additionalOutputTags = new ArrayList<>();
+ for (String outputTag : outputTags) {
+ additionalOutputTags.add(new TupleTag<>(outputTag));
+ }
+ return TupleTagList.of(additionalOutputTags);
+ }
+
+ public static List<PCollectionView<?>> getSideInputs(AppliedPTransform<?, ?, ?> application)
+ throws IOException {
+ PTransform<?, ?> transform = application.getTransform();
+ if (transform instanceof ParDo.MultiOutput) {
+ return ((ParDo.MultiOutput<?, ?>) transform).getSideInputs();
+ }
+
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PTransform parDoProto = PTransformTranslation.toProto(application, sdkComponents);
+ ParDoPayload payload = ParDoPayload.parseFrom(parDoProto.getSpec().getPayload());
+
+ List<PCollectionView<?>> views = new ArrayList<>();
+ RehydratedComponents components =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+ for (Map.Entry<String, SideInput> sideInputEntry : payload.getSideInputsMap().entrySet()) {
+ String sideInputTag = sideInputEntry.getKey();
+ RunnerApi.SideInput sideInput = sideInputEntry.getValue();
+ PCollection<?> originalPCollection =
+ checkNotNull(
+ (PCollection<?>) application.getInputs().get(new TupleTag<>(sideInputTag)),
+ "no input with tag %s",
+ sideInputTag);
+ views.add(
+ viewFromProto(sideInput, sideInputTag, originalPCollection, parDoProto, components));
+ }
+ return views;
+ }
+
+ public static RunnerApi.PCollection getMainInput(
+ RunnerApi.PTransform ptransform, Components components) throws IOException {
+ checkArgument(
+ ptransform.getSpec().getUrn().equals(PAR_DO_TRANSFORM_URN),
+ "Unexpected payload type %s",
+ ptransform.getSpec().getUrn());
+ ParDoPayload payload = ParDoPayload.parseFrom(ptransform.getSpec().getPayload());
+ String mainInputId =
+ Iterables.getOnlyElement(
+ Sets.difference(
+ ptransform.getInputsMap().keySet(), payload.getSideInputsMap().keySet()));
+ return components.getPcollectionsOrThrow(ptransform.getInputsOrThrow(mainInputId));
+ }
+
+ @VisibleForTesting
+ static RunnerApi.StateSpec toProto(StateSpec<?> stateSpec, final SdkComponents components)
+ throws IOException {
+ final RunnerApi.StateSpec.Builder builder = RunnerApi.StateSpec.newBuilder();
+
+ return stateSpec.match(
+ new StateSpec.Cases<RunnerApi.StateSpec>() {
+ @Override
+ public RunnerApi.StateSpec dispatchValue(Coder<?> valueCoder) {
+ return builder
+ .setValueSpec(
+ RunnerApi.ValueStateSpec.newBuilder()
+ .setCoderId(registerCoderOrThrow(components, valueCoder)))
+ .build();
+ }
+
+ @Override
+ public RunnerApi.StateSpec dispatchBag(Coder<?> elementCoder) {
+ return builder
+ .setBagSpec(
+ RunnerApi.BagStateSpec.newBuilder()
+ .setElementCoderId(registerCoderOrThrow(components, elementCoder)))
+ .build();
+ }
+
+ @Override
+ public RunnerApi.StateSpec dispatchCombining(
+ Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder) {
+ return builder
+ .setCombiningSpec(
+ RunnerApi.CombiningStateSpec.newBuilder()
+ .setAccumulatorCoderId(registerCoderOrThrow(components, accumCoder))
+ .setCombineFn(CombineTranslation.toProto(combineFn)))
+ .build();
+ }
+
+ @Override
+ public RunnerApi.StateSpec dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder) {
+ return builder
+ .setMapSpec(
+ RunnerApi.MapStateSpec.newBuilder()
+ .setKeyCoderId(registerCoderOrThrow(components, keyCoder))
+ .setValueCoderId(registerCoderOrThrow(components, valueCoder)))
+ .build();
+ }
+
+ @Override
+ public RunnerApi.StateSpec dispatchSet(Coder<?> elementCoder) {
+ return builder
+ .setSetSpec(
+ RunnerApi.SetStateSpec.newBuilder()
+ .setElementCoderId(registerCoderOrThrow(components, elementCoder)))
+ .build();
+ }
+ });
+ }
+
+ @VisibleForTesting
+ static StateSpec<?> fromProto(RunnerApi.StateSpec stateSpec, RehydratedComponents components)
+ throws IOException {
+ switch (stateSpec.getSpecCase()) {
+ case VALUE_SPEC:
+ return StateSpecs.value(components.getCoder(stateSpec.getValueSpec().getCoderId()));
+ case BAG_SPEC:
+ return StateSpecs.bag(components.getCoder(stateSpec.getBagSpec().getElementCoderId()));
+ case COMBINING_SPEC:
+ FunctionSpec combineFnSpec = stateSpec.getCombiningSpec().getCombineFn().getSpec();
+
+ if (!combineFnSpec.getUrn().equals(CombineTranslation.JAVA_SERIALIZED_COMBINE_FN_URN)) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Cannot create %s from non-Java %s: %s",
+ StateSpec.class.getSimpleName(),
+ Combine.CombineFn.class.getSimpleName(),
+ combineFnSpec.getUrn()));
+ }
+
+ Combine.CombineFn<?, ?, ?> combineFn =
+ (Combine.CombineFn<?, ?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ combineFnSpec.getPayload().toByteArray(),
+ Combine.CombineFn.class.getSimpleName());
+
+ // Rawtype coder cast because it is required to be a valid accumulator coder
+ // for the CombineFn, by construction
+ return StateSpecs.combining(
+ (Coder) components.getCoder(stateSpec.getCombiningSpec().getAccumulatorCoderId()),
+ combineFn);
+
+ case MAP_SPEC:
+ return StateSpecs.map(
+ components.getCoder(stateSpec.getMapSpec().getKeyCoderId()),
+ components.getCoder(stateSpec.getMapSpec().getValueCoderId()));
+
+ case SET_SPEC:
+ return StateSpecs.set(components.getCoder(stateSpec.getSetSpec().getElementCoderId()));
+
+ case SPEC_NOT_SET:
+ default:
+ throw new IllegalArgumentException(
+ String.format("Unknown %s: %s", RunnerApi.StateSpec.class.getName(), stateSpec));
+ }
+ }
+
+ private static String registerCoderOrThrow(SdkComponents components, Coder coder) {
+ try {
+ return components.registerCoder(coder);
+ } catch (IOException exc) {
+ throw new RuntimeException("Failure to register coder", exc);
+ }
+ }
+
+ private static RunnerApi.TimerSpec toProto(TimerSpec timer) {
+ return RunnerApi.TimerSpec.newBuilder().setTimeDomain(toProto(timer.getTimeDomain())).build();
+ }
+
+ private static RunnerApi.TimeDomain.Enum toProto(TimeDomain timeDomain) {
+ switch (timeDomain) {
+ case EVENT_TIME:
+ return RunnerApi.TimeDomain.Enum.EVENT_TIME;
+ case PROCESSING_TIME:
+ return RunnerApi.TimeDomain.Enum.PROCESSING_TIME;
+ case SYNCHRONIZED_PROCESSING_TIME:
+ return RunnerApi.TimeDomain.Enum.SYNCHRONIZED_PROCESSING_TIME;
+ default:
+ throw new IllegalArgumentException("Unknown time domain");
+ }
+ }
+
+ @AutoValue
+ abstract static class DoFnAndMainOutput implements Serializable {
+ public static DoFnAndMainOutput of(DoFn<?, ?> fn, TupleTag<?> tag) {
+ return new AutoValue_ParDoTranslation_DoFnAndMainOutput(fn, tag);
+ }
+
+ abstract DoFn<?, ?> getDoFn();
+
+ abstract TupleTag<?> getMainOutputTag();
+ }
+
+ private static SdkFunctionSpec toProto(DoFn<?, ?> fn, TupleTag<?> tag) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(CUSTOM_JAVA_DO_FN_URN)
+ .setPayload(
+ ByteString.copyFrom(
+ SerializableUtils.serializeToByteArray(DoFnAndMainOutput.of(fn, tag))))
+ .build())
+ .build();
+ }
+
+ private static DoFnAndMainOutput doFnAndMainOutputTagFromProto(SdkFunctionSpec fnSpec)
+ throws InvalidProtocolBufferException {
+ checkArgument(
+ fnSpec.getSpec().getUrn().equals(CUSTOM_JAVA_DO_FN_URN),
+ "Expected %s to be %s with URN %s, but URN was %s",
+ DoFn.class.getSimpleName(),
+ FunctionSpec.class.getSimpleName(),
+ CUSTOM_JAVA_DO_FN_URN,
+ fnSpec.getSpec().getUrn());
+ byte[] serializedFn = fnSpec.getSpec().getPayload().toByteArray();
+ return (DoFnAndMainOutput)
+ SerializableUtils.deserializeFromByteArray(serializedFn, "Custom DoFn And Main Output tag");
+ }
+
+ private static Optional<RunnerApi.Parameter> toProto(Parameter parameter) {
+ return parameter.match(
+ new Cases.WithDefault<Optional<RunnerApi.Parameter>>() {
+ @Override
+ public Optional<RunnerApi.Parameter> dispatch(WindowParameter p) {
+ return Optional.of(RunnerApi.Parameter.newBuilder().setType(Type.Enum.WINDOW).build());
+ }
+
+ @Override
+ public Optional<RunnerApi.Parameter> dispatch(RestrictionTrackerParameter p) {
+ return Optional.of(
+ RunnerApi.Parameter.newBuilder().setType(Type.Enum.RESTRICTION_TRACKER).build());
+ }
+
+ @Override
+ protected Optional<RunnerApi.Parameter> dispatchDefault(Parameter p) {
+ return Optional.absent();
+ }
+ });
+ }
+
+ public static SideInput toProto(PCollectionView<?> view) {
+ Builder builder = SideInput.newBuilder();
+ builder.setAccessPattern(
+ FunctionSpec.newBuilder().setUrn(view.getViewFn().getMaterialization().getUrn()).build());
+ builder.setViewFn(toProto(view.getViewFn()));
+ builder.setWindowMappingFn(toProto(view.getWindowMappingFn()));
+ return builder.build();
+ }
+
+ /**
+ * Create a {@link PCollectionView} from a side input spec and an already-deserialized {@link
+ * PCollection} that should be wired up.
+ */
+ public static PCollectionView<?> viewFromProto(
+ SideInput sideInput,
+ String localName,
+ PCollection<?> pCollection,
+ RunnerApi.PTransform parDoTransform,
+ RehydratedComponents components)
+ throws IOException {
+ checkArgument(
+ localName != null,
+ "%s.viewFromProto: localName must not be null",
+ ParDoTranslation.class.getSimpleName());
+ TupleTag<?> tag = new TupleTag<>(localName);
+ WindowMappingFn<?> windowMappingFn = windowMappingFnFromProto(sideInput.getWindowMappingFn());
+ ViewFn<?, ?> viewFn = viewFnFromProto(sideInput.getViewFn());
+
+ WindowingStrategy<?, ?> windowingStrategy = pCollection.getWindowingStrategy().fixDefaults();
+ Coder<Iterable<WindowedValue<?>>> coder =
+ (Coder)
+ IterableCoder.of(
+ FullWindowedValueCoder.of(
+ pCollection.getCoder(),
+ pCollection.getWindowingStrategy().getWindowFn().windowCoder()));
+ checkArgument(
+ sideInput.getAccessPattern().getUrn().equals(Materializations.ITERABLE_MATERIALIZATION_URN),
+ "Unknown View Materialization URN %s",
+ sideInput.getAccessPattern().getUrn());
+
+ PCollectionView<?> view =
+ new RunnerPCollectionView<>(
+ pCollection,
+ (TupleTag<Iterable<WindowedValue<?>>>) tag,
+ (ViewFn<Iterable<WindowedValue<?>>, ?>) viewFn,
+ windowMappingFn,
+ windowingStrategy,
+ coder);
+ return view;
+ }
+
+ private static SdkFunctionSpec toProto(ViewFn<?, ?> viewFn) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(CUSTOM_JAVA_VIEW_FN_URN)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn)))
+ .build())
+ .build();
+ }
+
+ private static <T> ParDoPayload getParDoPayload(AppliedPTransform<?, ?, ?> transform)
+ throws IOException {
+ RunnerApi.PTransform parDoPTransform =
+ PTransformTranslation.toProto(
+ transform, Collections.<AppliedPTransform<?, ?, ?>>emptyList(), SdkComponents.create());
+ return ParDoPayload.parseFrom(parDoPTransform.getSpec().getPayload());
+ }
+
+ public static boolean usesStateOrTimers(AppliedPTransform<?, ?, ?> transform) throws IOException {
+ ParDoPayload payload = getParDoPayload(transform);
+ return payload.getStateSpecsCount() > 0 || payload.getTimerSpecsCount() > 0;
+ }
+
+ public static boolean isSplittable(AppliedPTransform<?, ?, ?> transform) throws IOException {
+ ParDoPayload payload = getParDoPayload(transform);
+ return payload.getSplittable();
+ }
+
+ private static ViewFn<?, ?> viewFnFromProto(SdkFunctionSpec viewFn)
+ throws InvalidProtocolBufferException {
+ FunctionSpec spec = viewFn.getSpec();
+ checkArgument(
+ spec.getUrn().equals(CUSTOM_JAVA_VIEW_FN_URN),
+ "Can't deserialize unknown %s type %s",
+ ViewFn.class.getSimpleName(),
+ spec.getUrn());
+ return (ViewFn<?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ spec.getPayload().toByteArray(), "Custom ViewFn");
+ }
+
+ private static SdkFunctionSpec toProto(WindowMappingFn<?> windowMappingFn) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN)
+ .setPayload(
+ ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn)))
+ .build())
+ .build();
+ }
+
+ private static WindowMappingFn<?> windowMappingFnFromProto(SdkFunctionSpec windowMappingFn)
+ throws InvalidProtocolBufferException {
+ FunctionSpec spec = windowMappingFn.getSpec();
+ checkArgument(
+ spec.getUrn().equals(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN),
+ "Can't deserialize unknown %s type %s",
+ WindowMappingFn.class.getSimpleName(),
+ spec.getUrn());
+ return (WindowMappingFn<?>)
+ SerializableUtils.deserializeFromByteArray(
+ spec.getPayload().toByteArray(), "Custom WinodwMappingFn");
+ }
+
+ static class RawParDo<InputT, OutputT>
+ extends PTransformTranslation.RawPTransform<PCollection<InputT>, PCollection<OutputT>>
+ implements ParDoLike {
+
+ private final RunnerApi.PTransform protoTransform;
+ private final transient RehydratedComponents rehydratedComponents;
+
+ // Parsed from protoTransform and cached
+ private final FunctionSpec spec;
+ private final ParDoPayload payload;
+
+ public RawParDo(RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ this.rehydratedComponents = rehydratedComponents;
+ this.protoTransform = protoTransform;
+ this.spec = protoTransform.getSpec();
+ this.payload = ParDoPayload.parseFrom(spec.getPayload());
+ }
+
+ @Override
+ public FunctionSpec getSpec() {
+ return spec;
+ }
+
+ @Override
+ public FunctionSpec migrate(SdkComponents components) throws IOException {
+ return FunctionSpec.newBuilder()
+ .setUrn(PAR_DO_TRANSFORM_URN)
+ .setPayload(payloadForParDoLike(this, components).toByteString())
+ .build();
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ for (Map.Entry<String, SideInput> sideInputEntry : payload.getSideInputsMap().entrySet()) {
+ try {
+ additionalInputs.put(
+ new TupleTag<>(sideInputEntry.getKey()),
+ rehydratedComponents.getPCollection(
+ protoTransform.getInputsOrThrow(sideInputEntry.getKey())));
+ } catch (IOException exc) {
+ throw new IllegalStateException(
+ String.format(
+ "Could not find input with name %s for %s transform",
+ sideInputEntry.getKey(), ParDo.class.getSimpleName()));
+ }
+ }
+ return additionalInputs;
+ }
+
+ @Override
+ public SdkFunctionSpec translateDoFn(SdkComponents newComponents) {
+ // TODO: re-register the environment with the new components
+ return payload.getDoFn();
+ }
+
+ @Override
+ public List<RunnerApi.Parameter> translateParameters() {
+ return MoreObjects.firstNonNull(
+ payload.getParametersList(), Collections.<RunnerApi.Parameter>emptyList());
+ }
+
+ @Override
+ public Map<String, SideInput> translateSideInputs(SdkComponents components) {
+ // TODO: re-register the PCollections and UDF environments
+ return MoreObjects.firstNonNull(
+ payload.getSideInputsMap(), Collections.<String, SideInput>emptyMap());
+ }
+
+ @Override
+ public Map<String, RunnerApi.StateSpec> translateStateSpecs(SdkComponents components) {
+ // TODO: re-register the coders
+ return MoreObjects.firstNonNull(
+ payload.getStateSpecsMap(), Collections.<String, RunnerApi.StateSpec>emptyMap());
+ }
+
+ @Override
+ public Map<String, RunnerApi.TimerSpec> translateTimerSpecs(SdkComponents newComponents) {
+ return MoreObjects.firstNonNull(
+ payload.getTimerSpecsMap(), Collections.<String, RunnerApi.TimerSpec>emptyMap());
+ }
+
+ @Override
+ public boolean isSplittable() {
+ return payload.getSplittable();
+ }
+ }
+
+ /** These methods drive to-proto translation from Java and from rehydrated ParDos. */
+ private interface ParDoLike {
+ SdkFunctionSpec translateDoFn(SdkComponents newComponents);
+
+ List<RunnerApi.Parameter> translateParameters();
+
+ Map<String, RunnerApi.SideInput> translateSideInputs(SdkComponents components);
+
+ Map<String, RunnerApi.StateSpec> translateStateSpecs(SdkComponents components)
+ throws IOException;
+
+ Map<String, RunnerApi.TimerSpec> translateTimerSpecs(SdkComponents newComponents);
+
+ boolean isSplittable();
+ }
+
+ public static ParDoPayload payloadForParDoLike(ParDoLike parDo, SdkComponents components)
+ throws IOException {
+
+ return ParDoPayload.newBuilder()
+ .setDoFn(parDo.translateDoFn(components))
+ .addAllParameters(parDo.translateParameters())
+ .putAllStateSpecs(parDo.translateStateSpecs(components))
+ .putAllTimerSpecs(parDo.translateTimerSpecs(components))
+ .putAllSideInputs(parDo.translateSideInputs(components))
+ .setSplittable(parDo.isSplittable())
+ .build();
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java
new file mode 100644
index 0000000..4cdca61
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.protobuf.Struct;
+import com.google.protobuf.util.JsonFormat;
+import java.io.IOException;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
+
+/** Utilities for going to/from Runner API pipeline options. */
+public class PipelineOptionsTranslation {
+ private static final ObjectMapper MAPPER =
+ new ObjectMapper()
+ .registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+
+ /** Converts the provided {@link PipelineOptions} to a {@link Struct}. */
+ public static Struct toProto(PipelineOptions options) {
+ Struct.Builder builder = Struct.newBuilder();
+ try {
+ // The JSON format of a Protobuf Struct is the JSON object that is equivalent to that struct
+ // (with values encoded in a standard json-codeable manner). See Beam PR 3719 for more.
+ JsonFormat.parser().merge(MAPPER.writeValueAsString(options), builder);
+ return builder.build();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Converts the provided {@link Struct} into {@link PipelineOptions}. */
+ public static PipelineOptions fromProto(Struct protoOptions) throws IOException {
+ return MAPPER.readValue(JsonFormat.printer().print(protoOptions), PipelineOptions.class);
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
new file mode 100644
index 0000000..c8d38eb7
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Utilities for going to/from Runner API pipelines. */
+public class PipelineTranslation {
+
+ public static RunnerApi.Pipeline toProto(final Pipeline pipeline) {
+ final SdkComponents components = SdkComponents.create();
+ final Collection<String> rootIds = new HashSet<>();
+ pipeline.traverseTopologically(
+ new PipelineVisitor.Defaults() {
+ private final ListMultimap<Node, AppliedPTransform<?, ?, ?>> children =
+ ArrayListMultimap.create();
+
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ if (node.isRootNode()) {
+ for (AppliedPTransform<?, ?, ?> pipelineRoot : children.get(node)) {
+ rootIds.add(components.getExistingPTransformId(pipelineRoot));
+ }
+ } else {
+ // TODO: Include DisplayData in the proto
+ children.put(node.getEnclosingNode(), node.toAppliedPTransform(pipeline));
+ try {
+ components.registerPTransform(
+ node.toAppliedPTransform(pipeline), children.get(node));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ // TODO: Include DisplayData in the proto
+ children.put(node.getEnclosingNode(), node.toAppliedPTransform(pipeline));
+ try {
+ components.registerPTransform(
+ node.toAppliedPTransform(pipeline),
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ });
+ return RunnerApi.Pipeline.newBuilder()
+ .setComponents(components.toComponents())
+ .addAllRootTransformIds(rootIds)
+ .build();
+ }
+
+ private static DisplayData evaluateDisplayData(HasDisplayData component) {
+ return DisplayData.from(component);
+ }
+
+ public static Pipeline fromProto(final RunnerApi.Pipeline pipelineProto) throws IOException {
+ TransformHierarchy transforms = new TransformHierarchy();
+ Pipeline pipeline = Pipeline.forTransformHierarchy(transforms, PipelineOptionsFactory.create());
+
+ // Keeping the PCollections straight is a semantic necessity, but being careful not to explode
+ // the number of coders and windowing strategies is also nice, and helps testing.
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(pipelineProto.getComponents()).withPipeline(pipeline);
+
+ for (String rootId : pipelineProto.getRootTransformIdsList()) {
+ addRehydratedTransform(
+ transforms,
+ pipelineProto.getComponents().getTransformsOrThrow(rootId),
+ pipeline,
+ pipelineProto.getComponents().getTransformsMap(),
+ rehydratedComponents);
+ }
+
+ return pipeline;
+ }
+
+ private static void addRehydratedTransform(
+ TransformHierarchy transforms,
+ RunnerApi.PTransform transformProto,
+ Pipeline pipeline,
+ Map<String, RunnerApi.PTransform> transformProtos,
+ RehydratedComponents rehydratedComponents)
+ throws IOException {
+
+ Map<TupleTag<?>, PValue> rehydratedInputs = new HashMap<>();
+ for (Map.Entry<String, String> inputEntry : transformProto.getInputsMap().entrySet()) {
+ rehydratedInputs.put(
+ new TupleTag<>(inputEntry.getKey()),
+ rehydratedComponents.getPCollection(inputEntry.getValue()));
+ }
+
+ Map<TupleTag<?>, PValue> rehydratedOutputs = new HashMap<>();
+ for (Map.Entry<String, String> outputEntry : transformProto.getOutputsMap().entrySet()) {
+ rehydratedOutputs.put(
+ new TupleTag<>(outputEntry.getKey()),
+ rehydratedComponents.getPCollection(outputEntry.getValue()));
+ }
+
+ RawPTransform<?, ?> transform =
+ PTransformTranslation.rehydrate(transformProto, rehydratedComponents);
+
+ if (isPrimitive(transformProto)) {
+ transforms.addFinalizedPrimitiveNode(
+ transformProto.getUniqueName(), rehydratedInputs, transform, rehydratedOutputs);
+ } else {
+ transforms.pushFinalizedNode(
+ transformProto.getUniqueName(), rehydratedInputs, transform, rehydratedOutputs);
+
+ for (String childTransformId : transformProto.getSubtransformsList()) {
+ addRehydratedTransform(
+ transforms,
+ transformProtos.get(childTransformId),
+ pipeline,
+ transformProtos,
+ rehydratedComponents);
+ }
+
+ transforms.popNode();
+ }
+ }
+
+ private static Map<TupleTag<?>, PValue> sideInputMapToAdditionalInputs(
+ RunnerApi.PTransform transformProto,
+ RehydratedComponents rehydratedComponents,
+ Map<TupleTag<?>, PValue> rehydratedInputs,
+ Map<String, RunnerApi.SideInput> sideInputsMap)
+ throws IOException {
+ List<PCollectionView<?>> views = new ArrayList<>();
+ for (Map.Entry<String, RunnerApi.SideInput> sideInputEntry : sideInputsMap.entrySet()) {
+ String localName = sideInputEntry.getKey();
+ RunnerApi.SideInput sideInput = sideInputEntry.getValue();
+ PCollection<?> pCollection =
+ (PCollection<?>) checkNotNull(rehydratedInputs.get(new TupleTag<>(localName)));
+ views.add(
+ ParDoTranslation.viewFromProto(
+ sideInput, localName, pCollection, transformProto, rehydratedComponents));
+ }
+ return PCollectionViews.toAdditionalInputs(views);
+ }
+
+ // A primitive transform is one with outputs that are not in its input and also
+ // not produced by a subtransform.
+ private static boolean isPrimitive(RunnerApi.PTransform transformProto) {
+ return transformProto.getSubtransformsCount() == 0
+ && !transformProto
+ .getInputsMap()
+ .values()
+ .containsAll(transformProto.getOutputsMap().values());
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
index f43d23b..62b6d0a 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
@@ -18,7 +18,9 @@
package org.apache.beam.runners.core.construction;
+import com.google.common.collect.Iterables;
import java.util.Map;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.Create;
@@ -36,15 +38,17 @@
*/
public class PrimitiveCreate<T> extends PTransform<PBegin, PCollection<T>> {
private final Create.Values<T> transform;
+ private final Coder<T> coder;
- private PrimitiveCreate(Create.Values<T> transform) {
+ private PrimitiveCreate(Create.Values<T> transform, Coder<T> coder) {
this.transform = transform;
+ this.coder = coder;
}
@Override
public PCollection<T> expand(PBegin input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, coder);
}
public Iterable<T> getElements() {
@@ -60,7 +64,11 @@
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
AppliedPTransform<PBegin, PCollection<T>, Values<T>> transform) {
return PTransformReplacement.of(
- transform.getPipeline().begin(), new PrimitiveCreate<T>(transform.getTransform()));
+ transform.getPipeline().begin(),
+ new PrimitiveCreate<T>(
+ transform.getTransform(),
+ ((PCollection<T>) Iterables.getOnlyElement(transform.getOutputs().values()))
+ .getCoder()));
}
@Override
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
new file mode 100644
index 0000000..ee89562
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.IsBounded;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ReadPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Methods for translating {@link Read.Bounded} and {@link Read.Unbounded} {@link PTransform
+ * PTransformTranslation} into {@link ReadPayload} protos.
+ */
+public class ReadTranslation {
+ private static final String JAVA_SERIALIZED_BOUNDED_SOURCE = "urn:beam:java:boundedsource:v1";
+ private static final String JAVA_SERIALIZED_UNBOUNDED_SOURCE = "urn:beam:java:unboundedsource:v1";
+
+ public static ReadPayload toProto(Read.Bounded<?> read) {
+ return ReadPayload.newBuilder()
+ .setIsBounded(IsBounded.Enum.BOUNDED)
+ .setSource(toProto(read.getSource()))
+ .build();
+ }
+
+ public static ReadPayload toProto(Read.Unbounded<?> read) {
+ return ReadPayload.newBuilder()
+ .setIsBounded(IsBounded.Enum.UNBOUNDED)
+ .setSource(toProto(read.getSource()))
+ .build();
+ }
+
+ public static SdkFunctionSpec toProto(Source<?> source) {
+ if (source instanceof BoundedSource) {
+ return toProto((BoundedSource) source);
+ } else if (source instanceof UnboundedSource) {
+ return toProto((UnboundedSource<?, ?>) source);
+ } else {
+ throw new IllegalArgumentException(
+ String.format("Unknown %s type %s", Source.class.getSimpleName(), source.getClass()));
+ }
+ }
+
+ private static SdkFunctionSpec toProto(BoundedSource<?> source) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source)))
+ .build())
+ .build();
+ }
+
+ public static BoundedSource<?> boundedSourceFromProto(ReadPayload payload)
+ throws InvalidProtocolBufferException {
+ checkArgument(payload.getIsBounded().equals(IsBounded.Enum.BOUNDED));
+ return (BoundedSource<?>)
+ SerializableUtils.deserializeFromByteArray(
+ payload.getSource().getSpec().getPayload().toByteArray(), "BoundedSource");
+ }
+
+ public static <T> BoundedSource<T> boundedSourceFromTransform(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+ throws IOException {
+ return (BoundedSource<T>) boundedSourceFromProto(getReadPayload(transform));
+ }
+
+ public static <T, CheckpointT extends UnboundedSource.CheckpointMark>
+ UnboundedSource<T, CheckpointT> unboundedSourceFromTransform(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+ throws IOException {
+ return (UnboundedSource<T, CheckpointT>) unboundedSourceFromProto(getReadPayload(transform));
+ }
+
+ private static <T> ReadPayload getReadPayload(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+ throws IOException {
+ return ReadPayload.parseFrom(
+ PTransformTranslation.toProto(
+ transform,
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+ SdkComponents.create())
+ .getSpec()
+ .getPayload());
+ }
+
+ private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(JAVA_SERIALIZED_UNBOUNDED_SOURCE)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source)))
+ .build())
+ .build();
+ }
+
+ public static UnboundedSource<?, ?> unboundedSourceFromProto(ReadPayload payload)
+ throws InvalidProtocolBufferException {
+ checkArgument(payload.getIsBounded().equals(IsBounded.Enum.UNBOUNDED));
+ return (UnboundedSource<?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ payload.getSource().getSpec().getPayload().toByteArray(), "UnboundedSource");
+ }
+
+ public static PCollection.IsBounded sourceIsBounded(AppliedPTransform<?, ?, ?> transform) {
+ try {
+ return PCollectionTranslation.fromProto(
+ ReadPayload.parseFrom(
+ PTransformTranslation.toProto(
+ transform,
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+ SdkComponents.create())
+ .getSpec()
+ .getPayload())
+ .getIsBounded());
+ } catch (IOException e) {
+ throw new RuntimeException("Internal error determining boundedness of Read", e);
+ }
+ }
+
+ /** A {@link TransformPayloadTranslator} for {@link Read.Unbounded}. */
+ public static class UnboundedReadPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.WithDefaultRehydration<
+ Read.Unbounded<?>> {
+ public static TransformPayloadTranslator create() {
+ return new UnboundedReadPayloadTranslator();
+ }
+
+ private UnboundedReadPayloadTranslator() {}
+
+ @Override
+ public String getUrn(Read.Unbounded<?> transform) {
+ return PTransformTranslation.READ_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) {
+ ReadPayload payload = toProto(transform.getTransform());
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(getUrn(transform.getTransform()))
+ .setPayload(payload.toByteString())
+ .build();
+ }
+ }
+
+ /** A {@link TransformPayloadTranslator} for {@link Read.Bounded}. */
+ public static class BoundedReadPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.WithDefaultRehydration<
+ Read.Bounded<?>> {
+ public static TransformPayloadTranslator create() {
+ return new BoundedReadPayloadTranslator();
+ }
+
+ private BoundedReadPayloadTranslator() {}
+
+ @Override
+ public String getUrn(Read.Bounded<?> transform) {
+ return PTransformTranslation.READ_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) {
+ ReadPayload payload = toProto(transform.getTransform());
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(getUrn(transform.getTransform()))
+ .setPayload(payload.toByteString())
+ .build();
+ }
+ }
+
+ /** Registers {@link UnboundedReadPayloadTranslator} and {@link BoundedReadPayloadTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+ .put(Read.Unbounded.class, new UnboundedReadPayloadTranslator())
+ .put(Read.Bounded.class, new BoundedReadPayloadTranslator())
+ .build();
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
new file mode 100644
index 0000000..09457a3
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * Vends Java SDK objects rehydrated from a Runner API {@link Components} collection.
+ *
+ * <p>This ensures maximum memoization of rehydrated components, which is semantically necessary for
+ * {@link PCollection} and nice-to-have for other objects.
+ */
+public class RehydratedComponents {
+ private final Components components;
+
+ /**
+ * This class may be used in the context of a pipeline or not. If not, then it cannot
+ * rehydrated {@link PCollection PCollections}.
+ */
+ @Nullable
+ private final Pipeline pipeline;
+
+ /**
+ * A non-evicting cache, serving as a memo table for rehydrated {@link WindowingStrategy
+ * WindowingStrategies}.
+ */
+ private final LoadingCache<String, WindowingStrategy<?, ?>> windowingStrategies =
+ CacheBuilder.newBuilder()
+ .build(
+ new CacheLoader<String, WindowingStrategy<?, ?>>() {
+ @Override
+ public WindowingStrategy<?, ?> load(String id) throws Exception {
+ return WindowingStrategyTranslation.fromProto(
+ components.getWindowingStrategiesOrThrow(id), RehydratedComponents.this);
+ }
+ });
+
+ /** A non-evicting cache, serving as a memo table for rehydrated {@link Coder Coders}. */
+ private final LoadingCache<String, Coder<?>> coders =
+ CacheBuilder.newBuilder()
+ .build(
+ new CacheLoader<String, Coder<?>>() {
+ @Override
+ public Coder<?> load(String id) throws Exception {
+ @Nullable RunnerApi.Coder coder = components.getCodersOrDefault(id, null);
+ checkState(coder != null, "No coder with id '%s' in serialized components", id);
+ return CoderTranslation.fromProto(coder, RehydratedComponents.this);
+ }
+ });
+
+ /**
+ * A non-evicting cache, serving as a memo table for rehydrated {@link PCollection PCollections}.
+ */
+ private final LoadingCache<String, PCollection<?>> pCollections =
+ CacheBuilder.newBuilder()
+ .build(
+ new CacheLoader<String, PCollection<?>>() {
+ @Override
+ public PCollection<?> load(String id) throws Exception {
+ checkState(
+ pipeline != null,
+ "%s Cannot rehydrate %s without a %s:"
+ + " provide one via .withPipeline(...)",
+ RehydratedComponents.class.getSimpleName(),
+ PCollection.class.getSimpleName(),
+ Pipeline.class.getSimpleName());
+ return PCollectionTranslation.fromProto(
+ components.getPcollectionsOrThrow(id), pipeline, RehydratedComponents.this)
+ .setName(id);
+ }
+ });
+
+
+ /** Create a new {@link RehydratedComponents} from a Runner API {@link Components}. */
+ public static RehydratedComponents forComponents(RunnerApi.Components components) {
+ return new RehydratedComponents(components, null);
+ }
+
+ /** Create a new {@link RehydratedComponents} with a pipeline attached. */
+ public RehydratedComponents withPipeline(Pipeline pipeline) {
+ return new RehydratedComponents(components, pipeline);
+ }
+
+ private RehydratedComponents(RunnerApi.Components components, @Nullable Pipeline pipeline) {
+ this.components = components;
+ this.pipeline = pipeline;
+ }
+
+ /**
+ * Returns a {@link PCollection} rehydrated from the Runner API component with the given ID.
+ *
+ * <p>For a single instance of {@link RehydratedComponents}, this always returns the same instance
+ * for a particular id.
+ */
+ public PCollection<?> getPCollection(String pCollectionId) throws IOException {
+ try {
+ return pCollections.get(pCollectionId);
+ } catch (ExecutionException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ /**
+ * Returns a {@link WindowingStrategy} rehydrated from the Runner API component with the given ID.
+ *
+ * <p>For a single instance of {@link RehydratedComponents}, this always returns the same instance
+ * for a particular id.
+ */
+ public WindowingStrategy<?, ?> getWindowingStrategy(String windowingStrategyId)
+ throws IOException {
+ try {
+ return windowingStrategies.get(windowingStrategyId);
+ } catch (ExecutionException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ /**
+ * Returns a {@link Coder} rehydrated from the Runner API component with the given ID.
+ *
+ * <p>For a single instance of {@link RehydratedComponents}, this always returns the same instance
+ * for a particular id.
+ */
+ public Coder<?> getCoder(String coderId) throws IOException {
+ try {
+ return coders.get(coderId);
+ } catch (ExecutionException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
new file mode 100644
index 0000000..c676c97
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.PValueBase;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/** A {@link PCollectionView} created from the components of a {@link SideInput}. */
+class RunnerPCollectionView<T> extends PValueBase implements PCollectionView<T> {
+ private final TupleTag<Iterable<WindowedValue<?>>> tag;
+ private final ViewFn<Iterable<WindowedValue<?>>, T> viewFn;
+ private final WindowMappingFn<?> windowMappingFn;
+ private final WindowingStrategy<?, ?> windowingStrategy;
+ private final Coder<Iterable<WindowedValue<?>>> coder;
+ private final transient PCollection<?> pCollection;
+
+ /**
+ * Create a new {@link RunnerPCollectionView} from the provided components.
+ */
+ RunnerPCollectionView(
+ PCollection<?> pCollection,
+ TupleTag<Iterable<WindowedValue<?>>> tag,
+ ViewFn<Iterable<WindowedValue<?>>, T> viewFn,
+ WindowMappingFn<?> windowMappingFn,
+ @Nullable WindowingStrategy<?, ?> windowingStrategy,
+ @Nullable Coder<Iterable<WindowedValue<?>>> coder) {
+ this.pCollection = pCollection;
+ this.tag = tag;
+ this.viewFn = viewFn;
+ this.windowMappingFn = windowMappingFn;
+ this.windowingStrategy = windowingStrategy;
+ this.coder = coder;
+ }
+
+ @Override
+ public PCollection<?> getPCollection() {
+ return pCollection;
+ }
+
+ @Override
+ public TupleTag<Iterable<WindowedValue<?>>> getTagInternal() {
+ return tag;
+ }
+
+ @Override
+ public ViewFn<Iterable<WindowedValue<?>>, T> getViewFn() {
+ return viewFn;
+ }
+
+ @Override
+ public WindowMappingFn<?> getWindowMappingFn() {
+ return windowMappingFn;
+ }
+
+ @Override
+ public WindowingStrategy<?, ?> getWindowingStrategyInternal() {
+ return windowingStrategy;
+ }
+
+ @Override
+ public Coder<Iterable<WindowedValue<?>>> getCoderInternal() {
+ return coder;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ throw new UnsupportedOperationException(String.format(
+ "A %s cannot be expanded", RunnerPCollectionView.class.getSimpleName()));
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PCollectionView)) {
+ return false;
+ }
+ @SuppressWarnings("unchecked")
+ PCollectionView<?> otherView = (PCollectionView<?>) other;
+ return tag.equals(otherView.getTagInternal());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tag);
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index eb29b9a..0a8ffb6 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -22,31 +22,23 @@
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.base.Equivalence;
-import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
-import com.google.common.collect.ListMultimap;
import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.TransformHierarchy.Node;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.util.NameUtils;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.WindowingStrategy;
/** SDK objects that will be represented at some later point within a {@link Components} object. */
-class SdkComponents {
+public class SdkComponents {
private final RunnerApi.Components.Builder componentsBuilder;
private final BiMap<AppliedPTransform<?, ?, ?>, String> transformIds;
@@ -58,52 +50,10 @@
// TODO: Specify environments
/** Create a new {@link SdkComponents} with no components. */
- static SdkComponents create() {
+ public static SdkComponents create() {
return new SdkComponents();
}
- public static RunnerApi.Pipeline translatePipeline(Pipeline p) {
- final SdkComponents components = create();
- final Collection<String> rootIds = new HashSet<>();
- p.traverseTopologically(
- new PipelineVisitor.Defaults() {
- private final ListMultimap<Node, AppliedPTransform<?, ?, ?>> children =
- ArrayListMultimap.create();
-
- @Override
- public void leaveCompositeTransform(Node node) {
- if (node.isRootNode()) {
- for (AppliedPTransform<?, ?, ?> pipelineRoot : children.get(node)) {
- rootIds.add(components.getExistingPTransformId(pipelineRoot));
- }
- } else {
- children.put(node.getEnclosingNode(), node.toAppliedPTransform());
- try {
- components.registerPTransform(node.toAppliedPTransform(), children.get(node));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void visitPrimitiveTransform(Node node) {
- children.put(node.getEnclosingNode(), node.toAppliedPTransform());
- try {
- components.registerPTransform(
- node.toAppliedPTransform(), Collections.<AppliedPTransform<?, ?, ?>>emptyList());
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- }
- });
- // TODO: Display Data
- return RunnerApi.Pipeline.newBuilder()
- .setComponents(components.toComponents())
- .addAllRootTransformIds(rootIds)
- .build();
- }
-
private SdkComponents() {
this.componentsBuilder = RunnerApi.Components.newBuilder();
this.transformIds = HashBiMap.create();
@@ -119,7 +69,7 @@
*
* <p>All of the children must already be registered within this {@link SdkComponents}.
*/
- String registerPTransform(
+ public String registerPTransform(
AppliedPTransform<?, ?, ?> appliedPTransform, List<AppliedPTransform<?, ?, ?>> children)
throws IOException {
String name = getApplicationName(appliedPTransform);
@@ -129,7 +79,8 @@
return name;
}
checkNotNull(children, "child nodes may not be null");
- componentsBuilder.putTransforms(name, PTransforms.toProto(appliedPTransform, children, this));
+ componentsBuilder.putTransforms(name, PTransformTranslation
+ .toProto(appliedPTransform, children, this));
return name;
}
@@ -167,14 +118,15 @@
* ID for the {@link PCollection}. Multiple registrations of the same {@link PCollection} will
* return the same unique ID.
*/
- String registerPCollection(PCollection<?> pCollection) throws IOException {
+ public String registerPCollection(PCollection<?> pCollection) throws IOException {
String existing = pCollectionIds.get(pCollection);
if (existing != null) {
return existing;
}
String uniqueName = uniqify(pCollection.getName(), pCollectionIds.values());
pCollectionIds.put(pCollection, uniqueName);
- componentsBuilder.putPcollections(uniqueName, PCollections.toProto(pCollection, this));
+ componentsBuilder.putPcollections(
+ uniqueName, PCollectionTranslation.toProto(pCollection, this));
return uniqueName;
}
@@ -183,7 +135,8 @@
* unique ID for the {@link WindowingStrategy}. Multiple registrations of the same {@link
* WindowingStrategy} will return the same unique ID.
*/
- String registerWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) throws IOException {
+ public String registerWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy)
+ throws IOException {
String existing = windowingStrategyIds.get(windowingStrategy);
if (existing != null) {
return existing;
@@ -196,7 +149,7 @@
String name = uniqify(baseName, windowingStrategyIds.values());
windowingStrategyIds.put(windowingStrategy, name);
RunnerApi.WindowingStrategy windowingStrategyProto =
- WindowingStrategies.toProto(windowingStrategy, this);
+ WindowingStrategyTranslation.toProto(windowingStrategy, this);
componentsBuilder.putWindowingStrategies(name, windowingStrategyProto);
return name;
}
@@ -210,7 +163,7 @@
* #equals(Object)} and {@link #hashCode()} but incompatible binary formats are not considered the
* same coder.
*/
- String registerCoder(Coder<?> coder) throws IOException {
+ public String registerCoder(Coder<?> coder) throws IOException {
String existing = coderIds.get(Equivalence.identity().wrap(coder));
if (existing != null) {
return existing;
@@ -218,7 +171,7 @@
String baseName = NameUtils.approximateSimpleName(coder);
String name = uniqify(baseName, coderIds.values());
coderIds.put(Equivalence.identity().wrap(coder), name);
- RunnerApi.Coder coderProto = Coders.toProto(coder, this);
+ RunnerApi.Coder coderProto = CoderTranslation.toProto(coder, this);
componentsBuilder.putCoders(name, coderProto);
return name;
}
@@ -239,7 +192,7 @@
* PCollection PCollections}, and {@link PTransform PTransforms}.
*/
@Experimental
- RunnerApi.Components toComponents() {
+ public RunnerApi.Components toComponents() {
return componentsBuilder.build();
}
}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SerializablePipelineOptions.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SerializablePipelineOptions.java
new file mode 100644
index 0000000..e697fb2
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SerializablePipelineOptions.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
+
+/**
+ * Holds a {@link PipelineOptions} in JSON serialized form and calls {@link
+ * FileSystems#setDefaultPipelineOptions(PipelineOptions)} on construction or on deserialization.
+ */
+public class SerializablePipelineOptions implements Serializable {
+ private static final ObjectMapper MAPPER =
+ new ObjectMapper()
+ .registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+
+ private final String serializedPipelineOptions;
+ private transient PipelineOptions options;
+
+ public SerializablePipelineOptions(PipelineOptions options) {
+ this.serializedPipelineOptions = serializeToJson(options);
+ this.options = options;
+ FileSystems.setDefaultPipelineOptions(options);
+ }
+
+ public PipelineOptions get() {
+ return options;
+ }
+
+ private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
+ is.defaultReadObject();
+ this.options = deserializeFromJson(serializedPipelineOptions);
+ // TODO https://issues.apache.org/jira/browse/BEAM-2712: remove this call.
+ FileSystems.setDefaultPipelineOptions(options);
+ }
+
+ private static String serializeToJson(PipelineOptions options) {
+ try {
+ return MAPPER.writeValueAsString(options);
+ } catch (JsonProcessingException e) {
+ throw new IllegalArgumentException("Failed to serialize PipelineOptions", e);
+ }
+ }
+
+ private static PipelineOptions deserializeFromJson(String options) {
+ try {
+ return MAPPER.readValue(options, PipelineOptions.class);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Failed to deserialize PipelineOptions", e);
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
new file mode 100644
index 0000000..ab66e84
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * A utility transform that executes a <a
+ * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn} by expanding it into a
+ * network of simpler transforms:
+ *
+ * <ol>
+ * <li>Pair each element with an initial restriction
+ * <li>Split each restriction into sub-restrictions
+ * <li>Explode windows, since splitting within each window has to happen independently
+ * <li>Assign a unique key to each element/restriction pair
+ * <li>Process the keyed element/restriction pairs in a runner-specific way with the splittable
+ * {@link DoFn}'s {@link DoFn.ProcessElement} method.
+ * </ol>
+ *
+ * <p>This transform is intended as a helper for internal use by runners when implementing {@code
+ * ParDo.of(splittable DoFn)}, but not for direct use by pipeline writers.
+ */
+@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+public class SplittableParDo<InputT, OutputT, RestrictionT>
+ extends PTransform<PCollection<InputT>, PCollectionTuple> {
+
+ private final DoFn<InputT, OutputT> doFn;
+ private final List<PCollectionView<?>> sideInputs;
+ private final TupleTag<OutputT> mainOutputTag;
+ private final TupleTagList additionalOutputTags;
+ private final Map<TupleTag<?>, Coder<?>> outputTagsToCoders;
+
+ public static final String SPLITTABLE_PROCESS_URN =
+ "urn:beam:runners_core:transforms:splittable_process:v1";
+
+ public static final String SPLITTABLE_PROCESS_KEYED_ELEMENTS_URN =
+ "urn:beam:runners_core:transforms:splittable_process_keyed_elements:v1";
+
+ public static final String SPLITTABLE_GBKIKWI_URN =
+ "urn:beam:runners_core:transforms:splittable_gbkikwi:v1";
+
+ private SplittableParDo(
+ DoFn<InputT, OutputT> doFn,
+ List<PCollectionView<?>> sideInputs,
+ TupleTag<OutputT> mainOutputTag,
+ TupleTagList additionalOutputTags,
+ Map<TupleTag<?>, Coder<?>> outputTagsToCoders) {
+ checkArgument(
+ DoFnSignatures.getSignature(doFn.getClass()).processElement().isSplittable(),
+ "fn must be a splittable DoFn");
+ this.doFn = doFn;
+ this.sideInputs = sideInputs;
+ this.mainOutputTag = mainOutputTag;
+ this.additionalOutputTags = additionalOutputTags;
+ this.outputTagsToCoders = outputTagsToCoders;
+ }
+
+ /**
+ * Creates the transform for a {@link ParDo}-compatible {@link AppliedPTransform}.
+ *
+ * <p>The input may generally be a deserialized transform so it may not actually be a {@link
+ * ParDo}. Instead {@link ParDoTranslation} will be used to extract fields.
+ */
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static <InputT, OutputT> SplittableParDo<InputT, OutputT, ?> forAppliedParDo(
+ AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> parDo) {
+ checkArgument(parDo != null, "parDo must not be null");
+
+ try {
+ Map<TupleTag<?>, Coder<?>> outputTagsToCoders = Maps.newHashMap();
+ for (Map.Entry<TupleTag<?>, PValue> entry : parDo.getOutputs().entrySet()) {
+ outputTagsToCoders.put(entry.getKey(), ((PCollection) entry.getValue()).getCoder());
+ }
+ return new SplittableParDo(
+ ParDoTranslation.getDoFn(parDo),
+ ParDoTranslation.getSideInputs(parDo),
+ ParDoTranslation.getMainOutputTag(parDo),
+ ParDoTranslation.getAdditionalOutputTags(parDo),
+ outputTagsToCoders);
+ } catch (IOException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ @Override
+ public PCollectionTuple expand(PCollection<InputT> input) {
+ Coder<RestrictionT> restrictionCoder =
+ DoFnInvokers.invokerFor(doFn)
+ .invokeGetRestrictionCoder(input.getPipeline().getCoderRegistry());
+ Coder<KV<InputT, RestrictionT>> splitCoder = KvCoder.of(input.getCoder(), restrictionCoder);
+
+ PCollection<KV<String, KV<InputT, RestrictionT>>> keyedRestrictions =
+ input
+ .apply(
+ "Pair with initial restriction",
+ ParDo.of(new PairWithRestrictionFn<InputT, OutputT, RestrictionT>(doFn)))
+ .setCoder(splitCoder)
+ .apply(
+ "Split restriction", ParDo.of(new SplitRestrictionFn<InputT, RestrictionT>(doFn)))
+ .setCoder(splitCoder)
+ // ProcessFn requires all input elements to be in a single window and have a single
+ // element per work item. This must precede the unique keying so each key has a single
+ // associated element.
+ .apply("Explode windows", ParDo.of(new ExplodeWindowsFn<KV<InputT, RestrictionT>>()))
+ .apply(
+ "Assign unique key",
+ WithKeys.of(new RandomUniqueKeyFn<KV<InputT, RestrictionT>>()));
+
+ return keyedRestrictions.apply(
+ "ProcessKeyedElements",
+ new ProcessKeyedElements<>(
+ doFn,
+ input.getCoder(),
+ restrictionCoder,
+ (WindowingStrategy<InputT, ?>) input.getWindowingStrategy(),
+ sideInputs,
+ mainOutputTag,
+ additionalOutputTags,
+ outputTagsToCoders));
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return PCollectionViews.toAdditionalInputs(sideInputs);
+ }
+
+ /**
+ * A {@link DoFn} that forces each of its outputs to be in a single window, by indicating to the
+ * runner that it observes the window of its input element, so the runner is forced to apply it to
+ * each input in a single window and thus its output is also in a single window.
+ */
+ private static class ExplodeWindowsFn<InputT> extends DoFn<InputT, InputT> {
+ @ProcessElement
+ public void process(ProcessContext c, BoundedWindow window) {
+ c.output(c.element());
+ }
+ }
+
+ /**
+ * Runner-specific primitive {@link PTransform} that invokes the {@link DoFn.ProcessElement}
+ * method for a splittable {@link DoFn} on each {@link KV} of the input {@link PCollection} of
+ * {@link KV KVs} keyed with arbitrary but globally unique keys.
+ */
+ public static class ProcessKeyedElements<InputT, OutputT, RestrictionT>
+ extends RawPTransform<PCollection<KV<String, KV<InputT, RestrictionT>>>, PCollectionTuple> {
+ private final DoFn<InputT, OutputT> fn;
+ private final Coder<InputT> elementCoder;
+ private final Coder<RestrictionT> restrictionCoder;
+ private final WindowingStrategy<InputT, ?> windowingStrategy;
+ private final List<PCollectionView<?>> sideInputs;
+ private final TupleTag<OutputT> mainOutputTag;
+ private final TupleTagList additionalOutputTags;
+ private final Map<TupleTag<?>, Coder<?>> outputTagsToCoders;
+
+ /**
+ * @param fn the splittable {@link DoFn}.
+ * @param windowingStrategy the {@link WindowingStrategy} of the input collection.
+ * @param sideInputs list of side inputs that should be available to the {@link DoFn}.
+ * @param mainOutputTag {@link TupleTag Tag} of the {@link DoFn DoFn's} main output.
+ * @param additionalOutputTags {@link TupleTagList Tags} of the {@link DoFn DoFn's} additional
+ * @param outputTagsToCoders A map from output tag to the coder for that output, which should
+ * provide mappings for the main and all additional tags.
+ */
+ public ProcessKeyedElements(
+ DoFn<InputT, OutputT> fn,
+ Coder<InputT> elementCoder,
+ Coder<RestrictionT> restrictionCoder,
+ WindowingStrategy<InputT, ?> windowingStrategy,
+ List<PCollectionView<?>> sideInputs,
+ TupleTag<OutputT> mainOutputTag,
+ TupleTagList additionalOutputTags,
+ Map<TupleTag<?>, Coder<?>> outputTagsToCoders) {
+ this.fn = fn;
+ this.elementCoder = elementCoder;
+ this.restrictionCoder = restrictionCoder;
+ this.windowingStrategy = windowingStrategy;
+ this.sideInputs = sideInputs;
+ this.mainOutputTag = mainOutputTag;
+ this.additionalOutputTags = additionalOutputTags;
+ this.outputTagsToCoders = outputTagsToCoders;
+ }
+
+ public DoFn<InputT, OutputT> getFn() {
+ return fn;
+ }
+
+ public Coder<InputT> getElementCoder() {
+ return elementCoder;
+ }
+
+ public Coder<RestrictionT> getRestrictionCoder() {
+ return restrictionCoder;
+ }
+
+ public WindowingStrategy<InputT, ?> getInputWindowingStrategy() {
+ return windowingStrategy;
+ }
+
+ public List<PCollectionView<?>> getSideInputs() {
+ return sideInputs;
+ }
+
+ public TupleTag<OutputT> getMainOutputTag() {
+ return mainOutputTag;
+ }
+
+ public TupleTagList getAdditionalOutputTags() {
+ return additionalOutputTags;
+ }
+
+ public Map<TupleTag<?>, Coder<?>> getOutputTagsToCoders() {
+ return outputTagsToCoders;
+ }
+
+ @Override
+ public PCollectionTuple expand(PCollection<KV<String, KV<InputT, RestrictionT>>> input) {
+ return createPrimitiveOutputFor(
+ input, fn, mainOutputTag, additionalOutputTags, outputTagsToCoders, windowingStrategy);
+ }
+
+ public static <OutputT> PCollectionTuple createPrimitiveOutputFor(
+ PCollection<?> input,
+ DoFn<?, OutputT> fn,
+ TupleTag<OutputT> mainOutputTag,
+ TupleTagList additionalOutputTags,
+ Map<TupleTag<?>, Coder<?>> outputTagsToCoders,
+ WindowingStrategy<?, ?> windowingStrategy) {
+ DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
+ PCollectionTuple outputs =
+ PCollectionTuple.ofPrimitiveOutputsInternal(
+ input.getPipeline(),
+ TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
+ outputTagsToCoders,
+ windowingStrategy,
+ input.isBounded().and(signature.isBoundedPerElement()));
+
+ // Set output type descriptor similarly to how ParDo.MultiOutput does it.
+ outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor());
+
+ return outputs;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return PCollectionViews.toAdditionalInputs(sideInputs);
+ }
+
+ @Override
+ public String getUrn() {
+ return SPLITTABLE_PROCESS_KEYED_ELEMENTS_URN;
+ }
+
+ @Nullable
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ return null;
+ }
+ }
+
+ /**
+ * Assigns a random unique key to each element of the input collection, so that the output
+ * collection is effectively the same elements as input, but the per-key state and timers are now
+ * effectively per-element.
+ */
+ private static class RandomUniqueKeyFn<T> implements SerializableFunction<T, String> {
+ @Override
+ public String apply(T input) {
+ return UUID.randomUUID().toString();
+ }
+ }
+
+ /**
+ * Pairs each input element with its initial restriction using the given splittable {@link DoFn}.
+ */
+ private static class PairWithRestrictionFn<InputT, OutputT, RestrictionT>
+ extends DoFn<InputT, KV<InputT, RestrictionT>> {
+ private DoFn<InputT, OutputT> fn;
+ private transient DoFnInvoker<InputT, OutputT> invoker;
+
+ PairWithRestrictionFn(DoFn<InputT, OutputT> fn) {
+ this.fn = fn;
+ }
+
+ @Setup
+ public void setup() {
+ invoker = DoFnInvokers.invokerFor(fn);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) {
+ context.output(
+ KV.of(
+ context.element(),
+ invoker.<RestrictionT>invokeGetInitialRestriction(context.element())));
+ }
+ }
+
+ /** Splits the restriction using the given {@link SplitRestriction} method. */
+ private static class SplitRestrictionFn<InputT, RestrictionT>
+ extends DoFn<KV<InputT, RestrictionT>, KV<InputT, RestrictionT>> {
+ private final DoFn<InputT, ?> splittableFn;
+ private transient DoFnInvoker<InputT, ?> invoker;
+
+ SplitRestrictionFn(DoFn<InputT, ?> splittableFn) {
+ this.splittableFn = splittableFn;
+ }
+
+ @Setup
+ public void setup() {
+ invoker = DoFnInvokers.invokerFor(splittableFn);
+ }
+
+ @ProcessElement
+ public void processElement(final ProcessContext c) {
+ final InputT element = c.element().getKey();
+ invoker.invokeSplitRestriction(
+ element,
+ c.element().getValue(),
+ new OutputReceiver<RestrictionT>() {
+ @Override
+ public void output(RestrictionT part) {
+ c.output(KV.of(element, part));
+ }
+ });
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
new file mode 100644
index 0000000..1b18844
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * Utility methods for translating a {@link TestStream} to and from {@link RunnerApi}
+ * representations.
+ */
+public class TestStreamTranslation {
+
+ private interface TestStreamLike {
+ Coder<?> getValueCoder();
+
+ List<RunnerApi.TestStreamPayload.Event> getEvents();
+ }
+
+ @VisibleForTesting
+ static class RawTestStream<T> extends PTransformTranslation.RawPTransform<PBegin, PCollection<T>>
+ implements TestStreamLike {
+
+ private final transient RehydratedComponents rehydratedComponents;
+ private final RunnerApi.TestStreamPayload payload;
+ private final Coder<T> valueCoder;
+ private final RunnerApi.FunctionSpec spec;
+
+ public RawTestStream(
+ RunnerApi.TestStreamPayload payload, RehydratedComponents rehydratedComponents) {
+ this.payload = payload;
+ this.spec =
+ RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(TEST_STREAM_TRANSFORM_URN)
+ .setPayload(payload.toByteString())
+ .build();
+ this.rehydratedComponents = rehydratedComponents;
+
+ // Eagerly extract the coder to throw a good exception here
+ try {
+ this.valueCoder = (Coder<T>) rehydratedComponents.getCoder(payload.getCoderId());
+ } catch (IOException exc) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Failure extracting coder with id '%s' for %s",
+ payload.getCoderId(), TestStream.class.getSimpleName()),
+ exc);
+ }
+ }
+
+ @Override
+ public String getUrn() {
+ return TEST_STREAM_TRANSFORM_URN;
+ }
+
+ @Nonnull
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ return spec;
+ }
+
+ @Override
+ public RunnerApi.FunctionSpec migrate(SdkComponents components) throws IOException {
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(TEST_STREAM_TRANSFORM_URN)
+ .setPayload(payloadForTestStreamLike(this, components).toByteString())
+ .build();
+ }
+
+ @Override
+ public Coder<T> getValueCoder() {
+ return valueCoder;
+ }
+
+ @Override
+ public List<RunnerApi.TestStreamPayload.Event> getEvents() {
+ return payload.getEventsList();
+ }
+ }
+
+ private static TestStream<?> testStreamFromProtoPayload(
+ RunnerApi.TestStreamPayload testStreamPayload, RehydratedComponents components)
+ throws IOException {
+
+ Coder<Object> coder = (Coder<Object>) components.getCoder(testStreamPayload.getCoderId());
+
+ List<TestStream.Event<Object>> events = new ArrayList<>();
+
+ for (RunnerApi.TestStreamPayload.Event event : testStreamPayload.getEventsList()) {
+ events.add(eventFromProto(event, coder));
+ }
+ return TestStream.fromRawEvents(coder, events);
+ }
+
+ /**
+ * Converts an {@link AppliedPTransform}, which may be a rehydrated transform or an original
+ * {@link TestStream}, to a {@link TestStream}.
+ */
+ public static <T> TestStream<T> getTestStream(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> application)
+ throws IOException {
+ // For robustness, we don't take this shortcut:
+ // if (application.getTransform() instanceof TestStream) {
+ // return application.getTransform()
+ // }
+
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PTransform transformProto = PTransformTranslation.toProto(application, sdkComponents);
+ checkArgument(
+ TEST_STREAM_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
+ "Attempt to get %s from a transform with wrong URN %s",
+ TestStream.class.getSimpleName(),
+ transformProto.getSpec().getUrn());
+ RunnerApi.TestStreamPayload testStreamPayload =
+ RunnerApi.TestStreamPayload.parseFrom(transformProto.getSpec().getPayload());
+
+ return (TestStream<T>)
+ testStreamFromProtoPayload(
+ testStreamPayload, RehydratedComponents.forComponents(sdkComponents.toComponents()));
+ }
+
+ static <T> RunnerApi.TestStreamPayload.Event eventToProto(
+ TestStream.Event<T> event, Coder<T> coder) throws IOException {
+ switch (event.getType()) {
+ case WATERMARK:
+ return RunnerApi.TestStreamPayload.Event.newBuilder()
+ .setWatermarkEvent(
+ RunnerApi.TestStreamPayload.Event.AdvanceWatermark.newBuilder()
+ .setNewWatermark(
+ ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis()))
+ .build();
+
+ case PROCESSING_TIME:
+ return RunnerApi.TestStreamPayload.Event.newBuilder()
+ .setProcessingTimeEvent(
+ RunnerApi.TestStreamPayload.Event.AdvanceProcessingTime.newBuilder()
+ .setAdvanceDuration(
+ ((TestStream.ProcessingTimeEvent<T>) event)
+ .getProcessingTimeAdvance()
+ .getMillis()))
+ .build();
+
+ case ELEMENT:
+ RunnerApi.TestStreamPayload.Event.AddElements.Builder builder =
+ RunnerApi.TestStreamPayload.Event.AddElements.newBuilder();
+ for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) {
+ builder.addElements(
+ RunnerApi.TestStreamPayload.TimestampedElement.newBuilder()
+ .setTimestamp(element.getTimestamp().getMillis())
+ .setEncodedElement(
+ ByteString.copyFrom(
+ CoderUtils.encodeToByteArray(coder, element.getValue()))));
+ }
+ return RunnerApi.TestStreamPayload.Event.newBuilder().setElementEvent(builder).build();
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Unsupported type of %s: %s",
+ TestStream.Event.class.getCanonicalName(), event.getType()));
+ }
+ }
+
+ static <T> TestStream.Event<T> eventFromProto(
+ RunnerApi.TestStreamPayload.Event protoEvent, Coder<T> coder) throws IOException {
+ switch (protoEvent.getEventCase()) {
+ case WATERMARK_EVENT:
+ return TestStream.WatermarkEvent.advanceTo(
+ new Instant(protoEvent.getWatermarkEvent().getNewWatermark()));
+ case PROCESSING_TIME_EVENT:
+ return TestStream.ProcessingTimeEvent.advanceBy(
+ Duration.millis(protoEvent.getProcessingTimeEvent().getAdvanceDuration()));
+ case ELEMENT_EVENT:
+ List<TimestampedValue<T>> decodedElements = new ArrayList<>();
+ for (RunnerApi.TestStreamPayload.TimestampedElement element :
+ protoEvent.getElementEvent().getElementsList()) {
+ decodedElements.add(
+ TimestampedValue.of(
+ CoderUtils.decodeFromByteArray(coder, element.getEncodedElement().toByteArray()),
+ new Instant(element.getTimestamp())));
+ }
+ return TestStream.ElementEvent.add(decodedElements);
+ case EVENT_NOT_SET:
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Unsupported type of %s: %s",
+ RunnerApi.TestStreamPayload.Event.class.getCanonicalName(),
+ protoEvent.getEventCase()));
+ }
+ }
+
+ /** A translator registered to translate {@link TestStream} objects to protobuf representation. */
+ static class TestStreamTranslator implements TransformPayloadTranslator<TestStream<?>> {
+ @Override
+ public String getUrn(TestStream<?> transform) {
+ return TEST_STREAM_TRANSFORM_URN;
+ }
+
+ @Override
+ public RunnerApi.FunctionSpec translate(
+ final AppliedPTransform<?, ?, TestStream<?>> transform, SdkComponents components)
+ throws IOException {
+ return translateTyped(transform.getTransform(), components);
+ }
+
+ @Override
+ public PTransformTranslation.RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ checkArgument(
+ protoTransform.getSpec() != null,
+ "%s received transform with null spec",
+ getClass().getSimpleName());
+ checkArgument(protoTransform.getSpec().getUrn().equals(TEST_STREAM_TRANSFORM_URN));
+ return new RawTestStream<>(
+ RunnerApi.TestStreamPayload.parseFrom(protoTransform.getSpec().getPayload()),
+ rehydratedComponents);
+ }
+
+ private <T> RunnerApi.FunctionSpec translateTyped(
+ final TestStream<T> testStream, SdkComponents components) throws IOException {
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(TEST_STREAM_TRANSFORM_URN)
+ .setPayload(payloadForTestStream(testStream, components).toByteString())
+ .build();
+ }
+
+ /** Registers {@link TestStreamTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(TestStream.class, new TestStreamTranslator());
+ }
+
+ @Override
+ public Map<String, ? extends TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.singletonMap(TEST_STREAM_TRANSFORM_URN, new TestStreamTranslator());
+ }
+ }
+ }
+
+ /** Produces a {@link RunnerApi.TestStreamPayload} from a portable {@link RawTestStream}. */
+ static RunnerApi.TestStreamPayload payloadForTestStreamLike(
+ TestStreamLike transform, SdkComponents components) throws IOException {
+ return RunnerApi.TestStreamPayload.newBuilder()
+ .setCoderId(components.registerCoder(transform.getValueCoder()))
+ .addAllEvents(transform.getEvents())
+ .build();
+ }
+
+ @VisibleForTesting
+ static <T> RunnerApi.TestStreamPayload payloadForTestStream(
+ final TestStream<T> testStream, SdkComponents components) throws IOException {
+ return payloadForTestStreamLike(
+ new TestStreamLike() {
+ @Override
+ public Coder<T> getValueCoder() {
+ return testStream.getValueCoder();
+ }
+
+ @Override
+ public List<RunnerApi.TestStreamPayload.Event> getEvents() {
+ try {
+ List<RunnerApi.TestStreamPayload.Event> protoEvents = new ArrayList<>();
+ for (TestStream.Event<T> event : testStream.getEvents()) {
+ protoEvents.add(eventToProto(event, testStream.getValueCoder()));
+ }
+ return protoEvents;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ },
+ components);
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
new file mode 100644
index 0000000..2baf93a
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
+public class TransformInputs {
+ /**
+ * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
+ * PTransform#getAdditionalInputs()}.
+ */
+ public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
+ ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
+ PTransform<?, ?> transform = application.getTransform();
+ for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
+ if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
+ mainInputs.add(input.getValue());
+ }
+ }
+ checkArgument(
+ !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
+ "Expected at least one main input if any inputs exist");
+ return mainInputs.build();
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
new file mode 100644
index 0000000..58417a8
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.transforms.PTransform;
+
+/** A registrar of TransformPayloadTranslator. */
+public interface TransformPayloadTranslatorRegistrar {
+ Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators();
+
+ Map<String, ? extends TransformPayloadTranslator> getTransformRehydrators();
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
new file mode 100644
index 0000000..6b2a182
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import java.io.Serializable;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.transforms.windowing.AfterAll;
+import org.apache.beam.sdk.transforms.windowing.AfterEach;
+import org.apache.beam.sdk.transforms.windowing.AfterFirst;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterSynchronizedProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark.AfterWatermarkEarlyAndLate;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark.FromEndOfWindow;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.Never;
+import org.apache.beam.sdk.transforms.windowing.Never.NeverTrigger;
+import org.apache.beam.sdk.transforms.windowing.OrFinallyTrigger;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
+import org.apache.beam.sdk.transforms.windowing.TimestampTransform;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/** Utilities for working with {@link TriggerTranslation Triggers}. */
+@Experimental(Experimental.Kind.TRIGGER)
+public class TriggerTranslation implements Serializable {
+
+ @VisibleForTesting static final ProtoConverter CONVERTER = new ProtoConverter();
+
+ public static RunnerApi.Trigger toProto(Trigger trigger) {
+ return CONVERTER.convertTrigger(trigger);
+ }
+
+ @VisibleForTesting
+ static class ProtoConverter {
+
+ public RunnerApi.Trigger convertTrigger(Trigger trigger) {
+ Method evaluationMethod = getEvaluationMethod(trigger.getClass());
+ return tryConvert(evaluationMethod, trigger);
+ }
+
+ private RunnerApi.Trigger tryConvert(Method evaluationMethod, Trigger trigger) {
+ try {
+ return (RunnerApi.Trigger) evaluationMethod.invoke(this, trigger);
+ } catch (InvocationTargetException exc) {
+ if (exc.getCause() instanceof RuntimeException) {
+ throw (RuntimeException) exc.getCause();
+ } else {
+ throw new RuntimeException(exc.getCause());
+ }
+ } catch (IllegalAccessException exc) {
+ throw new IllegalStateException(
+ String.format("Internal error: could not invoke %s", evaluationMethod));
+ }
+ }
+
+ private Method getEvaluationMethod(Class<?> clazz) {
+ try {
+ return getClass().getDeclaredMethod("convertSpecific", clazz);
+ } catch (NoSuchMethodException exc) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot translate trigger class %s to a runner-API proto.",
+ clazz.getCanonicalName()),
+ exc);
+ }
+ }
+
+ private RunnerApi.Trigger convertSpecific(DefaultTrigger v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setDefault(RunnerApi.Trigger.Default.getDefaultInstance())
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(FromEndOfWindow v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setAfterEndOfWindow(RunnerApi.Trigger.AfterEndOfWindow.newBuilder())
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(NeverTrigger v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setNever(RunnerApi.Trigger.Never.getDefaultInstance())
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(ReshuffleTrigger v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setAlways(RunnerApi.Trigger.Always.getDefaultInstance())
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterSynchronizedProcessingTime v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setAfterSynchronizedProcessingTime(
+ RunnerApi.Trigger.AfterSynchronizedProcessingTime.getDefaultInstance())
+ .build();
+ }
+
+ private RunnerApi.TimeDomain.Enum convertTimeDomain(TimeDomain timeDomain) {
+ switch (timeDomain) {
+ case EVENT_TIME:
+ return RunnerApi.TimeDomain.Enum.EVENT_TIME;
+ case PROCESSING_TIME:
+ return RunnerApi.TimeDomain.Enum.PROCESSING_TIME;
+ case SYNCHRONIZED_PROCESSING_TIME:
+ return RunnerApi.TimeDomain.Enum.SYNCHRONIZED_PROCESSING_TIME;
+ default:
+ throw new IllegalArgumentException(String.format("Unknown time domain: %s", timeDomain));
+ }
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterFirst v) {
+ RunnerApi.Trigger.AfterAny.Builder builder = RunnerApi.Trigger.AfterAny.newBuilder();
+
+ for (Trigger subtrigger : v.subTriggers()) {
+ builder.addSubtriggers(toProto(subtrigger));
+ }
+
+ return RunnerApi.Trigger.newBuilder().setAfterAny(builder).build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterAll v) {
+ RunnerApi.Trigger.AfterAll.Builder builder = RunnerApi.Trigger.AfterAll.newBuilder();
+
+ for (Trigger subtrigger : v.subTriggers()) {
+ builder.addSubtriggers(toProto(subtrigger));
+ }
+
+ return RunnerApi.Trigger.newBuilder().setAfterAll(builder).build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterPane v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setElementCount(
+ RunnerApi.Trigger.ElementCount.newBuilder().setElementCount(v.getElementCount()))
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterWatermarkEarlyAndLate v) {
+ RunnerApi.Trigger.AfterEndOfWindow.Builder builder =
+ RunnerApi.Trigger.AfterEndOfWindow.newBuilder();
+
+ builder.setEarlyFirings(toProto(v.getEarlyTrigger()));
+ if (v.getLateTrigger() != null) {
+ builder.setLateFirings(toProto(v.getLateTrigger()));
+ }
+
+ return RunnerApi.Trigger.newBuilder().setAfterEndOfWindow(builder).build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterEach v) {
+ RunnerApi.Trigger.AfterEach.Builder builder = RunnerApi.Trigger.AfterEach.newBuilder();
+
+ for (Trigger subtrigger : v.subTriggers()) {
+ builder.addSubtriggers(toProto(subtrigger));
+ }
+
+ return RunnerApi.Trigger.newBuilder().setAfterEach(builder).build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(Repeatedly v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setRepeat(
+ RunnerApi.Trigger.Repeat.newBuilder()
+ .setSubtrigger(toProto(v.getRepeatedTrigger())))
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(OrFinallyTrigger v) {
+ return RunnerApi.Trigger.newBuilder()
+ .setOrFinally(
+ RunnerApi.Trigger.OrFinally.newBuilder()
+ .setMain(toProto(v.getMainTrigger()))
+ .setFinally(toProto(v.getUntilTrigger())))
+ .build();
+ }
+
+ private RunnerApi.Trigger convertSpecific(AfterProcessingTime v) {
+ RunnerApi.Trigger.AfterProcessingTime.Builder builder =
+ RunnerApi.Trigger.AfterProcessingTime.newBuilder();
+
+ for (TimestampTransform transform : v.getTimestampTransforms()) {
+ builder.addTimestampTransforms(convertTimestampTransform(transform));
+ }
+
+ return RunnerApi.Trigger.newBuilder().setAfterProcessingTime(builder).build();
+ }
+
+ private RunnerApi.TimestampTransform convertTimestampTransform(TimestampTransform transform) {
+ if (transform instanceof TimestampTransform.Delay) {
+ return RunnerApi.TimestampTransform.newBuilder()
+ .setDelay(
+ RunnerApi.TimestampTransform.Delay.newBuilder()
+ .setDelayMillis(((TimestampTransform.Delay) transform).getDelay().getMillis()))
+ .build();
+ } else if (transform instanceof TimestampTransform.AlignTo) {
+ TimestampTransform.AlignTo alignTo = (TimestampTransform.AlignTo) transform;
+ return RunnerApi.TimestampTransform.newBuilder()
+ .setAlignTo(
+ RunnerApi.TimestampTransform.AlignTo.newBuilder()
+ .setPeriod(alignTo.getPeriod().getMillis())
+ .setOffset(alignTo.getOffset().getMillis()))
+ .build();
+
+ } else {
+ throw new IllegalArgumentException(
+ String.format("Unknown %s: %s", TimestampTransform.class.getSimpleName(), transform));
+ }
+ }
+ }
+
+ public static Trigger fromProto(RunnerApi.Trigger triggerProto) {
+ switch (triggerProto.getTriggerCase()) {
+ case AFTER_ALL:
+ return AfterAll.of(protosToTriggers(triggerProto.getAfterAll().getSubtriggersList()));
+ case AFTER_ANY:
+ return AfterFirst.of(protosToTriggers(triggerProto.getAfterAny().getSubtriggersList()));
+ case AFTER_EACH:
+ return AfterEach.inOrder(
+ protosToTriggers(triggerProto.getAfterEach().getSubtriggersList()));
+ case AFTER_END_OF_WINDOW:
+ RunnerApi.Trigger.AfterEndOfWindow eowProto = triggerProto.getAfterEndOfWindow();
+
+ if (!eowProto.hasEarlyFirings() && !eowProto.hasLateFirings()) {
+ return AfterWatermark.pastEndOfWindow();
+ }
+
+ // It either has early or late firings or both; our typing in Java makes this a smidge
+ // annoying
+ if (triggerProto.getAfterEndOfWindow().hasEarlyFirings()) {
+ AfterWatermarkEarlyAndLate trigger =
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(
+ (OnceTrigger)
+ fromProto(triggerProto.getAfterEndOfWindow().getEarlyFirings()));
+
+ if (triggerProto.getAfterEndOfWindow().hasLateFirings()) {
+ trigger =
+ trigger.withLateFirings(
+ (OnceTrigger)
+ fromProto(triggerProto.getAfterEndOfWindow().getLateFirings()));
+ }
+ return trigger;
+ } else {
+ // only late firings, so return directly
+ return AfterWatermark.pastEndOfWindow()
+ .withLateFirings((OnceTrigger) fromProto(eowProto.getLateFirings()));
+ }
+ case AFTER_PROCESSING_TIME:
+ AfterProcessingTime trigger = AfterProcessingTime.pastFirstElementInPane();
+ for (RunnerApi.TimestampTransform transform :
+ triggerProto.getAfterProcessingTime().getTimestampTransformsList()) {
+ switch (transform.getTimestampTransformCase()) {
+ case ALIGN_TO:
+ trigger =
+ trigger.alignedTo(
+ Duration.millis(transform.getAlignTo().getPeriod()),
+ new Instant(transform.getAlignTo().getOffset()));
+ break;
+ case DELAY:
+ trigger = trigger.plusDelayOf(Duration.millis(transform.getDelay().getDelayMillis()));
+ break;
+ case TIMESTAMPTRANSFORM_NOT_SET:
+ throw new IllegalArgumentException(
+ String.format(
+ "Required field 'timestamp_transform' not set in %s", transform));
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Unknown timestamp transform case: %s",
+ transform.getTimestampTransformCase()));
+ }
+ }
+ return trigger;
+ case AFTER_SYNCHRONIZED_PROCESSING_TIME:
+ return AfterSynchronizedProcessingTime.ofFirstElement();
+ case ALWAYS:
+ return new ReshuffleTrigger();
+ case ELEMENT_COUNT:
+ return AfterPane.elementCountAtLeast(triggerProto.getElementCount().getElementCount());
+ case NEVER:
+ return Never.ever();
+ case OR_FINALLY:
+ return fromProto(triggerProto.getOrFinally().getMain())
+ .orFinally((OnceTrigger) fromProto(triggerProto.getOrFinally().getFinally()));
+ case REPEAT:
+ return Repeatedly.forever(fromProto(triggerProto.getRepeat().getSubtrigger()));
+ case DEFAULT:
+ return DefaultTrigger.of();
+ case TRIGGER_NOT_SET:
+ throw new IllegalArgumentException(
+ String.format("Required field 'trigger' not set in %s", triggerProto));
+ default:
+ throw new IllegalArgumentException(
+ String.format("Unknown trigger case: %s", triggerProto.getTriggerCase()));
+ }
+ }
+
+ private static List<Trigger> protosToTriggers(List<RunnerApi.Trigger> triggers) {
+ List<Trigger> result = Lists.newArrayList();
+ for (RunnerApi.Trigger trigger : triggers) {
+ result.add(fromProto(trigger));
+ }
+ return result;
+ }
+
+ // Do not instantiate
+ private TriggerTranslation() {}
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
deleted file mode 100644
index df6c9ed..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core.construction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.List;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.state.TimeDomain;
-import org.apache.beam.sdk.transforms.windowing.AfterAll;
-import org.apache.beam.sdk.transforms.windowing.AfterEach;
-import org.apache.beam.sdk.transforms.windowing.AfterFirst;
-import org.apache.beam.sdk.transforms.windowing.AfterPane;
-import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
-import org.apache.beam.sdk.transforms.windowing.AfterSynchronizedProcessingTime;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark.AfterWatermarkEarlyAndLate;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark.FromEndOfWindow;
-import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
-import org.apache.beam.sdk.transforms.windowing.Never;
-import org.apache.beam.sdk.transforms.windowing.Never.NeverTrigger;
-import org.apache.beam.sdk.transforms.windowing.OrFinallyTrigger;
-import org.apache.beam.sdk.transforms.windowing.Repeatedly;
-import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
-import org.apache.beam.sdk.transforms.windowing.TimestampTransform;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-
-/** Utilities for working with {@link Triggers Triggers}. */
-@Experimental(Experimental.Kind.TRIGGER)
-public class Triggers implements Serializable {
-
- @VisibleForTesting static final ProtoConverter CONVERTER = new ProtoConverter();
-
- public static RunnerApi.Trigger toProto(Trigger trigger) {
- return CONVERTER.convertTrigger(trigger);
- }
-
- @VisibleForTesting
- static class ProtoConverter {
-
- public RunnerApi.Trigger convertTrigger(Trigger trigger) {
- Method evaluationMethod = getEvaluationMethod(trigger.getClass());
- return tryConvert(evaluationMethod, trigger);
- }
-
- private RunnerApi.Trigger tryConvert(Method evaluationMethod, Trigger trigger) {
- try {
- return (RunnerApi.Trigger) evaluationMethod.invoke(this, trigger);
- } catch (InvocationTargetException exc) {
- if (exc.getCause() instanceof RuntimeException) {
- throw (RuntimeException) exc.getCause();
- } else {
- throw new RuntimeException(exc.getCause());
- }
- } catch (IllegalAccessException exc) {
- throw new IllegalStateException(
- String.format("Internal error: could not invoke %s", evaluationMethod));
- }
- }
-
- private Method getEvaluationMethod(Class<?> clazz) {
- try {
- return getClass().getDeclaredMethod("convertSpecific", clazz);
- } catch (NoSuchMethodException exc) {
- throw new IllegalArgumentException(
- String.format(
- "Cannot translate trigger class %s to a runner-API proto.",
- clazz.getCanonicalName()),
- exc);
- }
- }
-
- private RunnerApi.Trigger convertSpecific(DefaultTrigger v) {
- return RunnerApi.Trigger.newBuilder()
- .setDefault(RunnerApi.Trigger.Default.getDefaultInstance())
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(FromEndOfWindow v) {
- return RunnerApi.Trigger.newBuilder()
- .setAfterEndOfWindow(RunnerApi.Trigger.AfterEndOfWindow.newBuilder())
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(NeverTrigger v) {
- return RunnerApi.Trigger.newBuilder()
- .setNever(RunnerApi.Trigger.Never.getDefaultInstance())
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(ReshuffleTrigger v) {
- return RunnerApi.Trigger.newBuilder()
- .setAlways(RunnerApi.Trigger.Always.getDefaultInstance())
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterSynchronizedProcessingTime v) {
- return RunnerApi.Trigger.newBuilder()
- .setAfterSynchronizedProcessingTime(
- RunnerApi.Trigger.AfterSynchronizedProcessingTime.getDefaultInstance())
- .build();
- }
-
- private RunnerApi.TimeDomain convertTimeDomain(TimeDomain timeDomain) {
- switch (timeDomain) {
- case EVENT_TIME:
- return RunnerApi.TimeDomain.EVENT_TIME;
- case PROCESSING_TIME:
- return RunnerApi.TimeDomain.PROCESSING_TIME;
- case SYNCHRONIZED_PROCESSING_TIME:
- return RunnerApi.TimeDomain.SYNCHRONIZED_PROCESSING_TIME;
- default:
- throw new IllegalArgumentException(String.format("Unknown time domain: %s", timeDomain));
- }
- }
-
- private RunnerApi.Trigger convertSpecific(AfterFirst v) {
- RunnerApi.Trigger.AfterAny.Builder builder = RunnerApi.Trigger.AfterAny.newBuilder();
-
- for (Trigger subtrigger : v.subTriggers()) {
- builder.addSubtriggers(toProto(subtrigger));
- }
-
- return RunnerApi.Trigger.newBuilder().setAfterAny(builder).build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterAll v) {
- RunnerApi.Trigger.AfterAll.Builder builder = RunnerApi.Trigger.AfterAll.newBuilder();
-
- for (Trigger subtrigger : v.subTriggers()) {
- builder.addSubtriggers(toProto(subtrigger));
- }
-
- return RunnerApi.Trigger.newBuilder().setAfterAll(builder).build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterPane v) {
- return RunnerApi.Trigger.newBuilder()
- .setElementCount(
- RunnerApi.Trigger.ElementCount.newBuilder().setElementCount(v.getElementCount()))
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterWatermarkEarlyAndLate v) {
- RunnerApi.Trigger.AfterEndOfWindow.Builder builder =
- RunnerApi.Trigger.AfterEndOfWindow.newBuilder();
-
- builder.setEarlyFirings(toProto(v.getEarlyTrigger()));
- if (v.getLateTrigger() != null) {
- builder.setLateFirings(toProto(v.getLateTrigger()));
- }
-
- return RunnerApi.Trigger.newBuilder().setAfterEndOfWindow(builder).build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterEach v) {
- RunnerApi.Trigger.AfterEach.Builder builder = RunnerApi.Trigger.AfterEach.newBuilder();
-
- for (Trigger subtrigger : v.subTriggers()) {
- builder.addSubtriggers(toProto(subtrigger));
- }
-
- return RunnerApi.Trigger.newBuilder().setAfterEach(builder).build();
- }
-
- private RunnerApi.Trigger convertSpecific(Repeatedly v) {
- return RunnerApi.Trigger.newBuilder()
- .setRepeat(
- RunnerApi.Trigger.Repeat.newBuilder()
- .setSubtrigger(toProto(v.getRepeatedTrigger())))
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(OrFinallyTrigger v) {
- return RunnerApi.Trigger.newBuilder()
- .setOrFinally(
- RunnerApi.Trigger.OrFinally.newBuilder()
- .setMain(toProto(v.getMainTrigger()))
- .setFinally(toProto(v.getUntilTrigger())))
- .build();
- }
-
- private RunnerApi.Trigger convertSpecific(AfterProcessingTime v) {
- RunnerApi.Trigger.AfterProcessingTime.Builder builder =
- RunnerApi.Trigger.AfterProcessingTime.newBuilder();
-
- for (TimestampTransform transform : v.getTimestampTransforms()) {
- builder.addTimestampTransforms(convertTimestampTransform(transform));
- }
-
- return RunnerApi.Trigger.newBuilder().setAfterProcessingTime(builder).build();
- }
-
- private RunnerApi.TimestampTransform convertTimestampTransform(TimestampTransform transform) {
- if (transform instanceof TimestampTransform.Delay) {
- return RunnerApi.TimestampTransform.newBuilder()
- .setDelay(
- RunnerApi.TimestampTransform.Delay.newBuilder()
- .setDelayMillis(((TimestampTransform.Delay) transform).getDelay().getMillis()))
- .build();
- } else if (transform instanceof TimestampTransform.AlignTo) {
- TimestampTransform.AlignTo alignTo = (TimestampTransform.AlignTo) transform;
- return RunnerApi.TimestampTransform.newBuilder()
- .setAlignTo(
- RunnerApi.TimestampTransform.AlignTo.newBuilder()
- .setPeriod(alignTo.getPeriod().getMillis())
- .setOffset(alignTo.getOffset().getMillis()))
- .build();
-
- } else {
- throw new IllegalArgumentException(
- String.format("Unknown %s: %s", TimestampTransform.class.getSimpleName(), transform));
- }
- }
- }
-
- public static Trigger fromProto(RunnerApi.Trigger triggerProto) {
- switch (triggerProto.getTriggerCase()) {
- case AFTER_ALL:
- return AfterAll.of(protosToTriggers(triggerProto.getAfterAll().getSubtriggersList()));
- case AFTER_ANY:
- return AfterFirst.of(protosToTriggers(triggerProto.getAfterAny().getSubtriggersList()));
- case AFTER_EACH:
- return AfterEach.inOrder(
- protosToTriggers(triggerProto.getAfterEach().getSubtriggersList()));
- case AFTER_END_OF_WINDOW:
- RunnerApi.Trigger.AfterEndOfWindow eowProto = triggerProto.getAfterEndOfWindow();
-
- if (!eowProto.hasEarlyFirings() && !eowProto.hasLateFirings()) {
- return AfterWatermark.pastEndOfWindow();
- }
-
- // It either has early or late firings or both; our typing in Java makes this a smidge
- // annoying
- if (triggerProto.getAfterEndOfWindow().hasEarlyFirings()) {
- AfterWatermarkEarlyAndLate trigger =
- AfterWatermark.pastEndOfWindow()
- .withEarlyFirings(
- (OnceTrigger)
- fromProto(triggerProto.getAfterEndOfWindow().getEarlyFirings()));
-
- if (triggerProto.getAfterEndOfWindow().hasLateFirings()) {
- trigger =
- trigger.withLateFirings(
- (OnceTrigger)
- fromProto(triggerProto.getAfterEndOfWindow().getLateFirings()));
- }
- return trigger;
- } else {
- // only late firings, so return directly
- return AfterWatermark.pastEndOfWindow()
- .withLateFirings((OnceTrigger) fromProto(eowProto.getLateFirings()));
- }
- case AFTER_PROCESSING_TIME:
- AfterProcessingTime trigger = AfterProcessingTime.pastFirstElementInPane();
- for (RunnerApi.TimestampTransform transform :
- triggerProto.getAfterProcessingTime().getTimestampTransformsList()) {
- switch (transform.getTimestampTransformCase()) {
- case ALIGN_TO:
- trigger =
- trigger.alignedTo(
- Duration.millis(transform.getAlignTo().getPeriod()),
- new Instant(transform.getAlignTo().getOffset()));
- break;
- case DELAY:
- trigger = trigger.plusDelayOf(Duration.millis(transform.getDelay().getDelayMillis()));
- break;
- case TIMESTAMPTRANSFORM_NOT_SET:
- throw new IllegalArgumentException(
- String.format(
- "Required field 'timestamp_transform' not set in %s", transform));
- default:
- throw new IllegalArgumentException(
- String.format(
- "Unknown timestamp transform case: %s",
- transform.getTimestampTransformCase()));
- }
- }
- return trigger;
- case AFTER_SYNCHRONIZED_PROCESSING_TIME:
- return AfterSynchronizedProcessingTime.ofFirstElement();
- case ALWAYS:
- return new ReshuffleTrigger();
- case ELEMENT_COUNT:
- return AfterPane.elementCountAtLeast(triggerProto.getElementCount().getElementCount());
- case NEVER:
- return Never.ever();
- case OR_FINALLY:
- return fromProto(triggerProto.getOrFinally().getMain())
- .orFinally((OnceTrigger) fromProto(triggerProto.getOrFinally().getFinally()));
- case REPEAT:
- return Repeatedly.forever(fromProto(triggerProto.getRepeat().getSubtrigger()));
- case DEFAULT:
- return DefaultTrigger.of();
- case TRIGGER_NOT_SET:
- throw new IllegalArgumentException(
- String.format("Required field 'trigger' not set in %s", triggerProto));
- default:
- throw new IllegalArgumentException(
- String.format("Unknown trigger case: %s", triggerProto.getTriggerCase()));
- }
- }
-
- private static List<Trigger> protosToTriggers(List<RunnerApi.Trigger> triggers) {
- List<Trigger> result = Lists.newArrayList();
- for (RunnerApi.Trigger trigger : triggers) {
- result.add(fromProto(trigger));
- }
- return result;
- }
-
- // Do not instantiate
- private Triggers() {}
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
new file mode 100644
index 0000000..9158aba
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.service.AutoService;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WindowIntoPayload;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+
+/**
+ * Utility methods for translating a {@link Window.Assign} to and from {@link RunnerApi}
+ * representations.
+ */
+public class WindowIntoTranslation {
+
+ static class WindowAssignTranslator
+ extends TransformPayloadTranslator.WithDefaultRehydration<Window.Assign<?>> {
+
+ @Override
+ public String getUrn(Assign<?> transform) {
+ return PTransformTranslation.WINDOW_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
+ return FunctionSpec.newBuilder()
+ .setUrn("urn:beam:transform:window:v1")
+ .setPayload(
+ WindowIntoTranslation.toProto(transform.getTransform(), components).toByteString())
+ .build();
+ }
+ }
+
+ public static WindowIntoPayload toProto(Window.Assign<?> transform, SdkComponents components) {
+ return WindowIntoPayload.newBuilder()
+ .setWindowFn(WindowingStrategyTranslation.toProto(transform.getWindowFn(), components))
+ .build();
+ }
+
+ public static WindowIntoPayload getWindowIntoPayload(AppliedPTransform<?, ?, ?> application) {
+ RunnerApi.PTransform transformProto;
+ try {
+ transformProto =
+ PTransformTranslation.toProto(
+ application,
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+ SdkComponents.create());
+ } catch (IOException exc) {
+ throw new RuntimeException(exc);
+ }
+
+ checkArgument(
+ PTransformTranslation.WINDOW_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
+ "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"",
+ Window.Assign.class.getSimpleName(),
+ application.getTransform(),
+ application.getFullName(),
+ transformProto.getSpec().getUrn());
+
+ WindowIntoPayload windowIntoPayload;
+ try {
+ return WindowIntoPayload.parseFrom(transformProto.getSpec().getPayload());
+ } catch (InvalidProtocolBufferException exc) {
+ throw new IllegalStateException(
+ String.format(
+ "%s translated %s with URN '%s' but payload was not a %s",
+ PTransformTranslation.class.getSimpleName(),
+ application,
+ PTransformTranslation.WINDOW_TRANSFORM_URN,
+ WindowIntoPayload.class.getSimpleName()),
+ exc);
+ }
+ }
+
+ public static WindowFn<?, ?> getWindowFn(AppliedPTransform<?, ?, ?> application) {
+ return WindowingStrategyTranslation.windowFnFromProto(
+ getWindowIntoPayload(application).getWindowFn());
+ }
+
+ /** A {@link TransformPayloadTranslator} for {@link Window}. */
+ public static class WindowIntoPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.WithDefaultRehydration<
+ Window.Assign<?>> {
+ public static TransformPayloadTranslator create() {
+ return new WindowIntoPayloadTranslator();
+ }
+
+ private WindowIntoPayloadTranslator() {}
+
+ @Override
+ public String getUrn(Window.Assign<?> transform) {
+ return PTransformTranslation.WINDOW_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
+ WindowIntoPayload payload = toProto(transform.getTransform(), components);
+ return RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(getUrn(transform.getTransform()))
+ .setPayload(payload.toByteString())
+ .build();
+ }
+ }
+
+ /** Registers {@link WindowIntoPayloadTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.singletonMap(Window.Assign.class, new WindowIntoPayloadTranslator());
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
deleted file mode 100644
index 395702f..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core.construction;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import com.google.protobuf.InvalidProtocolBufferException;
-import java.io.IOException;
-import java.io.Serializable;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.OutputTime;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
-import org.joda.time.Duration;
-
-/** Utilities for working with {@link WindowingStrategy WindowingStrategies}. */
-public class WindowingStrategies implements Serializable {
-
- public static AccumulationMode fromProto(RunnerApi.AccumulationMode proto) {
- switch (proto) {
- case DISCARDING:
- return AccumulationMode.DISCARDING_FIRED_PANES;
- case ACCUMULATING:
- return AccumulationMode.ACCUMULATING_FIRED_PANES;
- case UNRECOGNIZED:
- default:
- // Whether or not it is proto that cannot recognize it (due to the version of the
- // generated code we link to) or the switch hasn't been updated to handle it,
- // the situation is the same: we don't know what this OutputTime means
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- RunnerApi.AccumulationMode.class.getCanonicalName(),
- AccumulationMode.class.getCanonicalName(),
- proto));
- }
- }
-
- public static RunnerApi.AccumulationMode toProto(AccumulationMode accumulationMode) {
- switch (accumulationMode) {
- case DISCARDING_FIRED_PANES:
- return RunnerApi.AccumulationMode.DISCARDING;
- case ACCUMULATING_FIRED_PANES:
- return RunnerApi.AccumulationMode.ACCUMULATING;
- default:
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- AccumulationMode.class.getCanonicalName(),
- RunnerApi.AccumulationMode.class.getCanonicalName(),
- accumulationMode));
- }
- }
-
- public static RunnerApi.ClosingBehavior toProto(ClosingBehavior closingBehavior) {
- switch (closingBehavior) {
- case FIRE_ALWAYS:
- return RunnerApi.ClosingBehavior.EMIT_ALWAYS;
- case FIRE_IF_NON_EMPTY:
- return RunnerApi.ClosingBehavior.EMIT_IF_NONEMPTY;
- default:
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- ClosingBehavior.class.getCanonicalName(),
- RunnerApi.ClosingBehavior.class.getCanonicalName(),
- closingBehavior));
- }
- }
-
- public static ClosingBehavior fromProto(RunnerApi.ClosingBehavior proto) {
- switch (proto) {
- case EMIT_ALWAYS:
- return ClosingBehavior.FIRE_ALWAYS;
- case EMIT_IF_NONEMPTY:
- return ClosingBehavior.FIRE_IF_NON_EMPTY;
- case UNRECOGNIZED:
- default:
- // Whether or not it is proto that cannot recognize it (due to the version of the
- // generated code we link to) or the switch hasn't been updated to handle it,
- // the situation is the same: we don't know what this OutputTime means
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- RunnerApi.ClosingBehavior.class.getCanonicalName(),
- ClosingBehavior.class.getCanonicalName(),
- proto));
- }
- }
-
- public static RunnerApi.OutputTime toProto(TimestampCombiner timestampCombiner) {
- switch(timestampCombiner) {
- case EARLIEST:
- return OutputTime.EARLIEST_IN_PANE;
- case END_OF_WINDOW:
- return OutputTime.END_OF_WINDOW;
- case LATEST:
- return OutputTime.LATEST_IN_PANE;
- default:
- throw new IllegalArgumentException(
- String.format(
- "Unknown %s: %s",
- TimestampCombiner.class.getSimpleName(),
- timestampCombiner));
- }
- }
-
- public static TimestampCombiner timestampCombinerFromProto(RunnerApi.OutputTime proto) {
- switch (proto) {
- case EARLIEST_IN_PANE:
- return TimestampCombiner.EARLIEST;
- case END_OF_WINDOW:
- return TimestampCombiner.END_OF_WINDOW;
- case LATEST_IN_PANE:
- return TimestampCombiner.LATEST;
- case UNRECOGNIZED:
- default:
- // Whether or not it is proto that cannot recognize it (due to the version of the
- // generated code we link to) or the switch hasn't been updated to handle it,
- // the situation is the same: we don't know what this OutputTime means
- throw new IllegalArgumentException(
- String.format(
- "Cannot convert unknown %s to %s: %s",
- RunnerApi.OutputTime.class.getCanonicalName(),
- OutputTime.class.getCanonicalName(),
- proto));
- }
- }
-
- // This URN says that the WindowFn is just a UDF blob the indicated SDK understands
- // TODO: standardize such things
- public static final String CUSTOM_WINDOWFN_URN = "urn:beam:windowfn:javasdk:0.1";
-
- /**
- * Converts a {@link WindowFn} into a {@link RunnerApi.MessageWithComponents} where {@link
- * RunnerApi.MessageWithComponents#getFunctionSpec()} is a {@link RunnerApi.FunctionSpec} for the
- * input {@link WindowFn}.
- */
- public static SdkFunctionSpec toProto(
- WindowFn<?, ?> windowFn, @SuppressWarnings("unused") SdkComponents components)
- throws IOException {
- return SdkFunctionSpec.newBuilder()
- // TODO: Set environment ID
- .setSpec(
- FunctionSpec.newBuilder()
- .setUrn(CUSTOM_WINDOWFN_URN)
- .setParameter(
- Any.pack(
- BytesValue.newBuilder()
- .setValue(
- ByteString.copyFrom(
- SerializableUtils.serializeToByteArray(windowFn)))
- .build())))
- .build();
- }
-
- /**
- * Converts a {@link WindowingStrategy} into a {@link RunnerApi.MessageWithComponents} where
- * {@link RunnerApi.MessageWithComponents#getWindowingStrategy()} ()} is a {@link
- * RunnerApi.WindowingStrategy RunnerApi.WindowingStrategy (proto)} for the input {@link
- * WindowingStrategy}.
- */
- public static RunnerApi.MessageWithComponents toProto(WindowingStrategy<?, ?> windowingStrategy)
- throws IOException {
- SdkComponents components = SdkComponents.create();
- RunnerApi.WindowingStrategy windowingStrategyProto = toProto(windowingStrategy, components);
-
- return RunnerApi.MessageWithComponents.newBuilder()
- .setWindowingStrategy(windowingStrategyProto)
- .setComponents(components.toComponents())
- .build();
- }
-
- /**
- * Converts a {@link WindowingStrategy} into a {@link RunnerApi.WindowingStrategy}, registering
- * any components in the provided {@link SdkComponents}.
- */
- public static RunnerApi.WindowingStrategy toProto(
- WindowingStrategy<?, ?> windowingStrategy, SdkComponents components) throws IOException {
- SdkFunctionSpec windowFnSpec = toProto(windowingStrategy.getWindowFn(), components);
-
- RunnerApi.WindowingStrategy.Builder windowingStrategyProto =
- RunnerApi.WindowingStrategy.newBuilder()
- .setOutputTime(toProto(windowingStrategy.getTimestampCombiner()))
- .setAccumulationMode(toProto(windowingStrategy.getMode()))
- .setClosingBehavior(toProto(windowingStrategy.getClosingBehavior()))
- .setAllowedLateness(windowingStrategy.getAllowedLateness().getMillis())
- .setTrigger(Triggers.toProto(windowingStrategy.getTrigger()))
- .setWindowFn(windowFnSpec)
- .setWindowCoderId(
- components.registerCoder(windowingStrategy.getWindowFn().windowCoder()));
-
- return windowingStrategyProto.build();
- }
-
- /**
- * Converts from a {@link RunnerApi.WindowingStrategy} accompanied by {@link Components}
- * to the SDK's {@link WindowingStrategy}.
- */
- public static WindowingStrategy<?, ?> fromProto(RunnerApi.MessageWithComponents proto)
- throws InvalidProtocolBufferException {
- switch (proto.getRootCase()) {
- case WINDOWING_STRATEGY:
- return fromProto(proto.getWindowingStrategy(), proto.getComponents());
- default:
- throw new IllegalArgumentException(
- String.format(
- "Expected a %s with components but received %s",
- RunnerApi.WindowingStrategy.class.getCanonicalName(), proto));
- }
- }
-
- /**
- * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
- * the provided components to dereferences identifiers found in the proto.
- */
- public static WindowingStrategy<?, ?> fromProto(
- RunnerApi.WindowingStrategy proto, Components components)
- throws InvalidProtocolBufferException {
-
- SdkFunctionSpec windowFnSpec = proto.getWindowFn();
-
- checkArgument(
- windowFnSpec.getSpec().getUrn().equals(CUSTOM_WINDOWFN_URN),
- "Only Java-serialized %s instances are supported, with URN %s. But found URN %s",
- WindowFn.class.getSimpleName(),
- CUSTOM_WINDOWFN_URN,
- windowFnSpec.getSpec().getUrn());
-
- Object deserializedWindowFn =
- SerializableUtils.deserializeFromByteArray(
- windowFnSpec.getSpec().getParameter().unpack(BytesValue.class).getValue().toByteArray(),
- "WindowFn");
-
- WindowFn<?, ?> windowFn = (WindowFn<?, ?>) deserializedWindowFn;
- TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
- AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
- Trigger trigger = Triggers.fromProto(proto.getTrigger());
- ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
- Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
-
- return WindowingStrategy.of(windowFn)
- .withAllowedLateness(allowedLateness)
- .withMode(accumulationMode)
- .withTrigger(trigger)
- .withTimestampCombiner(timestampCombiner)
- .withClosingBehavior(closingBehavior);
- }
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
new file mode 100644
index 0000000..893fbe5
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.Durations;
+import com.google.protobuf.util.Timestamps;
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.OutputTime;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.FixedWindowsPayload;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.SessionsPayload;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.SlidingWindowsPayload;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
+import org.apache.beam.sdk.transforms.windowing.Window.OnTimeBehavior;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
+import org.joda.time.Duration;
+
+/** Utilities for working with {@link WindowingStrategy WindowingStrategies}. */
+public class WindowingStrategyTranslation implements Serializable {
+
+ public static AccumulationMode fromProto(RunnerApi.AccumulationMode.Enum proto) {
+ switch (proto) {
+ case DISCARDING:
+ return AccumulationMode.DISCARDING_FIRED_PANES;
+ case ACCUMULATING:
+ return AccumulationMode.ACCUMULATING_FIRED_PANES;
+ case UNRECOGNIZED:
+ default:
+ // Whether or not it is proto that cannot recognize it (due to the version of the
+ // generated code we link to) or the switch hasn't been updated to handle it,
+ // the situation is the same: we don't know what this OutputTime means
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ RunnerApi.AccumulationMode.class.getCanonicalName(),
+ AccumulationMode.class.getCanonicalName(),
+ proto));
+ }
+ }
+
+ public static RunnerApi.AccumulationMode.Enum toProto(AccumulationMode accumulationMode) {
+ switch (accumulationMode) {
+ case DISCARDING_FIRED_PANES:
+ return RunnerApi.AccumulationMode.Enum.DISCARDING;
+ case ACCUMULATING_FIRED_PANES:
+ return RunnerApi.AccumulationMode.Enum.ACCUMULATING;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ AccumulationMode.class.getCanonicalName(),
+ RunnerApi.AccumulationMode.class.getCanonicalName(),
+ accumulationMode));
+ }
+ }
+
+ public static RunnerApi.ClosingBehavior.Enum toProto(ClosingBehavior closingBehavior) {
+ switch (closingBehavior) {
+ case FIRE_ALWAYS:
+ return RunnerApi.ClosingBehavior.Enum.EMIT_ALWAYS;
+ case FIRE_IF_NON_EMPTY:
+ return RunnerApi.ClosingBehavior.Enum.EMIT_IF_NONEMPTY;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ ClosingBehavior.class.getCanonicalName(),
+ RunnerApi.ClosingBehavior.class.getCanonicalName(),
+ closingBehavior));
+ }
+ }
+
+ public static ClosingBehavior fromProto(RunnerApi.ClosingBehavior.Enum proto) {
+ switch (proto) {
+ case EMIT_ALWAYS:
+ return ClosingBehavior.FIRE_ALWAYS;
+ case EMIT_IF_NONEMPTY:
+ return ClosingBehavior.FIRE_IF_NON_EMPTY;
+ case UNRECOGNIZED:
+ default:
+ // Whether or not it is proto that cannot recognize it (due to the version of the
+ // generated code we link to) or the switch hasn't been updated to handle it,
+ // the situation is the same: we don't know what this OutputTime means
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ RunnerApi.ClosingBehavior.class.getCanonicalName(),
+ ClosingBehavior.class.getCanonicalName(),
+ proto));
+ }
+ }
+
+ public static RunnerApi.OnTimeBehavior.Enum toProto(OnTimeBehavior onTimeBehavior) {
+ switch (onTimeBehavior) {
+ case FIRE_ALWAYS:
+ return RunnerApi.OnTimeBehavior.Enum.FIRE_ALWAYS;
+ case FIRE_IF_NON_EMPTY:
+ return RunnerApi.OnTimeBehavior.Enum.FIRE_IF_NONEMPTY;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ OnTimeBehavior.class.getCanonicalName(),
+ RunnerApi.OnTimeBehavior.class.getCanonicalName(),
+ onTimeBehavior));
+ }
+ }
+
+ public static OnTimeBehavior fromProto(RunnerApi.OnTimeBehavior.Enum proto) {
+ switch (proto) {
+ case FIRE_ALWAYS:
+ return OnTimeBehavior.FIRE_ALWAYS;
+ case FIRE_IF_NONEMPTY:
+ return OnTimeBehavior.FIRE_IF_NON_EMPTY;
+ case UNRECOGNIZED:
+ default:
+ // Whether or not it is proto that cannot recognize it (due to the version of the
+ // generated code we link to) or the switch hasn't been updated to handle it,
+ // the situation is the same: we don't know what this OutputTime means
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ RunnerApi.OnTimeBehavior.class.getCanonicalName(),
+ OnTimeBehavior.class.getCanonicalName(),
+ proto));
+ }
+ }
+
+ public static RunnerApi.OutputTime.Enum toProto(TimestampCombiner timestampCombiner) {
+ switch(timestampCombiner) {
+ case EARLIEST:
+ return OutputTime.Enum.EARLIEST_IN_PANE;
+ case END_OF_WINDOW:
+ return OutputTime.Enum.END_OF_WINDOW;
+ case LATEST:
+ return OutputTime.Enum.LATEST_IN_PANE;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Unknown %s: %s",
+ TimestampCombiner.class.getSimpleName(),
+ timestampCombiner));
+ }
+ }
+
+ public static TimestampCombiner timestampCombinerFromProto(RunnerApi.OutputTime.Enum proto) {
+ switch (proto) {
+ case EARLIEST_IN_PANE:
+ return TimestampCombiner.EARLIEST;
+ case END_OF_WINDOW:
+ return TimestampCombiner.END_OF_WINDOW;
+ case LATEST_IN_PANE:
+ return TimestampCombiner.LATEST;
+ case UNRECOGNIZED:
+ default:
+ // Whether or not it is proto that cannot recognize it (due to the version of the
+ // generated code we link to) or the switch hasn't been updated to handle it,
+ // the situation is the same: we don't know what this OutputTime means
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot convert unknown %s to %s: %s",
+ RunnerApi.OutputTime.class.getCanonicalName(),
+ OutputTime.class.getCanonicalName(),
+ proto));
+ }
+ }
+
+ public static final String GLOBAL_WINDOWS_FN = "beam:windowfn:global_windows:v0.1";
+ public static final String FIXED_WINDOWS_FN = "beam:windowfn:fixed_windows:v0.1";
+ public static final String SLIDING_WINDOWS_FN = "beam:windowfn:sliding_windows:v0.1";
+ public static final String SESSION_WINDOWS_FN = "beam:windowfn:session_windows:v0.1";
+ // This URN says that the WindowFn is just a UDF blob the Java SDK understands
+ // TODO: standardize such things
+ public static final String SERIALIZED_JAVA_WINDOWFN_URN = "beam:windowfn:javasdk:v0.1";
+ public static final String OLD_SERIALIZED_JAVA_WINDOWFN_URN = "urn:beam:windowfn:javasdk:0.1";
+ // Remove this once the dataflow worker understands all the above formats.
+ private static final boolean USE_OLD_SERIALIZED_JAVA_WINDOWFN_URN = true;
+
+ /**
+ * Converts a {@link WindowFn} into a {@link RunnerApi.MessageWithComponents} where {@link
+ * RunnerApi.MessageWithComponents#getFunctionSpec()} is a {@link RunnerApi.FunctionSpec} for the
+ * input {@link WindowFn}.
+ */
+ public static SdkFunctionSpec toProto(
+ WindowFn<?, ?> windowFn, @SuppressWarnings("unused") SdkComponents components) {
+ // TODO: Set environment IDs
+ ByteString serializedFn = ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowFn));
+ if (USE_OLD_SERIALIZED_JAVA_WINDOWFN_URN) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(OLD_SERIALIZED_JAVA_WINDOWFN_URN)
+ .setPayload(serializedFn)
+ .build())
+ .build();
+ } else if (windowFn instanceof GlobalWindows) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(FunctionSpec.newBuilder().setUrn(GLOBAL_WINDOWS_FN))
+ .build();
+ } else if (windowFn instanceof FixedWindows) {
+ FixedWindowsPayload fixedWindowsPayload =
+ FixedWindowsPayload.newBuilder()
+ .setSize(Durations.fromMillis(((FixedWindows) windowFn).getSize().getMillis()))
+ .setOffset(Timestamps.fromMillis(((FixedWindows) windowFn).getOffset().getMillis()))
+ .build();
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(FIXED_WINDOWS_FN)
+ .setPayload(fixedWindowsPayload.toByteString()))
+ .build();
+ } else if (windowFn instanceof SlidingWindows) {
+ SlidingWindowsPayload slidingWindowsPayload = SlidingWindowsPayload.newBuilder()
+ .setSize(Durations.fromMillis(((SlidingWindows) windowFn).getSize().getMillis()))
+ .setOffset(Timestamps.fromMillis(((SlidingWindows) windowFn).getOffset().getMillis()))
+ .setPeriod(Durations.fromMillis(((SlidingWindows) windowFn).getPeriod().getMillis()))
+ .build();
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(SLIDING_WINDOWS_FN)
+ .setPayload(slidingWindowsPayload.toByteString()))
+ .build();
+ } else if (windowFn instanceof Sessions) {
+ SessionsPayload sessionsPayload =
+ SessionsPayload.newBuilder()
+ .setGapSize(Durations.fromMillis(((Sessions) windowFn).getGapDuration().getMillis()))
+ .build();
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(SESSION_WINDOWS_FN)
+ .setPayload(sessionsPayload.toByteString()))
+ .build();
+ } else {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(SERIALIZED_JAVA_WINDOWFN_URN)
+ .setPayload(serializedFn))
+ .build();
+ }
+ }
+
+ /**
+ * Converts a {@link WindowingStrategy} into a {@link RunnerApi.MessageWithComponents} where
+ * {@link RunnerApi.MessageWithComponents#getWindowingStrategy()} ()} is a {@link
+ * RunnerApi.WindowingStrategy RunnerApi.WindowingStrategy (proto)} for the input {@link
+ * WindowingStrategy}.
+ */
+ public static RunnerApi.MessageWithComponents toProto(WindowingStrategy<?, ?> windowingStrategy)
+ throws IOException {
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.WindowingStrategy windowingStrategyProto = toProto(windowingStrategy, components);
+
+ return RunnerApi.MessageWithComponents.newBuilder()
+ .setWindowingStrategy(windowingStrategyProto)
+ .setComponents(components.toComponents())
+ .build();
+ }
+
+ /**
+ * Converts a {@link WindowingStrategy} into a {@link RunnerApi.WindowingStrategy}, registering
+ * any components in the provided {@link SdkComponents}.
+ */
+ public static RunnerApi.WindowingStrategy toProto(
+ WindowingStrategy<?, ?> windowingStrategy, SdkComponents components) throws IOException {
+ SdkFunctionSpec windowFnSpec = toProto(windowingStrategy.getWindowFn(), components);
+
+ RunnerApi.WindowingStrategy.Builder windowingStrategyProto =
+ RunnerApi.WindowingStrategy.newBuilder()
+ .setOutputTime(toProto(windowingStrategy.getTimestampCombiner()))
+ .setAccumulationMode(toProto(windowingStrategy.getMode()))
+ .setClosingBehavior(toProto(windowingStrategy.getClosingBehavior()))
+ .setAllowedLateness(windowingStrategy.getAllowedLateness().getMillis())
+ .setTrigger(TriggerTranslation.toProto(windowingStrategy.getTrigger()))
+ .setWindowFn(windowFnSpec)
+ .setAssignsToOneWindow(windowingStrategy.getWindowFn().assignsToOneWindow())
+ .setOnTimeBehavior(toProto(windowingStrategy.getOnTimeBehavior()))
+ .setWindowCoderId(
+ components.registerCoder(windowingStrategy.getWindowFn().windowCoder()));
+
+ return windowingStrategyProto.build();
+ }
+
+ /**
+ * Converts from a {@link RunnerApi.WindowingStrategy} accompanied by {@link Components}
+ * to the SDK's {@link WindowingStrategy}.
+ */
+ public static WindowingStrategy<?, ?> fromProto(RunnerApi.MessageWithComponents proto)
+ throws InvalidProtocolBufferException {
+ switch (proto.getRootCase()) {
+ case WINDOWING_STRATEGY:
+ return fromProto(
+ proto.getWindowingStrategy(),
+ RehydratedComponents.forComponents(proto.getComponents()));
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Expected a %s with components but received %s",
+ RunnerApi.WindowingStrategy.class.getCanonicalName(), proto));
+ }
+ }
+
+ /**
+ * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
+ * the provided components to dereferences identifiers found in the proto.
+ */
+ public static WindowingStrategy<?, ?> fromProto(
+ RunnerApi.WindowingStrategy proto, RehydratedComponents components)
+ throws InvalidProtocolBufferException {
+
+ SdkFunctionSpec windowFnSpec = proto.getWindowFn();
+ WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
+ TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
+ AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
+ Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
+ ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
+ Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
+ OnTimeBehavior onTimeBehavior = fromProto(proto.getOnTimeBehavior());
+
+ return WindowingStrategy.of(windowFn)
+ .withAllowedLateness(allowedLateness)
+ .withMode(accumulationMode)
+ .withTrigger(trigger)
+ .withTimestampCombiner(timestampCombiner)
+ .withClosingBehavior(closingBehavior)
+ .withOnTimeBehavior(onTimeBehavior);
+ }
+
+ public static WindowFn<?, ?> windowFnFromProto(SdkFunctionSpec windowFnSpec) {
+ try {
+ switch (windowFnSpec.getSpec().getUrn()) {
+ case GLOBAL_WINDOWS_FN:
+ return new GlobalWindows();
+ case FIXED_WINDOWS_FN:
+ StandardWindowFns.FixedWindowsPayload fixedParams = null;
+ fixedParams =
+ StandardWindowFns.FixedWindowsPayload.parseFrom(
+ windowFnSpec.getSpec().getPayload());
+ return FixedWindows.of(Duration.millis(Durations.toMillis(fixedParams.getSize())))
+ .withOffset(Duration.millis(Timestamps.toMillis(fixedParams.getOffset())));
+ case SLIDING_WINDOWS_FN:
+ StandardWindowFns.SlidingWindowsPayload slidingParams =
+ StandardWindowFns.SlidingWindowsPayload.parseFrom(
+ windowFnSpec.getSpec().getPayload());
+ return SlidingWindows.of(Duration.millis(Durations.toMillis(slidingParams.getSize())))
+ .every(Duration.millis(Durations.toMillis(slidingParams.getPeriod())))
+ .withOffset(Duration.millis(Timestamps.toMillis(slidingParams.getOffset())));
+ case SESSION_WINDOWS_FN:
+ StandardWindowFns.SessionsPayload sessionParams =
+ StandardWindowFns.SessionsPayload.parseFrom(windowFnSpec.getSpec().getPayload());
+ return Sessions.withGapDuration(
+ Duration.millis(Durations.toMillis(sessionParams.getGapSize())));
+ case SERIALIZED_JAVA_WINDOWFN_URN:
+ case OLD_SERIALIZED_JAVA_WINDOWFN_URN:
+ return (WindowFn<?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ windowFnSpec.getSpec().getPayload().toByteArray(), "WindowFn");
+ default:
+ throw new IllegalArgumentException(
+ "Unknown or unsupported WindowFn: " + windowFnSpec.getSpec().getUrn());
+ }
+ } catch (InvalidProtocolBufferException e) {
+ throw new IllegalArgumentException(
+ String.format(
+ "%s for %s with URN %s did not contain expected proto message for payload",
+ FunctionSpec.class.getSimpleName(),
+ WindowFn.class.getSimpleName(),
+ windowFnSpec.getSpec().getUrn()),
+ e);
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
new file mode 100644
index 0000000..d0b2182
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.WRITE_FILES_TRANSFORM_URN;
+
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WriteFilesPayload;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.io.FileBasedSink;
+import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Utility methods for translating a {@link WriteFiles} to and from {@link RunnerApi}
+ * representations.
+ */
+public class WriteFilesTranslation {
+
+ /** The URN for an unknown Java {@link FileBasedSink}. */
+ public static final String CUSTOM_JAVA_FILE_BASED_SINK_URN =
+ "urn:beam:file_based_sink:javasdk:0.1";
+
+ @VisibleForTesting
+ static WriteFilesPayload payloadForWriteFiles(
+ final WriteFiles<?, ?, ?> transform, SdkComponents components) throws IOException {
+ return payloadForWriteFilesLike(
+ new WriteFilesLike() {
+ @Override
+ public SdkFunctionSpec translateSink(SdkComponents newComponents) {
+ // TODO: register the environment
+ return toProto(transform.getSink());
+ }
+
+ @Override
+ public Map<String, SideInput> translateSideInputs(SdkComponents components) {
+ Map<String, SideInput> sideInputs = new HashMap<>();
+ for (PCollectionView<?> view :
+ transform.getSink().getDynamicDestinations().getSideInputs()) {
+ sideInputs.put(view.getTagInternal().getId(), ParDoTranslation.toProto(view));
+ }
+ return sideInputs;
+ }
+
+ @Override
+ public boolean isWindowedWrites() {
+ return transform.isWindowedWrites();
+ }
+
+ @Override
+ public boolean isRunnerDeterminedSharding() {
+ return transform.getNumShards() == null && transform.getSharding() == null;
+ }
+ },
+ components);
+ }
+
+ private static SdkFunctionSpec toProto(FileBasedSink<?, ?, ?> sink) {
+ return toProto(CUSTOM_JAVA_FILE_BASED_SINK_URN, sink);
+ }
+
+ private static SdkFunctionSpec toProto(String urn, Serializable serializable) {
+ return SdkFunctionSpec.newBuilder()
+ .setSpec(
+ FunctionSpec.newBuilder()
+ .setUrn(urn)
+ .setPayload(
+ ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable)))
+ .build())
+ .build();
+ }
+
+ @VisibleForTesting
+ static FileBasedSink<?, ?, ?> sinkFromProto(SdkFunctionSpec sinkProto) throws IOException {
+ checkArgument(
+ sinkProto.getSpec().getUrn().equals(CUSTOM_JAVA_FILE_BASED_SINK_URN),
+ "Cannot extract %s instance from %s with URN %s",
+ FileBasedSink.class.getSimpleName(),
+ FunctionSpec.class.getSimpleName(),
+ sinkProto.getSpec().getUrn());
+
+ byte[] serializedSink = sinkProto.getSpec().getPayload().toByteArray();
+
+ return (FileBasedSink<?, ?, ?>)
+ SerializableUtils.deserializeFromByteArray(
+ serializedSink, FileBasedSink.class.getSimpleName());
+ }
+
+ public static <UserT, DestinationT, OutputT> FileBasedSink<UserT, DestinationT, OutputT> getSink(
+ AppliedPTransform<
+ PCollection<UserT>, WriteFilesResult<DestinationT>,
+ ? extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>>>
+ transform)
+ throws IOException {
+ return (FileBasedSink<UserT, DestinationT, OutputT>)
+ sinkFromProto(getWriteFilesPayload(transform).getSink());
+ }
+
+ public static <UserT, DestinationT> List<PCollectionView<?>> getDynamicDestinationSideInputs(
+ AppliedPTransform<
+ PCollection<UserT>, WriteFilesResult<DestinationT>,
+ ? extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>>>
+ transform)
+ throws IOException {
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PTransform transformProto = PTransformTranslation.toProto(transform, sdkComponents);
+ List<PCollectionView<?>> views = Lists.newArrayList();
+ Map<String, SideInput> sideInputs = getWriteFilesPayload(transform).getSideInputsMap();
+ for (Map.Entry<String, SideInput> entry : sideInputs.entrySet()) {
+ PCollection<?> originalPCollection =
+ checkNotNull(
+ (PCollection<?>) transform.getInputs().get(new TupleTag<>(entry.getKey())),
+ "no input with tag %s",
+ entry.getKey());
+ views.add(
+ ParDoTranslation.viewFromProto(
+ entry.getValue(),
+ entry.getKey(),
+ originalPCollection,
+ transformProto,
+ RehydratedComponents.forComponents(sdkComponents.toComponents())));
+ }
+ return views;
+ }
+
+ public static <T, DestinationT> boolean isWindowedWrites(
+ AppliedPTransform<
+ PCollection<T>, WriteFilesResult<DestinationT>,
+ ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>>
+ transform)
+ throws IOException {
+ return getWriteFilesPayload(transform).getWindowedWrites();
+ }
+
+ public static <T, DestinationT> boolean isRunnerDeterminedSharding(
+ AppliedPTransform<
+ PCollection<T>, WriteFilesResult<DestinationT>,
+ ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>>
+ transform)
+ throws IOException {
+ return getWriteFilesPayload(transform).getRunnerDeterminedSharding();
+ }
+
+ private static <T, DestinationT> WriteFilesPayload getWriteFilesPayload(
+ AppliedPTransform<
+ PCollection<T>, WriteFilesResult<DestinationT>,
+ ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>>
+ transform)
+ throws IOException {
+ return WriteFilesPayload.parseFrom(
+ PTransformTranslation.toProto(
+ transform,
+ Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+ SdkComponents.create())
+ .getSpec()
+ .getPayload());
+ }
+
+ static class RawWriteFiles extends PTransformTranslation.RawPTransform<PInput, POutput>
+ implements WriteFilesLike {
+
+ private final RunnerApi.PTransform protoTransform;
+ private final transient RehydratedComponents rehydratedComponents;
+
+ // Parsed from protoTransform and cached
+ private final FunctionSpec spec;
+ private final RunnerApi.WriteFilesPayload payload;
+
+ public RawWriteFiles(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ this.rehydratedComponents = rehydratedComponents;
+ this.protoTransform = protoTransform;
+ this.spec = protoTransform.getSpec();
+ this.payload = RunnerApi.WriteFilesPayload.parseFrom(spec.getPayload());
+ }
+
+ @Override
+ public FunctionSpec getSpec() {
+ return spec;
+ }
+
+ @Override
+ public FunctionSpec migrate(SdkComponents components) throws IOException {
+ return FunctionSpec.newBuilder()
+ .setUrn(WRITE_FILES_TRANSFORM_URN)
+ .setPayload(payloadForWriteFilesLike(this, components).toByteString())
+ .build();
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ for (Map.Entry<String, SideInput> sideInputEntry : payload.getSideInputsMap().entrySet()) {
+ try {
+ additionalInputs.put(
+ new TupleTag<>(sideInputEntry.getKey()),
+ rehydratedComponents.getPCollection(
+ protoTransform.getInputsOrThrow(sideInputEntry.getKey())));
+ } catch (IOException exc) {
+ throw new IllegalStateException(
+ String.format(
+ "Could not find input with name %s for %s transform",
+ sideInputEntry.getKey(), WriteFiles.class.getSimpleName()));
+ }
+ }
+ return additionalInputs;
+ }
+
+ @Override
+ public SdkFunctionSpec translateSink(SdkComponents newComponents) {
+ // TODO: re-register the environment with the new components
+ return payload.getSink();
+ }
+
+ @Override
+ public Map<String, SideInput> translateSideInputs(SdkComponents components) {
+ // TODO: re-register the PCollections and UDF environments
+ return MoreObjects.firstNonNull(
+ payload.getSideInputsMap(), Collections.<String, SideInput>emptyMap());
+ }
+
+ @Override
+ public boolean isWindowedWrites() {
+ return payload.getWindowedWrites();
+ }
+
+ @Override
+ public boolean isRunnerDeterminedSharding() {
+ return payload.getRunnerDeterminedSharding();
+ }
+ }
+
+ static class WriteFilesTranslator implements TransformPayloadTranslator<WriteFiles<?, ?, ?>> {
+ @Override
+ public String getUrn(WriteFiles<?, ?, ?> transform) {
+ return WRITE_FILES_TRANSFORM_URN;
+ }
+
+ @Override
+ public FunctionSpec translate(
+ AppliedPTransform<?, ?, WriteFiles<?, ?, ?>> transform, SdkComponents components)
+ throws IOException {
+ return FunctionSpec.newBuilder()
+ .setUrn(getUrn(transform.getTransform()))
+ .setPayload(payloadForWriteFiles(transform.getTransform(), components).toByteString())
+ .build();
+ }
+
+ @Override
+ public PTransformTranslation.RawPTransform<?, ?> rehydrate(
+ RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents)
+ throws IOException {
+ return new RawWriteFiles(protoTransform, rehydratedComponents);
+ }
+ }
+ /** Registers {@link WriteFilesTranslator}. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class Registrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<Class<? extends PTransform>, TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return Collections.<Class<? extends PTransform>, TransformPayloadTranslator>singletonMap(
+ WriteFiles.class, new WriteFilesTranslator());
+ }
+
+ @Override
+ public Map<String, ? extends TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.singletonMap(WRITE_FILES_TRANSFORM_URN, new WriteFilesTranslator());
+ }
+ }
+
+ /** These methods drive to-proto translation from Java and from rehydrated WriteFiles. */
+ private interface WriteFilesLike {
+ SdkFunctionSpec translateSink(SdkComponents newComponents);
+
+ Map<String, RunnerApi.SideInput> translateSideInputs(SdkComponents components);
+
+ boolean isWindowedWrites();
+
+ boolean isRunnerDeterminedSharding();
+ }
+
+ public static WriteFilesPayload payloadForWriteFilesLike(
+ WriteFilesLike writeFiles, SdkComponents components) throws IOException {
+
+ return WriteFilesPayload.newBuilder()
+ .setSink(writeFiles.translateSink(components))
+ .putAllSideInputs(writeFiles.translateSideInputs(components))
+ .setWindowedWrites(writeFiles.isWindowedWrites())
+ .setRunnerDeterminedSharding(writeFiles.isRunnerDeterminedSharding())
+ .build();
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricFiltering.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricFiltering.java
new file mode 100644
index 0000000..99d8f0f
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricFiltering.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction.metrics;
+
+import com.google.common.base.Objects;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricsFilter;
+
+/**
+ * Implements matching for metrics filters. Specifically, matching for metric name,
+ * namespace, and step name.
+ */
+public class MetricFiltering {
+
+ private MetricFiltering() { }
+
+ /** Matching logic is implemented here rather than in MetricsFilter because we would like
+ * MetricsFilter to act as a "dumb" value-object, with the possibility of replacing it with
+ * a Proto/JSON/etc. schema object.
+ * @param filter {@link MetricsFilter} with the matching information of an actual metric
+ * @param key {@link MetricKey} with the information of a metric
+ * @return whether the filter matches the key or not
+ */
+ public static boolean matches(MetricsFilter filter, MetricKey key) {
+ return filter == null
+ || (matchesName(key.metricName(), filter.names())
+ && matchesScope(key.stepName(), filter.steps()));
+ }
+
+ /**
+ * {@code subPathMatches(haystack, needle)} returns true if {@code needle}
+ * represents a path within {@code haystack}. For example, "foo/bar" is in "a/foo/bar/b",
+ * but not "a/fool/bar/b" or "a/foo/bart/b".
+ */
+ public static boolean subPathMatches(String haystack, String needle) {
+ int location = haystack.indexOf(needle);
+ int end = location + needle.length();
+ if (location == -1) {
+ return false; // needle not found
+ } else if (location != 0 && haystack.charAt(location - 1) != '/') {
+ return false; // the first entry in needle wasn't exactly matched
+ } else if (end != haystack.length() && haystack.charAt(end) != '/') {
+ return false; // the last entry in needle wasn't exactly matched
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * {@code matchesScope(actualScope, scopes)} returns true if the scope of a metric is matched
+ * by any of the filters in {@code scopes}. A metric scope is a path of type "A/B/D". A
+ * path is matched by a filter if the filter is equal to the path (e.g. "A/B/D", or
+ * if it represents a subpath within it (e.g. "A/B" or "B/D", but not "A/D"). */
+ public static boolean matchesScope(String actualScope, Set<String> scopes) {
+ if (scopes.isEmpty() || scopes.contains(actualScope)) {
+ return true;
+ }
+
+ // If there is no perfect match, a stage name-level match is tried.
+ // This is done by a substring search over the levels of the scope.
+ // e.g. a scope "A/B/C/D" is matched by "A/B", but not by "A/C".
+ for (String scope : scopes) {
+ if (subPathMatches(actualScope, scope)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private static boolean matchesName(MetricName metricName, Set<MetricNameFilter> nameFilters) {
+ if (nameFilters.isEmpty()) {
+ return true;
+ }
+ for (MetricNameFilter nameFilter : nameFilters) {
+ if ((nameFilter.getName() == null || nameFilter.getName().equals(metricName.name()))
+ && Objects.equal(metricName.namespace(), nameFilter.getNamespace())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricKey.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricKey.java
new file mode 100644
index 0000000..f4521a3
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricKey.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction.metrics;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.metrics.MetricName;
+
+/**
+ * Metrics are keyed by the step name they are associated with and the name of the metric.
+ */
+@Experimental(Kind.METRICS)
+@AutoValue
+public abstract class MetricKey implements Serializable {
+
+ /** The step name that is associated with this metric. */
+ public abstract String stepName();
+
+ /** The name of the metric. */
+ public abstract MetricName metricName();
+
+ public static MetricKey create(String stepName, MetricName metricName) {
+ return new AutoValue_MetricKey(stepName, metricName);
+ }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/package-info.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/package-info.java
new file mode 100644
index 0000000..ceffd73
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for runners to implement metrics.
+ */
+package org.apache.beam.runners.core.construction.metrics;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
deleted file mode 100644
index d469d20..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.metrics;
-
-import com.google.common.base.Objects;
-import java.util.Set;
-import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricNameFilter;
-import org.apache.beam.sdk.metrics.MetricsFilter;
-
-/**
- * Implements matching for metrics filters. Specifically, matching for metric name,
- * namespace, and step name.
- */
-public class MetricFiltering {
-
- private MetricFiltering() { }
-
- /** Matching logic is implemented here rather than in MetricsFilter because we would like
- * MetricsFilter to act as a "dumb" value-object, with the possibility of replacing it with
- * a Proto/JSON/etc. schema object.
- * @param filter {@link MetricsFilter} with the matching information of an actual metric
- * @param key {@link MetricKey} with the information of a metric
- * @return whether the filter matches the key or not
- */
- public static boolean matches(MetricsFilter filter, MetricKey key) {
- return filter == null
- || (matchesName(key.metricName(), filter.names())
- && matchesScope(key.stepName(), filter.steps()));
- }
-
- /**
- * {@code subPathMatches(haystack, needle)} returns true if {@code needle}
- * represents a path within {@code haystack}. For example, "foo/bar" is in "a/foo/bar/b",
- * but not "a/fool/bar/b" or "a/foo/bart/b".
- */
- public static boolean subPathMatches(String haystack, String needle) {
- int location = haystack.indexOf(needle);
- int end = location + needle.length();
- if (location == -1) {
- return false; // needle not found
- } else if (location != 0 && haystack.charAt(location - 1) != '/') {
- return false; // the first entry in needle wasn't exactly matched
- } else if (end != haystack.length() && haystack.charAt(end) != '/') {
- return false; // the last entry in needle wasn't exactly matched
- } else {
- return true;
- }
- }
-
- /**
- * {@code matchesScope(actualScope, scopes)} returns true if the scope of a metric is matched
- * by any of the filters in {@code scopes}. A metric scope is a path of type "A/B/D". A
- * path is matched by a filter if the filter is equal to the path (e.g. "A/B/D", or
- * if it represents a subpath within it (e.g. "A/B" or "B/D", but not "A/D"). */
- public static boolean matchesScope(String actualScope, Set<String> scopes) {
- if (scopes.isEmpty() || scopes.contains(actualScope)) {
- return true;
- }
-
- // If there is no perfect match, a stage name-level match is tried.
- // This is done by a substring search over the levels of the scope.
- // e.g. a scope "A/B/C/D" is matched by "A/B", but not by "A/C".
- for (String scope : scopes) {
- if (subPathMatches(actualScope, scope)) {
- return true;
- }
- }
-
- return false;
- }
-
- private static boolean matchesName(MetricName metricName, Set<MetricNameFilter> nameFilters) {
- if (nameFilters.isEmpty()) {
- return true;
- }
- for (MetricNameFilter nameFilter : nameFilters) {
- if ((nameFilter.getName() == null || nameFilter.getName().equals(metricName.name()))
- && Objects.equal(metricName.namespace(), nameFilter.getNamespace())) {
- return true;
- }
- }
- return false;
- }
-
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
deleted file mode 100644
index 58d4055..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.metrics;
-
-import com.google.auto.value.AutoValue;
-import java.io.Serializable;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.metrics.MetricName;
-
-/**
- * Metrics are keyed by the step name they are associated with and the name of the metric.
- */
-@Experimental(Kind.METRICS)
-@AutoValue
-public abstract class MetricKey implements Serializable {
-
- /** The step name that is associated with this metric. */
- public abstract String stepName();
-
- /** The name of the metric. */
- public abstract MetricName metricName();
-
- public static MetricKey create(String stepName, MetricName metricName) {
- return new AutoValue_MetricKey(stepName, metricName);
- }
-}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
deleted file mode 100644
index 263a705..0000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Utilities for runners to implement metrics.
- */
-package org.apache.beam.runners.core.metrics;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
new file mode 100644
index 0000000..ffd023e
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.io.BaseEncoding;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
+import io.grpc.internal.ServerImpl;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.MessageDigest;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link ArtifactServiceStager}.
+ */
+@RunWith(JUnit4.class)
+public class ArtifactServiceStagerTest {
+ @Rule public TemporaryFolder temp = new TemporaryFolder();
+
+ private ServerImpl server;
+ private InMemoryArtifactStagerService service;
+ private ArtifactServiceStager stager;
+
+ @Before
+ public void setup() throws IOException {
+ stager =
+ ArtifactServiceStager.overChannel(
+ InProcessChannelBuilder.forName("service_stager").build(), 6);
+ service = new InMemoryArtifactStagerService();
+ server =
+ InProcessServerBuilder.forName("service_stager")
+ .directExecutor()
+ .addService(service)
+ .build()
+ .start();
+ }
+
+ @After
+ public void teardown() {
+ server.shutdownNow();
+ }
+
+ @Test
+ public void testStage() throws Exception {
+ File file = temp.newFile();
+ byte[] content = "foo-bar-baz".getBytes();
+ byte[] contentMd5 = MessageDigest.getInstance("MD5").digest(content);
+ try (FileChannel contentChannel = new FileOutputStream(file).getChannel()) {
+ contentChannel.write(ByteBuffer.wrap(content));
+ }
+
+ stager.stage(Collections.singleton(file));
+
+ assertThat(service.getStagedArtifacts().entrySet(), hasSize(1));
+ byte[] stagedContent = Iterables.getOnlyElement(service.getStagedArtifacts().values());
+ assertThat(stagedContent, equalTo(content));
+
+ ArtifactMetadata staged = service.getManifest().getArtifact(0);
+ assertThat(staged.getName(), equalTo(file.getName()));
+ byte[] manifestMd5 = BaseEncoding.base64().decode(staged.getMd5());
+ assertArrayEquals(contentMd5, manifestMd5);
+
+ assertThat(service.getManifest().getArtifactCount(), equalTo(1));
+ assertThat(staged, equalTo(Iterables.getOnlyElement(service.getStagedArtifacts().keySet())));
+ }
+
+ @Test
+ public void testStagingMultipleFiles() throws Exception {
+ File file = temp.newFile();
+ byte[] content = "foo-bar-baz".getBytes();
+ try (FileChannel contentChannel = new FileOutputStream(file).getChannel()) {
+ contentChannel.write(ByteBuffer.wrap(content));
+ }
+
+ File otherFile = temp.newFile();
+ byte[] otherContent = "spam-ham-eggs".getBytes();
+ try (FileChannel contentChannel = new FileOutputStream(otherFile).getChannel()) {
+ contentChannel.write(ByteBuffer.wrap(otherContent));
+ }
+
+ File thirdFile = temp.newFile();
+ byte[] thirdContent = "up, down, charm, top, bottom, strange".getBytes();
+ try (FileChannel contentChannel = new FileOutputStream(thirdFile).getChannel()) {
+ contentChannel.write(ByteBuffer.wrap(thirdContent));
+ }
+
+ stager.stage(ImmutableList.<File>of(file, otherFile, thirdFile));
+
+ assertThat(service.getManifest().getArtifactCount(), equalTo(3));
+ assertThat(service.getStagedArtifacts().entrySet(), hasSize(3));
+ Set<File> stagedFiles = new HashSet<>();
+ for (byte[] staged : service.getStagedArtifacts().values()) {
+ if (Arrays.equals(staged, content)) {
+ stagedFiles.add(file);
+ } else if (Arrays.equals(staged, otherContent)) {
+ stagedFiles.add(otherFile);
+ } else if (Arrays.equals(staged, thirdContent)) {
+ stagedFiles.add(thirdFile);
+ }
+ }
+ assertThat("All of the files contents should be staged", stagedFiles, hasSize(3));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
new file mode 100644
index 0000000..12ff9d6
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.LengthPrefixCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for {@link CoderTranslation}. */
+@RunWith(Enclosed.class)
+public class CoderTranslationTest {
+ private static final Set<StructuredCoder<?>> KNOWN_CODERS =
+ ImmutableSet.<StructuredCoder<?>>builder()
+ .add(ByteArrayCoder.of())
+ .add(KvCoder.of(VarLongCoder.of(), VarLongCoder.of()))
+ .add(VarLongCoder.of())
+ .add(IntervalWindowCoder.of())
+ .add(IterableCoder.of(ByteArrayCoder.of()))
+ .add(LengthPrefixCoder.of(IterableCoder.of(VarLongCoder.of())))
+ .add(GlobalWindow.Coder.INSTANCE)
+ .add(
+ FullWindowedValueCoder.of(
+ IterableCoder.of(VarLongCoder.of()), IntervalWindowCoder.of()))
+ .build();
+
+ /**
+ * Tests that all known coders are present in the parameters that will be used by
+ * {@link ToFromProtoTest}.
+ */
+ @RunWith(JUnit4.class)
+ public static class ValidateKnownCodersPresentTest {
+ @Test
+ public void validateKnownCoders() {
+ // Validates that every known coder in the Coders class is represented in a "Known Coder"
+ // tests, which demonstrates that they are serialized via components and specified URNs rather
+ // than java serialized
+ Set<Class<? extends StructuredCoder>> knownCoderClasses =
+ CoderTranslation.KNOWN_CODER_URNS.keySet();
+ Set<Class<? extends StructuredCoder>> knownCoderTests = new HashSet<>();
+ for (StructuredCoder<?> coder : KNOWN_CODERS) {
+ knownCoderTests.add(coder.getClass());
+ }
+ Set<Class<? extends StructuredCoder>> missingKnownCoders = new HashSet<>(knownCoderClasses);
+ missingKnownCoders.removeAll(knownCoderTests);
+ assertThat(
+ String.format(
+ "Missing validation of known coder %s in %s",
+ missingKnownCoders, CoderTranslationTest.class.getSimpleName()),
+ missingKnownCoders,
+ Matchers.empty());
+ }
+
+ @Test
+ public void validateCoderTranslators() {
+ assertThat(
+ "Every Known Coder must have a Known Translator",
+ CoderTranslation.KNOWN_CODER_URNS.keySet(),
+ equalTo(CoderTranslation.KNOWN_TRANSLATORS.keySet()));
+ }
+ }
+
+
+ /**
+ * Tests round-trip coder encodings for both known and unknown {@link Coder coders}.
+ */
+ @RunWith(Parameterized.class)
+ public static class ToFromProtoTest {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<Coder<?>> data() {
+ return ImmutableList.<Coder<?>>builder()
+ .addAll(KNOWN_CODERS)
+ .add(
+ StringUtf8Coder.of(),
+ SerializableCoder.of(Record.class),
+ new RecordCoder(),
+ KvCoder.of(new RecordCoder(), AvroCoder.of(Record.class)))
+ .build();
+ }
+
+ @Parameter(0)
+ public Coder<?> coder;
+
+ @Test
+ public void toAndFromProto() throws Exception {
+ SdkComponents componentsBuilder = SdkComponents.create();
+ RunnerApi.Coder coderProto = CoderTranslation.toProto(coder, componentsBuilder);
+
+ Components encodedComponents = componentsBuilder.toComponents();
+ Coder<?> decodedCoder =
+ CoderTranslation.fromProto(
+ coderProto, RehydratedComponents.forComponents(encodedComponents));
+ assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(coder));
+
+ if (KNOWN_CODERS.contains(coder)) {
+ for (RunnerApi.Coder encodedCoder : encodedComponents.getCodersMap().values()) {
+ assertThat(
+ encodedCoder.getSpec().getSpec().getUrn(),
+ not(equalTo(CoderTranslation.JAVA_SERIALIZED_CODER_URN)));
+ }
+ }
+ }
+
+ static class Record implements Serializable {}
+
+ private static class RecordCoder extends AtomicCoder<Record> {
+ @Override
+ public void encode(Record value, OutputStream outStream)
+ throws CoderException, IOException {}
+
+ @Override
+ public Record decode(InputStream inStream)
+ throws CoderException, IOException {
+ return new Record();
+ }
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
deleted file mode 100644
index 42fba7c..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertThat;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.beam.sdk.coders.AtomicCoder;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.LengthPrefixCoder;
-import org.apache.beam.sdk.coders.SerializableCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.StructuredCoder;
-import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
-import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-import org.hamcrest.Matchers;
-import org.junit.Test;
-import org.junit.experimental.runners.Enclosed;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-/** Tests for {@link Coders}. */
-@RunWith(Enclosed.class)
-public class CodersTest {
- private static final Set<StructuredCoder<?>> KNOWN_CODERS =
- ImmutableSet.<StructuredCoder<?>>builder()
- .add(ByteArrayCoder.of())
- .add(KvCoder.of(VarLongCoder.of(), VarLongCoder.of()))
- .add(VarLongCoder.of())
- .add(IntervalWindowCoder.of())
- .add(IterableCoder.of(ByteArrayCoder.of()))
- .add(LengthPrefixCoder.of(IterableCoder.of(VarLongCoder.of())))
- .add(GlobalWindow.Coder.INSTANCE)
- .add(
- FullWindowedValueCoder.of(
- IterableCoder.of(VarLongCoder.of()), IntervalWindowCoder.of()))
- .build();
-
- /**
- * Tests that all known coders are present in the parameters that will be used by
- * {@link ToFromProtoTest}.
- */
- @RunWith(JUnit4.class)
- public static class ValidateKnownCodersPresentTest {
- @Test
- public void validateKnownCoders() {
- // Validates that every known coder in the Coders class is represented in a "Known Coder"
- // tests, which demonstrates that they are serialized via components and specified URNs rather
- // than java serialized
- Set<Class<? extends StructuredCoder>> knownCoderClasses = Coders.KNOWN_CODER_URNS.keySet();
- Set<Class<? extends StructuredCoder>> knownCoderTests = new HashSet<>();
- for (StructuredCoder<?> coder : KNOWN_CODERS) {
- knownCoderTests.add(coder.getClass());
- }
- Set<Class<? extends StructuredCoder>> missingKnownCoders = new HashSet<>(knownCoderClasses);
- missingKnownCoders.removeAll(knownCoderTests);
- assertThat(
- String.format(
- "Missing validation of known coder %s in %s",
- missingKnownCoders, CodersTest.class.getSimpleName()),
- missingKnownCoders,
- Matchers.empty());
- }
-
- @Test
- public void validateCoderTranslators() {
- assertThat(
- "Every Known Coder must have a Known Translator",
- Coders.KNOWN_CODER_URNS.keySet(),
- equalTo(Coders.KNOWN_TRANSLATORS.keySet()));
- }
- }
-
-
- /**
- * Tests round-trip coder encodings for both known and unknown {@link Coder coders}.
- */
- @RunWith(Parameterized.class)
- public static class ToFromProtoTest {
- @Parameters(name = "{index}: {0}")
- public static Iterable<Coder<?>> data() {
- return ImmutableList.<Coder<?>>builder()
- .addAll(KNOWN_CODERS)
- .add(
- StringUtf8Coder.of(),
- SerializableCoder.of(Record.class),
- new RecordCoder(),
- KvCoder.of(new RecordCoder(), AvroCoder.of(Record.class)))
- .build();
- }
-
- @Parameter(0)
- public Coder<?> coder;
-
- @Test
- public void toAndFromProto() throws Exception {
- SdkComponents componentsBuilder = SdkComponents.create();
- RunnerApi.Coder coderProto = Coders.toProto(coder, componentsBuilder);
-
- Components encodedComponents = componentsBuilder.toComponents();
- Coder<?> decodedCoder = Coders.fromProto(coderProto, encodedComponents);
- assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(coder));
-
- if (KNOWN_CODERS.contains(coder)) {
- for (RunnerApi.Coder encodedCoder : encodedComponents.getCodersMap().values()) {
- assertThat(
- encodedCoder.getSpec().getSpec().getUrn(),
- not(equalTo(Coders.JAVA_SERIALIZED_CODER_URN)));
- }
- }
- }
-
- static class Record implements Serializable {}
-
- private static class RecordCoder extends AtomicCoder<Record> {
- @Override
- public void encode(Record value, OutputStream outStream)
- throws CoderException, IOException {}
-
- @Override
- public Record decode(InputStream inStream)
- throws CoderException, IOException {
- return new Record();
- }
- }
- }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
new file mode 100644
index 0000000..af162d3
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.CombinePayload;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.BinaryCombineIntegerFn;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
+import org.apache.beam.sdk.transforms.CombineWithContext.Context;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for {@link CombineTranslation}. */
+@RunWith(Enclosed.class)
+public class CombineTranslationTest {
+
+ /** Tests that simple {@link CombineFn CombineFns} can be translated to and from proto. */
+ @RunWith(Parameterized.class)
+ public static class TranslateSimpleCombinesTest {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<Combine.CombineFn<Integer, ?, ?>> params() {
+ BinaryCombineIntegerFn sum = Sum.ofIntegers();
+ CombineFn<Integer, ?, Long> count = Count.combineFn();
+ TestCombineFn test = new TestCombineFn();
+ return ImmutableList.<CombineFn<Integer, ?, ?>>builder()
+ .add(sum)
+ .add(count)
+ .add(test)
+ .build();
+ }
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @Parameter(0)
+ public Combine.CombineFn<Integer, ?, ?> combineFn;
+
+ @Test
+ public void testToFromProto() throws Exception {
+ PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3));
+ input.apply(Combine.globally(combineFn));
+ final AtomicReference<AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>>> combine =
+ new AtomicReference<>();
+ pipeline.traverseTopologically(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ if (node.getTransform() instanceof Combine.PerKey) {
+ checkState(combine.get() == null);
+ combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline()));
+ }
+ }
+ });
+ checkState(combine.get() != null);
+ assertEquals(combineFn, CombineTranslation.getCombineFn(combine.get()));
+
+ SdkComponents sdkComponents = SdkComponents.create();
+ CombinePayload combineProto = CombineTranslation.toProto(combine.get(), sdkComponents);
+ RunnerApi.Components componentsProto = sdkComponents.toComponents();
+
+ assertEquals(
+ combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()),
+ CombineTranslation.getAccumulatorCoder(
+ combineProto, RehydratedComponents.forComponents(componentsProto)));
+ assertEquals(combineFn, CombineTranslation.getCombineFn(combineProto));
+ }
+ }
+
+ /** Tests that a {@link CombineFnWithContext} can be translated. */
+ @RunWith(JUnit4.class)
+ public static class ValidateCombineWithContextTest {
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ public void testToFromProtoWithSideInputs() throws Exception {
+ PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3));
+ final PCollectionView<Iterable<String>> sideInput =
+ pipeline.apply(Create.of("foo")).apply(View.<String>asIterable());
+ CombineFnWithContext<Integer, int[], Integer> combineFn = new TestCombineFnWithContext();
+ input.apply(Combine.globally(combineFn).withSideInputs(sideInput).withoutDefaults());
+ final AtomicReference<AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>>> combine =
+ new AtomicReference<>();
+ pipeline.traverseTopologically(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ if (node.getTransform() instanceof Combine.PerKey) {
+ checkState(combine.get() == null);
+ combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline()));
+ }
+ }
+ });
+ checkState(combine.get() != null);
+ assertEquals(combineFn, CombineTranslation.getCombineFn(combine.get()));
+
+ SdkComponents sdkComponents = SdkComponents.create();
+ CombinePayload combineProto = CombineTranslation.toProto(combine.get(), sdkComponents);
+ RunnerApi.Components componentsProto = sdkComponents.toComponents();
+
+ assertEquals(
+ combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()),
+ CombineTranslation.getAccumulatorCoder(
+ combineProto, RehydratedComponents.forComponents(componentsProto)));
+ assertEquals(combineFn, CombineTranslation.getCombineFn(combineProto));
+ }
+ }
+
+ private static class TestCombineFn extends Combine.CombineFn<Integer, Void, Void> {
+ @Override
+ public Void createAccumulator() {
+ return null;
+ }
+
+ @Override
+ public Coder<Void> getAccumulatorCoder(CoderRegistry registry, Coder<Integer> inputCoder) {
+ return (Coder) VoidCoder.of();
+ }
+
+ @Override
+ public Void extractOutput(Void accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Void mergeAccumulators(Iterable<Void> accumulators) {
+ return null;
+ }
+
+ @Override
+ public Void addInput(Void accumulator, Integer input) {
+ return accumulator;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(TestCombineFn.class);
+ }
+
+ @Override
+ public int hashCode() {
+ return TestCombineFn.class.hashCode();
+ }
+ }
+
+ private static class TestCombineFnWithContext
+ extends CombineFnWithContext<Integer, int[], Integer> {
+
+ @Override
+ public int[] createAccumulator(Context c) {
+ return new int[1];
+ }
+
+ @Override
+ public int[] addInput(int[] accumulator, Integer input, Context c) {
+ accumulator[0] += input;
+ return accumulator;
+ }
+
+ @Override
+ public int[] mergeAccumulators(Iterable<int[]> accumulators, Context c) {
+ int[] res = new int[1];
+ for (int[] accum : accumulators) {
+ res[0] += accum[0];
+ }
+ return res;
+ }
+
+ @Override
+ public Integer extractOutput(int[] accumulator, Context c) {
+ return accumulator[0];
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof TestCombineFnWithContext;
+ }
+
+ @Override
+ public int hashCode() {
+ return TestCombineFnWithContext.class.hashCode();
+ }
+ };
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
new file mode 100644
index 0000000..df659a8
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
+
+/** Tests for {@link CreatePCollectionViewTranslation}. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ CreatePCollectionViewTranslationTest.TestCreatePCollectionViewPayloadTranslation.class,
+})
+public class CreatePCollectionViewTranslationTest {
+
+ /** Tests for translating various {@link ParDo} transforms to/from {@link ParDoPayload} protos. */
+ @RunWith(Parameterized.class)
+ public static class TestCreatePCollectionViewPayloadTranslation {
+
+ // Two parameters suffices because the nature of the serialization/deserialization of
+ // the view is not what is being tested; it is just important that the round trip
+ // is not vacuous.
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<CreatePCollectionView<?, ?>> data() {
+ return ImmutableList.<CreatePCollectionView<?, ?>>of(
+ CreatePCollectionView.of(
+ PCollectionViews.singletonView(
+ testPCollection,
+ testPCollection.getWindowingStrategy(),
+ false,
+ null,
+ testPCollection.getCoder())),
+ CreatePCollectionView.of(
+ PCollectionViews.listView(
+ testPCollection,
+ testPCollection.getWindowingStrategy(),
+ testPCollection.getCoder())));
+ }
+
+ @Parameter(0)
+ public CreatePCollectionView<?, ?> createViewTransform;
+
+ public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ private static final PCollection<String> testPCollection = p.apply(Create.of("one"));
+
+ @Test
+ public void testEncodedProto() throws Exception {
+ SdkComponents components = SdkComponents.create();
+ components.registerPCollection(testPCollection);
+
+ AppliedPTransform<?, ?, ?> appliedPTransform =
+ AppliedPTransform.of(
+ "foo",
+ testPCollection.expand(),
+ createViewTransform.getView().expand(),
+ createViewTransform,
+ p);
+
+ FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec();
+
+ // Checks that the payload is what it should be
+ PCollectionView<?> deserializedView =
+ (PCollectionView<?>)
+ SerializableUtils.deserializeFromByteArray(
+ payload.getPayload().toByteArray(), PCollectionView.class.getSimpleName());
+
+ assertThat(
+ deserializedView, Matchers.<PCollectionView<?>>equalTo(createViewTransform.getView()));
+ }
+
+ @Test
+ public void testExtractionDirectFromTransform() throws Exception {
+ SdkComponents components = SdkComponents.create();
+ components.registerPCollection(testPCollection);
+
+ AppliedPTransform<?, ?, ?> appliedPTransform =
+ AppliedPTransform.of(
+ "foo",
+ testPCollection.expand(),
+ createViewTransform.getView().expand(),
+ createViewTransform,
+ p);
+
+ CreatePCollectionViewTranslation.getView((AppliedPTransform) appliedPTransform);
+
+ FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec();
+
+ // Checks that the payload is what it should be
+ PCollectionView<?> deserializedView =
+ (PCollectionView<?>)
+ SerializableUtils.deserializeFromByteArray(
+ payload.getPayload().toByteArray(),
+ PCollectionView.class.getSimpleName());
+
+ assertThat(
+ deserializedView, Matchers.<PCollectionView<?>>equalTo(createViewTransform.getView()));
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java
index 74c056c..4741b6b 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java
@@ -26,6 +26,7 @@
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -90,14 +91,24 @@
@Test
public void getDefaultOutputCoderDelegates() throws Exception {
@SuppressWarnings("unchecked")
- PCollection<Integer> input = Mockito.mock(PCollection.class);
+ PCollection<Integer> input =
+ PCollection.createPrimitiveOutputInternal(
+ null /* pipeline */,
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.BOUNDED,
+ null /* coder */);
@SuppressWarnings("unchecked")
- PCollection<String> output = Mockito.mock(PCollection.class);
+ PCollection<String> output = PCollection.createPrimitiveOutputInternal(
+ null /* pipeline */,
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.BOUNDED,
+ null /* coder */);
@SuppressWarnings("unchecked")
Coder<String> outputCoder = Mockito.mock(Coder.class);
+ Mockito.when(delegate.expand(input)).thenReturn(output);
Mockito.when(delegate.getDefaultOutputCoder(input, output)).thenReturn(outputCoder);
- assertThat(forwarding.getDefaultOutputCoder(input, output), equalTo(outputCoder));
+ assertThat(forwarding.expand(input).getCoder(), equalTo(outputCoder));
}
@Test
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupByKeyTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupByKeyTranslationTest.java
new file mode 100644
index 0000000..22681f7
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupByKeyTranslationTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.apache.beam.runners.core.construction.PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link GroupByKeyTranslation}. */
+@RunWith(JUnit4.class)
+public class GroupByKeyTranslationTest {
+
+ /**
+ * Tests that the translator is registered so the URN can be retrieved (the only thing you can
+ * meaningfully do with a {@link GroupByKey}).
+ */
+ @Test
+ public void testUrnRetrievable() throws Exception {
+ assertThat(
+ PTransformTranslation.urnForTransform(GroupByKey.create()),
+ equalTo(GROUP_BY_KEY_TRANSFORM_URN));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
new file mode 100644
index 0000000..1f7a4fb
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.io.BaseEncoding;
+import io.grpc.stub.StreamObserver;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.CommitManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest.ContentCase;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase;
+
+/**
+ * An {@link ArtifactStagingServiceImplBase ArtifactStagingService} which stores the bytes of the
+ * artifacts in memory..
+ */
+public class InMemoryArtifactStagerService extends ArtifactStagingServiceImplBase {
+ private final Map<ArtifactMetadata, byte[]> artifactBytes;
+ private Manifest manifest;
+
+ public InMemoryArtifactStagerService() {
+ artifactBytes = new HashMap<>();
+ }
+
+ @Override
+ public StreamObserver<ArtifactApi.PutArtifactRequest> putArtifact(
+ StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
+ return new BufferingObserver(responseObserver);
+ }
+
+ @Override
+ public void commitManifest(
+ ArtifactApi.CommitManifestRequest request,
+ StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver) {
+ this.manifest = request.getManifest();
+ responseObserver.onNext(CommitManifestResponse.getDefaultInstance());
+ responseObserver.onCompleted();
+ }
+
+ public Map<ArtifactMetadata, byte[]> getStagedArtifacts() {
+ return Collections.unmodifiableMap(artifactBytes);
+ }
+
+ public Manifest getManifest() {
+ return manifest;
+ }
+
+ private class BufferingObserver implements StreamObserver<PutArtifactRequest> {
+ private final StreamObserver<PutArtifactResponse> responseObserver;
+ private ArtifactMetadata destination = null;
+ private BufferWritingObserver writer = null;
+
+ public BufferingObserver(StreamObserver<PutArtifactResponse> responseObserver) {
+ this.responseObserver = responseObserver;
+ }
+
+ @Override
+ public void onNext(PutArtifactRequest value) {
+ if (writer == null) {
+ checkArgument(value.getContentCase().equals(ContentCase.METADATA));
+ writer = new BufferWritingObserver();
+ destination = value.getMetadata();
+ } else {
+ writer.onNext(value);
+ }
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ if (writer != null) {
+ writer.onError(t);
+ }
+ onCompleted();
+ }
+
+ @Override
+ public void onCompleted() {
+ if (writer != null) {
+ writer.onCompleted();
+ try {
+ artifactBytes.put(
+ destination
+ .toBuilder()
+ .setMd5(
+ BaseEncoding.base64()
+ .encode(
+ MessageDigest.getInstance("MD5").digest(writer.stream.toByteArray())))
+ .build(),
+ writer.stream.toByteArray());
+ } catch (NoSuchAlgorithmException e) {
+ throw new AssertionError("The Java Spec requires all JVMs to support MD5", e);
+ }
+ }
+ responseObserver.onNext(PutArtifactResponse.getDefaultInstance());
+ responseObserver.onCompleted();
+ }
+ }
+
+ private static class BufferWritingObserver implements StreamObserver<PutArtifactRequest> {
+ private final ByteArrayOutputStream stream;
+
+ BufferWritingObserver() {
+ stream = new ByteArrayOutputStream();
+ }
+
+ @Override
+ public void onNext(PutArtifactRequest value) {
+ try {
+ stream.write(value.getData().getData().toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ onCompleted();
+ }
+
+ @Override
+ public void onCompleted() {
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
new file mode 100644
index 0000000..6c641bb
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.windowing.AfterFirst;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
+import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.util.VarInt;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link PCollectionTranslation}.
+ */
+@RunWith(Parameterized.class)
+public class PCollectionTranslationTest {
+ // Each spec activates tests of all subsets of its fields
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<PCollection<?>> data() {
+ Pipeline pipeline = TestPipeline.create();
+ PCollection<Integer> ints = pipeline.apply("ints", Create.of(1, 2, 3));
+ PCollection<Long> longs = pipeline.apply("unbounded longs", GenerateSequence.from(0));
+ PCollection<Long> windowedLongs =
+ longs.apply(
+ "into fixed windows",
+ Window.<Long>into(FixedWindows.of(Duration.standardMinutes(10L))));
+ PCollection<KV<String, Iterable<String>>> groupedStrings =
+ pipeline
+ .apply(
+ "kvs", Create.of(KV.of("foo", "spam"), KV.of("bar", "ham"), KV.of("baz", "eggs")))
+ .apply("group", GroupByKey.<String, String>create());
+ PCollection<Long> coderLongs =
+ pipeline
+ .apply("counts with alternative coder", GenerateSequence.from(0).to(10))
+ .setCoder(BigEndianLongCoder.of());
+ PCollection<Integer> allCustomInts =
+ pipeline
+ .apply(
+ "intsWithCustomCoder",
+ Create.of(1, 2)
+ .withCoder(new AutoValue_PCollectionTranslationTest_CustomIntCoder()))
+ .apply(
+ "into custom windows",
+ Window.<Integer>into(new CustomWindows())
+ .triggering(
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(
+ AfterFirst.of(
+ AfterPane.elementCountAtLeast(5),
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(Duration.millis(227L)))))
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.standardMinutes(12L)));
+ return ImmutableList.<PCollection<?>>of(ints, longs, windowedLongs, coderLongs, groupedStrings);
+ }
+
+ @Parameter(0)
+ public PCollection<?> testCollection;
+
+ @Test
+ public void testEncodeDecodeCycle() throws Exception {
+ // Encode
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PCollection protoCollection =
+ PCollectionTranslation.toProto(testCollection, sdkComponents);
+ RehydratedComponents protoComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+
+ // Decode
+ Pipeline pipeline = Pipeline.create();
+ PCollection<?> decodedCollection =
+ PCollectionTranslation.fromProto(protoCollection, pipeline, protoComponents);
+
+ // Verify
+ assertThat(decodedCollection.getCoder(), Matchers.<Coder<?>>equalTo(testCollection.getCoder()));
+ assertThat(
+ decodedCollection.getWindowingStrategy(),
+ Matchers.<WindowingStrategy<?, ?>>equalTo(
+ testCollection.getWindowingStrategy().fixDefaults()));
+ assertThat(decodedCollection.isBounded(), equalTo(testCollection.isBounded()));
+ }
+
+ @Test
+ public void testEncodeDecodeFields() throws Exception {
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PCollection protoCollection = PCollectionTranslation
+ .toProto(testCollection, sdkComponents);
+ RehydratedComponents protoComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+ Coder<?> decodedCoder = protoComponents.getCoder(protoCollection.getCoderId());
+ WindowingStrategy<?, ?> decodedStrategy =
+ protoComponents.getWindowingStrategy(protoCollection.getWindowingStrategyId());
+ IsBounded decodedIsBounded = PCollectionTranslation.isBounded(protoCollection);
+
+ assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(testCollection.getCoder()));
+ assertThat(
+ decodedStrategy,
+ Matchers.<WindowingStrategy<?, ?>>equalTo(
+ testCollection.getWindowingStrategy().fixDefaults()));
+ assertThat(decodedIsBounded, equalTo(testCollection.isBounded()));
+ }
+
+ @AutoValue
+ abstract static class CustomIntCoder extends CustomCoder<Integer> {
+ @Override
+ public Integer decode(InputStream inStream) throws IOException {
+ return VarInt.decodeInt(inStream);
+ }
+
+ @Override
+ public void encode(Integer value, OutputStream outStream) throws IOException {
+ VarInt.encode(value, outStream);
+ }
+ }
+
+ private static class CustomWindows extends NonMergingWindowFn<Integer, BoundedWindow> {
+ @Override
+ public Collection<BoundedWindow> assignWindows(final AssignContext c) throws Exception {
+ return Collections.<BoundedWindow>singleton(
+ new BoundedWindow() {
+ @Override
+ public Instant maxTimestamp() {
+ return new Instant(c.element().longValue());
+ }
+ });
+ }
+
+ @Override
+ public boolean isCompatible(WindowFn<?, ?> other) {
+ return other != null && this.getClass().equals(other.getClass());
+ }
+
+ @Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is only compatible with %s.",
+ CustomWindows.class.getSimpleName(), CustomWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
+ public Coder<BoundedWindow> windowCoder() {
+ return new AtomicCoder<BoundedWindow>() {
+ @Override public void verifyDeterministic() {}
+
+ @Override
+ public void encode(BoundedWindow value, OutputStream outStream)
+ throws IOException {
+ VarInt.encode(value.maxTimestamp().getMillis(), outStream);
+ }
+
+ @Override
+ public BoundedWindow decode(InputStream inStream) throws IOException {
+ final Instant ts = new Instant(VarInt.decodeLong(inStream));
+ return new BoundedWindow() {
+ @Override
+ public Instant maxTimestamp() {
+ return ts;
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public WindowMappingFn<BoundedWindow> getDefaultWindowMappingFn() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
deleted file mode 100644
index c38dbc0..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Collection;
-import java.util.Collections;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.AtomicCoder;
-import org.apache.beam.sdk.coders.BigEndianLongCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.io.GenerateSequence;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.windowing.AfterFirst;
-import org.apache.beam.sdk.transforms.windowing.AfterPane;
-import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
-import org.apache.beam.sdk.util.VarInt;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.hamcrest.Matchers;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * Tests for {@link PCollections}.
- */
-@RunWith(Parameterized.class)
-public class PCollectionsTest {
- // Each spec activates tests of all subsets of its fields
- @Parameters(name = "{index}: {0}")
- public static Iterable<PCollection<?>> data() {
- Pipeline pipeline = TestPipeline.create();
- PCollection<Integer> ints = pipeline.apply("ints", Create.of(1, 2, 3));
- PCollection<Long> longs = pipeline.apply("unbounded longs", GenerateSequence.from(0));
- PCollection<Long> windowedLongs =
- longs.apply(
- "into fixed windows",
- Window.<Long>into(FixedWindows.of(Duration.standardMinutes(10L))));
- PCollection<KV<String, Iterable<String>>> groupedStrings =
- pipeline
- .apply(
- "kvs", Create.of(KV.of("foo", "spam"), KV.of("bar", "ham"), KV.of("baz", "eggs")))
- .apply("group", GroupByKey.<String, String>create());
- PCollection<Long> coderLongs =
- pipeline
- .apply("counts with alternative coder", GenerateSequence.from(0).to(10))
- .setCoder(BigEndianLongCoder.of());
- PCollection<Integer> allCustomInts =
- pipeline
- .apply(
- "intsWithCustomCoder",
- Create.of(1, 2).withCoder(new AutoValue_PCollectionsTest_CustomIntCoder()))
- .apply(
- "into custom windows",
- Window.<Integer>into(new CustomWindows())
- .triggering(
- AfterWatermark.pastEndOfWindow()
- .withEarlyFirings(
- AfterFirst.of(
- AfterPane.elementCountAtLeast(5),
- AfterProcessingTime.pastFirstElementInPane()
- .plusDelayOf(Duration.millis(227L)))))
- .accumulatingFiredPanes()
- .withAllowedLateness(Duration.standardMinutes(12L)));
- return ImmutableList.<PCollection<?>>of(ints, longs, windowedLongs, coderLongs, groupedStrings);
- }
-
- @Parameter(0)
- public PCollection<?> testCollection;
-
- @Test
- public void testEncodeDecodeCycle() throws Exception {
- SdkComponents sdkComponents = SdkComponents.create();
- RunnerApi.PCollection protoCollection = PCollections.toProto(testCollection, sdkComponents);
- RunnerApi.Components protoComponents = sdkComponents.toComponents();
- Coder<?> decodedCoder = PCollections.getCoder(protoCollection, protoComponents);
- WindowingStrategy<?, ?> decodedStrategy =
- PCollections.getWindowingStrategy(protoCollection, protoComponents);
- IsBounded decodedIsBounded = PCollections.isBounded(protoCollection);
-
- assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(testCollection.getCoder()));
- assertThat(
- decodedStrategy,
- Matchers.<WindowingStrategy<?, ?>>equalTo(
- testCollection.getWindowingStrategy().fixDefaults()));
- assertThat(decodedIsBounded, equalTo(testCollection.isBounded()));
- }
-
- @AutoValue
- abstract static class CustomIntCoder extends CustomCoder<Integer> {
- @Override
- public Integer decode(InputStream inStream) throws IOException {
- return VarInt.decodeInt(inStream);
- }
-
- @Override
- public void encode(Integer value, OutputStream outStream) throws IOException {
- VarInt.encode(value, outStream);
- }
- }
-
- private static class CustomWindows extends NonMergingWindowFn<Integer, BoundedWindow> {
- @Override
- public Collection<BoundedWindow> assignWindows(final AssignContext c) throws Exception {
- return Collections.<BoundedWindow>singleton(
- new BoundedWindow() {
- @Override
- public Instant maxTimestamp() {
- return new Instant(c.element().longValue());
- }
- });
- }
-
- @Override
- public boolean isCompatible(WindowFn<?, ?> other) {
- return other != null && this.getClass().equals(other.getClass());
- }
-
- @Override
- public Coder<BoundedWindow> windowCoder() {
- return new AtomicCoder<BoundedWindow>() {
- @Override public void verifyDeterministic() {}
-
- @Override
- public void encode(BoundedWindow value, OutputStream outStream)
- throws IOException {
- VarInt.encode(value.maxTimestamp().getMillis(), outStream);
- }
-
- @Override
- public BoundedWindow decode(InputStream inStream) throws IOException {
- final Instant ts = new Instant(VarInt.decodeLong(inStream));
- return new BoundedWindow() {
- @Override
- public Instant maxTimestamp() {
- return ts;
- }
- };
- }
- };
- }
-
- @Override
- public WindowMappingFn<BoundedWindow> getDefaultWindowMappingFn() {
- throw new UnsupportedOperationException();
- }
- }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index cfea62f..324e38d 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -27,13 +27,18 @@
import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
import java.util.Collections;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.DefaultFilenamePolicy;
+import org.apache.beam.sdk.io.DynamicFileDestinations;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
import org.apache.beam.sdk.io.LocalResources;
import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -58,7 +63,9 @@
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
@@ -67,7 +74,6 @@
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
-import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
@@ -94,10 +100,16 @@
private AppliedPTransform<?, ?, ?> getAppliedTransform(PTransform pardo) {
PCollection<KV<String, Integer>> input =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p,
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
+ input.setName("dummy input");
+
PCollection<Integer> output =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
+ output.setName("dummy output");
return AppliedPTransform.of("pardo", input.expand(), output.expand(), pardo, p);
}
@@ -122,7 +134,7 @@
@Override
public PCollection<Integer> expand(PCollection<KV<String, Integer>> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), VarIntCoder.of());
}
}
PTransformMatcher matcher = PTransformMatchers.classEqualTo(MyPTransform.class);
@@ -272,6 +284,18 @@
}
@Test
+ public void parDoSplittable() {
+ AppliedPTransform<?, ?, ?> parDoApplication =
+ getAppliedTransform(
+ ParDo.of(splittableDoFn).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.splittableParDo().matches(parDoApplication), is(true));
+
+ assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false));
+ assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false));
+ assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false));
+ }
+
+ @Test
public void parDoMultiWithState() {
AppliedPTransform<?, ?, ?> parDoApplication =
getAppliedTransform(
@@ -284,6 +308,19 @@
}
@Test
+ public void parDoWithState() {
+ AppliedPTransform<?, ?, ?> statefulApplication =
+ getAppliedTransform(
+ ParDo.of(doFnWithState).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.stateOrTimerParDo().matches(statefulApplication), is(true));
+
+ AppliedPTransform<?, ?, ?> splittableApplication =
+ getAppliedTransform(
+ ParDo.of(splittableDoFn).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.stateOrTimerParDo().matches(splittableApplication), is(false));
+ }
+
+ @Test
public void parDoMultiWithTimers() {
AppliedPTransform<?, ?, ?> parDoApplication =
getAppliedTransform(
@@ -389,14 +426,14 @@
public void emptyFlattenWithEmptyFlatten() {
AppliedPTransform application =
AppliedPTransform
- .<PCollectionList<Object>, PCollection<Object>, Flatten.PCollections<Object>>of(
+ .<PCollectionList<Integer>, PCollection<Integer>, Flatten.PCollections<Integer>>of(
"EmptyFlatten",
Collections.<TupleTag<?>, PValue>emptyMap(),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.pCollections(),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ Flatten.<Integer>pCollections(),
p);
assertThat(PTransformMatchers.emptyFlatten().matches(application), is(true));
@@ -406,17 +443,17 @@
public void emptyFlattenWithNonEmptyFlatten() {
AppliedPTransform application =
AppliedPTransform
- .<PCollectionList<Object>, PCollection<Object>, Flatten.PCollections<Object>>of(
+ .<PCollectionList<Integer>, PCollection<Integer>, Flatten.PCollections<Integer>>of(
"Flatten",
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.pCollections(),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ Flatten.<Integer>pCollections(),
p);
assertThat(PTransformMatchers.emptyFlatten().matches(application), is(false));
@@ -426,15 +463,15 @@
public void emptyFlattenWithNonFlatten() {
AppliedPTransform application =
AppliedPTransform
- .<PCollection<Iterable<Object>>, PCollection<Object>, Flatten.Iterables<Object>>of(
+ .<PCollection<Iterable<Integer>>, PCollection<Integer>, Flatten.Iterables<Integer>>of(
"EmptyFlatten",
Collections.<TupleTag<?>, PValue>emptyMap(),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.iterables() /* This isn't actually possible to construct,
- * but for the sake of example */,
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ /* This isn't actually possible to construct, but for the sake of example */
+ Flatten.<Integer>iterables(),
p);
assertThat(PTransformMatchers.emptyFlatten().matches(application), is(false));
@@ -444,17 +481,17 @@
public void flattenWithDuplicateInputsWithoutDuplicates() {
AppliedPTransform application =
AppliedPTransform
- .<PCollectionList<Object>, PCollection<Object>, Flatten.PCollections<Object>>of(
+ .<PCollectionList<Integer>, PCollection<Integer>, Flatten.PCollections<Integer>>of(
"Flatten",
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.pCollections(),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ Flatten.<Integer>pCollections(),
p);
assertThat(PTransformMatchers.flattenWithDuplicateInputs().matches(application), is(false));
@@ -462,22 +499,22 @@
@Test
public void flattenWithDuplicateInputsWithDuplicates() {
- PCollection<Object> duplicate =
+ PCollection<Integer> duplicate =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
AppliedPTransform application =
AppliedPTransform
- .<PCollectionList<Object>, PCollection<Object>, Flatten.PCollections<Object>>of(
+ .<PCollectionList<Integer>, PCollection<Integer>, Flatten.PCollections<Integer>>of(
"Flatten",
ImmutableMap.<TupleTag<?>, PValue>builder()
- .put(new TupleTag<Object>(), duplicate)
- .put(new TupleTag<Object>(), duplicate)
+ .put(new TupleTag<Integer>(), duplicate)
+ .put(new TupleTag<Integer>(), duplicate)
.build(),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.pCollections(),
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ Flatten.<Integer>pCollections(),
p);
assertThat(PTransformMatchers.flattenWithDuplicateInputs().matches(application), is(true));
@@ -487,15 +524,15 @@
public void flattenWithDuplicateInputsNonFlatten() {
AppliedPTransform application =
AppliedPTransform
- .<PCollection<Iterable<Object>>, PCollection<Object>, Flatten.Iterables<Object>>of(
+ .<PCollection<Iterable<Integer>>, PCollection<Integer>, Flatten.Iterables<Integer>>of(
"EmptyFlatten",
Collections.<TupleTag<?>, PValue>emptyMap(),
Collections.<TupleTag<?>, PValue>singletonMap(
- new TupleTag<Object>(),
+ new TupleTag<Integer>(),
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)),
- Flatten.iterables() /* This isn't actually possible to construct,
- * but for the sake of example */,
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
+ /* This isn't actually possible to construct, but for the sake of example */
+ Flatten.<Integer>iterables(),
p);
assertThat(PTransformMatchers.flattenWithDuplicateInputs().matches(application), is(false));
@@ -504,13 +541,19 @@
@Test
public void writeWithRunnerDeterminedSharding() {
ResourceId outputDirectory = LocalResources.fromString("/foo/bar", true /* isDirectory */);
- FilenamePolicy policy = DefaultFilenamePolicy.constructUsingStandardParameters(
- StaticValueProvider.of(outputDirectory), DefaultFilenamePolicy.DEFAULT_SHARD_TEMPLATE, "");
- WriteFiles<Integer> write =
+ FilenamePolicy policy =
+ DefaultFilenamePolicy.fromStandardParameters(
+ StaticValueProvider.of(outputDirectory),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE,
+ "",
+ false);
+ WriteFiles<Integer, Void, Integer> write =
WriteFiles.to(
- new FileBasedSink<Integer>(StaticValueProvider.of(outputDirectory), policy) {
+ new FileBasedSink<Integer, Void, Integer>(
+ StaticValueProvider.of(outputDirectory),
+ DynamicFileDestinations.<Integer>constant(new FakeFilenamePolicy())) {
@Override
- public WriteOperation<Integer> createWriteOperation() {
+ public WriteOperation<Void, Integer> createWriteOperation() {
return null;
}
});
@@ -518,13 +561,13 @@
PTransformMatchers.writeWithRunnerDeterminedSharding().matches(appliedWrite(write)),
is(true));
- WriteFiles<Integer> withStaticSharding = write.withNumShards(3);
+ WriteFiles<Integer, Void, Integer> withStaticSharding = write.withNumShards(3);
assertThat(
PTransformMatchers.writeWithRunnerDeterminedSharding()
.matches(appliedWrite(withStaticSharding)),
is(false));
- WriteFiles<Integer> withCustomSharding =
+ WriteFiles<Integer, Void, Integer> withCustomSharding =
write.withSharding(Sum.integersGlobally().asSingletonView());
assertThat(
PTransformMatchers.writeWithRunnerDeterminedSharding()
@@ -532,12 +575,32 @@
is(false));
}
- private AppliedPTransform<?, ?, ?> appliedWrite(WriteFiles<Integer> write) {
- return AppliedPTransform.<PCollection<Integer>, PDone, WriteFiles<Integer>>of(
- "WriteFiles",
- Collections.<TupleTag<?>, PValue>emptyMap(),
- Collections.<TupleTag<?>, PValue>emptyMap(),
- write,
- p);
+ private AppliedPTransform<?, ?, ?> appliedWrite(WriteFiles<Integer, Void, Integer> write) {
+ return AppliedPTransform
+ .<PCollection<Integer>, WriteFilesResult<Void>, WriteFiles<Integer, Void, Integer>>of(
+ "WriteFiles",
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ write,
+ p);
+ }
+
+ private static class FakeFilenamePolicy extends FilenamePolicy {
+ @Override
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ FileBasedSink.OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("should not be called");
+ }
+
+ @Nullable
+ @Override
+ public ResourceId unwindowedFilename(
+ int shardNumber, int numShards, FileBasedSink.OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("should not be called");
+ }
}
}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
new file mode 100644
index 0000000..36f912c
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link PTransformTranslation}.
+ */
+@RunWith(Parameterized.class)
+public class PTransformTranslationTest {
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<ToAndFromProtoSpec> data() {
+ // This pipeline exists for construction, not to run any test.
+ // TODO: Leaf node with understood payload - i.e. validate payloads
+ ToAndFromProtoSpec readLeaf = ToAndFromProtoSpec.leaf(read(TestPipeline.create()));
+
+ ToAndFromProtoSpec readMultipleInAndOut =
+ ToAndFromProtoSpec.leaf(multiMultiParDo(TestPipeline.create()));
+
+ TestPipeline compositeReadPipeline = TestPipeline.create();
+ ToAndFromProtoSpec compositeRead =
+ ToAndFromProtoSpec.composite(
+ generateSequence(compositeReadPipeline),
+ ToAndFromProtoSpec.leaf(read(compositeReadPipeline)));
+
+ ToAndFromProtoSpec rawLeafNullSpec =
+ ToAndFromProtoSpec.leaf(rawPTransformWithNullSpec(TestPipeline.create()));
+
+ return ImmutableList.<ToAndFromProtoSpec>builder()
+ .add(readLeaf)
+ .add(readMultipleInAndOut)
+ .add(compositeRead)
+ .add(rawLeafNullSpec)
+ // TODO: Composite with multiple children
+ // TODO: Composite with a composite child
+ .build();
+ }
+
+ @AutoValue
+ abstract static class ToAndFromProtoSpec {
+ public static ToAndFromProtoSpec leaf(AppliedPTransform<?, ?, ?> transform) {
+ return new AutoValue_PTransformTranslationTest_ToAndFromProtoSpec(
+ transform, Collections.<ToAndFromProtoSpec>emptyList());
+ }
+
+ public static ToAndFromProtoSpec composite(
+ AppliedPTransform<?, ?, ?> topLevel, ToAndFromProtoSpec spec, ToAndFromProtoSpec... specs) {
+ List<ToAndFromProtoSpec> childSpecs = new ArrayList<>();
+ childSpecs.add(spec);
+ childSpecs.addAll(Arrays.asList(specs));
+ return new AutoValue_PTransformTranslationTest_ToAndFromProtoSpec(topLevel, childSpecs);
+ }
+
+ abstract AppliedPTransform<?, ?, ?> getTransform();
+ abstract Collection<ToAndFromProtoSpec> getChildren();
+ }
+
+ @Parameter(0)
+ public ToAndFromProtoSpec spec;
+
+ @Test
+ public void toAndFromProto() throws IOException {
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.PTransform converted = convert(spec, components);
+ Components protoComponents = components.toComponents();
+
+ // Sanity checks
+ assertThat(converted.getInputsCount(), equalTo(spec.getTransform().getInputs().size()));
+ assertThat(converted.getOutputsCount(), equalTo(spec.getTransform().getOutputs().size()));
+ assertThat(converted.getSubtransformsCount(), equalTo(spec.getChildren().size()));
+
+ assertThat(converted.getUniqueName(), equalTo(spec.getTransform().getFullName()));
+ for (PValue inputValue : spec.getTransform().getInputs().values()) {
+ PCollection<?> inputPc = (PCollection<?>) inputValue;
+ protoComponents.getPcollectionsOrThrow(components.registerPCollection(inputPc));
+ }
+ for (PValue outputValue : spec.getTransform().getOutputs().values()) {
+ PCollection<?> outputPc = (PCollection<?>) outputValue;
+ protoComponents.getPcollectionsOrThrow(components.registerPCollection(outputPc));
+ }
+ }
+
+ private RunnerApi.PTransform convert(ToAndFromProtoSpec spec, SdkComponents components)
+ throws IOException {
+ List<AppliedPTransform<?, ?, ?>> childTransforms = new ArrayList<>();
+ for (ToAndFromProtoSpec child : spec.getChildren()) {
+ childTransforms.add(child.getTransform());
+ System.out.println("Converting child " + child);
+ convert(child, components);
+ // Sanity call
+ components.getExistingPTransformId(child.getTransform());
+ }
+ RunnerApi.PTransform convert = PTransformTranslation
+ .toProto(spec.getTransform(), childTransforms, components);
+ // Make sure the converted transform is registered. Convert it independently, but if this is a
+ // child spec, the child must be in the components.
+ components.registerPTransform(spec.getTransform(), childTransforms);
+ return convert;
+ }
+
+ private static class TestDoFn extends DoFn<Long, KV<Long, String>> {
+ // Exists to stop the ParDo application from throwing
+ @ProcessElement public void process(ProcessContext context) {}
+ }
+
+ private static AppliedPTransform<?, ?, ?> generateSequence(Pipeline pipeline) {
+ GenerateSequence sequence = GenerateSequence.from(0);
+ PCollection<Long> pcollection = pipeline.apply(sequence);
+ return AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of(
+ "Count", pipeline.begin().expand(), pcollection.expand(), sequence, pipeline);
+ }
+
+ private static AppliedPTransform<?, ?, ?> read(Pipeline pipeline) {
+ Read.Unbounded<Long> transform = Read.from(CountingSource.unbounded());
+ PCollection<Long> pcollection = pipeline.apply(transform);
+ return AppliedPTransform.<PBegin, PCollection<Long>, Read.Unbounded<Long>>of(
+ "ReadTheCount", pipeline.begin().expand(), pcollection.expand(), transform, pipeline);
+ }
+
+ private static AppliedPTransform<?, ?, ?> rawPTransformWithNullSpec(Pipeline pipeline) {
+ PTransformTranslation.RawPTransform<PBegin, PDone> rawPTransform =
+ new PTransformTranslation.RawPTransform<PBegin, PDone>() {
+ @Override
+ public String getUrn() {
+ return "fake/urn";
+ }
+
+ @Nullable
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ return null;
+ }
+ };
+ return AppliedPTransform.<PBegin, PDone, PTransform<PBegin, PDone>>of(
+ "RawPTransformWithNoSpec",
+ pipeline.begin().expand(),
+ PDone.in(pipeline).expand(),
+ rawPTransform,
+ pipeline);
+ }
+
+ private static AppliedPTransform<?, ?, ?> multiMultiParDo(Pipeline pipeline) {
+ PCollectionView<String> view =
+ pipeline.apply(Create.of("foo")).apply(View.<String>asSingleton());
+ PCollection<Long> input = pipeline.apply(GenerateSequence.from(0));
+ ParDo.MultiOutput<Long, KV<Long, String>> parDo =
+ ParDo.of(new TestDoFn())
+ .withSideInputs(view)
+ .withOutputTags(
+ new TupleTag<KV<Long, String>>() {},
+ TupleTagList.of(new TupleTag<KV<String, Long>>() {}));
+ PCollectionTuple output = input.apply(parDo);
+
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ inputs.putAll(parDo.getAdditionalInputs());
+ inputs.putAll(input.expand());
+
+ return AppliedPTransform
+ .<PCollection<Long>, PCollectionTuple, ParDo.MultiOutput<Long, KV<Long, String>>>of(
+ "MultiParDoInAndOut", inputs, output.expand(), parDo, pipeline);
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
deleted file mode 100644
index 4125544..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.PTransform;
-import org.apache.beam.sdk.io.CountingSource;
-import org.apache.beam.sdk.io.GenerateSequence;
-import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PBegin;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * Tests for {@link PTransforms}.
- */
-@RunWith(Parameterized.class)
-public class PTransformsTest {
-
- @Parameters(name = "{index}: {0}")
- public static Iterable<ToAndFromProtoSpec> data() {
- // This pipeline exists for construction, not to run any test.
- // TODO: Leaf node with understood payload - i.e. validate payloads
- ToAndFromProtoSpec readLeaf = ToAndFromProtoSpec.leaf(read(TestPipeline.create()));
- ToAndFromProtoSpec readMultipleInAndOut =
- ToAndFromProtoSpec.leaf(multiMultiParDo(TestPipeline.create()));
- TestPipeline compositeReadPipeline = TestPipeline.create();
- ToAndFromProtoSpec compositeRead =
- ToAndFromProtoSpec.composite(
- generateSequence(compositeReadPipeline),
- ToAndFromProtoSpec.leaf(read(compositeReadPipeline)));
- return ImmutableList.<ToAndFromProtoSpec>builder()
- .add(readLeaf)
- .add(readMultipleInAndOut)
- .add(compositeRead)
- // TODO: Composite with multiple children
- // TODO: Composite with a composite child
- .build();
- }
-
- @AutoValue
- abstract static class ToAndFromProtoSpec {
- public static ToAndFromProtoSpec leaf(AppliedPTransform<?, ?, ?> transform) {
- return new AutoValue_PTransformsTest_ToAndFromProtoSpec(
- transform, Collections.<ToAndFromProtoSpec>emptyList());
- }
-
- public static ToAndFromProtoSpec composite(
- AppliedPTransform<?, ?, ?> topLevel, ToAndFromProtoSpec spec, ToAndFromProtoSpec... specs) {
- List<ToAndFromProtoSpec> childSpecs = new ArrayList<>();
- childSpecs.add(spec);
- childSpecs.addAll(Arrays.asList(specs));
- return new AutoValue_PTransformsTest_ToAndFromProtoSpec(topLevel, childSpecs);
- }
-
- abstract AppliedPTransform<?, ?, ?> getTransform();
- abstract Collection<ToAndFromProtoSpec> getChildren();
- }
-
- @Parameter(0)
- public ToAndFromProtoSpec spec;
-
- @Test
- public void toAndFromProto() throws IOException {
- SdkComponents components = SdkComponents.create();
- RunnerApi.PTransform converted = convert(spec, components);
- Components protoComponents = components.toComponents();
-
- // Sanity checks
- assertThat(converted.getInputsCount(), equalTo(spec.getTransform().getInputs().size()));
- assertThat(converted.getOutputsCount(), equalTo(spec.getTransform().getOutputs().size()));
- assertThat(converted.getSubtransformsCount(), equalTo(spec.getChildren().size()));
-
- assertThat(converted.getUniqueName(), equalTo(spec.getTransform().getFullName()));
- for (PValue inputValue : spec.getTransform().getInputs().values()) {
- PCollection<?> inputPc = (PCollection<?>) inputValue;
- protoComponents.getPcollectionsOrThrow(components.registerPCollection(inputPc));
- }
- for (PValue outputValue : spec.getTransform().getOutputs().values()) {
- PCollection<?> outputPc = (PCollection<?>) outputValue;
- protoComponents.getPcollectionsOrThrow(components.registerPCollection(outputPc));
- }
- }
-
- private RunnerApi.PTransform convert(ToAndFromProtoSpec spec, SdkComponents components)
- throws IOException {
- List<AppliedPTransform<?, ?, ?>> childTransforms = new ArrayList<>();
- for (ToAndFromProtoSpec child : spec.getChildren()) {
- childTransforms.add(child.getTransform());
- System.out.println("Converting child " + child);
- convert(child, components);
- // Sanity call
- components.getExistingPTransformId(child.getTransform());
- }
- PTransform convert = PTransforms.toProto(spec.getTransform(), childTransforms, components);
- // Make sure the converted transform is registered. Convert it independently, but if this is a
- // child spec, the child must be in the components.
- components.registerPTransform(spec.getTransform(), childTransforms);
- return convert;
- }
-
- private static class TestDoFn extends DoFn<Long, KV<Long, String>> {
- // Exists to stop the ParDo application from throwing
- @ProcessElement public void process(ProcessContext context) {}
- }
-
- private static AppliedPTransform<?, ?, ?> generateSequence(Pipeline pipeline) {
- GenerateSequence sequence = GenerateSequence.from(0);
- PCollection<Long> pcollection = pipeline.apply(sequence);
- return AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of(
- "Count", pipeline.begin().expand(), pcollection.expand(), sequence, pipeline);
- }
-
- private static AppliedPTransform<?, ?, ?> read(Pipeline pipeline) {
- Read.Unbounded<Long> transform = Read.from(CountingSource.unbounded());
- PCollection<Long> pcollection = pipeline.apply(transform);
- return AppliedPTransform.<PBegin, PCollection<Long>, Read.Unbounded<Long>>of(
- "ReadTheCount", pipeline.begin().expand(), pcollection.expand(), transform, pipeline);
- }
-
- private static AppliedPTransform<?, ?, ?> multiMultiParDo(Pipeline pipeline) {
- PCollectionView<String> view =
- pipeline.apply(Create.of("foo")).apply(View.<String>asSingleton());
- PCollection<Long> input = pipeline.apply(GenerateSequence.from(0));
- ParDo.MultiOutput<Long, KV<Long, String>> parDo =
- ParDo.of(new TestDoFn())
- .withSideInputs(view)
- .withOutputTags(
- new TupleTag<KV<Long, String>>() {},
- TupleTagList.of(new TupleTag<KV<String, Long>>() {}));
- PCollectionTuple output = input.apply(parDo);
-
- Map<TupleTag<?>, PValue> inputs = new HashMap<>();
- inputs.putAll(parDo.getAdditionalInputs());
- inputs.putAll(input.expand());
-
- return AppliedPTransform
- .<PCollection<Long>, PCollectionTuple, ParDo.MultiOutput<Long, KV<Long, String>>>of(
- "MultiParDoInAndOut", inputs, output.expand(), parDo, pipeline);
- }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
new file mode 100644
index 0000000..b79947e
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine.BinaryCombineLongFn;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
+
+/** Tests for {@link ParDoTranslation}. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ ParDoTranslationTest.TestParDoPayloadTranslation.class,
+ ParDoTranslationTest.TestStateAndTimerTranslation.class
+})
+public class ParDoTranslationTest {
+
+ /**
+ * Tests for translating various {@link ParDo} transforms to/from {@link ParDoPayload} protos.
+ */
+ @RunWith(Parameterized.class)
+ public static class TestParDoPayloadTranslation {
+ public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ private static PCollectionView<Long> singletonSideInput =
+ p.apply("GenerateSingleton", GenerateSequence.from(0L).to(1L))
+ .apply(View.<Long>asSingleton());
+ private static PCollectionView<Map<Long, Iterable<String>>> multimapSideInput =
+ p.apply("CreateMultimap", Create.of(KV.of(1L, "foo"), KV.of(1L, "bar"), KV.of(2L, "spam")))
+ .setCoder(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
+ .apply(View.<Long, String>asMultimap());
+
+ private static PCollection<KV<Long, String>> mainInput =
+ p.apply(
+ "CreateMainInput", Create.empty(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of())));
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<ParDo.MultiOutput<?, ?>> data() {
+ return ImmutableList.<ParDo.MultiOutput<?, ?>>of(
+ ParDo.of(new DropElementsFn()).withOutputTags(new TupleTag<Void>(), TupleTagList.empty()),
+ ParDo.of(new DropElementsFn())
+ .withOutputTags(new TupleTag<Void>(), TupleTagList.empty())
+ .withSideInputs(singletonSideInput, multimapSideInput),
+ ParDo.of(new DropElementsFn())
+ .withOutputTags(
+ new TupleTag<Void>(),
+ TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {}))
+ .withSideInputs(singletonSideInput, multimapSideInput),
+ ParDo.of(new DropElementsFn())
+ .withOutputTags(
+ new TupleTag<Void>(),
+ TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})),
+ ParDo.of(new SplittableDropElementsFn())
+ .withOutputTags(
+ new TupleTag<Void>(),
+ TupleTagList.empty()));
+ }
+
+ @Parameter(0)
+ public ParDo.MultiOutput<KV<Long, String>, Void> parDo;
+
+ @Test
+ public void testToAndFromProto() throws Exception {
+ SdkComponents components = SdkComponents.create();
+ ParDoPayload payload = ParDoTranslation.toProto(parDo, components);
+
+ assertThat(ParDoTranslation.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
+ assertThat(
+ ParDoTranslation.getMainOutputTag(payload),
+ Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
+ for (PCollectionView<?> view : parDo.getSideInputs()) {
+ payload.getSideInputsOrThrow(view.getTagInternal().getId());
+ }
+ }
+
+ @Test
+ public void toAndFromTransformProto() throws Exception {
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ inputs.put(new TupleTag<KV<Long, String>>() {}, mainInput);
+ inputs.putAll(parDo.getAdditionalInputs());
+ PCollectionTuple output = mainInput.apply(parDo);
+
+ SdkComponents sdkComponents = SdkComponents.create();
+
+ // Encode
+ RunnerApi.PTransform protoTransform =
+ PTransformTranslation.toProto(
+ AppliedPTransform.<PCollection<KV<Long, String>>, PCollection<Void>, MultiOutput>of(
+ "foo", inputs, output.expand(), parDo, p),
+ sdkComponents);
+ RunnerApi.Components components = sdkComponents.toComponents();
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(components);
+
+ // Decode
+ Pipeline rehydratedPipeline = Pipeline.create();
+
+ ParDoPayload parDoPayload = ParDoPayload.parseFrom(protoTransform.getSpec().getPayload());
+ for (PCollectionView<?> view : parDo.getSideInputs()) {
+ SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId());
+ PCollectionView<?> restoredView =
+ ParDoTranslation.viewFromProto(
+ sideInput,
+ view.getTagInternal().getId(),
+ view.getPCollection(),
+ protoTransform,
+ rehydratedComponents);
+ assertThat(restoredView.getTagInternal(), equalTo(view.getTagInternal()));
+ assertThat(restoredView.getViewFn(), instanceOf(view.getViewFn().getClass()));
+ assertThat(
+ restoredView.getWindowMappingFn(), instanceOf(view.getWindowMappingFn().getClass()));
+ assertThat(
+ restoredView.getWindowingStrategyInternal(),
+ Matchers.<WindowingStrategy<?, ?>>equalTo(
+ view.getWindowingStrategyInternal().fixDefaults()));
+ assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
+ }
+ String mainInputId = sdkComponents.registerPCollection(mainInput);
+ assertThat(
+ ParDoTranslation.getMainInput(protoTransform, components),
+ equalTo(components.getPcollectionsOrThrow(mainInputId)));
+ }
+ }
+
+ /**
+ * Tests for translating state and timer bits to/from protos.
+ */
+ @RunWith(Parameterized.class)
+ public static class TestStateAndTimerTranslation {
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<StateSpec<?>> stateSpecs() {
+ return ImmutableList.of(
+ StateSpecs.value(VarIntCoder.of()),
+ StateSpecs.bag(VarIntCoder.of()),
+ StateSpecs.set(VarIntCoder.of()),
+ StateSpecs.map(StringUtf8Coder.of(), VarIntCoder.of()));
+ }
+
+ @Parameter
+ public StateSpec<?> stateSpec;
+
+ @Test
+ public void testStateSpecToFromProto() throws Exception {
+ // Encode
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.StateSpec stateSpecProto = ParDoTranslation.toProto(stateSpec, sdkComponents);
+
+ // Decode
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+ StateSpec<?> deserializedStateSpec =
+ ParDoTranslation.fromProto(stateSpecProto, rehydratedComponents);
+
+ assertThat(stateSpec, Matchers.<StateSpec<?>>equalTo(deserializedStateSpec));
+ }
+ }
+
+ private static class DropElementsFn extends DoFn<KV<Long, String>, Void> {
+ @ProcessElement
+ public void proc(ProcessContext context, BoundedWindow window) {
+ context.output(null);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof DropElementsFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return DropElementsFn.class.hashCode();
+ }
+ }
+
+ private static class SplittableDropElementsFn extends DoFn<KV<Long, String>, Void> {
+ @ProcessElement
+ public void proc(ProcessContext context, RestrictionTracker<Integer> restriction) {
+ context.output(null);
+ }
+
+ @GetInitialRestriction
+ public Integer restriction(KV<Long, String> elem) {
+ return 42;
+ }
+
+ @NewTracker
+ public RestrictionTracker<Integer> newTracker(Integer restriction) {
+ throw new UnsupportedOperationException("Should never be called; only to test translation");
+ }
+
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof SplittableDropElementsFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return SplittableDropElementsFn.class.hashCode();
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class StateTimerDropElementsFn extends DoFn<KV<Long, String>, Void> {
+ private static final String BAG_STATE_ID = "bagState";
+ private static final String COMBINING_STATE_ID = "combiningState";
+ private static final String EVENT_TIMER_ID = "eventTimer";
+ private static final String PROCESSING_TIMER_ID = "processingTimer";
+
+ @StateId(BAG_STATE_ID)
+ private final StateSpec<BagState<String>> bagState = StateSpecs.bag(StringUtf8Coder.of());
+
+ @StateId(COMBINING_STATE_ID)
+ private final StateSpec<CombiningState<Long, long[], Long>> combiningState =
+ StateSpecs.combining(
+ new BinaryCombineLongFn() {
+ @Override
+ public long apply(long left, long right) {
+ return Math.max(left, right);
+ }
+
+ @Override
+ public long identity() {
+ return Long.MIN_VALUE;
+ }
+ });
+
+ @TimerId(EVENT_TIMER_ID)
+ private final TimerSpec eventTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @TimerId(PROCESSING_TIMER_ID)
+ private final TimerSpec processingTimer = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);
+
+ @ProcessElement
+ public void dropInput(
+ ProcessContext context,
+ BoundedWindow window,
+ @StateId(BAG_STATE_ID) BagState<String> bagStateState,
+ @StateId(COMBINING_STATE_ID) CombiningState<Long, long[], Long> combiningStateState,
+ @TimerId(EVENT_TIMER_ID) Timer eventTimerTimer,
+ @TimerId(PROCESSING_TIMER_ID) Timer processingTimerTimer) {
+ context.output(null);
+ }
+
+ @OnTimer(EVENT_TIMER_ID)
+ public void onEventTime(OnTimerContext context) {}
+
+ @OnTimer(PROCESSING_TIMER_ID)
+ public void onProcessingTime(OnTimerContext context) {}
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof StateTimerDropElementsFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return StateTimerDropElementsFn.class.hashCode();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java
new file mode 100644
index 0000000..eb59bac
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.Struct;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for {@link PipelineOptionsTranslation}. */
+@RunWith(Enclosed.class)
+public class PipelineOptionsTranslationTest {
+ /** Tests that translations can round-trip through the proto format. */
+ @RunWith(Parameterized.class)
+ public static class ToFromProtoTest {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<? extends PipelineOptions> options() {
+ PipelineOptionsFactory.register(TestUnserializableOptions.class);
+ PipelineOptionsFactory.register(TestDefaultOptions.class);
+ PipelineOptionsFactory.register(TestOptions.class);
+ PipelineOptions emptyOptions = PipelineOptionsFactory.create();
+
+ TestUnserializableOptions withNonSerializable =
+ PipelineOptionsFactory.as(TestUnserializableOptions.class);
+ withNonSerializable.setUnserializable(new Object());
+
+ TestOptions withCustomField = PipelineOptionsFactory.as(TestOptions.class);
+ withCustomField.setExample(99);
+
+ PipelineOptions withSettings = PipelineOptionsFactory.create();
+ withSettings.as(ApplicationNameOptions.class).setAppName("my_app");
+ withSettings.setJobName("my_job");
+
+ PipelineOptions withParsedSettings =
+ PipelineOptionsFactory.fromArgs("--jobName=my_job --appName=my_app").create();
+
+ return ImmutableList.of(
+ emptyOptions, withNonSerializable, withCustomField, withSettings, withParsedSettings);
+ }
+
+ @Parameter(0)
+ public PipelineOptions options;
+
+ @Test
+ public void testToFromProto() throws Exception {
+ options.getOptionsId();
+ Struct originalStruct = PipelineOptionsTranslation.toProto(options);
+ PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct);
+
+ Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct);
+ assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap()));
+ }
+ }
+
+ /** Tests that translations contain the correct contents. */
+ @RunWith(JUnit4.class)
+ public static class TranslationTest {
+ @Test
+ public void customSettingsRetained() throws Exception {
+ TestOptions options = PipelineOptionsFactory.as(TestOptions.class);
+ options.setExample(23);
+ Struct serialized = PipelineOptionsTranslation.toProto(options);
+ PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized);
+
+ assertThat(deserialized.as(TestOptions.class).getExample(), equalTo(23));
+ }
+
+ @Test
+ public void ignoredSettingsNotSerialized() throws Exception {
+ TestUnserializableOptions opts = PipelineOptionsFactory.as(TestUnserializableOptions.class);
+ opts.setUnserializable(new Object());
+
+ Struct serialized = PipelineOptionsTranslation.toProto(opts);
+ PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized);
+
+ assertThat(
+ deserialized.as(TestUnserializableOptions.class).getUnserializable(), is(nullValue()));
+ }
+
+ @Test
+ public void defaultsRestored() throws Exception {
+ Struct serialized =
+ PipelineOptionsTranslation.toProto(PipelineOptionsFactory.as(TestDefaultOptions.class));
+ PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized);
+
+ assertThat(deserialized.as(TestDefaultOptions.class).getDefault(), equalTo(19));
+ }
+ }
+
+ /** {@link PipelineOptions} with an unserializable option. */
+ public interface TestUnserializableOptions extends PipelineOptions {
+ @JsonIgnore
+ Object getUnserializable();
+
+ void setUnserializable(Object unserializable);
+ }
+
+ /** {@link PipelineOptions} with an default option. */
+ public interface TestDefaultOptions extends PipelineOptions {
+ @Default.Integer(19)
+ int getDefault();
+
+ void setDefault(int example);
+ }
+
+ /** {@link PipelineOptions} for testing. */
+ public interface TestOptions extends PipelineOptions {
+ int getExample();
+
+ void setExample(int example);
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
new file mode 100644
index 0000000..66fe686
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.base.Equivalence;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for {@link PipelineTranslation}. */
+@RunWith(Parameterized.class)
+public class PipelineTranslationTest {
+ @Parameter(0)
+ public Pipeline pipeline;
+
+ @Parameters(name = "{index}")
+ public static Iterable<Pipeline> testPipelines() {
+ Pipeline trivialPipeline = Pipeline.create();
+ trivialPipeline.apply(Create.of(1, 2, 3));
+
+ Pipeline sideInputPipeline = Pipeline.create();
+ final PCollectionView<String> singletonView =
+ sideInputPipeline.apply(Create.of("foo")).apply(View.<String>asSingleton());
+ sideInputPipeline
+ .apply(Create.of("main input"))
+ .apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ // actually never executed and no effect on translation
+ c.sideInput(singletonView);
+ }
+ })
+ .withSideInputs(singletonView));
+
+ Pipeline complexPipeline = Pipeline.create();
+ BigEndianLongCoder customCoder = BigEndianLongCoder.of();
+ PCollection<Long> elems = complexPipeline.apply(GenerateSequence.from(0L).to(207L));
+ PCollection<Long> counted = elems.apply(Count.<Long>globally()).setCoder(customCoder);
+ PCollection<Long> windowed =
+ counted.apply(
+ Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7)))
+ .triggering(
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(AfterPane.elementCountAtLeast(19)))
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.standardMinutes(3L)));
+ final WindowingStrategy<?, ?> windowedStrategy = windowed.getWindowingStrategy();
+ PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.<String, Long>of("foo"));
+ PCollection<KV<String, Iterable<Long>>> grouped =
+ keyed.apply(GroupByKey.<String, Long>create());
+
+ return ImmutableList.of(trivialPipeline, sideInputPipeline, complexPipeline);
+ }
+
+ @Test
+ public void testProtoDirectly() {
+ final RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
+ pipeline.traverseTopologically(
+ new PipelineProtoVerificationVisitor(pipelineProto));
+ }
+
+ @Test
+ public void testProtoAgainstRehydrated() throws Exception {
+ RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
+ Pipeline rehydrated = PipelineTranslation.fromProto(pipelineProto);
+
+ rehydrated.traverseTopologically(
+ new PipelineProtoVerificationVisitor(pipelineProto));
+ }
+
+ private static class PipelineProtoVerificationVisitor extends PipelineVisitor.Defaults {
+
+ private final RunnerApi.Pipeline pipelineProto;
+ Set<Node> transforms;
+ Set<PCollection<?>> pcollections;
+ Set<Equivalence.Wrapper<? extends Coder<?>>> coders;
+ Set<WindowingStrategy<?, ?>> windowingStrategies;
+
+ public PipelineProtoVerificationVisitor(RunnerApi.Pipeline pipelineProto) {
+ this.pipelineProto = pipelineProto;
+ transforms = new HashSet<>();
+ pcollections = new HashSet<>();
+ coders = new HashSet<>();
+ windowingStrategies = new HashSet<>();
+ }
+
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ if (node.isRootNode()) {
+ assertThat(
+ "Unexpected number of PTransforms",
+ pipelineProto.getComponents().getTransformsCount(),
+ equalTo(transforms.size()));
+ assertThat(
+ "Unexpected number of PCollections",
+ pipelineProto.getComponents().getPcollectionsCount(),
+ equalTo(pcollections.size()));
+ assertThat(
+ "Unexpected number of Coders",
+ pipelineProto.getComponents().getCodersCount(),
+ equalTo(coders.size()));
+ assertThat(
+ "Unexpected number of Windowing Strategies",
+ pipelineProto.getComponents().getWindowingStrategiesCount(),
+ equalTo(windowingStrategies.size()));
+ } else {
+ transforms.add(node);
+ if (PTransformTranslation.COMBINE_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(node.getTransform()))) {
+ // Combine translation introduces a coder that is not assigned to any PCollection
+ // in the default expansion, and must be explicitly added here.
+ try {
+ addCoders(
+ CombineTranslation.getAccumulatorCoder(node.toAppliedPTransform(getPipeline())));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ transforms.add(node);
+ }
+
+ @Override
+ public void visitValue(PValue value, Node producer) {
+ if (value instanceof PCollection) {
+ PCollection pc = (PCollection) value;
+ pcollections.add(pc);
+ addCoders(pc.getCoder());
+ windowingStrategies.add(pc.getWindowingStrategy());
+ addCoders(pc.getWindowingStrategy().getWindowFn().windowCoder());
+ }
+ }
+
+ private void addCoders(Coder<?> coder) {
+ coders.add(Equivalence.<Coder<?>>identity().wrap(coder));
+ if (CoderTranslation.KNOWN_CODER_URNS.containsKey(coder.getClass())) {
+ for (Coder<?> component : ((StructuredCoder<?>) coder).getComponents()) {
+ addCoders(component);
+ }
+ }
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
new file mode 100644
index 0000000..56cf5f3
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assume.assumeThat;
+
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ReadPayload;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link ReadTranslation}.
+ */
+@RunWith(Parameterized.class)
+public class ReadTranslationTest {
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<Source<?>> data() {
+ return ImmutableList.<Source<?>>of(
+ CountingSource.unbounded(),
+ CountingSource.upTo(100L),
+ new TestBoundedSource(),
+ new TestUnboundedSource());
+ }
+
+ @Parameter(0)
+ public Source<?> source;
+
+ @Test
+ public void testToFromProtoBounded() throws Exception {
+ // TODO: Split into two tests.
+ assumeThat(source, instanceOf(BoundedSource.class));
+ BoundedSource<?> boundedSource = (BoundedSource<?>) this.source;
+ Read.Bounded<?> boundedRead = Read.from(boundedSource);
+ ReadPayload payload = ReadTranslation.toProto(boundedRead);
+ assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.BOUNDED));
+ BoundedSource<?> deserializedSource = ReadTranslation.boundedSourceFromProto(payload);
+ assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
+ }
+
+ @Test
+ public void testToFromProtoUnbounded() throws Exception {
+ assumeThat(source, instanceOf(UnboundedSource.class));
+ UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
+ Read.Unbounded<?> unboundedRead = Read.from(unboundedSource);
+ ReadPayload payload = ReadTranslation.toProto(unboundedRead);
+ assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.UNBOUNDED));
+ UnboundedSource<?, ?> deserializedSource = ReadTranslation.unboundedSourceFromProto(payload);
+ assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
+ }
+
+ private static class TestBoundedSource extends BoundedSource<String> {
+ @Override
+ public List<? extends BoundedSource<String>> split(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BoundedReader<String> createReader(PipelineOptions options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Coder<String> getOutputCoder() {
+ return StringUtf8Coder.of();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(TestBoundedSource.class);
+ }
+
+ @Override
+ public int hashCode() {
+ return TestBoundedSource.class.hashCode();
+ }
+ }
+
+ private static class TestUnboundedSource extends UnboundedSource<byte[], CheckpointMark> {
+ @Override
+ public Coder<byte[]> getOutputCoder() {
+ return ByteArrayCoder.of();
+ }
+
+ @Override
+ public List<? extends UnboundedSource<byte[], CheckpointMark>> split(
+ int desiredNumSplits, PipelineOptions options) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public UnboundedReader<byte[]> createReader(
+ PipelineOptions options, @Nullable CheckpointMark checkpointMark) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Coder<CheckpointMark> getCheckpointMarkCoder() {
+ return new TestCheckpointMarkCoder();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(TestUnboundedSource.class);
+ }
+
+ @Override
+ public int hashCode() {
+ return TestUnboundedSource.class.hashCode();
+ }
+
+ private class TestCheckpointMarkCoder extends AtomicCoder<CheckpointMark> {
+ @Override
+ public void encode(CheckpointMark value, OutputStream outStream)
+ throws CoderException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public CheckpointMark decode(InputStream inStream) throws CoderException, IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/RehydratedComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/RehydratedComponentsTest.java
new file mode 100644
index 0000000..0da487b
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/RehydratedComponentsTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.theInstance;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link RehydratedComponents}.
+ *
+ * <p>These are basic sanity checks. The most thorough testing of this is by extensive use in all
+ * other rehydration. The two are tightly coupled, as they recursively invoke each other.
+ */
+@RunWith(JUnit4.class)
+public class RehydratedComponentsTest {
+
+ @Test
+ public void testSimpleCoder() throws Exception {
+ SdkComponents sdkComponents = SdkComponents.create();
+ Coder<?> coder = VarIntCoder.of();
+ String id = sdkComponents.registerCoder(coder);
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+
+ Coder<?> rehydratedCoder = rehydratedComponents.getCoder(id);
+ assertThat(rehydratedCoder, equalTo((Coder) coder));
+ assertThat(rehydratedComponents.getCoder(id), theInstance((Coder) rehydratedCoder));
+ }
+
+ @Test
+ public void testCompoundCoder() throws Exception {
+ SdkComponents sdkComponents = SdkComponents.create();
+ Coder<?> coder = VarIntCoder.of();
+ Coder<?> compoundCoder = NullableCoder.of(coder);
+ String compoundCoderId = sdkComponents.registerCoder(compoundCoder);
+ String coderId = sdkComponents.registerCoder(coder);
+
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+
+ Coder<?> rehydratedCoder = rehydratedComponents.getCoder(coderId);
+ Coder<?> rehydratedCompoundCoder = rehydratedComponents.getCoder(compoundCoderId);
+
+ assertThat(rehydratedCoder, equalTo((Coder) coder));
+ assertThat(rehydratedCompoundCoder, equalTo((Coder) compoundCoder));
+
+ assertThat(rehydratedComponents.getCoder(coderId), theInstance((Coder) rehydratedCoder));
+ assertThat(
+ rehydratedComponents.getCoder(compoundCoderId),
+ theInstance((Coder) rehydratedCompoundCoder));
+ }
+
+ @Test
+ public void testWindowingStrategy() throws Exception {
+ SdkComponents sdkComponents = SdkComponents.create();
+ WindowingStrategy windowingStrategy =
+ WindowingStrategy.of(FixedWindows.of(Duration.millis(1)))
+ .withAllowedLateness(Duration.standardSeconds(4));
+ String id = sdkComponents.registerWindowingStrategy(windowingStrategy);
+ RehydratedComponents rehydratedComponents =
+ RehydratedComponents.forComponents(sdkComponents.toComponents());
+
+ WindowingStrategy<?, ?> rehydratedStrategy = rehydratedComponents.getWindowingStrategy(id);
+ assertThat(rehydratedStrategy, equalTo((WindowingStrategy) windowingStrategy.fixDefaults()));
+ assertThat(
+ rehydratedComponents.getWindowingStrategy(id),
+ theInstance((WindowingStrategy) rehydratedStrategy));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
index f8d01e9..0165e4b 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
@@ -24,6 +24,8 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import java.util.Map;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.values.PCollection;
@@ -50,23 +52,23 @@
private PCollection<Integer> ints =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
private PCollection<Integer> moreInts =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
private PCollection<String> strs =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of());
private PCollection<Integer> replacementInts =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
private PCollection<Integer> moreReplacementInts =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
private PCollection<String> replacementStrs =
PCollection.createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of());
@Test
public void singletonSucceeds() {
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index 7424886..44c42cc 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -24,43 +24,25 @@
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
-import com.google.common.base.Equivalence;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.beam.sdk.Pipeline.PipelineVisitor;
-import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.SetCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.TransformHierarchy.Node;
import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.windowing.AfterPane;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
import org.hamcrest.Matchers;
-import org.joda.time.Duration;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -78,88 +60,6 @@
private SdkComponents components = SdkComponents.create();
@Test
- public void translatePipeline() {
- BigEndianLongCoder customCoder = BigEndianLongCoder.of();
- PCollection<Long> elems = pipeline.apply(GenerateSequence.from(0L).to(207L));
- PCollection<Long> counted = elems.apply(Count.<Long>globally()).setCoder(customCoder);
- PCollection<Long> windowed =
- counted.apply(
- Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7)))
- .triggering(
- AfterWatermark.pastEndOfWindow()
- .withEarlyFirings(AfterPane.elementCountAtLeast(19)))
- .accumulatingFiredPanes()
- .withAllowedLateness(Duration.standardMinutes(3L)));
- final WindowingStrategy<?, ?> windowedStrategy = windowed.getWindowingStrategy();
- PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.<String, Long>of("foo"));
- PCollection<KV<String, Iterable<Long>>> grouped =
- keyed.apply(GroupByKey.<String, Long>create());
-
- final RunnerApi.Pipeline pipelineProto = SdkComponents.translatePipeline(pipeline);
- pipeline.traverseTopologically(
- new PipelineVisitor() {
- Set<Node> transforms = new HashSet<>();
- Set<PCollection<?>> pcollections = new HashSet<>();
- Set<Equivalence.Wrapper<? extends Coder<?>>> coders = new HashSet<>();
- Set<WindowingStrategy<?, ?>> windowingStrategies = new HashSet<>();
-
- @Override
- public CompositeBehavior enterCompositeTransform(Node node) {
- return CompositeBehavior.ENTER_TRANSFORM;
- }
-
- @Override
- public void leaveCompositeTransform(Node node) {
- if (node.isRootNode()) {
- assertThat(
- "Unexpected number of PTransforms",
- pipelineProto.getComponents().getTransformsCount(),
- equalTo(transforms.size()));
- assertThat(
- "Unexpected number of PCollections",
- pipelineProto.getComponents().getPcollectionsCount(),
- equalTo(pcollections.size()));
- assertThat(
- "Unexpected number of Coders",
- pipelineProto.getComponents().getCodersCount(),
- equalTo(coders.size()));
- assertThat(
- "Unexpected number of Windowing Strategies",
- pipelineProto.getComponents().getWindowingStrategiesCount(),
- equalTo(windowingStrategies.size()));
- } else {
- transforms.add(node);
- }
- }
-
- @Override
- public void visitPrimitiveTransform(Node node) {
- transforms.add(node);
- }
-
- @Override
- public void visitValue(PValue value, Node producer) {
- if (value instanceof PCollection) {
- PCollection pc = (PCollection) value;
- pcollections.add(pc);
- addCoders(pc.getCoder());
- windowingStrategies.add(pc.getWindowingStrategy());
- addCoders(pc.getWindowingStrategy().getWindowFn().windowCoder());
- }
- }
-
- private void addCoders(Coder<?> coder) {
- coders.add(Equivalence.<Coder<?>>identity().wrap(coder));
- if (coder instanceof StructuredCoder) {
- for (Coder<?> component : ((StructuredCoder <?>) coder).getComponents()) {
- addCoders(component);
- }
- }
- }
- });
- }
-
- @Test
public void registerCoder() throws IOException {
Coder<?> coder =
KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of())));
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SerializablePipelineOptionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SerializablePipelineOptionsTest.java
new file mode 100644
index 0000000..cd470b2
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SerializablePipelineOptionsTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SerializablePipelineOptions}. */
+@RunWith(JUnit4.class)
+public class SerializablePipelineOptionsTest {
+ /** Options for testing. */
+ public interface MyOptions extends PipelineOptions {
+ String getFoo();
+
+ void setFoo(String foo);
+
+ @JsonIgnore
+ @Default.String("not overridden")
+ String getIgnoredField();
+
+ void setIgnoredField(String value);
+ }
+
+ @Test
+ public void testSerializationAndDeserialization() throws Exception {
+ PipelineOptions options =
+ PipelineOptionsFactory.fromArgs("--foo=testValue", "--ignoredField=overridden")
+ .as(MyOptions.class);
+
+ SerializablePipelineOptions serializableOptions = new SerializablePipelineOptions(options);
+ assertEquals("testValue", serializableOptions.get().as(MyOptions.class).getFoo());
+ assertEquals("overridden", serializableOptions.get().as(MyOptions.class).getIgnoredField());
+
+ SerializablePipelineOptions copy = SerializableUtils.clone(serializableOptions);
+ assertEquals("testValue", copy.get().as(MyOptions.class).getFoo());
+ assertEquals("not overridden", copy.get().as(MyOptions.class).getIgnoredField());
+ }
+
+ @Test
+ public void testIndependence() throws Exception {
+ SerializablePipelineOptions first =
+ new SerializablePipelineOptions(
+ PipelineOptionsFactory.fromArgs("--foo=first").as(MyOptions.class));
+ SerializablePipelineOptions firstCopy = SerializableUtils.clone(first);
+ SerializablePipelineOptions second =
+ new SerializablePipelineOptions(
+ PipelineOptionsFactory.fromArgs("--foo=second").as(MyOptions.class));
+ SerializablePipelineOptions secondCopy = SerializableUtils.clone(second);
+
+ assertEquals("first", first.get().as(MyOptions.class).getFoo());
+ assertEquals("first", firstCopy.get().as(MyOptions.class).getFoo());
+ assertEquals("second", second.get().as(MyOptions.class).getFoo());
+ assertEquals("second", secondCopy.get().as(MyOptions.class).getFoo());
+
+ first.get().as(MyOptions.class).setFoo("new first");
+ firstCopy.get().as(MyOptions.class).setFoo("new firstCopy");
+ second.get().as(MyOptions.class).setFoo("new second");
+ secondCopy.get().as(MyOptions.class).setFoo("new secondCopy");
+
+ assertEquals("new first", first.get().as(MyOptions.class).getFoo());
+ assertEquals("new firstCopy", firstCopy.get().as(MyOptions.class).getFoo());
+ assertEquals("new second", second.get().as(MyOptions.class).getFoo());
+ assertEquals("new secondCopy", secondCopy.get().as(MyOptions.class).getFoo());
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
new file mode 100644
index 0000000..05c471d
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SplittableParDo}. */
+@RunWith(JUnit4.class)
+public class SplittableParDoTest {
+ // ----------------- Tests for whether the transform sets boundedness correctly --------------
+ private static class SomeRestriction
+ implements Serializable, HasDefaultTracker<SomeRestriction, SomeRestrictionTracker> {
+ @Override
+ public SomeRestrictionTracker newTracker() {
+ return new SomeRestrictionTracker(this);
+ }
+ }
+
+ private static class SomeRestrictionTracker implements RestrictionTracker<SomeRestriction> {
+ private final SomeRestriction someRestriction;
+
+ public SomeRestrictionTracker(SomeRestriction someRestriction) {
+ this.someRestriction = someRestriction;
+ }
+
+ @Override
+ public SomeRestriction currentRestriction() {
+ return someRestriction;
+ }
+
+ @Override
+ public SomeRestriction checkpoint() {
+ return someRestriction;
+ }
+
+ @Override
+ public void checkDone() {}
+ }
+
+ private static class BoundedFakeFn extends DoFn<Integer, String> {
+ @ProcessElement
+ public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return null;
+ }
+ }
+
+ private static class UnboundedFakeFn extends DoFn<Integer, String> {
+ @ProcessElement
+ public ProcessContinuation processElement(
+ ProcessContext context, SomeRestrictionTracker tracker) {
+ return stop();
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return null;
+ }
+ }
+
+ private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) {
+ return pipeline
+ .apply("unbounded", Create.of(1, 2, 3))
+ .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
+ }
+
+ private static PCollection<Integer> makeBoundedCollection(Pipeline pipeline) {
+ return pipeline
+ .apply("bounded", Create.of(1, 2, 3))
+ .setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
+ }
+
+ private static final TupleTag<String> MAIN_OUTPUT_TAG = new TupleTag<String>() {};
+
+ private PCollection<String> applySplittableParDo(
+ String name, PCollection<Integer> input, DoFn<Integer, String> fn) {
+ ParDo.MultiOutput<Integer, String> multiOutput =
+ ParDo.of(fn).withOutputTags(MAIN_OUTPUT_TAG, TupleTagList.empty());
+ PCollectionTuple output = multiOutput.expand(input);
+ output.get(MAIN_OUTPUT_TAG).setName("main");
+ AppliedPTransform<PCollection<Integer>, PCollectionTuple, ?> transform =
+ AppliedPTransform.of("ParDo", input.expand(), output.expand(), multiOutput, pipeline);
+ return input.apply(name, SplittableParDo.forAppliedParDo(transform)).get(MAIN_OUTPUT_TAG);
+ }
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ public void testBoundednessForBoundedFn() {
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ DoFn<Integer, String> boundedFn = new BoundedFakeFn();
+ assertEquals(
+ "Applying a bounded SDF to a bounded collection produces a bounded collection",
+ PCollection.IsBounded.BOUNDED,
+ applySplittableParDo("bounded to bounded", makeBoundedCollection(pipeline), boundedFn)
+ .isBounded());
+ assertEquals(
+ "Applying a bounded SDF to an unbounded collection produces an unbounded collection",
+ PCollection.IsBounded.UNBOUNDED,
+ applySplittableParDo("bounded to unbounded", makeUnboundedCollection(pipeline), boundedFn)
+ .isBounded());
+ }
+
+ @Test
+ public void testBoundednessForUnboundedFn() {
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ DoFn<Integer, String> unboundedFn = new UnboundedFakeFn();
+ assertEquals(
+ "Applying an unbounded SDF to a bounded collection produces a bounded collection",
+ PCollection.IsBounded.UNBOUNDED,
+ applySplittableParDo("unbounded to bounded", makeBoundedCollection(pipeline), unboundedFn)
+ .isBounded());
+ assertEquals(
+ "Applying an unbounded SDF to an unbounded collection produces an unbounded collection",
+ PCollection.IsBounded.UNBOUNDED,
+ applySplittableParDo(
+ "unbounded to unbounded", makeUnboundedCollection(pipeline), unboundedFn)
+ .isBounded());
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
new file mode 100644
index 0000000..fc30552
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.TestStreamPayload;
+import org.apache.beam.runners.core.construction.TestStreamTranslationTest.TestStreamPayloadTranslation;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
+
+/** Tests for {@link TestStreamTranslation}. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ TestStreamPayloadTranslation.class,
+})
+public class TestStreamTranslationTest {
+
+ /** Tests for translating various {@link ParDo} transforms to/from {@link ParDoPayload} protos. */
+ @RunWith(Parameterized.class)
+ public static class TestStreamPayloadTranslation {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<TestStream<?>> data() {
+ return ImmutableList.<TestStream<?>>of(
+ TestStream.create(VarIntCoder.of()).advanceWatermarkToInfinity(),
+ TestStream.create(VarIntCoder.of())
+ .advanceWatermarkTo(new Instant(42))
+ .advanceWatermarkToInfinity(),
+ TestStream.create(VarIntCoder.of())
+ .addElements(TimestampedValue.of(3, new Instant(17)))
+ .advanceWatermarkToInfinity(),
+ TestStream.create(StringUtf8Coder.of())
+ .advanceProcessingTime(Duration.millis(82))
+ .advanceWatermarkToInfinity());
+ }
+
+ @Parameter(0)
+ public TestStream<String> testStream;
+
+ public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ @Test
+ public void testEncodedProto() throws Exception {
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.TestStreamPayload payload =
+ TestStreamTranslation.payloadForTestStream(testStream, components);
+
+ verifyTestStreamEncoding(
+ testStream, payload, RehydratedComponents.forComponents(components.toComponents()));
+ }
+
+ @Test
+ public void testRegistrarEncodedProto() throws Exception {
+ PCollection<String> output = p.apply(testStream);
+
+ AppliedPTransform<PBegin, PCollection<String>, TestStream<String>> appliedTestStream =
+ AppliedPTransform.<PBegin, PCollection<String>, TestStream<String>>of(
+ "fakeName", PBegin.in(p).expand(), output.expand(), testStream, p);
+
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.FunctionSpec spec =
+ PTransformTranslation.toProto(appliedTestStream, components).getSpec();
+
+ assertThat(spec.getUrn(), equalTo(TEST_STREAM_TRANSFORM_URN));
+
+ RunnerApi.TestStreamPayload payload = TestStreamPayload.parseFrom(spec.getPayload());
+
+ verifyTestStreamEncoding(
+ testStream, payload, RehydratedComponents.forComponents(components.toComponents()));
+ }
+
+ private static <T> void verifyTestStreamEncoding(
+ TestStream<T> testStream,
+ RunnerApi.TestStreamPayload payload,
+ RehydratedComponents protoComponents)
+ throws Exception {
+
+ // This reverse direction is only valid for Java-based coders
+ assertThat(
+ protoComponents.getCoder(payload.getCoderId()),
+ Matchers.<Coder<?>>equalTo(testStream.getValueCoder()));
+
+ assertThat(payload.getEventsList().size(), equalTo(testStream.getEvents().size()));
+
+ for (int i = 0; i < payload.getEventsList().size(); ++i) {
+ assertThat(
+ TestStreamTranslation.eventFromProto(payload.getEvents(i), testStream.getValueCoder()),
+ equalTo(testStream.getEvents().get(i)));
+ }
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
new file mode 100644
index 0000000..f5b2c11
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link TransformInputs}. */
+@RunWith(JUnit4.class)
+public class TransformInputsTest {
+ @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void nonAdditionalInputsWithNoInputSucceeds() {
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "input-free",
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
+ }
+
+ @Test
+ public void nonAdditionalInputsWithOneMainInputSucceeds() {
+ PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "input-single",
+ Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
+ Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+ PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+ allInputs.put(new TupleTag<Integer>() {}, mainInts);
+ PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+ allInputs.put(new TupleTag<Void>() {}, voids);
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional-free",
+ allInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform),
+ Matchers.<PValue>containsInAnyOrder(voids, mainInts));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+ additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+ Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+ PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+ allInputs.put(new TupleTag<Integer>() {}, mainInts);
+ PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+ allInputs.put(
+ new TupleTag<Void>() {}, voids);
+ allInputs.putAll(additionalInputs);
+
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional",
+ allInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(additionalInputs),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform),
+ Matchers.<PValue>containsInAnyOrder(mainInts, voids));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+ additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional-only",
+ additionalInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(additionalInputs),
+ pipeline);
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("at least one");
+ TransformInputs.nonAdditionalInputs(transform);
+ }
+
+ private static class TestTransform extends PTransform<PInput, POutput> {
+ private final Map<TupleTag<?>, PValue> additionalInputs;
+
+ private TestTransform() {
+ this(Collections.<TupleTag<?>, PValue>emptyMap());
+ }
+
+ private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
+ this.additionalInputs = additionalInputs;
+ }
+
+ @Override
+ public POutput expand(PInput input) {
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return additionalInputs;
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
new file mode 100644
index 0000000..55ea87b
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.transforms.windowing.AfterAll;
+import org.apache.beam.sdk.transforms.windowing.AfterEach;
+import org.apache.beam.sdk.transforms.windowing.AfterFirst;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterSynchronizedProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.Never;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for utilities in {@link TriggerTranslation}. */
+@RunWith(Parameterized.class)
+public class TriggerTranslationTest {
+
+ @AutoValue
+ abstract static class ToProtoAndBackSpec {
+ abstract Trigger getTrigger();
+ }
+
+ private static ToProtoAndBackSpec toProtoAndBackSpec(Trigger trigger) {
+ return new AutoValue_TriggerTranslationTest_ToProtoAndBackSpec(trigger);
+ }
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<ToProtoAndBackSpec> data() {
+ return ImmutableList.of(
+ // Atomic triggers
+ toProtoAndBackSpec(AfterWatermark.pastEndOfWindow()),
+ toProtoAndBackSpec(AfterPane.elementCountAtLeast(73)),
+ toProtoAndBackSpec(AfterSynchronizedProcessingTime.ofFirstElement()),
+ toProtoAndBackSpec(Never.ever()),
+ toProtoAndBackSpec(DefaultTrigger.of()),
+ toProtoAndBackSpec(AfterProcessingTime.pastFirstElementInPane()),
+ toProtoAndBackSpec(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(23))),
+ toProtoAndBackSpec(
+ AfterProcessingTime.pastFirstElementInPane()
+ .alignedTo(Duration.millis(5), new Instant(27))),
+ toProtoAndBackSpec(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(Duration.standardSeconds(3))
+ .alignedTo(Duration.millis(5), new Instant(27))
+ .plusDelayOf(Duration.millis(13))),
+
+ // Composite triggers
+
+ toProtoAndBackSpec(
+ AfterAll.of(AfterPane.elementCountAtLeast(79), AfterWatermark.pastEndOfWindow())),
+ toProtoAndBackSpec(
+ AfterEach.inOrder(AfterPane.elementCountAtLeast(79), AfterPane.elementCountAtLeast(3))),
+ toProtoAndBackSpec(
+ AfterFirst.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(3))),
+ toProtoAndBackSpec(
+ AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(3))),
+ toProtoAndBackSpec(
+ AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(3))),
+ toProtoAndBackSpec(
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(42)))
+ .withLateFirings(AfterPane.elementCountAtLeast(3))),
+ toProtoAndBackSpec(Repeatedly.forever(AfterWatermark.pastEndOfWindow())),
+ toProtoAndBackSpec(
+ Repeatedly.forever(AfterPane.elementCountAtLeast(1))
+ .orFinally(AfterWatermark.pastEndOfWindow())));
+ }
+
+ @Parameter(0)
+ public ToProtoAndBackSpec toProtoAndBackSpec;
+
+ @Test
+ public void testToProtoAndBack() throws Exception {
+ Trigger trigger = toProtoAndBackSpec.getTrigger();
+ Trigger toProtoAndBackTrigger =
+ TriggerTranslation.fromProto(TriggerTranslation.toProto(trigger));
+
+ assertThat(toProtoAndBackTrigger, equalTo(trigger));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java
deleted file mode 100644
index cf9d40c..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core.construction;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.transforms.windowing.AfterAll;
-import org.apache.beam.sdk.transforms.windowing.AfterEach;
-import org.apache.beam.sdk.transforms.windowing.AfterFirst;
-import org.apache.beam.sdk.transforms.windowing.AfterPane;
-import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
-import org.apache.beam.sdk.transforms.windowing.AfterSynchronizedProcessingTime;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
-import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
-import org.apache.beam.sdk.transforms.windowing.Never;
-import org.apache.beam.sdk.transforms.windowing.Repeatedly;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-/** Tests for utilities in {@link Triggers}. */
-@RunWith(Parameterized.class)
-public class TriggersTest {
-
- @AutoValue
- abstract static class ToProtoAndBackSpec {
- abstract Trigger getTrigger();
- }
-
- private static ToProtoAndBackSpec toProtoAndBackSpec(Trigger trigger) {
- return new AutoValue_TriggersTest_ToProtoAndBackSpec(trigger);
- }
-
- @Parameters(name = "{index}: {0}")
- public static Iterable<ToProtoAndBackSpec> data() {
- return ImmutableList.of(
- // Atomic triggers
- toProtoAndBackSpec(AfterWatermark.pastEndOfWindow()),
- toProtoAndBackSpec(AfterPane.elementCountAtLeast(73)),
- toProtoAndBackSpec(AfterSynchronizedProcessingTime.ofFirstElement()),
- toProtoAndBackSpec(Never.ever()),
- toProtoAndBackSpec(DefaultTrigger.of()),
- toProtoAndBackSpec(AfterProcessingTime.pastFirstElementInPane()),
- toProtoAndBackSpec(
- AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(23))),
- toProtoAndBackSpec(
- AfterProcessingTime.pastFirstElementInPane()
- .alignedTo(Duration.millis(5), new Instant(27))),
- toProtoAndBackSpec(
- AfterProcessingTime.pastFirstElementInPane()
- .plusDelayOf(Duration.standardSeconds(3))
- .alignedTo(Duration.millis(5), new Instant(27))
- .plusDelayOf(Duration.millis(13))),
-
- // Composite triggers
-
- toProtoAndBackSpec(
- AfterAll.of(AfterPane.elementCountAtLeast(79), AfterWatermark.pastEndOfWindow())),
- toProtoAndBackSpec(
- AfterEach.inOrder(AfterPane.elementCountAtLeast(79), AfterPane.elementCountAtLeast(3))),
- toProtoAndBackSpec(
- AfterFirst.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(3))),
- toProtoAndBackSpec(
- AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(3))),
- toProtoAndBackSpec(
- AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(3))),
- toProtoAndBackSpec(
- AfterWatermark.pastEndOfWindow()
- .withEarlyFirings(
- AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(42)))
- .withLateFirings(AfterPane.elementCountAtLeast(3))),
- toProtoAndBackSpec(Repeatedly.forever(AfterWatermark.pastEndOfWindow())),
- toProtoAndBackSpec(
- Repeatedly.forever(AfterPane.elementCountAtLeast(1))
- .orFinally(AfterWatermark.pastEndOfWindow())));
- }
-
- @Parameter(0)
- public ToProtoAndBackSpec toProtoAndBackSpec;
-
- @Test
- public void testToProtoAndBack() throws Exception {
- Trigger trigger = toProtoAndBackSpec.getTrigger();
- Trigger toProtoAndBackTrigger = Triggers.fromProto(Triggers.toProto(trigger));
-
- assertThat(toProtoAndBackTrigger, equalTo(trigger));
- }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java
index 0e48a9d..62b06b7 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java
@@ -320,7 +320,7 @@
}
@Override
- public Coder<Byte> getDefaultOutputCoder() {
+ public Coder<Byte> getOutputCoder() {
return SerializableCoder.of(Byte.class);
}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
new file mode 100644
index 0000000..b40ccb3
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WindowIntoPayload;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.PartitioningWindowFn;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link WindowIntoTranslation}.
+ */
+@RunWith(Parameterized.class)
+public class WindowIntoTranslationTest {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<WindowFn<?, ?>> data() {
+ // This pipeline exists for construction, not to run any test.
+ return ImmutableList.<WindowFn<?, ?>>builder()
+ .add(FixedWindows.of(Duration.standardMinutes(10L)))
+ .add(new GlobalWindows())
+ .add(Sessions.withGapDuration(Duration.standardMinutes(15L)))
+ .add(SlidingWindows.of(Duration.standardMinutes(5L)).every(Duration.standardMinutes(1L)))
+ .add(new CustomWindows())
+ .build();
+ }
+
+ @Parameter(0)
+ public WindowFn<?, ?> windowFn;
+
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ public void testToFromProto() throws InvalidProtocolBufferException {
+ pipeline.apply(GenerateSequence.from(0)).apply(Window.<Long>into((WindowFn) windowFn));
+
+ final AtomicReference<AppliedPTransform<?, ?, Assign<?>>> assign = new AtomicReference<>(null);
+ pipeline.traverseTopologically(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ if (node.getTransform() instanceof Window.Assign) {
+ checkState(assign.get() == null);
+ assign.set(
+ (AppliedPTransform<?, ?, Assign<?>>) node.toAppliedPTransform(getPipeline()));
+ }
+ }
+ });
+ checkState(assign.get() != null);
+
+ SdkComponents components = SdkComponents.create();
+ WindowIntoPayload payload =
+ WindowIntoTranslation.toProto(assign.get().getTransform(), components);
+
+ assertEquals(windowFn, WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn()));
+ }
+
+ private static class CustomWindows extends PartitioningWindowFn<String, BoundedWindow> {
+ @Override
+ public BoundedWindow assignWindow(Instant timestamp) {
+ return GlobalWindow.INSTANCE;
+ }
+
+ @Override
+ public boolean isCompatible(WindowFn<?, ?> other) {
+ return getClass().equals(other.getClass());
+ }
+
+ @Override
+ public Coder<BoundedWindow> windowCoder() {
+ return (Coder) GlobalWindow.Coder.INSTANCE;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(this.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
deleted file mode 100644
index 7296a77..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core.construction;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
-import org.hamcrest.Matchers;
-import org.joda.time.Duration;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-/** Unit tests for {@link WindowingStrategy}. */
-@RunWith(Parameterized.class)
-public class WindowingStrategiesTest {
-
- // Each spec activates tests of all subsets of its fields
- @AutoValue
- abstract static class ToProtoAndBackSpec {
- abstract WindowingStrategy getWindowingStrategy();
- }
-
- private static ToProtoAndBackSpec toProtoAndBackSpec(WindowingStrategy windowingStrategy) {
- return new AutoValue_WindowingStrategiesTest_ToProtoAndBackSpec(windowingStrategy);
- }
-
- private static final WindowFn<?, ?> REPRESENTATIVE_WINDOW_FN =
- FixedWindows.of(Duration.millis(12));
-
- private static final Trigger REPRESENTATIVE_TRIGGER = AfterWatermark.pastEndOfWindow();
-
- @Parameters(name = "{index}: {0}")
- public static Iterable<ToProtoAndBackSpec> data() {
- return ImmutableList.of(
- toProtoAndBackSpec(WindowingStrategy.globalDefault()),
- toProtoAndBackSpec(
- WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN)
- .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)
- .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
- .withTrigger(REPRESENTATIVE_TRIGGER)
- .withAllowedLateness(Duration.millis(71))
- .withTimestampCombiner(TimestampCombiner.EARLIEST)),
- toProtoAndBackSpec(
- WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN)
- .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY)
- .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
- .withTrigger(REPRESENTATIVE_TRIGGER)
- .withAllowedLateness(Duration.millis(93))
- .withTimestampCombiner(TimestampCombiner.LATEST)));
- }
-
- @Parameter(0)
- public ToProtoAndBackSpec toProtoAndBackSpec;
-
- @Test
- public void testToProtoAndBack() throws Exception {
- WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
- WindowingStrategy<?, ?> toProtoAndBackWindowingStrategy =
- WindowingStrategies.fromProto(WindowingStrategies.toProto(windowingStrategy));
-
- assertThat(
- toProtoAndBackWindowingStrategy,
- equalTo((WindowingStrategy) windowingStrategy.fixDefaults()));
- }
-
- @Test
- public void testToProtoAndBackWithComponents() throws Exception {
- WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
- SdkComponents components = SdkComponents.create();
- RunnerApi.WindowingStrategy proto =
- WindowingStrategies.toProto(windowingStrategy, components);
- RunnerApi.Components protoComponents = components.toComponents();
-
- assertThat(
- WindowingStrategies.fromProto(proto, protoComponents).fixDefaults(),
- Matchers.<WindowingStrategy<?, ?>>equalTo(windowingStrategy.fixDefaults()));
-
- protoComponents.getCodersOrThrow(
- components.registerCoder(windowingStrategy.getWindowFn().windowCoder()));
- }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
new file mode 100644
index 0000000..ddf0316
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Unit tests for {@link WindowingStrategy}. */
+@RunWith(Parameterized.class)
+public class WindowingStrategyTranslationTest {
+
+ // Each spec activates tests of all subsets of its fields
+ @AutoValue
+ abstract static class ToProtoAndBackSpec {
+ abstract WindowingStrategy getWindowingStrategy();
+ }
+
+ private static ToProtoAndBackSpec toProtoAndBackSpec(WindowingStrategy windowingStrategy) {
+ return new AutoValue_WindowingStrategyTranslationTest_ToProtoAndBackSpec(windowingStrategy);
+ }
+
+ private static final WindowFn<?, ?> REPRESENTATIVE_WINDOW_FN =
+ FixedWindows.of(Duration.millis(12));
+
+ private static final Trigger REPRESENTATIVE_TRIGGER = AfterWatermark.pastEndOfWindow();
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<ToProtoAndBackSpec> data() {
+ return ImmutableList.of(
+ toProtoAndBackSpec(WindowingStrategy.globalDefault()),
+ toProtoAndBackSpec(WindowingStrategy.of(
+ FixedWindows.of(Duration.millis(11)).withOffset(Duration.millis(3)))),
+ toProtoAndBackSpec(WindowingStrategy.of(
+ SlidingWindows.of(Duration.millis(37)).every(Duration.millis(3))
+ .withOffset(Duration.millis(2)))),
+ toProtoAndBackSpec(WindowingStrategy.of(
+ Sessions.withGapDuration(Duration.millis(389)))),
+ toProtoAndBackSpec(
+ WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN)
+ .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withTrigger(REPRESENTATIVE_TRIGGER)
+ .withAllowedLateness(Duration.millis(71))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)),
+ toProtoAndBackSpec(
+ WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN)
+ .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY)
+ .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
+ .withTrigger(REPRESENTATIVE_TRIGGER)
+ .withAllowedLateness(Duration.millis(93))
+ .withTimestampCombiner(TimestampCombiner.LATEST)));
+ }
+
+ @Parameter(0)
+ public ToProtoAndBackSpec toProtoAndBackSpec;
+
+ @Test
+ public void testToProtoAndBack() throws Exception {
+ WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
+ WindowingStrategy<?, ?> toProtoAndBackWindowingStrategy =
+ WindowingStrategyTranslation.fromProto(
+ WindowingStrategyTranslation.toProto(windowingStrategy));
+
+ assertThat(
+ toProtoAndBackWindowingStrategy,
+ equalTo((WindowingStrategy) windowingStrategy.fixDefaults()));
+ }
+
+ @Test
+ public void testToProtoAndBackWithComponents() throws Exception {
+ WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
+ SdkComponents components = SdkComponents.create();
+ RunnerApi.WindowingStrategy proto =
+ WindowingStrategyTranslation.toProto(windowingStrategy, components);
+ RehydratedComponents protoComponents =
+ RehydratedComponents.forComponents(components.toComponents());
+
+ assertThat(
+ WindowingStrategyTranslation.fromProto(proto, protoComponents).fixDefaults(),
+ Matchers.<WindowingStrategy<?, ?>>equalTo(windowingStrategy.fixDefaults()));
+
+ protoComponents.getCoder(
+ components.registerCoder(windowingStrategy.getWindowFn().windowCoder()));
+ assertThat(
+ proto.getAssignsToOneWindow(),
+ equalTo(windowingStrategy.getWindowFn().assignsToOneWindow()));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
new file mode 100644
index 0000000..4bc61d4
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.sdk.io.DynamicFileDestinations;
+import org.apache.beam.sdk.io.FileBasedSink;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
+
+/** Tests for {@link WriteFilesTranslation}. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ WriteFilesTranslationTest.TestWriteFilesPayloadTranslation.class,
+})
+public class WriteFilesTranslationTest {
+
+ /** Tests for translating various {@link ParDo} transforms to/from {@link ParDoPayload} protos. */
+ @RunWith(Parameterized.class)
+ public static class TestWriteFilesPayloadTranslation {
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<WriteFiles<Object, Void, Object>> data() {
+ return ImmutableList.of(
+ WriteFiles.to(new DummySink()),
+ WriteFiles.to(new DummySink()).withWindowedWrites(),
+ WriteFiles.to(new DummySink()).withNumShards(17),
+ WriteFiles.to(new DummySink()).withWindowedWrites().withNumShards(42));
+ }
+
+ @Parameter(0)
+ public WriteFiles<String, Void, String> writeFiles;
+
+ public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ @Test
+ public void testEncodedProto() throws Exception {
+ RunnerApi.WriteFilesPayload payload =
+ WriteFilesTranslation.payloadForWriteFiles(writeFiles, SdkComponents.create());
+
+ assertThat(
+ payload.getRunnerDeterminedSharding(),
+ equalTo(writeFiles.getNumShards() == null && writeFiles.getSharding() == null));
+
+ assertThat(payload.getWindowedWrites(), equalTo(writeFiles.isWindowedWrites()));
+
+ assertThat(
+ (FileBasedSink<String, Void, String>)
+ WriteFilesTranslation.sinkFromProto(payload.getSink()),
+ equalTo(writeFiles.getSink()));
+ }
+
+ @Test
+ public void testExtractionDirectFromTransform() throws Exception {
+ PCollection<String> input = p.apply(Create.of("hello"));
+ WriteFilesResult<Void> output = input.apply(writeFiles);
+
+ AppliedPTransform<
+ PCollection<String>, WriteFilesResult<Void>, WriteFiles<String, Void, String>>
+ appliedPTransform =
+ AppliedPTransform.of("foo", input.expand(), output.expand(), writeFiles, p);
+
+ assertThat(
+ WriteFilesTranslation.isRunnerDeterminedSharding(appliedPTransform),
+ equalTo(writeFiles.getNumShards() == null && writeFiles.getSharding() == null));
+
+ assertThat(
+ WriteFilesTranslation.isWindowedWrites(appliedPTransform),
+ equalTo(writeFiles.isWindowedWrites()));
+ assertThat(
+ WriteFilesTranslation.<String, Void, String>getSink(appliedPTransform),
+ equalTo(writeFiles.getSink()));
+ }
+ }
+
+ /**
+ * A simple {@link FileBasedSink} for testing serialization/deserialization. Not mocked to avoid
+ * any issues serializing mocks.
+ */
+ private static class DummySink extends FileBasedSink<Object, Void, Object> {
+
+ DummySink() {
+ super(
+ StaticValueProvider.of(FileSystems.matchNewResource("nowhere", false)),
+ DynamicFileDestinations.constant(
+ new DummyFilenamePolicy(), SerializableFunctions.constant(null)));
+ }
+
+ @Override
+ public WriteOperation<Void, Object> createWriteOperation() {
+ return new DummyWriteOperation(this);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof DummySink)) {
+ return false;
+ }
+
+ DummySink that = (DummySink) other;
+
+ return getTempDirectoryProvider().isAccessible()
+ && that.getTempDirectoryProvider().isAccessible()
+ && getTempDirectoryProvider().get().equals(that.getTempDirectoryProvider().get());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ DummySink.class,
+ getTempDirectoryProvider().isAccessible() ? getTempDirectoryProvider().get() : null);
+ }
+ }
+
+ private static class DummyWriteOperation extends FileBasedSink.WriteOperation<Void, Object> {
+ public DummyWriteOperation(FileBasedSink<Object, Void, Object> sink) {
+ super(sink);
+ }
+
+ @Override
+ public FileBasedSink.Writer<Void, Object> createWriter() throws Exception {
+ throw new UnsupportedOperationException("Should never be called.");
+ }
+ }
+
+ private static class DummyFilenamePolicy extends FilenamePolicy {
+ @Override
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("Should never be called.");
+ }
+
+ @Nullable
+ @Override
+ public ResourceId unwindowedFilename(
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("Should never be called.");
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof DummyFilenamePolicy;
+ }
+
+ @Override
+ public int hashCode() {
+ return DummyFilenamePolicy.class.hashCode();
+ }
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/metrics/MetricFilteringTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/metrics/MetricFilteringTest.java
new file mode 100644
index 0000000..5814551
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/metrics/MetricFilteringTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction.metrics;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricsFilter;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link MetricFiltering}.
+ */
+@RunWith(JUnit4.class)
+public class MetricFilteringTest {
+ private static final MetricName NAME1 = MetricName.named("ns1", "name1");
+
+
+ private boolean matchesSubPath(String actualScope, String subPath) {
+ return MetricFiltering.subPathMatches(actualScope, subPath);
+ }
+
+ @Test
+ public void testMatchCompositeStepNameFilters() {
+ // MetricsFilter with a Class-namespace + name filter + step filter.
+ // Successful match.
+ assertTrue(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
+ .addStep("myStep").build(),
+ MetricKey.create(
+ "myBigStep/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
+
+ // Unsuccessful match.
+ assertFalse(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
+ .addStep("myOtherStep").build(),
+ MetricKey.create(
+ "myOtherStepNoMatch/myStep",
+ MetricName.named(MetricFilteringTest.class, "myMetricName"))));
+ }
+
+ @Test
+ public void testMatchStepNameFilters() {
+ // MetricsFilter with a Class-namespace + name filter + step filter.
+ // Successful match.
+ assertTrue(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
+ .addStep("myStep").build(),
+ MetricKey.create("myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
+
+ // Unsuccessful match.
+ assertFalse(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
+ .addStep("myOtherStep").build(),
+ MetricKey.create("myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
+ }
+
+ @Test
+ public void testMatchClassNamespaceFilters() {
+ // MetricsFilter with a Class-namespace + name filter. Without step filter.
+ // Successful match.
+ assertTrue(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")).build(),
+ MetricKey.create("anyStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
+
+ // Unsuccessful match.
+ assertFalse(MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")).build(),
+ MetricKey.create("anyStep", MetricName.named(MetricFiltering.class, "myMetricName"))));
+ }
+
+ @Test
+ public void testMatchStringNamespaceFilters() {
+ // MetricsFilter with a String-namespace + name filter. Without step filter.
+ // Successful match.
+ assertTrue(
+ MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named("myNamespace", "myMetricName")).build(),
+ MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricName"))));
+
+ // Unsuccessful match.
+ assertFalse(
+ MetricFiltering.matches(
+ MetricsFilter.builder().addNameFilter(
+ MetricNameFilter.named("myOtherNamespace", "myMetricName")).build(),
+ MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricname"))));
+ }
+
+ @Test
+ public void testMatchesSubPath() {
+ assertTrue("Match of the first element",
+ matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top1"));
+ assertTrue("Match of the first elements",
+ matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1"));
+ assertTrue("Match of the last elements",
+ matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Inner1/Bottom1"));
+ assertFalse("Substring match but no subpath match",
+ matchesSubPath("Top1/Outer1/Inner1/Bottom1", "op1/Outer1/Inner1"));
+ assertFalse("Substring match from start - but no subpath match",
+ matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top"));
+ }
+
+ private boolean matchesScopeWithSingleFilter(String actualScope, String filter) {
+ Set<String> scopeFilter = new HashSet<String>();
+ scopeFilter.add(filter);
+ return MetricFiltering.matchesScope(actualScope, scopeFilter);
+ }
+
+ @Test
+ public void testMatchesScope() {
+ assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1"));
+ assertTrue(matchesScopeWithSingleFilter(
+ "Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inner1/Bottom1"));
+ assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1"));
+ assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inner1"));
+ assertFalse(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Inner1"));
+ assertFalse(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inn"));
+ }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
deleted file mode 100644
index 8953f21..0000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.metrics;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricNameFilter;
-import org.apache.beam.sdk.metrics.MetricsFilter;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link MetricFiltering}.
- */
-@RunWith(JUnit4.class)
-public class MetricFilteringTest {
- private static final MetricName NAME1 = MetricName.named("ns1", "name1");
-
-
- private boolean matchesSubPath(String actualScope, String subPath) {
- return MetricFiltering.subPathMatches(actualScope, subPath);
- }
-
- @Test
- public void testMatchCompositeStepNameFilters() {
- // MetricsFilter with a Class-namespace + name filter + step filter.
- // Successful match.
- assertTrue(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
- .addStep("myStep").build(),
- MetricKey.create(
- "myBigStep/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
-
- // Unsuccessful match.
- assertFalse(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
- .addStep("myOtherStep").build(),
- MetricKey.create(
- "myOtherStepNoMatch/myStep",
- MetricName.named(MetricFilteringTest.class, "myMetricName"))));
- }
-
- @Test
- public void testMatchStepNameFilters() {
- // MetricsFilter with a Class-namespace + name filter + step filter.
- // Successful match.
- assertTrue(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
- .addStep("myStep").build(),
- MetricKey.create("myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
-
- // Unsuccessful match.
- assertFalse(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName"))
- .addStep("myOtherStep").build(),
- MetricKey.create("myStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
- }
-
- @Test
- public void testMatchClassNamespaceFilters() {
- // MetricsFilter with a Class-namespace + name filter. Without step filter.
- // Successful match.
- assertTrue(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")).build(),
- MetricKey.create("anyStep", MetricName.named(MetricFilteringTest.class, "myMetricName"))));
-
- // Unsuccessful match.
- assertFalse(MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")).build(),
- MetricKey.create("anyStep", MetricName.named(MetricFiltering.class, "myMetricName"))));
- }
-
- @Test
- public void testMatchStringNamespaceFilters() {
- // MetricsFilter with a String-namespace + name filter. Without step filter.
- // Successful match.
- assertTrue(
- MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named("myNamespace", "myMetricName")).build(),
- MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricName"))));
-
- // Unsuccessful match.
- assertFalse(
- MetricFiltering.matches(
- MetricsFilter.builder().addNameFilter(
- MetricNameFilter.named("myOtherNamespace", "myMetricName")).build(),
- MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricname"))));
- }
-
- @Test
- public void testMatchesSubPath() {
- assertTrue("Match of the first element",
- matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top1"));
- assertTrue("Match of the first elements",
- matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1"));
- assertTrue("Match of the last elements",
- matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Inner1/Bottom1"));
- assertFalse("Substring match but no subpath match",
- matchesSubPath("Top1/Outer1/Inner1/Bottom1", "op1/Outer1/Inner1"));
- assertFalse("Substring match from start - but no subpath match",
- matchesSubPath("Top1/Outer1/Inner1/Bottom1", "Top"));
- }
-
- private boolean matchesScopeWithSingleFilter(String actualScope, String filter) {
- Set<String> scopeFilter = new HashSet<String>();
- scopeFilter.add(filter);
- return MetricFiltering.matchesScope(actualScope, scopeFilter);
- }
-
- @Test
- public void testMatchesScope() {
- assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1"));
- assertTrue(matchesScopeWithSingleFilter(
- "Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inner1/Bottom1"));
- assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1"));
- assertTrue(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inner1"));
- assertFalse(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Inner1"));
- assertFalse(matchesScopeWithSingleFilter("Top1/Outer1/Inner1/Bottom1", "Top1/Outer1/Inn"));
- }
-}
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index c3a8d25..087e24d 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -59,12 +59,17 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
</dependency>
<dependency>
@@ -91,10 +96,25 @@
</dependency>
<dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
<!-- test dependencies -->
<!-- Utilities such as WindowMatchers -->
@@ -135,12 +155,5 @@
<artifactId>jackson-dataformat-yaml</artifactId>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
deleted file mode 100644
index 23d61f8..0000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/**
- * Base class for implementations of {@link ExecutionContext}.
- *
- * <p>A concrete subclass should implement {@link #createStepContext} to create the appropriate
- * {@link StepContext} implementation. Any {@code StepContext} created will
- * be cached for the lifetime of this {@link ExecutionContext}.
- *
- * <p>BaseExecutionContext is generic to allow implementing subclasses to return a concrete subclass
- * of {@link StepContext} from {@link #getOrCreateStepContext(String, String)} and
- * {@link #getAllStepContexts()} without forcing each subclass to override the method, e.g.
- * <pre>{@code
- * {@literal @}Override
- * StreamingModeExecutionContext.StepContext getOrCreateStepContext(...) {
- * return (StreamingModeExecutionContext.StepContext) super.getOrCreateStepContext(...);
- * }
- * }</pre>
- *
- * <p>When a subclass of {@code BaseExecutionContext} has been downcast, the return types of
- * {@link #createStepContext(String, String)},
- * {@link #getOrCreateStepContext(String, String)}, and {@link #getAllStepContexts()}
- * will be appropriately specialized.
- */
-public abstract class BaseExecutionContext<T extends ExecutionContext.StepContext>
- implements ExecutionContext {
-
- private Map<String, T> cachedStepContexts = new LinkedHashMap<>();
-
- /**
- * Implementations should override this to create the specific type
- * of {@link StepContext} they need.
- */
- protected abstract T createStepContext(String stepName, String transformName);
-
- /**
- * Returns the {@link StepContext} associated with the given step.
- */
- @Override
- public T getOrCreateStepContext(String stepName, String transformName) {
- final String finalStepName = stepName;
- final String finalTransformName = transformName;
- return getOrCreateStepContext(
- stepName,
- new CreateStepContextFunction<T>() {
- @Override
- public T create() {
- return createStepContext(finalStepName, finalTransformName);
- }
- });
- }
-
- /**
- * Factory method interface to create an execution context if none exists during
- * {@link #getOrCreateStepContext(String, CreateStepContextFunction)}.
- */
- protected interface CreateStepContextFunction<T extends ExecutionContext.StepContext> {
- T create();
- }
-
- protected final T getOrCreateStepContext(String stepName,
- CreateStepContextFunction<T> createContextFunc) {
- T context = cachedStepContexts.get(stepName);
- if (context == null) {
- context = createContextFunc.create();
- cachedStepContexts.put(stepName, context);
- }
-
- return context;
- }
-
- /**
- * Returns a collection view of all of the {@link StepContext}s.
- */
- @Override
- public Collection<? extends T> getAllStepContexts() {
- return Collections.unmodifiableCollection(cachedStepContexts.values());
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {}
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
-
- /**
- * Base class for implementations of {@link ExecutionContext.StepContext}.
- *
- * <p>To complete a concrete subclass, implement {@link #timerInternals} and
- * {@link #stateInternals}.
- */
- public abstract static class StepContext implements ExecutionContext.StepContext {
- private final ExecutionContext executionContext;
- private final String stepName;
- private final String transformName;
-
- public StepContext(ExecutionContext executionContext, String stepName, String transformName) {
- this.executionContext = executionContext;
- this.stepName = stepName;
- this.transformName = transformName;
- }
-
- @Override
- public String getStepName() {
- return stepName;
- }
-
- @Override
- public String getTransformName() {
- return transformName;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {
- executionContext.noteOutput(output);
- }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
- executionContext.noteOutput(tag, output);
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window, Coder<W> windowCoder) throws IOException {
- throw new UnsupportedOperationException("Not implemented.");
- }
-
- @Override
- public abstract StateInternals stateInternals();
-
- @Override
- public abstract TimerInternals timerInternals();
- }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index f3cca6f..9d3e25d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -19,8 +19,7 @@
import java.util.Collection;
import java.util.List;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
-import org.apache.beam.runners.core.SplittableParDo.ProcessFn;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
import org.apache.beam.runners.core.StatefulDoFnRunner.CleanupTimer;
import org.apache.beam.runners.core.StatefulDoFnRunner.StateCleaner;
import org.apache.beam.sdk.options.PipelineOptions;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
deleted file mode 100644
index 4a5d0c4..0000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import com.google.auto.value.AutoValue;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.transforms.DoFn;
-
-/**
- * A tuple of an element and a restriction applied to processing it with a
- * <a href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn}.
- */
-@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
-@AutoValue
-public abstract class ElementAndRestriction<ElementT, RestrictionT> {
- /** The element to process. */
- public abstract ElementT element();
-
- /** The restriction applied to processing the element. */
- public abstract RestrictionT restriction();
-
- /** Constructs the {@link ElementAndRestriction}. */
- public static <InputT, RestrictionT> ElementAndRestriction<InputT, RestrictionT> of(
- InputT element, RestrictionT restriction) {
- return new AutoValue_ElementAndRestriction<>(element, restriction);
- }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
deleted file mode 100644
index 4440b85..0000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.List;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.StructuredCoder;
-
-/** A {@link Coder} for {@link ElementAndRestriction}. */
-@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
-public class ElementAndRestrictionCoder<ElementT, RestrictionT>
- extends StructuredCoder<ElementAndRestriction<ElementT, RestrictionT>> {
- private final Coder<ElementT> elementCoder;
- private final Coder<RestrictionT> restrictionCoder;
-
- /**
- * Creates an {@link ElementAndRestrictionCoder} from an element coder and a restriction coder.
- */
- public static <ElementT, RestrictionT> ElementAndRestrictionCoder<ElementT, RestrictionT> of(
- Coder<ElementT> elementCoder, Coder<RestrictionT> restrictionCoder) {
- return new ElementAndRestrictionCoder<>(elementCoder, restrictionCoder);
- }
-
- private ElementAndRestrictionCoder(
- Coder<ElementT> elementCoder, Coder<RestrictionT> restrictionCoder) {
- this.elementCoder = elementCoder;
- this.restrictionCoder = restrictionCoder;
- }
-
- @Override
- public void encode(
- ElementAndRestriction<ElementT, RestrictionT> value, OutputStream outStream)
- throws IOException {
- if (value == null) {
- throw new CoderException("cannot encode a null ElementAndRestriction");
- }
- elementCoder.encode(value.element(), outStream);
- restrictionCoder.encode(value.restriction(), outStream);
- }
-
- @Override
- public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream)
- throws IOException {
- ElementT key = elementCoder.decode(inStream);
- RestrictionT value = restrictionCoder.decode(inStream);
- return ElementAndRestriction.of(key, value);
- }
-
- @Override
- public List<? extends Coder<?>> getCoderArguments() {
- return ImmutableList.of(elementCoder, restrictionCoder);
- }
-
- @Override
- public void verifyDeterministic() throws NonDeterministicException {
- elementCoder.verifyDeterministic();
- restrictionCoder.verifyDeterministic();
- }
-
- public Coder<ElementT> getElementCoder() {
- return elementCoder;
- }
-
- public Coder<RestrictionT> getRestrictionCoder() {
- return restrictionCoder;
- }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
deleted file mode 100644
index d2fdaac..0000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.io.IOException;
-import java.util.Collection;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/**
- * Context for the current execution. This is guaranteed to exist during processing,
- * but does not necessarily persist between different batches of work.
- */
-public interface ExecutionContext {
- /**
- * Returns the {@link StepContext} associated with the given step.
- */
- StepContext getOrCreateStepContext(String stepName, String transformName);
-
- /**
- * Returns a collection view of all of the {@link StepContext}s.
- */
- Collection<? extends StepContext> getAllStepContexts();
-
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link WindowedContext#output(TupleTag, Object)} is called.
- */
- void noteOutput(WindowedValue<?> output);
-
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link WindowedContext#output(TupleTag, Object)} is called.
- */
- void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
-
- /**
- * Per-step, per-key context used for retrieving state.
- */
- public interface StepContext {
-
- /**
- * The name of the step.
- */
- String getStepName();
-
- /**
- * The name of the transform for the step.
- */
- String getTransformName();
-
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link WindowedContext#output}
- * is called.
- */
- void noteOutput(WindowedValue<?> output);
-
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link WindowedContext#output}
- * is called.
- */
- void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
-
- /**
- * Writes the given {@code PCollectionView} data to a globally accessible location.
- */
- <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window,
- Coder<W> windowCoder)
- throws IOException;
-
- StateInternals stateInternals();
-
- TimerInternals timerInternals();
- }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
index 744d162..0a520bd 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.core;
import java.util.Collection;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.sdk.transforms.DoFn;
@@ -122,7 +122,7 @@
windowingStrategy,
ExecutableTriggerStateMachine.create(
TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger()))),
+ TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
stateInternals,
timerInternals,
outputWindowedValue(),
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
index fca3c76..1fdf07c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
@@ -111,12 +111,10 @@
@Override
public PCollection<KV<K, Iterable<WindowedValue<V>>>> expand(PCollection<KV<K, V>> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
- }
-
- @Override
- public Coder<KV<K, Iterable<V>>> getDefaultOutputCoder(PCollection<KV<K, V>> input) {
- return GroupByKey.getOutputKvCoder(input.getCoder());
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.isBounded(),
+ (Coder) GroupByKey.getOutputKvCoder(input.getCoder()));
}
}
@@ -244,9 +242,8 @@
Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder);
Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder);
- return PCollection.<KV<K, Iterable<V>>>createPrimitiveOutputInternal(
- input.getPipeline(), windowingStrategy, input.isBounded())
- .setCoder(outputKvCoder);
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder);
}
}
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
index 59814bc..075e264 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
@@ -17,8 +17,12 @@
*/
package org.apache.beam.runners.core;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -326,7 +330,8 @@
@Override
public OutputT read() {
- return combineFn.extractOutput(accum);
+ return combineFn.extractOutput(
+ combineFn.mergeAccumulators(Arrays.asList(combineFn.createAccumulator(), accum)));
}
@Override
@@ -407,7 +412,7 @@
@Override
public Iterable<T> read() {
- return contents;
+ return Iterables.limit(contents, contents.size());
}
@Override
@@ -478,7 +483,7 @@
@Override
public Iterable<T> read() {
- return contents;
+ return ImmutableSet.copyOf(contents);
}
@Override
@@ -551,19 +556,41 @@
contents.remove(key);
}
+ private static class CollectionViewState<T> implements ReadableState<Iterable<T>> {
+ private final Collection<T> collection;
+
+ private CollectionViewState(Collection<T> collection) {
+ this.collection = collection;
+ }
+
+ public static <T> CollectionViewState<T> of(Collection<T> collection) {
+ return new CollectionViewState<>(collection);
+ }
+
+ @Override
+ public Iterable<T> read() {
+ return ImmutableList.copyOf(collection);
+ }
+
+ @Override
+ public ReadableState<Iterable<T>> readLater() {
+ return this;
+ }
+ }
+
@Override
public ReadableState<Iterable<K>> keys() {
- return ReadableStates.immediate((Iterable<K>) contents.keySet());
+ return CollectionViewState.of(contents.keySet());
}
@Override
public ReadableState<Iterable<V>> values() {
- return ReadableStates.immediate((Iterable<V>) contents.values());
+ return CollectionViewState.of(contents.values());
}
@Override
public ReadableState<Iterable<Map.Entry<K, V>>> entries() {
- return ReadableStates.immediate((Iterable<Map.Entry<K, V>>) contents.entrySet());
+ return CollectionViewState.of(contents.entrySet());
}
@Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
index e68bb24..c7b4ac6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
@@ -25,6 +25,7 @@
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table;
import java.util.NavigableSet;
+import java.util.NoSuchElementException;
import java.util.TreeSet;
import javax.annotation.Nullable;
import org.apache.beam.sdk.state.TimeDomain;
@@ -71,21 +72,20 @@
*/
@Nullable
public Instant getNextTimer(TimeDomain domain) {
- final TimerData data;
- switch (domain) {
- case EVENT_TIME:
- data = watermarkTimers.first();
- break;
- case PROCESSING_TIME:
- data = processingTimers.first();
- break;
- case SYNCHRONIZED_PROCESSING_TIME:
- data = synchronizedProcessingTimers.first();
- break;
- default:
- throw new IllegalArgumentException("Unexpected time domain: " + domain);
+ try {
+ switch (domain) {
+ case EVENT_TIME:
+ return watermarkTimers.first().getTimestamp();
+ case PROCESSING_TIME:
+ return processingTimers.first().getTimestamp();
+ case SYNCHRONIZED_PROCESSING_TIME:
+ return synchronizedProcessingTimers.first().getTimestamp();
+ default:
+ throw new IllegalArgumentException("Unexpected time domain: " + domain);
+ }
+ } catch (NoSuchElementException exc) {
+ return null;
}
- return (data == null) ? null : data.getTimestamp();
}
private NavigableSet<TimerData> timersForDomain(TimeDomain domain) {
@@ -107,6 +107,9 @@
setTimer(TimerData.of(timerId, namespace, target, timeDomain));
}
+ /**
+ * @deprecated use {@link #setTimer(StateNamespace, String, Instant, TimeDomain)}.
+ */
@Deprecated
@Override
public void setTimer(TimerData timerData) {
@@ -136,6 +139,9 @@
throw new UnsupportedOperationException("Canceling a timer by ID is not yet supported.");
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(StateNamespace namespace, String timerId) {
@@ -145,6 +151,9 @@
}
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(TimerData timer) {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
index 1cf1509..28938c1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
@@ -134,26 +134,27 @@
// The element is too late for this window.
droppedDueToLateness.inc();
WindowTracing.debug(
- "ReduceFnRunner.processElement: Dropping element at {} for key:{}; window:{} "
- + "since too far behind inputWatermark:{}; outputWatermark:{}",
- input.getTimestamp(), key, window, timerInternals.currentInputWatermarkTime(),
+ "{}: Dropping element at {} for key:{}; window:{} "
+ + "since too far behind inputWatermark:{}; outputWatermark:{}",
+ LateDataFilter.class.getSimpleName(),
+ input.getTimestamp(),
+ key,
+ window,
+ timerInternals.currentInputWatermarkTime(),
timerInternals.currentOutputWatermarkTime());
}
}
- Iterable<WindowedValue<InputT>> nonLateElements = Iterables.filter(
- concatElements,
- new Predicate<WindowedValue<InputT>>() {
- @Override
- public boolean apply(WindowedValue<InputT> input) {
- BoundedWindow window = Iterables.getOnlyElement(input.getWindows());
- if (canDropDueToExpiredWindow(window)) {
- return false;
- } else {
- return true;
- }
- }
- });
+ Iterable<WindowedValue<InputT>> nonLateElements =
+ Iterables.filter(
+ concatElements,
+ new Predicate<WindowedValue<InputT>>() {
+ @Override
+ public boolean apply(WindowedValue<InputT> input) {
+ BoundedWindow window = Iterables.getOnlyElement(input.getWindows());
+ return !canDropDueToExpiredWindow(window);
+ }
+ });
return nonLateElements;
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 2db6531..d830db5 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.core;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Futures;
@@ -37,6 +38,7 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.joda.time.Duration;
@@ -96,7 +98,7 @@
final WindowedValue<InputT> element,
final TrackerT tracker) {
final ProcessContext processContext = new ProcessContext(element, tracker);
- invoker.invokeProcessElement(
+ DoFn.ProcessContinuation cont = invoker.invokeProcessElement(
new DoFnInvoker.ArgumentProvider<InputT, OutputT>() {
@Override
public DoFn<InputT, OutputT>.ProcessContext processContext(
@@ -118,6 +120,11 @@
}
@Override
+ public PipelineOptions pipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
throw new IllegalStateException(
"Should not access startBundleContext() from @"
@@ -150,10 +157,44 @@
"Access to timers not supported in Splittable DoFn");
}
});
-
+ // TODO: verify that if there was a failed tryClaim() call, then cont.shouldResume() is false.
+ // Currently we can't verify this because there are no hooks into tryClaim().
+ // See https://issues.apache.org/jira/browse/BEAM-2607
+ processContext.cancelScheduledCheckpoint();
+ KV<RestrictionT, Instant> residual = processContext.getTakenCheckpoint();
+ if (cont.shouldResume()) {
+ if (residual == null) {
+ // No checkpoint had been taken by the runner while the ProcessElement call ran, however
+ // the call says that not the whole restriction has been processed. So we need to take
+ // a checkpoint now: checkpoint() guarantees that the primary restriction describes exactly
+ // the work that was done in the current ProcessElement call, and returns a residual
+ // restriction that describes exactly the work that wasn't done in the current call.
+ residual = checkNotNull(processContext.takeCheckpointNow());
+ } else {
+ // A checkpoint was taken by the runner, and then the ProcessElement call returned resume()
+ // without making more tryClaim() calls (since no tryClaim() calls can succeed after
+ // checkpoint(), and since if it had made a failed tryClaim() call, it should have returned
+ // stop()).
+ // This means that the resulting primary restriction and the taken checkpoint already
+ // accurately describe respectively the work that was and wasn't done in the current
+ // ProcessElement call.
+ // In other words, if we took a checkpoint *after* ProcessElement completed (like in the
+ // branch above), it would have been equivalent to this one.
+ }
+ } else {
+ // The ProcessElement call returned stop() - that means the tracker's current restriction
+ // has been fully processed by the call. A checkpoint may or may not have been taken in
+ // "residual"; if it was, then we'll need to process it; if no, then we don't - nothing
+ // special needs to be done.
+ }
tracker.checkDone();
- return new Result(
- processContext.extractCheckpoint(), processContext.getLastReportedWatermark());
+ if (residual == null) {
+ // Can only be true if cont.shouldResume() is false and no checkpoint was taken.
+ // This means the restriction has been fully processed.
+ checkState(!cont.shouldResume());
+ return new Result(null, cont, BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+ return new Result(residual.getKey(), cont, residual.getValue());
}
private class ProcessContext extends DoFn<InputT, OutputT>.ProcessContext {
@@ -167,6 +208,9 @@
// This is either the result of the sole tracker.checkpoint() call, or null if
// the call completed before reaching the given number of outputs or duration.
private RestrictionT checkpoint;
+ // Watermark captured at the moment before checkpoint was taken, describing a lower bound
+ // on the output from "checkpoint".
+ private Instant residualWatermark;
// A handle on the scheduled action to take a checkpoint.
private Future<?> scheduledCheckpoint;
private Instant lastReportedWatermark;
@@ -181,34 +225,36 @@
new Runnable() {
@Override
public void run() {
- initiateCheckpoint();
+ takeCheckpointNow();
}
},
maxDuration.getMillis(),
TimeUnit.MILLISECONDS);
}
- @Nullable
- RestrictionT extractCheckpoint() {
+ void cancelScheduledCheckpoint() {
scheduledCheckpoint.cancel(true);
try {
Futures.getUnchecked(scheduledCheckpoint);
} catch (CancellationException e) {
// This is expected if the call took less than the maximum duration.
}
- // By now, a checkpoint may or may not have been taken;
- // via .output() or via scheduledCheckpoint.
- synchronized (this) {
- return checkpoint;
- }
}
- private synchronized void initiateCheckpoint() {
+ synchronized KV<RestrictionT, Instant> takeCheckpointNow() {
// This method may be entered either via .output(), or via scheduledCheckpoint.
// Only one of them "wins" - tracker.checkpoint() must be called only once.
if (checkpoint == null) {
+ residualWatermark = lastReportedWatermark;
checkpoint = checkNotNull(tracker.checkpoint());
}
+ return getTakenCheckpoint();
+ }
+
+ @Nullable
+ synchronized KV<RestrictionT, Instant> getTakenCheckpoint() {
+ // The checkpoint may or may not have been taken.
+ return (checkpoint == null) ? null : KV.of(checkpoint, residualWatermark);
}
@Override
@@ -239,10 +285,6 @@
lastReportedWatermark = watermark;
}
- public synchronized Instant getLastReportedWatermark() {
- return lastReportedWatermark;
- }
-
@Override
public PipelineOptions getPipelineOptions() {
return pipelineOptions;
@@ -274,7 +316,7 @@
private void noteOutput() {
++numOutputs;
if (numOutputs >= maxNumOutputs) {
- initiateCheckpoint();
+ takeCheckpointNow();
}
}
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
index 61f413f..88275d6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
@@ -18,7 +18,6 @@
package org.apache.beam.runners.core;
import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.beam.runners.core.SplittableParDo.ProcessFn;
import com.google.common.collect.Iterables;
import java.util.Collection;
@@ -29,22 +28,23 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.joda.time.Instant;
-/** Runs a {@link ProcessFn} by constructing the appropriate contexts and passing them in. */
+/**
+ * Runs a {@link SplittableParDoViaKeyedWorkItems.ProcessFn} by constructing the appropriate
+ * contexts and passing them in.
+ */
public class ProcessFnRunner<InputT, OutputT, RestrictionT>
implements PushbackSideInputDoFnRunner<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT> {
- private final DoFnRunner<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
- underlying;
+ KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> {
+ private final DoFnRunner<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> underlying;
private final Collection<PCollectionView<?>> views;
private final ReadyCheckingSideInputReader sideInputReader;
ProcessFnRunner(
- DoFnRunner<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
- underlying,
+ DoFnRunner<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> underlying,
Collection<PCollectionView<?>> views,
ReadyCheckingSideInputReader sideInputReader) {
this.underlying = underlying;
@@ -58,10 +58,9 @@
}
@Override
- public Iterable<WindowedValue<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>>
+ public Iterable<WindowedValue<KeyedWorkItem<String, KV<InputT, RestrictionT>>>>
processElementInReadyWindows(
- WindowedValue<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
- windowedKWI) {
+ WindowedValue<KeyedWorkItem<String, KV<InputT, RestrictionT>>> windowedKWI) {
checkTrivialOuterWindows(windowedKWI);
BoundedWindow window = getUnderlyingWindow(windowedKWI.getValue());
if (!isReady(window)) {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
index 62d519f..634a2d1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
@@ -29,7 +29,6 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -51,6 +50,7 @@
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowTracing;
@@ -637,11 +637,9 @@
}
/**
- * Enriches TimerData with state necessary for processing a timer as well as
- * common queries about a timer.
+ * A descriptor of the activation for a window based on a timer.
*/
- private class EnrichedTimerData {
- public final Instant timestamp;
+ private class WindowActivation {
public final ReduceFn<K, InputT, OutputT, W>.Context directContext;
public final ReduceFn<K, InputT, OutputT, W>.Context renamedContext;
// If this is an end-of-window timer then we may need to set a garbage collection timer
@@ -652,18 +650,34 @@
// end-of-window time to be a signal to garbage collect.
public final boolean isGarbageCollection;
- EnrichedTimerData(
- TimerData timer,
+ WindowActivation(
ReduceFn<K, InputT, OutputT, W>.Context directContext,
ReduceFn<K, InputT, OutputT, W>.Context renamedContext) {
- this.timestamp = timer.getTimestamp();
this.directContext = directContext;
this.renamedContext = renamedContext;
W window = directContext.window();
- this.isEndOfWindow = TimeDomain.EVENT_TIME == timer.getDomain()
- && timer.getTimestamp().equals(window.maxTimestamp());
- Instant cleanupTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy);
- this.isGarbageCollection = !timer.getTimestamp().isBefore(cleanupTime);
+
+ // The output watermark is before the end of the window if it is either unknown
+ // or it is known to be before it. If it is unknown, that means that there hasn't been
+ // enough data to advance it.
+ boolean outputWatermarkBeforeEOW =
+ timerInternals.currentOutputWatermarkTime() == null
+ || !timerInternals.currentOutputWatermarkTime().isAfter(window.maxTimestamp());
+
+ // The "end of the window" is reached when the local input watermark (for this key) surpasses
+ // it but the local output watermark (also for this key) has not. After data is emitted and
+ // the output watermark hold is released, the output watermark on this key will immediately
+ // exceed the end of the window (otherwise we could see multiple ON_TIME outputs)
+ this.isEndOfWindow =
+ timerInternals.currentInputWatermarkTime().isAfter(window.maxTimestamp())
+ && outputWatermarkBeforeEOW;
+
+ // The "GC time" is reached when the input watermark surpasses the end of the window
+ // plus allowed lateness. After this, the window is expired and expunged.
+ this.isGarbageCollection =
+ timerInternals
+ .currentInputWatermarkTime()
+ .isAfter(LateDataUtils.garbageCollectionTime(window, windowingStrategy));
}
// Has this window had its trigger finish?
@@ -682,24 +696,47 @@
return;
}
- // Create a reusable context for each timer and begin prefetching necessary
+ // Create a reusable context for each window and begin prefetching necessary
// state.
- List<EnrichedTimerData> enrichedTimers = new LinkedList();
+ Map<BoundedWindow, WindowActivation> windowActivations = new HashMap();
+
for (TimerData timer : timers) {
checkArgument(timer.getNamespace() instanceof WindowNamespace,
"Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace());
@SuppressWarnings("unchecked")
WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace();
W window = windowNamespace.getWindow();
+
+ WindowTracing.debug("{}: Received timer key:{}; window:{}; data:{} with "
+ + "inputWatermark:{}; outputWatermark:{}",
+ ReduceFnRunner.class.getSimpleName(),
+ key, window, timer,
+ timerInternals.currentInputWatermarkTime(),
+ timerInternals.currentOutputWatermarkTime());
+
+ // Processing time timers for an expired window are ignored, just like elements
+ // that show up too late. Window GC is management by an event time timer
+ if (TimeDomain.EVENT_TIME != timer.getDomain() && windowIsExpired(window)) {
+ continue;
+ }
+
+ // How a window is processed is a function only of the current state, not the details
+ // of the timer. This makes us robust to large leaps in processing time and watermark
+ // time, where both EOW and GC timers come in together and we need to GC and emit
+ // the final pane.
+ if (windowActivations.containsKey(window)) {
+ continue;
+ }
+
ReduceFn<K, InputT, OutputT, W>.Context directContext =
contextFactory.base(window, StateStyle.DIRECT);
ReduceFn<K, InputT, OutputT, W>.Context renamedContext =
contextFactory.base(window, StateStyle.RENAMED);
- EnrichedTimerData enrichedTimer = new EnrichedTimerData(timer, directContext, renamedContext);
- enrichedTimers.add(enrichedTimer);
+ WindowActivation windowActivation = new WindowActivation(directContext, renamedContext);
+ windowActivations.put(window, windowActivation);
// Perform prefetching of state to determine if the trigger should fire.
- if (enrichedTimer.isGarbageCollection) {
+ if (windowActivation.isGarbageCollection) {
triggerRunner.prefetchIsClosed(directContext.state());
} else {
triggerRunner.prefetchShouldFire(directContext.window(), directContext.state());
@@ -707,7 +744,7 @@
}
// For those windows that are active and open, prefetch the triggering or emitting state.
- for (EnrichedTimerData timer : enrichedTimers) {
+ for (WindowActivation timer : windowActivations.values()) {
if (timer.windowIsActiveAndOpen()) {
ReduceFn<K, InputT, OutputT, W>.Context directContext = timer.directContext;
if (timer.isGarbageCollection) {
@@ -720,25 +757,27 @@
}
// Perform processing now that everything is prefetched.
- for (EnrichedTimerData timer : enrichedTimers) {
- ReduceFn<K, InputT, OutputT, W>.Context directContext = timer.directContext;
- ReduceFn<K, InputT, OutputT, W>.Context renamedContext = timer.renamedContext;
+ for (WindowActivation windowActivation : windowActivations.values()) {
+ ReduceFn<K, InputT, OutputT, W>.Context directContext = windowActivation.directContext;
+ ReduceFn<K, InputT, OutputT, W>.Context renamedContext = windowActivation.renamedContext;
- if (timer.isGarbageCollection) {
- WindowTracing.debug("ReduceFnRunner.onTimer: Cleaning up for key:{}; window:{} at {} with "
- + "inputWatermark:{}; outputWatermark:{}",
- key, directContext.window(), timer.timestamp,
+ if (windowActivation.isGarbageCollection) {
+ WindowTracing.debug(
+ "{}: Cleaning up for key:{}; window:{} with inputWatermark:{}; outputWatermark:{}",
+ ReduceFnRunner.class.getSimpleName(),
+ key,
+ directContext.window(),
timerInternals.currentInputWatermarkTime(),
timerInternals.currentOutputWatermarkTime());
- boolean windowIsActiveAndOpen = timer.windowIsActiveAndOpen();
+ boolean windowIsActiveAndOpen = windowActivation.windowIsActiveAndOpen();
if (windowIsActiveAndOpen) {
// We need to call onTrigger to emit the final pane if required.
// The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted,
// and the watermark has passed the end of the window.
@Nullable
Instant newHold = onTrigger(
- directContext, renamedContext, true /* isFinished */, timer.isEndOfWindow);
+ directContext, renamedContext, true /* isFinished */, windowActivation.isEndOfWindow);
checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold);
}
@@ -746,18 +785,20 @@
// see elements for it again.
clearAllState(directContext, renamedContext, windowIsActiveAndOpen);
} else {
- WindowTracing.debug("ReduceFnRunner.onTimer: Triggering for key:{}; window:{} at {} with "
+ WindowTracing.debug(
+ "{}.onTimers: Triggering for key:{}; window:{} at {} with "
+ "inputWatermark:{}; outputWatermark:{}",
- key, directContext.window(), timer.timestamp,
+ key,
+ directContext.window(),
timerInternals.currentInputWatermarkTime(),
timerInternals.currentOutputWatermarkTime());
- if (timer.windowIsActiveAndOpen()
+ if (windowActivation.windowIsActiveAndOpen()
&& triggerRunner.shouldFire(
directContext.window(), directContext.timers(), directContext.state())) {
emit(directContext, renamedContext);
}
- if (timer.isEndOfWindow) {
+ if (windowActivation.isEndOfWindow) {
// If the window strategy trigger includes a watermark trigger then at this point
// there should be no data holds, either because we'd already cleared them on an
// earlier onTrigger, or because we just cleared them on the above emit.
@@ -919,8 +960,9 @@
// The pane has elements.
return true;
}
- if (timing == Timing.ON_TIME) {
- // This is the unique ON_TIME pane.
+ if (timing == Timing.ON_TIME
+ && windowingStrategy.getOnTimeBehavior() == Window.OnTimeBehavior.FIRE_ALWAYS) {
+ // This is an empty ON_TIME pane.
return true;
}
if (isFinished && windowingStrategy.getClosingBehavior() == ClosingBehavior.FIRE_ALWAYS) {
@@ -948,13 +990,8 @@
private Instant onTrigger(
final ReduceFn<K, InputT, OutputT, W>.Context directContext,
ReduceFn<K, InputT, OutputT, W>.Context renamedContext,
- boolean isFinished, boolean isEndOfWindow)
+ final boolean isFinished, boolean isEndOfWindow)
throws Exception {
- Instant inputWM = timerInternals.currentInputWatermarkTime();
-
- // Calculate the pane info.
- final PaneInfo pane = paneInfoTracker.getNextPaneInfo(directContext, isFinished).read();
-
// Extract the window hold, and as a side effect clear it.
final WatermarkHold.OldAndNewHolds pair =
watermarkHold.extractAndRelease(renamedContext, isFinished).read();
@@ -963,7 +1000,13 @@
@Nullable Instant newHold = pair.newHold;
final boolean isEmpty = nonEmptyPanes.isEmpty(renamedContext.state()).read();
+ if (isEmpty
+ && windowingStrategy.getClosingBehavior() == ClosingBehavior.FIRE_IF_NON_EMPTY
+ && windowingStrategy.getOnTimeBehavior() == Window.OnTimeBehavior.FIRE_IF_NON_EMPTY) {
+ return newHold;
+ }
+ Instant inputWM = timerInternals.currentInputWatermarkTime();
if (newHold != null) {
// We can't be finished yet.
checkState(
@@ -995,6 +1038,9 @@
}
}
+ // Calculate the pane info.
+ final PaneInfo pane = paneInfoTracker.getNextPaneInfo(directContext, isFinished).read();
+
// Only emit a pane if it has data or empty panes are observable.
if (needToEmit(isEmpty, isFinished, pane.getTiming())) {
// Run reduceFn.onTrigger method.
@@ -1005,9 +1051,11 @@
@Override
public void output(OutputT toOutput) {
// We're going to output panes, so commit the (now used) PaneInfo.
- // TODO: This is unnecessary if the trigger isFinished since the saved
+ // This is unnecessary if the trigger isFinished since the saved
// state will be immediately deleted.
- paneInfoTracker.storeCurrentPaneInfo(directContext, pane);
+ if (!isFinished) {
+ paneInfoTracker.storeCurrentPaneInfo(directContext, pane);
+ }
// Output the actual value.
outputter.outputWindowedValue(
@@ -1081,4 +1129,9 @@
}
}
+ private boolean windowIsExpired(BoundedWindow w) {
+ return timerInternals
+ .currentInputWatermarkTime()
+ .isAfter(w.maxTimestamp().plus(windowingStrategy.getAllowedLateness()));
+ }
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
index 539b9f0..3b37702 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
@@ -174,7 +174,7 @@
ValueState<Iterable<WindowedValue<?>>> state =
stateInternals.state(StateNamespaces.window(windowCoder, window), stateTag);
- Iterable<WindowedValue<?>> elements = state.read();
+ @Nullable Iterable<WindowedValue<?>> elements = state.read();
if (elements == null) {
elements = Collections.emptyList();
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 65384da..c3bfef6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -20,16 +20,14 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.State;
@@ -38,20 +36,13 @@
import org.apache.beam.sdk.state.Timer;
import org.apache.beam.sdk.state.TimerSpec;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
-import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
-import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.SystemDoFnInternal;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
@@ -74,18 +65,19 @@
*/
public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
+ private final PipelineOptions options;
/** The {@link DoFn} being run. */
private final DoFn<InputT, OutputT> fn;
/** The {@link DoFnInvoker} being run. */
private final DoFnInvoker<InputT, OutputT> invoker;
- /** The context used for running the {@link DoFn}. */
- private final DoFnContext<InputT, OutputT> context;
-
+ private final SideInputReader sideInputReader;
private final OutputManager outputManager;
private final TupleTag<OutputT> mainOutputTag;
+ /** The set of known output tags. */
+ private final Set<TupleTag<?>> outputTags;
private final boolean observesWindow;
@@ -107,12 +99,16 @@
List<TupleTag<?>> additionalOutputTags,
StepContext stepContext,
WindowingStrategy<?, ?> windowingStrategy) {
+ this.options = options;
this.fn = fn;
this.signature = DoFnSignatures.getSignature(fn.getClass());
this.observesWindow = signature.processElement().observesWindow() || !sideInputReader.isEmpty();
this.invoker = DoFnInvokers.invokerFor(fn);
+ this.sideInputReader = sideInputReader;
this.outputManager = outputManager;
this.mainOutputTag = mainOutputTag;
+ this.outputTags =
+ Sets.newHashSet(FluentIterable.<TupleTag<?>>of(mainOutputTag).append(additionalOutputTags));
this.stepContext = stepContext;
// This is a cast of an _invariant_ coder. But we are assured by pipeline validation
@@ -122,26 +118,13 @@
(Coder<BoundedWindow>) windowingStrategy.getWindowFn().windowCoder();
this.windowCoder = untypedCoder;
this.allowedLateness = windowingStrategy.getAllowedLateness();
-
- this.context =
- new DoFnContext<>(
- options,
- fn,
- sideInputReader,
- outputManager,
- mainOutputTag,
- additionalOutputTags,
- stepContext,
- windowingStrategy.getWindowFn());
}
@Override
public void startBundle() {
- DoFnStartBundleContext<InputT, OutputT> startBundleContext =
- createStartBundleContext(fn, context);
// This can contain user code. Wrap it in case it throws an exception.
try {
- invoker.invokeStartBundle(startBundleContext);
+ invoker.invokeStartBundle(new DoFnStartBundleContext());
} catch (Throwable t) {
// Exception in user code.
throw wrapUserCodeException(t);
@@ -175,7 +158,7 @@
case PROCESSING_TIME:
case SYNCHRONIZED_PROCESSING_TIME:
- effectiveTimestamp = context.stepContext.timerInternals().currentInputWatermarkTime();
+ effectiveTimestamp = stepContext.timerInternals().currentInputWatermarkTime();
break;
default:
@@ -183,18 +166,15 @@
String.format("Unknown time domain: %s", timeDomain));
}
- OnTimerArgumentProvider<InputT, OutputT> argumentProvider =
- new OnTimerArgumentProvider<>(
- fn, context, window, allowedLateness, effectiveTimestamp, timeDomain);
+ OnTimerArgumentProvider argumentProvider =
+ new OnTimerArgumentProvider(window, effectiveTimestamp, timeDomain);
invoker.invokeOnTimer(timerId, argumentProvider);
}
private void invokeProcessElement(WindowedValue<InputT> elem) {
- final DoFnProcessContext<InputT, OutputT> processContext = createProcessContext(elem);
-
// This can contain user code. Wrap it in case it throws an exception.
try {
- invoker.invokeProcessElement(processContext);
+ invoker.invokeProcessElement(new DoFnProcessContext(elem));
} catch (Exception ex) {
throw wrapUserCodeException(ex);
}
@@ -202,32 +182,15 @@
@Override
public void finishBundle() {
- DoFnFinishBundleContext<InputT, OutputT> finishBundleContext =
- createFinishBundleContext(fn, context);
// This can contain user code. Wrap it in case it throws an exception.
try {
- invoker.invokeFinishBundle(finishBundleContext);
+ invoker.invokeFinishBundle(new DoFnFinishBundleContext());
} catch (Throwable t) {
// Exception in user code.
throw wrapUserCodeException(t);
}
}
- private DoFnStartBundleContext<InputT, OutputT> createStartBundleContext(
- DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
- return new DoFnStartBundleContext<>(fn, context);
- }
-
- private DoFnFinishBundleContext<InputT, OutputT> createFinishBundleContext(
- DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
- return new DoFnFinishBundleContext<>(fn, context);
- }
-
- /** Returns a new {@link DoFn.ProcessContext} for the given element. */
- private DoFnProcessContext<InputT, OutputT> createProcessContext(WindowedValue<InputT> elem) {
- return new DoFnProcessContext<InputT, OutputT>(fn, context, elem, allowedLateness);
- }
-
private RuntimeException wrapUserCodeException(Throwable t) {
throw UserCodeException.wrapIf(!isSystemDoFn(), t);
}
@@ -236,176 +199,31 @@
return invoker.getClass().isAnnotationPresent(SystemDoFnInternal.class);
}
- /**
- * A concrete implementation of {@code DoFn.Context} used for running a {@link DoFn}.
- *
- * @param <InputT> the type of the {@link DoFn} (main) input elements
- * @param <OutputT> the type of the {@link DoFn} (main) output elements
- */
- private static class DoFnContext<InputT, OutputT> {
- private static final int MAX_SIDE_OUTPUTS = 1000;
-
- final PipelineOptions options;
- final DoFn<InputT, OutputT> fn;
- final SideInputReader sideInputReader;
- final OutputManager outputManager;
- final TupleTag<OutputT> mainOutputTag;
- final StepContext stepContext;
- final WindowFn<?, ?> windowFn;
-
- /**
- * The set of known output tags, some of which may be undeclared, so we can throw an exception
- * when it exceeds {@link #MAX_SIDE_OUTPUTS}.
- */
- private Set<TupleTag<?>> outputTags;
-
- public DoFnContext(
- PipelineOptions options,
- DoFn<InputT, OutputT> fn,
- SideInputReader sideInputReader,
- OutputManager outputManager,
- TupleTag<OutputT> mainOutputTag,
- List<TupleTag<?>> additionalOutputTags,
- StepContext stepContext,
- WindowFn<?, ?> windowFn) {
- this.options = options;
- this.fn = fn;
- this.sideInputReader = sideInputReader;
- this.outputManager = outputManager;
- this.mainOutputTag = mainOutputTag;
- this.outputTags = Sets.newHashSet();
-
- outputTags.add(mainOutputTag);
- for (TupleTag<?> additionalOutputTag : additionalOutputTags) {
- outputTags.add(additionalOutputTag);
- }
-
- this.stepContext = stepContext;
- this.windowFn = windowFn;
+ private <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
+ if (!sideInputReader.contains(view)) {
+ throw new IllegalArgumentException("calling sideInput() with unknown view");
}
-
- //////////////////////////////////////////////////////////////////////////////
-
- public PipelineOptions getPipelineOptions() {
- return options;
- }
-
- <T, W extends BoundedWindow> WindowedValue<T> makeWindowedValue(
- T output, Instant timestamp, Collection<W> windows, PaneInfo pane) {
- final Instant inputTimestamp = timestamp;
-
- if (timestamp == null) {
- timestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
- }
-
- if (windows == null) {
- try {
- // The windowFn can never succeed at accessing the element, so its type does not
- // matter here
- @SuppressWarnings("unchecked")
- WindowFn<Object, W> objectWindowFn = (WindowFn<Object, W>) windowFn;
- windows =
- objectWindowFn.assignWindows(
- objectWindowFn.new AssignContext() {
- @Override
- public Object element() {
- throw new UnsupportedOperationException(
- "WindowFn attempted to access input element when none was available");
- }
-
- @Override
- public Instant timestamp() {
- if (inputTimestamp == null) {
- throw new UnsupportedOperationException(
- "WindowFn attempted to access input timestamp when none was available");
- }
- return inputTimestamp;
- }
-
- @Override
- public W window() {
- throw new UnsupportedOperationException(
- "WindowFn attempted to access input windows when none were available");
- }
- });
- } catch (Exception e) {
- throw UserCodeException.wrap(e);
- }
- }
-
- return WindowedValue.of(output, timestamp, windows, pane);
- }
-
- public <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
- if (!sideInputReader.contains(view)) {
- throw new IllegalArgumentException("calling sideInput() with unknown view");
- }
- return sideInputReader.get(view, sideInputWindow);
- }
-
- void outputWindowedValue(
- OutputT output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- outputWindowedValue(makeWindowedValue(output, timestamp, windows, pane));
- }
-
- void outputWindowedValue(WindowedValue<OutputT> windowedElem) {
- outputManager.output(mainOutputTag, windowedElem);
- if (stepContext != null) {
- stepContext.noteOutput(windowedElem);
- }
- }
-
- private <T> void outputWindowedValue(
- TupleTag<T> tag,
- T output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- outputWindowedValue(tag, makeWindowedValue(output, timestamp, windows, pane));
- }
-
- private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedElem) {
- if (!outputTags.contains(tag)) {
- // This tag wasn't declared nor was it seen before during this execution.
- // Thus, this must be a new, undeclared and unconsumed output.
- // To prevent likely user errors, enforce the limit on the number of side
- // outputs.
- if (outputTags.size() >= MAX_SIDE_OUTPUTS) {
- throw new IllegalArgumentException(
- "the number of outputs has exceeded a limit of " + MAX_SIDE_OUTPUTS);
- }
- outputTags.add(tag);
- }
-
- outputManager.output(tag, windowedElem);
- if (stepContext != null) {
- stepContext.noteOutput(tag, windowedElem);
- }
- }
+ return sideInputReader.get(view, sideInputWindow);
}
+ private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedElem) {
+ checkArgument(outputTags.contains(tag), "Unknown output tag %s", tag);
+ outputManager.output(tag, windowedElem);
+ }
/**
* A concrete implementation of {@link DoFn.StartBundleContext}.
*/
- private class DoFnStartBundleContext<InputT, OutputT>
+ private class DoFnStartBundleContext
extends DoFn<InputT, OutputT>.StartBundleContext
implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
- private final DoFn<InputT, OutputT> fn;
- private final DoFnContext<InputT, OutputT> context;
-
- private DoFnStartBundleContext(DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ private DoFnStartBundleContext() {
fn.super();
- this.fn = fn;
- this.context = context;
}
@Override
public PipelineOptions getPipelineOptions() {
- return context.getPipelineOptions();
+ return options;
}
@Override
@@ -415,24 +233,30 @@
}
@Override
- public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ public PipelineOptions pipelineOptions() {
+ return getPipelineOptions();
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
return this;
}
@Override
- public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access FinishBundleContext outside of @FinishBundle method.");
}
@Override
- public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access ProcessContext outside of @ProcessElement method.");
}
@Override
- public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access OnTimerContext outside of @OnTimer methods.");
}
@@ -460,20 +284,16 @@
* B
* A concrete implementation of {@link DoFn.FinishBundleContext}.
*/
- private class DoFnFinishBundleContext<InputT, OutputT>
+ private class DoFnFinishBundleContext
extends DoFn<InputT, OutputT>.FinishBundleContext
implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
- private final DoFnContext<InputT, OutputT> context;
-
- private DoFnFinishBundleContext(
- DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ private DoFnFinishBundleContext() {
fn.super();
- this.context = context;
}
@Override
public PipelineOptions getPipelineOptions() {
- return context.getPipelineOptions();
+ return options;
}
@Override
@@ -483,24 +303,30 @@
}
@Override
- public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ public PipelineOptions pipelineOptions() {
+ return getPipelineOptions();
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access StartBundleContext outside of @StartBundle method.");
}
@Override
- public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
return this;
}
@Override
- public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access ProcessContext outside of @ProcessElement method.");
}
@Override
- public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access OnTimerContext outside of @OnTimer methods.");
}
@@ -525,30 +351,22 @@
@Override
public void output(OutputT output, Instant timestamp, BoundedWindow window) {
- context.outputWindowedValue(WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ output(mainOutputTag, output, timestamp, window);
}
@Override
public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
- context.outputWindowedValue(
- tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ outputWindowedValue(tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
}
}
/**
* A concrete implementation of {@link DoFn.ProcessContext} used for running a {@link DoFn} over a
* single element.
- *
- * @param <InputT> the type of the {@link DoFn} (main) input elements
- * @param <OutputT> the type of the {@link DoFn} (main) output elements
*/
- private class DoFnProcessContext<InputT, OutputT> extends DoFn<InputT, OutputT>.ProcessContext
+ private class DoFnProcessContext extends DoFn<InputT, OutputT>.ProcessContext
implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
- final DoFn<InputT, OutputT> fn;
- final DoFnContext<InputT, OutputT> context;
- final WindowedValue<InputT> windowedValue;
- private final Duration allowedLateness;
+ final WindowedValue<InputT> elem;
/** Lazily initialized; should only be accessed via {@link #getNamespace()}. */
@Nullable private StateNamespace namespace;
@@ -556,7 +374,7 @@
/**
* The state namespace for this context.
*
- * <p>Any call to {@link #getNamespace()} when more than one window is present will crash; this
+ * <p>Any call to this method when more than one window is present will crash; this
* represents a bug in the runner or the {@link DoFnSignature}, since values must be in exactly
* one window when state or timers are relevant.
*/
@@ -568,55 +386,32 @@
}
private DoFnProcessContext(
- DoFn<InputT, OutputT> fn,
- DoFnContext<InputT, OutputT> context,
- WindowedValue<InputT> windowedValue,
- Duration allowedLateness) {
+ WindowedValue<InputT> elem) {
fn.super();
- this.fn = fn;
- this.context = context;
- this.windowedValue = windowedValue;
- this.allowedLateness = allowedLateness;
+ this.elem = elem;
}
@Override
public PipelineOptions getPipelineOptions() {
- return context.getPipelineOptions();
+ return options;
}
@Override
public InputT element() {
- return windowedValue.getValue();
+ return elem.getValue();
}
@Override
public <T> T sideInput(PCollectionView<T> view) {
checkNotNull(view, "View passed to sideInput cannot be null");
- Iterator<? extends BoundedWindow> windowIter = windows().iterator();
- BoundedWindow window;
- if (!windowIter.hasNext()) {
- if (context.windowFn instanceof GlobalWindows) {
- // TODO: Remove this once GroupByKeyOnly no longer outputs elements
- // without windows
- window = GlobalWindow.INSTANCE;
- } else {
- throw new IllegalStateException(
- "sideInput called when main input element is not in any windows");
- }
- } else {
- window = windowIter.next();
- if (windowIter.hasNext()) {
- throw new IllegalStateException(
- "sideInput called when main input element is in multiple windows");
- }
- }
- return context.sideInput(
+ BoundedWindow window = Iterables.getOnlyElement(windows());
+ return SimpleDoFnRunner.this.sideInput(
view, view.getWindowMappingFn().getSideInputWindow(window));
}
@Override
public PaneInfo pane() {
- return windowedValue.getPane();
+ return elem.getPane();
}
@Override
@@ -626,37 +421,36 @@
@Override
public void output(OutputT output) {
- context.outputWindowedValue(windowedValue.withValue(output));
+ output(mainOutputTag, output);
}
@Override
public void outputWithTimestamp(OutputT output, Instant timestamp) {
checkTimestamp(timestamp);
- context.outputWindowedValue(
- output, timestamp, windowedValue.getWindows(), windowedValue.getPane());
+ outputWithTimestamp(mainOutputTag, output, timestamp);
}
@Override
public <T> void output(TupleTag<T> tag, T output) {
checkNotNull(tag, "Tag passed to output cannot be null");
- context.outputWindowedValue(tag, windowedValue.withValue(output));
+ outputWindowedValue(tag, elem.withValue(output));
}
@Override
public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
checkNotNull(tag, "Tag passed to outputWithTimestamp cannot be null");
checkTimestamp(timestamp);
- context.outputWindowedValue(
- tag, output, timestamp, windowedValue.getWindows(), windowedValue.getPane());
+ outputWindowedValue(
+ tag, WindowedValue.of(output, timestamp, elem.getWindows(), elem.getPane()));
}
@Override
public Instant timestamp() {
- return windowedValue.getTimestamp();
+ return elem.getTimestamp();
}
public Collection<? extends BoundedWindow> windows() {
- return windowedValue.getWindows();
+ return elem.getWindows();
}
@SuppressWarnings("deprecation") // Allowed Skew is deprecated for users, but must be respected
@@ -664,7 +458,7 @@
// The documentation of getAllowedTimestampSkew explicitly permits Long.MAX_VALUE to be used
// for infinite skew. Defend against underflow in that case for timestamps before the epoch
if (fn.getAllowedTimestampSkew().getMillis() != Long.MAX_VALUE
- && timestamp.isBefore(windowedValue.getTimestamp().minus(fn.getAllowedTimestampSkew()))) {
+ && timestamp.isBefore(elem.getTimestamp().minus(fn.getAllowedTimestampSkew()))) {
throw new IllegalArgumentException(
String.format(
"Cannot output with timestamp %s. Output timestamps must be no earlier than the "
@@ -672,23 +466,29 @@
+ "DoFn#getAllowedTimestampSkew() Javadoc for details on changing the allowed "
+ "skew.",
timestamp,
- windowedValue.getTimestamp(),
+ elem.getTimestamp(),
PeriodFormat.getDefault().print(fn.getAllowedTimestampSkew().toPeriod())));
}
}
@Override
public BoundedWindow window() {
- return Iterables.getOnlyElement(windowedValue.getWindows());
+ return Iterables.getOnlyElement(elem.getWindows());
}
@Override
- public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ public PipelineOptions pipelineOptions() {
+ return getPipelineOptions();
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
}
@Override
- public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
}
@@ -698,7 +498,7 @@
}
@Override
- public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
"Cannot access OnTimerContext outside of @OnTimer methods.");
}
@@ -727,7 +527,7 @@
TimerSpec spec =
(TimerSpec) signature.timerDeclarations().get(timerId).field().get(fn);
return new TimerInternalsTimer(
- window(), getNamespace(), allowedLateness, timerId, spec, stepContext.timerInternals());
+ window(), getNamespace(), timerId, spec, stepContext.timerInternals());
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
@@ -737,20 +537,13 @@
/**
* A concrete implementation of {@link DoFnInvoker.ArgumentProvider} used for running a {@link
* DoFn} on a timer.
- *
- * @param <InputT> the type of the {@link DoFn} (main) input elements
- * @param <OutputT> the type of the {@link DoFn} (main) output elements
*/
- private class OnTimerArgumentProvider<InputT, OutputT>
+ private class OnTimerArgumentProvider
extends DoFn<InputT, OutputT>.OnTimerContext
implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
- final DoFn<InputT, OutputT> fn;
- final DoFnContext<InputT, OutputT> context;
private final BoundedWindow window;
private final Instant timestamp;
private final TimeDomain timeDomain;
- private final Duration allowedLateness;
/** Lazily initialized; should only be accessed via {@link #getNamespace()}. */
private StateNamespace namespace;
@@ -758,7 +551,7 @@
/**
* The state namespace for this context.
*
- * <p>Any call to {@link #getNamespace()} when more than one window is present will crash; this
+ * <p>Any call to this method when more than one window is present will crash; this
* represents a bug in the runner or the {@link DoFnSignature}, since values must be in exactly
* one window when state or timers are relevant.
*/
@@ -770,17 +563,11 @@
}
private OnTimerArgumentProvider(
- DoFn<InputT, OutputT> fn,
- DoFnContext<InputT, OutputT> context,
BoundedWindow window,
- Duration allowedLateness,
Instant timestamp,
TimeDomain timeDomain) {
fn.super();
- this.fn = fn;
- this.context = context;
this.window = window;
- this.allowedLateness = allowedLateness;
this.timestamp = timestamp;
this.timeDomain = timeDomain;
}
@@ -796,12 +583,18 @@
}
@Override
- public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ public PipelineOptions pipelineOptions() {
+ return getPipelineOptions();
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
}
@Override
- public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
}
@@ -812,12 +605,12 @@
@Override
- public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("ProcessContext parameters are not supported.");
}
@Override
- public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
return this;
}
@@ -845,7 +638,7 @@
TimerSpec spec =
(TimerSpec) signature.timerDeclarations().get(timerId).field().get(fn);
return new TimerInternalsTimer(
- window, getNamespace(), allowedLateness, timerId, spec, stepContext.timerInternals());
+ window, getNamespace(), timerId, spec, stepContext.timerInternals());
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
@@ -853,42 +646,37 @@
@Override
public PipelineOptions getPipelineOptions() {
- return context.getPipelineOptions();
+ return options;
}
@Override
public void output(OutputT output) {
- context.outputWindowedValue(
- output, timestamp(), Collections.singleton(window()), PaneInfo.NO_FIRING);
+ output(mainOutputTag, output);
}
@Override
public void outputWithTimestamp(OutputT output, Instant timestamp) {
- context.outputWindowedValue(
- output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+ outputWithTimestamp(mainOutputTag, output, timestamp);
}
@Override
public <T> void output(TupleTag<T> tag, T output) {
- context.outputWindowedValue(
- tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+ outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
}
@Override
public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
- context.outputWindowedValue(
- tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+ outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
}
}
- private static class TimerInternalsTimer implements Timer {
+ private class TimerInternalsTimer implements Timer {
private final TimerInternals timerInternals;
// The window and the namespace represent the same thing, but the namespace is a cached
// and specially encoded form. Since the namespace can be cached across timers, it is
// passed in whole rather than being computed here.
private final BoundedWindow window;
- private final Duration allowedLateness;
private final StateNamespace namespace;
private final String timerId;
private final TimerSpec spec;
@@ -898,12 +686,10 @@
public TimerInternalsTimer(
BoundedWindow window,
StateNamespace namespace,
- Duration allowedLateness,
String timerId,
TimerSpec spec,
TimerInternals timerInternals) {
this.window = window;
- this.allowedLateness = allowedLateness;
this.namespace = namespace;
this.timerId = timerId;
this.spec = spec;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
deleted file mode 100644
index 6503fa2..0000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ /dev/null
@@ -1,602 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
-import java.util.List;
-import java.util.UUID;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.state.TimeDomain;
-import org.apache.beam.sdk.state.ValueState;
-import org.apache.beam.sdk.state.WatermarkHoldState;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
-import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.joda.time.Instant;
-
-/**
- * A utility transform that executes a <a
- * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn} by expanding it into a
- * network of simpler transforms:
- *
- * <ol>
- * <li>Pair each element with an initial restriction
- * <li>Split each restriction into sub-restrictions
- * <li>Assign a unique key to each element/restriction pair
- * <li>Group by key (so that work is partitioned by key and we can access state/timers)
- * <li>Process each keyed element/restriction pair with the splittable {@link DoFn}'s {@link
- * DoFn.ProcessElement} method, using state and timers API.
- * </ol>
- *
- * <p>This transform is intended as a helper for internal use by runners when implementing {@code
- * ParDo.of(splittable DoFn)}, but not for direct use by pipeline writers.
- */
-@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
-public class SplittableParDo<InputT, OutputT, RestrictionT>
- extends PTransform<PCollection<InputT>, PCollectionTuple> {
- private final ParDo.MultiOutput<InputT, OutputT> parDo;
-
- /**
- * Creates the transform for the given original multi-output {@link ParDo}.
- *
- * @param parDo The splittable {@link ParDo} transform.
- */
- public SplittableParDo(ParDo.MultiOutput<InputT, OutputT> parDo) {
- checkNotNull(parDo, "parDo must not be null");
- this.parDo = parDo;
- checkArgument(
- DoFnSignatures.getSignature(parDo.getFn().getClass()).processElement().isSplittable(),
- "fn must be a splittable DoFn");
- }
-
- @Override
- public PCollectionTuple expand(PCollection<InputT> input) {
- return applyTyped(input);
- }
-
- private PCollectionTuple applyTyped(PCollection<InputT> input) {
- DoFn<InputT, OutputT> fn = parDo.getFn();
- Coder<RestrictionT> restrictionCoder =
- DoFnInvokers.invokerFor(fn)
- .invokeGetRestrictionCoder(input.getPipeline().getCoderRegistry());
- PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> keyedWorkItems =
- applySplitIntoKeyedWorkItems(input, fn, restrictionCoder);
- return keyedWorkItems.apply(
- "Process",
- new ProcessElements<>(
- fn,
- input.getCoder(),
- restrictionCoder,
- (WindowingStrategy<InputT, ?>) input.getWindowingStrategy(),
- parDo.getSideInputs(),
- parDo.getMainOutputTag(),
- parDo.getAdditionalOutputTags()));
- }
-
- private static <InputT, OutputT, RestrictionT>
- PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
- applySplitIntoKeyedWorkItems(
- PCollection<InputT> input,
- DoFn<InputT, OutputT> fn,
- Coder<RestrictionT> restrictionCoder) {
- Coder<ElementAndRestriction<InputT, RestrictionT>> splitCoder =
- ElementAndRestrictionCoder.of(input.getCoder(), restrictionCoder);
-
- PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> keyedWorkItems =
- input
- .apply(
- "Pair with initial restriction",
- ParDo.of(new PairWithRestrictionFn<InputT, OutputT, RestrictionT>(fn)))
- .setCoder(splitCoder)
- .apply("Split restriction", ParDo.of(new SplitRestrictionFn<InputT, RestrictionT>(fn)))
- .setCoder(splitCoder)
- // ProcessFn requires all input elements to be in a single window and have a single
- // element per work item. This must precede the unique keying so each key has a single
- // associated element.
- .apply(
- "Explode windows",
- ParDo.of(new ExplodeWindowsFn<ElementAndRestriction<InputT, RestrictionT>>()))
- .apply(
- "Assign unique key",
- WithKeys.of(new RandomUniqueKeyFn<ElementAndRestriction<InputT, RestrictionT>>()))
- .apply(
- "Group by key",
- new GBKIntoKeyedWorkItems<String, ElementAndRestriction<InputT, RestrictionT>>())
- .setCoder(
- KeyedWorkItemCoder.of(
- StringUtf8Coder.of(),
- splitCoder,
- input.getWindowingStrategy().getWindowFn().windowCoder()));
- checkArgument(
- keyedWorkItems.getWindowingStrategy().getWindowFn() instanceof GlobalWindows,
- "GBKIntoKeyedWorkItems must produce a globally windowed collection, "
- + "but windowing strategy was: %s",
- keyedWorkItems.getWindowingStrategy());
- return keyedWorkItems;
- }
-
- /**
- * A {@link DoFn} that forces each of its outputs to be in a single window, by indicating to the
- * runner that it observes the window of its input element, so the runner is forced to apply it to
- * each input in a single window and thus its output is also in a single window.
- */
- private static class ExplodeWindowsFn<InputT> extends DoFn<InputT, InputT> {
- @ProcessElement
- public void process(ProcessContext c, BoundedWindow window) {
- c.output(c.element());
- }
- }
-
- /**
- * Runner-specific primitive {@link GroupByKey GroupByKey-like} {@link PTransform} that produces
- * {@link KeyedWorkItem KeyedWorkItems} so that downstream transforms can access state and timers.
- *
- * <p>Unlike a real {@link GroupByKey}, ignores the input's windowing and triggering strategy and
- * emits output immediately.
- */
- public static class GBKIntoKeyedWorkItems<KeyT, InputT>
- extends PTransform<PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>> {
- @Override
- public PCollection<KeyedWorkItem<KeyT, InputT>> expand(PCollection<KV<KeyT, InputT>> input) {
- return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded());
- }
- }
-
- /**
- * Runner-specific primitive {@link PTransform} that invokes the {@link DoFn.ProcessElement}
- * method for a splittable {@link DoFn}.
- */
- public static class ProcessElements<
- InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
- extends PTransform<
- PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>,
- PCollectionTuple> {
- private final DoFn<InputT, OutputT> fn;
- private final Coder<InputT> elementCoder;
- private final Coder<RestrictionT> restrictionCoder;
- private final WindowingStrategy<InputT, ?> windowingStrategy;
- private final List<PCollectionView<?>> sideInputs;
- private final TupleTag<OutputT> mainOutputTag;
- private final TupleTagList additionalOutputTags;
-
- /**
- * @param fn the splittable {@link DoFn}.
- * @param windowingStrategy the {@link WindowingStrategy} of the input collection.
- * @param sideInputs list of side inputs that should be available to the {@link DoFn}.
- * @param mainOutputTag {@link TupleTag Tag} of the {@link DoFn DoFn's} main output.
- * @param additionalOutputTags {@link TupleTagList Tags} of the {@link DoFn DoFn's} additional
- * outputs.
- */
- public ProcessElements(
- DoFn<InputT, OutputT> fn,
- Coder<InputT> elementCoder,
- Coder<RestrictionT> restrictionCoder,
- WindowingStrategy<InputT, ?> windowingStrategy,
- List<PCollectionView<?>> sideInputs,
- TupleTag<OutputT> mainOutputTag,
- TupleTagList additionalOutputTags) {
- this.fn = fn;
- this.elementCoder = elementCoder;
- this.restrictionCoder = restrictionCoder;
- this.windowingStrategy = windowingStrategy;
- this.sideInputs = sideInputs;
- this.mainOutputTag = mainOutputTag;
- this.additionalOutputTags = additionalOutputTags;
- }
-
- public DoFn<InputT, OutputT> getFn() {
- return fn;
- }
-
- public List<PCollectionView<?>> getSideInputs() {
- return sideInputs;
- }
-
- public TupleTag<OutputT> getMainOutputTag() {
- return mainOutputTag;
- }
-
- public TupleTagList getAdditionalOutputTags() {
- return additionalOutputTags;
- }
-
- public ProcessFn<InputT, OutputT, RestrictionT, TrackerT> newProcessFn(
- DoFn<InputT, OutputT> fn) {
- return new SplittableParDo.ProcessFn<>(
- fn, elementCoder, restrictionCoder, windowingStrategy);
- }
-
- @Override
- public PCollectionTuple expand(
- PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
- input) {
- DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
- PCollectionTuple outputs =
- PCollectionTuple.ofPrimitiveOutputsInternal(
- input.getPipeline(),
- TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
- windowingStrategy,
- input.isBounded().and(signature.isBoundedPerElement()));
-
- // Set output type descriptor similarly to how ParDo.MultiOutput does it.
- outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor());
-
- return outputs;
- }
-
- @Override
- public <T> Coder<T> getDefaultOutputCoder(
- PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
- input,
- PCollection<T> output)
- throws CannotProvideCoderException {
- // Similar logic to ParDo.MultiOutput.getOutputCoder.
- @SuppressWarnings("unchecked")
- KeyedWorkItemCoder<String, ElementAndRestriction<InputT, RestrictionT>> kwiCoder =
- (KeyedWorkItemCoder) input.getCoder();
- Coder<InputT> inputCoder =
- ((ElementAndRestrictionCoder<InputT, RestrictionT>) kwiCoder.getElementCoder())
- .getElementCoder();
- return input
- .getPipeline()
- .getCoderRegistry()
- .getCoder(output.getTypeDescriptor(), fn.getInputTypeDescriptor(), inputCoder);
- }
- }
-
- /**
- * Assigns a random unique key to each element of the input collection, so that the output
- * collection is effectively the same elements as input, but the per-key state and timers are now
- * effectively per-element.
- */
- private static class RandomUniqueKeyFn<T> implements SerializableFunction<T, String> {
- @Override
- public String apply(T input) {
- return UUID.randomUUID().toString();
- }
- }
-
- /**
- * Pairs each input element with its initial restriction using the given splittable {@link DoFn}.
- */
- private static class PairWithRestrictionFn<InputT, OutputT, RestrictionT>
- extends DoFn<InputT, ElementAndRestriction<InputT, RestrictionT>> {
- private DoFn<InputT, OutputT> fn;
- private transient DoFnInvoker<InputT, OutputT> invoker;
-
- PairWithRestrictionFn(DoFn<InputT, OutputT> fn) {
- this.fn = fn;
- }
-
- @Setup
- public void setup() {
- invoker = DoFnInvokers.invokerFor(fn);
- }
-
- @ProcessElement
- public void processElement(ProcessContext context) {
- context.output(
- ElementAndRestriction.of(
- context.element(),
- invoker.<RestrictionT>invokeGetInitialRestriction(context.element())));
- }
- }
-
- /**
- * The heart of splittable {@link DoFn} execution: processes a single (element, restriction) pair
- * by creating a tracker for the restriction and checkpointing/resuming processing later if
- * necessary.
- *
- * <p>Takes {@link KeyedWorkItem} and assumes that the KeyedWorkItem contains a single element
- * (or a single timer set by {@link ProcessFn itself}, in a single window. This is necessary
- * because {@link ProcessFn} sets timers, and timers are namespaced to a single window and it
- * should be the window of the input element.
- *
- * <p>See also: https://issues.apache.org/jira/browse/BEAM-1983
- */
- @VisibleForTesting
- public static class ProcessFn<
- InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
- extends DoFn<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT> {
- /**
- * The state cell containing a watermark hold for the output of this {@link DoFn}. The hold is
- * acquired during the first {@link DoFn.ProcessElement} call for each element and restriction,
- * and is released when the {@link DoFn.ProcessElement} call returns and there is no residual
- * restriction captured by the {@link SplittableProcessElementInvoker}.
- *
- * <p>A hold is needed to avoid letting the output watermark immediately progress together with
- * the input watermark when the first {@link DoFn.ProcessElement} call for this element
- * completes.
- */
- private static final StateTag<WatermarkHoldState> watermarkHoldTag =
- StateTags.makeSystemTagInternal(
- StateTags.<GlobalWindow>watermarkStateInternal(
- "hold", TimestampCombiner.LATEST));
-
- /**
- * The state cell containing a copy of the element. Written during the first {@link
- * DoFn.ProcessElement} call and read during subsequent calls in response to timer firings, when
- * the original element is no longer available.
- */
- private final StateTag<ValueState<WindowedValue<InputT>>> elementTag;
-
- /**
- * The state cell containing a restriction representing the unprocessed part of work for this
- * element.
- */
- private StateTag<ValueState<RestrictionT>> restrictionTag;
-
- private final DoFn<InputT, OutputT> fn;
- private final Coder<InputT> elementCoder;
- private final Coder<RestrictionT> restrictionCoder;
- private final WindowingStrategy<InputT, ?> inputWindowingStrategy;
-
- private transient StateInternalsFactory<String> stateInternalsFactory;
- private transient TimerInternalsFactory<String> timerInternalsFactory;
- private transient SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT>
- processElementInvoker;
-
- private transient DoFnInvoker<InputT, OutputT> invoker;
-
- public ProcessFn(
- DoFn<InputT, OutputT> fn,
- Coder<InputT> elementCoder,
- Coder<RestrictionT> restrictionCoder,
- WindowingStrategy<InputT, ?> inputWindowingStrategy) {
- this.fn = fn;
- this.elementCoder = elementCoder;
- this.restrictionCoder = restrictionCoder;
- this.inputWindowingStrategy = inputWindowingStrategy;
- this.elementTag =
- StateTags.value(
- "element",
- WindowedValue.getFullCoder(
- elementCoder, inputWindowingStrategy.getWindowFn().windowCoder()));
- this.restrictionTag = StateTags.value("restriction", restrictionCoder);
- }
-
- public void setStateInternalsFactory(StateInternalsFactory<String> stateInternalsFactory) {
- this.stateInternalsFactory = stateInternalsFactory;
- }
-
- public void setTimerInternalsFactory(TimerInternalsFactory<String> timerInternalsFactory) {
- this.timerInternalsFactory = timerInternalsFactory;
- }
-
- public void setProcessElementInvoker(
- SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT> invoker) {
- this.processElementInvoker = invoker;
- }
-
- public DoFn<InputT, OutputT> getFn() {
- return fn;
- }
-
- public Coder<InputT> getElementCoder() {
- return elementCoder;
- }
-
- public Coder<RestrictionT> getRestrictionCoder() {
- return restrictionCoder;
- }
-
- public WindowingStrategy<InputT, ?> getInputWindowingStrategy() {
- return inputWindowingStrategy;
- }
-
- @Setup
- public void setup() throws Exception {
- invoker = DoFnInvokers.invokerFor(fn);
- invoker.invokeSetup();
- }
-
- @Teardown
- public void tearDown() throws Exception {
- invoker.invokeTeardown();
- }
-
- @StartBundle
- public void startBundle(StartBundleContext c) throws Exception {
- invoker.invokeStartBundle(wrapContextAsStartBundle(c));
- }
-
- @FinishBundle
- public void finishBundle(FinishBundleContext c) throws Exception {
- invoker.invokeFinishBundle(wrapContextAsFinishBundle(c));
- }
-
- @ProcessElement
- public void processElement(final ProcessContext c) {
- String key = c.element().key();
- StateInternals stateInternals =
- stateInternalsFactory.stateInternalsForKey(key);
- TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key);
-
- // Initialize state (element and restriction) depending on whether this is the seed call.
- // The seed call is the first call for this element, which actually has the element.
- // Subsequent calls are timer firings and the element has to be retrieved from the state.
- TimerInternals.TimerData timer = Iterables.getOnlyElement(c.element().timersIterable(), null);
- boolean isSeedCall = (timer == null);
- StateNamespace stateNamespace;
- if (isSeedCall) {
- WindowedValue<ElementAndRestriction<InputT, RestrictionT>> windowedValue =
- Iterables.getOnlyElement(c.element().elementsIterable());
- BoundedWindow window = Iterables.getOnlyElement(windowedValue.getWindows());
- stateNamespace =
- StateNamespaces.window(
- (Coder<BoundedWindow>) inputWindowingStrategy.getWindowFn().windowCoder(), window);
- } else {
- stateNamespace = timer.getNamespace();
- }
-
- ValueState<WindowedValue<InputT>> elementState =
- stateInternals.state(stateNamespace, elementTag);
- ValueState<RestrictionT> restrictionState =
- stateInternals.state(stateNamespace, restrictionTag);
- WatermarkHoldState holdState =
- stateInternals.state(stateNamespace, watermarkHoldTag);
-
- ElementAndRestriction<WindowedValue<InputT>, RestrictionT> elementAndRestriction;
- if (isSeedCall) {
- WindowedValue<ElementAndRestriction<InputT, RestrictionT>> windowedValue =
- Iterables.getOnlyElement(c.element().elementsIterable());
- WindowedValue<InputT> element = windowedValue.withValue(windowedValue.getValue().element());
- elementState.write(element);
- elementAndRestriction =
- ElementAndRestriction.of(element, windowedValue.getValue().restriction());
- } else {
- // This is not the first ProcessElement call for this element/restriction - rather,
- // this is a timer firing, so we need to fetch the element and restriction from state.
- elementState.readLater();
- restrictionState.readLater();
- elementAndRestriction =
- ElementAndRestriction.of(elementState.read(), restrictionState.read());
- }
-
- final TrackerT tracker = invoker.invokeNewTracker(elementAndRestriction.restriction());
- SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT>.Result result =
- processElementInvoker.invokeProcessElement(
- invoker, elementAndRestriction.element(), tracker);
-
- // Save state for resuming.
- if (result.getResidualRestriction() == null) {
- // All work for this element/restriction is completed. Clear state and release hold.
- elementState.clear();
- restrictionState.clear();
- holdState.clear();
- return;
- }
- restrictionState.write(result.getResidualRestriction());
- Instant futureOutputWatermark = result.getFutureOutputWatermark();
- if (futureOutputWatermark == null) {
- futureOutputWatermark = elementAndRestriction.element().getTimestamp();
- }
- holdState.add(futureOutputWatermark);
- // Set a timer to continue processing this element.
- timerInternals.setTimer(
- TimerInternals.TimerData.of(
- stateNamespace, timerInternals.currentProcessingTime(), TimeDomain.PROCESSING_TIME));
- }
-
- private DoFn<InputT, OutputT>.StartBundleContext wrapContextAsStartBundle(
- final StartBundleContext baseContext) {
- return fn.new StartBundleContext() {
- @Override
- public PipelineOptions getPipelineOptions() {
- return baseContext.getPipelineOptions();
- }
-
- private void throwUnsupportedOutput() {
- throw new UnsupportedOperationException(
- String.format(
- "Splittable DoFn can only output from @%s",
- ProcessElement.class.getSimpleName()));
- }
- };
- }
-
- private DoFn<InputT, OutputT>.FinishBundleContext wrapContextAsFinishBundle(
- final FinishBundleContext baseContext) {
- return fn.new FinishBundleContext() {
- @Override
- public void output(OutputT output, Instant timestamp, BoundedWindow window) {
- throwUnsupportedOutput();
- }
-
- @Override
- public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
- throwUnsupportedOutput();
- }
-
- @Override
- public PipelineOptions getPipelineOptions() {
- return baseContext.getPipelineOptions();
- }
-
- private void throwUnsupportedOutput() {
- throw new UnsupportedOperationException(
- String.format(
- "Splittable DoFn can only output from @%s",
- ProcessElement.class.getSimpleName()));
- }
- };
- }
-
- }
-
- /** Splits the restriction using the given {@link DoFn.SplitRestriction} method. */
- private static class SplitRestrictionFn<InputT, RestrictionT>
- extends DoFn<
- ElementAndRestriction<InputT, RestrictionT>,
- ElementAndRestriction<InputT, RestrictionT>> {
- private final DoFn<InputT, ?> splittableFn;
- private transient DoFnInvoker<InputT, ?> invoker;
-
- SplitRestrictionFn(DoFn<InputT, ?> splittableFn) {
- this.splittableFn = splittableFn;
- }
-
- @Setup
- public void setup() {
- invoker = DoFnInvokers.invokerFor(splittableFn);
- }
-
- @ProcessElement
- public void processElement(final ProcessContext c) {
- final InputT element = c.element().element();
- invoker.invokeSplitRestriction(
- element,
- c.element().restriction(),
- new OutputReceiver<RestrictionT>() {
- @Override
- public void output(RestrictionT part) {
- c.output(ElementAndRestriction.of(element, part));
- }
- });
- }
- }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
new file mode 100644
index 0000000..400df19
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.runners.core.construction.SplittableParDo.ProcessKeyedElements;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Instant;
+
+/**
+ * Utilities for implementing {@link ProcessKeyedElements} using {@link KeyedWorkItem} and
+ * runner-specific {@link StateInternals} and {@link TimerInternals}.
+ */
+public class SplittableParDoViaKeyedWorkItems {
+ /**
+ * Runner-specific primitive {@link GroupByKey GroupByKey-like} {@link PTransform} that produces
+ * {@link KeyedWorkItem KeyedWorkItems} so that downstream transforms can access state and timers.
+ *
+ * <p>Unlike a real {@link GroupByKey}, ignores the input's windowing and triggering strategy and
+ * emits output immediately.
+ */
+ public static class GBKIntoKeyedWorkItems<KeyT, InputT>
+ extends RawPTransform<
+ PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>> {
+ @Override
+ public PCollection<KeyedWorkItem<KeyT, InputT>> expand(PCollection<KV<KeyT, InputT>> input) {
+ KvCoder<KeyT, InputT> kvCoder = (KvCoder<KeyT, InputT>) input.getCoder();
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ input.isBounded(),
+ KeyedWorkItemCoder.of(
+ kvCoder.getKeyCoder(),
+ kvCoder.getValueCoder(),
+ input.getWindowingStrategy().getWindowFn().windowCoder()));
+ }
+
+ @Override
+ public String getUrn() {
+ return SplittableParDo.SPLITTABLE_GBKIKWI_URN;
+ }
+
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ throw new UnsupportedOperationException(
+ String.format("%s should never be serialized to proto", getClass().getSimpleName()));
+ }
+ }
+
+ /** Overrides a {@link ProcessKeyedElements} into {@link SplittableProcessViaKeyedWorkItems}. */
+ public static class OverrideFactory<InputT, OutputT, RestrictionT>
+ implements PTransformOverrideFactory<
+ PCollection<KV<String, KV<InputT, RestrictionT>>>, PCollectionTuple,
+ ProcessKeyedElements<InputT, OutputT, RestrictionT>> {
+ @Override
+ public PTransformReplacement<
+ PCollection<KV<String, KV<InputT, RestrictionT>>>, PCollectionTuple>
+ getReplacementTransform(
+ AppliedPTransform<
+ PCollection<KV<String, KV<InputT, RestrictionT>>>, PCollectionTuple,
+ ProcessKeyedElements<InputT, OutputT, RestrictionT>>
+ transform) {
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform),
+ new SplittableProcessViaKeyedWorkItems<>(transform.getTransform()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollectionTuple newOutput) {
+ return ReplacementOutputs.tagged(outputs, newOutput);
+ }
+ }
+
+ /**
+ * Runner-specific primitive {@link PTransform} that invokes the {@link DoFn.ProcessElement}
+ * method for a splittable {@link DoFn}.
+ */
+ public static class SplittableProcessViaKeyedWorkItems<InputT, OutputT, RestrictionT>
+ extends PTransform<PCollection<KV<String, KV<InputT, RestrictionT>>>, PCollectionTuple> {
+ private final ProcessKeyedElements<InputT, OutputT, RestrictionT> original;
+
+ public SplittableProcessViaKeyedWorkItems(
+ ProcessKeyedElements<InputT, OutputT, RestrictionT> original) {
+ this.original = original;
+ }
+
+ @Override
+ public PCollectionTuple expand(PCollection<KV<String, KV<InputT, RestrictionT>>> input) {
+ return input
+ .apply(new GBKIntoKeyedWorkItems<String, KV<InputT, RestrictionT>>())
+ .setCoder(
+ KeyedWorkItemCoder.of(
+ StringUtf8Coder.of(),
+ ((KvCoder<String, KV<InputT, RestrictionT>>) input.getCoder()).getValueCoder(),
+ input.getWindowingStrategy().getWindowFn().windowCoder()))
+ .apply(new ProcessElements<>(original));
+ }
+ }
+
+ /** A primitive transform wrapping around {@link ProcessFn}. */
+ public static class ProcessElements<
+ InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+ extends PTransform<
+ PCollection<KeyedWorkItem<String, KV<InputT, RestrictionT>>>, PCollectionTuple> {
+ private final ProcessKeyedElements<InputT, OutputT, RestrictionT> original;
+
+ public ProcessElements(ProcessKeyedElements<InputT, OutputT, RestrictionT> original) {
+ this.original = original;
+ }
+
+ public ProcessFn<InputT, OutputT, RestrictionT, TrackerT> newProcessFn(
+ DoFn<InputT, OutputT> fn) {
+ return new ProcessFn<>(
+ fn,
+ original.getElementCoder(),
+ original.getRestrictionCoder(),
+ original.getInputWindowingStrategy());
+ }
+
+ public DoFn<InputT, OutputT> getFn() {
+ return original.getFn();
+ }
+
+ public List<PCollectionView<?>> getSideInputs() {
+ return original.getSideInputs();
+ }
+
+ public TupleTag<OutputT> getMainOutputTag() {
+ return original.getMainOutputTag();
+ }
+
+ public TupleTagList getAdditionalOutputTags() {
+ return original.getAdditionalOutputTags();
+ }
+
+ @Override
+ public PCollectionTuple expand(
+ PCollection<KeyedWorkItem<String, KV<InputT, RestrictionT>>> input) {
+ return ProcessKeyedElements.createPrimitiveOutputFor(
+ input,
+ original.getFn(),
+ original.getMainOutputTag(),
+ original.getAdditionalOutputTags(),
+ original.getOutputTagsToCoders(),
+ original.getInputWindowingStrategy());
+ }
+ }
+
+ /**
+ * The heart of splittable {@link DoFn} execution: processes a single (element, restriction) pair
+ * by creating a tracker for the restriction and checkpointing/resuming processing later if
+ * necessary.
+ *
+ * <p>Takes {@link KeyedWorkItem} and assumes that the KeyedWorkItem contains a single element (or
+ * a single timer set by {@link ProcessFn itself}, in a single window. This is necessary because
+ * {@link ProcessFn} sets timers, and timers are namespaced to a single window and it should be
+ * the window of the input element.
+ *
+ * <p>See also: https://issues.apache.org/jira/browse/BEAM-1983
+ */
+ @VisibleForTesting
+ public static class ProcessFn<
+ InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+ extends DoFn<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> {
+ /**
+ * The state cell containing a watermark hold for the output of this {@link DoFn}. The hold is
+ * acquired during the first {@link DoFn.ProcessElement} call for each element and restriction,
+ * and is released when the {@link DoFn.ProcessElement} call returns {@link
+ * ProcessContinuation#stop()}.
+ *
+ * <p>A hold is needed to avoid letting the output watermark immediately progress together with
+ * the input watermark when the first {@link DoFn.ProcessElement} call for this element
+ * completes.
+ */
+ private static final StateTag<WatermarkHoldState> watermarkHoldTag =
+ StateTags.makeSystemTagInternal(
+ StateTags.<GlobalWindow>watermarkStateInternal("hold", TimestampCombiner.LATEST));
+
+ /**
+ * The state cell containing a copy of the element. Written during the first {@link
+ * DoFn.ProcessElement} call and read during subsequent calls in response to timer firings, when
+ * the original element is no longer available.
+ */
+ private final StateTag<ValueState<WindowedValue<InputT>>> elementTag;
+
+ /**
+ * The state cell containing a restriction representing the unprocessed part of work for this
+ * element.
+ */
+ private StateTag<ValueState<RestrictionT>> restrictionTag;
+
+ private final DoFn<InputT, OutputT> fn;
+ private final Coder<InputT> elementCoder;
+ private final Coder<RestrictionT> restrictionCoder;
+ private final WindowingStrategy<InputT, ?> inputWindowingStrategy;
+
+ private transient StateInternalsFactory<String> stateInternalsFactory;
+ private transient TimerInternalsFactory<String> timerInternalsFactory;
+ private transient SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT>
+ processElementInvoker;
+
+ private transient DoFnInvoker<InputT, OutputT> invoker;
+
+ public ProcessFn(
+ DoFn<InputT, OutputT> fn,
+ Coder<InputT> elementCoder,
+ Coder<RestrictionT> restrictionCoder,
+ WindowingStrategy<InputT, ?> inputWindowingStrategy) {
+ this.fn = fn;
+ this.elementCoder = elementCoder;
+ this.restrictionCoder = restrictionCoder;
+ this.inputWindowingStrategy = inputWindowingStrategy;
+ this.elementTag =
+ StateTags.value(
+ "element",
+ WindowedValue.getFullCoder(
+ elementCoder, inputWindowingStrategy.getWindowFn().windowCoder()));
+ this.restrictionTag = StateTags.value("restriction", restrictionCoder);
+ }
+
+ public void setStateInternalsFactory(StateInternalsFactory<String> stateInternalsFactory) {
+ this.stateInternalsFactory = stateInternalsFactory;
+ }
+
+ public void setTimerInternalsFactory(TimerInternalsFactory<String> timerInternalsFactory) {
+ this.timerInternalsFactory = timerInternalsFactory;
+ }
+
+ public void setProcessElementInvoker(
+ SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT> invoker) {
+ this.processElementInvoker = invoker;
+ }
+
+ public DoFn<InputT, OutputT> getFn() {
+ return fn;
+ }
+
+ public Coder<InputT> getElementCoder() {
+ return elementCoder;
+ }
+
+ public Coder<RestrictionT> getRestrictionCoder() {
+ return restrictionCoder;
+ }
+
+ public WindowingStrategy<InputT, ?> getInputWindowingStrategy() {
+ return inputWindowingStrategy;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ invoker = DoFnInvokers.invokerFor(fn);
+ invoker.invokeSetup();
+ }
+
+ @Teardown
+ public void tearDown() throws Exception {
+ invoker.invokeTeardown();
+ }
+
+ @StartBundle
+ public void startBundle(StartBundleContext c) throws Exception {
+ invoker.invokeStartBundle(wrapContextAsStartBundle(c));
+ }
+
+ @FinishBundle
+ public void finishBundle(FinishBundleContext c) throws Exception {
+ invoker.invokeFinishBundle(wrapContextAsFinishBundle(c));
+ }
+
+ @ProcessElement
+ public void processElement(final ProcessContext c) {
+ String key = c.element().key();
+ StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key);
+ TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key);
+
+ // Initialize state (element and restriction) depending on whether this is the seed call.
+ // The seed call is the first call for this element, which actually has the element.
+ // Subsequent calls are timer firings and the element has to be retrieved from the state.
+ TimerInternals.TimerData timer = Iterables.getOnlyElement(c.element().timersIterable(), null);
+ boolean isSeedCall = (timer == null);
+ StateNamespace stateNamespace;
+ if (isSeedCall) {
+ WindowedValue<KV<InputT, RestrictionT>> windowedValue =
+ Iterables.getOnlyElement(c.element().elementsIterable());
+ BoundedWindow window = Iterables.getOnlyElement(windowedValue.getWindows());
+ stateNamespace =
+ StateNamespaces.window(
+ (Coder<BoundedWindow>) inputWindowingStrategy.getWindowFn().windowCoder(), window);
+ } else {
+ stateNamespace = timer.getNamespace();
+ }
+
+ ValueState<WindowedValue<InputT>> elementState =
+ stateInternals.state(stateNamespace, elementTag);
+ ValueState<RestrictionT> restrictionState =
+ stateInternals.state(stateNamespace, restrictionTag);
+ WatermarkHoldState holdState = stateInternals.state(stateNamespace, watermarkHoldTag);
+
+ KV<WindowedValue<InputT>, RestrictionT> elementAndRestriction;
+ if (isSeedCall) {
+ WindowedValue<KV<InputT, RestrictionT>> windowedValue =
+ Iterables.getOnlyElement(c.element().elementsIterable());
+ WindowedValue<InputT> element = windowedValue.withValue(windowedValue.getValue().getKey());
+ elementState.write(element);
+ elementAndRestriction = KV.of(element, windowedValue.getValue().getValue());
+ } else {
+ // This is not the first ProcessElement call for this element/restriction - rather,
+ // this is a timer firing, so we need to fetch the element and restriction from state.
+ elementState.readLater();
+ restrictionState.readLater();
+ elementAndRestriction = KV.of(elementState.read(), restrictionState.read());
+ }
+
+ final TrackerT tracker = invoker.invokeNewTracker(elementAndRestriction.getValue());
+ SplittableProcessElementInvoker<InputT, OutputT, RestrictionT, TrackerT>.Result result =
+ processElementInvoker.invokeProcessElement(
+ invoker, elementAndRestriction.getKey(), tracker);
+
+ // Save state for resuming.
+ if (result.getResidualRestriction() == null) {
+ // All work for this element/restriction is completed. Clear state and release hold.
+ elementState.clear();
+ restrictionState.clear();
+ holdState.clear();
+ return;
+ }
+ restrictionState.write(result.getResidualRestriction());
+ Instant futureOutputWatermark = result.getFutureOutputWatermark();
+ if (futureOutputWatermark == null) {
+ futureOutputWatermark = elementAndRestriction.getKey().getTimestamp();
+ }
+ Instant wakeupTime =
+ timerInternals.currentProcessingTime().plus(result.getContinuation().resumeDelay());
+ holdState.add(futureOutputWatermark);
+ // Set a timer to continue processing this element.
+ timerInternals.setTimer(
+ TimerInternals.TimerData.of(stateNamespace, wakeupTime, TimeDomain.PROCESSING_TIME));
+ }
+
+ private DoFn<InputT, OutputT>.StartBundleContext wrapContextAsStartBundle(
+ final StartBundleContext baseContext) {
+ return fn.new StartBundleContext() {
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return baseContext.getPipelineOptions();
+ }
+
+ private void throwUnsupportedOutput() {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Splittable DoFn can only output from @%s",
+ ProcessElement.class.getSimpleName()));
+ }
+ };
+ }
+
+ private DoFn<InputT, OutputT>.FinishBundleContext wrapContextAsFinishBundle(
+ final FinishBundleContext baseContext) {
+ return fn.new FinishBundleContext() {
+ @Override
+ public void output(OutputT output, Instant timestamp, BoundedWindow window) {
+ throwUnsupportedOutput();
+ }
+
+ @Override
+ public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+ throwUnsupportedOutput();
+ }
+
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return baseContext.getPipelineOptions();
+ }
+
+ private void throwUnsupportedOutput() {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Splittable DoFn can only output from @%s",
+ ProcessElement.class.getSimpleName()));
+ }
+ };
+ }
+ }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java
index ced6c01..7732df3 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.runners.core;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import javax.annotation.Nullable;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
@@ -34,20 +36,35 @@
public class Result {
@Nullable
private final RestrictionT residualRestriction;
+ private final DoFn.ProcessContinuation continuation;
private final Instant futureOutputWatermark;
public Result(
- @Nullable RestrictionT residualRestriction, Instant futureOutputWatermark) {
+ @Nullable RestrictionT residualRestriction,
+ DoFn.ProcessContinuation continuation,
+ Instant futureOutputWatermark) {
+ this.continuation = checkNotNull(continuation);
+ if (continuation.shouldResume()) {
+ checkNotNull(residualRestriction);
+ }
this.residualRestriction = residualRestriction;
this.futureOutputWatermark = futureOutputWatermark;
}
- /** If {@code null}, means the call should not resume. */
+ /**
+ * Can be {@code null} only if {@link #getContinuation} specifies the call should not resume.
+ * However, the converse is not true: this can be non-null even if {@link #getContinuation}
+ * is {@link DoFn.ProcessContinuation#stop()}.
+ */
@Nullable
public RestrictionT getResidualRestriction() {
return residualRestriction;
}
+ public DoFn.ProcessContinuation getContinuation() {
+ return continuation;
+ }
+
public Instant getFutureOutputWatermark() {
return futureOutputWatermark;
}
@@ -57,8 +74,8 @@
* Invokes the {@link DoFn.ProcessElement} method using the given {@link DoFnInvoker} for the
* original {@link DoFn}, on the given element and with the given {@link RestrictionTracker}.
*
- * @return Information on how to resume the call: residual restriction and a
- * future output watermark.
+ * @return Information on how to resume the call: residual restriction, a {@link
+ * DoFn.ProcessContinuation}, and a future output watermark.
*/
public abstract Result invokeProcessElement(
DoFnInvoker<InputT, OutputT> invoker, WindowedValue<InputT> element, TrackerT tracker);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
index d996729..fa858b0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
@@ -17,20 +17,27 @@
*/
package org.apache.beam.runners.core;
+import com.google.common.base.Equivalence;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
import org.apache.beam.runners.core.StateTag.StateBinder;
import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.StateContext;
/**
* Table mapping {@code StateNamespace} and {@code StateTag<?>} to a {@code State} instance.
+ *
+ * <p>Two {@link StateTag StateTags} with the same ID are considered equivalent. The remaining
+ * information carried by the {@link StateTag} is used to configure an empty state cell if it is not
+ * yet initialized.
*/
public abstract class StateTable {
- private final Table<StateNamespace, StateTag<?>, State> stateTable =
+ private final Table<StateNamespace, Equivalence.Wrapper<StateTag>, State> stateTable =
HashBasedTable.create();
/**
@@ -40,7 +47,10 @@
*/
public <StateT extends State> StateT get(
StateNamespace namespace, StateTag<StateT> tag, StateContext<?> c) {
- State storage = stateTable.get(namespace, tag);
+
+ Equivalence.Wrapper<StateTag> tagById = StateTags.ID_EQUIVALENCE.wrap((StateTag) tag);
+
+ @Nullable State storage = getOrNull(namespace, tagById, c);
if (storage != null) {
@SuppressWarnings("unchecked")
StateT typedStorage = (StateT) storage;
@@ -48,10 +58,20 @@
}
StateT typedStorage = tag.bind(binderForNamespace(namespace, c));
- stateTable.put(namespace, tag, typedStorage);
+ stateTable.put(namespace, tagById, typedStorage);
return typedStorage;
}
+ /**
+ * Gets the {@link State} in the specified {@link StateNamespace} with the specified identifier or
+ * {@code null} if it is not yet present.
+ */
+ @Nullable
+ public State getOrNull(
+ StateNamespace namespace, Equivalence.Wrapper<StateTag> tag, StateContext<?> c) {
+ return stateTable.get(namespace, tag);
+ }
+
public void clearNamespace(StateNamespace namespace) {
stateTable.rowKeySet().remove(namespace);
}
@@ -68,8 +88,18 @@
return stateTable.containsRow(namespace);
}
- public Map<StateTag<?>, State> getTagsInUse(StateNamespace namespace) {
- return stateTable.row(namespace);
+ public Map<StateTag, State> getTagsInUse(StateNamespace namespace) {
+ // Because of shading, Equivalence.Wrapper cannot be on the API surface; it won't work.
+ // If runners-core ceases to shade Guava then it can (all runners should shade runners-core
+ // anyhow)
+ Map<Equivalence.Wrapper<StateTag>, State> row = stateTable.row(namespace);
+ HashMap<StateTag, State> result = new HashMap<>();
+
+ for (Map.Entry<Equivalence.Wrapper<StateTag>, State> entry : row.entrySet()) {
+ result.put(entry.getKey().get(), entry.getValue());
+ }
+
+ return result;
}
public Set<StateNamespace> getNamespacesInUse() {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
index 53f9edc..da94ef2 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.core;
+import com.google.common.base.Equivalence;
import com.google.common.base.MoreObjects;
import java.io.IOException;
import java.io.Serializable;
@@ -48,6 +49,18 @@
private static final CoderRegistry STANDARD_REGISTRY = CoderRegistry.createDefault();
+ public static final Equivalence<StateTag> ID_EQUIVALENCE = new Equivalence<StateTag>() {
+ @Override
+ protected boolean doEquivalent(StateTag a, StateTag b) {
+ return a.getId().equals(b.getId());
+ }
+
+ @Override
+ protected int doHash(StateTag stateTag) {
+ return stateTag.getId().hashCode();
+ }
+ };
+
/** @deprecated for migration purposes only */
@Deprecated
private static StateBinder adaptTagBinder(final StateTag.StateBinder binder) {
@@ -302,6 +315,9 @@
this.spec = spec;
}
+ /**
+ * @deprecated use {@link StateSpec#bind} method via {@link #getSpec} for now.
+ */
@Override
@Deprecated
public StateT bind(StateTag.StateBinder binder) {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
new file mode 100644
index 0000000..4d66d66
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+/**
+ * The context in which a specific step is executing, including access to state and timers.
+ *
+ * <p>This interface exists as the API between a runner and the support code, but is not user
+ * facing.
+ *
+ * <p>These will often be scoped to a particular step and key, though it is not required.
+ */
+public interface StepContext {
+
+ StateInternals stateInternals();
+
+ TimerInternals timerInternals();
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
index c189b0d..3144bd6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.core;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
@@ -103,6 +104,11 @@
this.bufferTag = bufferTag;
}
+ @VisibleForTesting
+ StateTag<? extends GroupingState<InputT, OutputT>> getBufferTag() {
+ return bufferTag;
+ }
+
@Override
public void processValue(ProcessValueContext c) throws Exception {
c.state().access(bufferTag).add(c.value());
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
index 2052c03..ee8d560 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
@@ -32,9 +32,9 @@
super(key);
}
- public Set<StateTag<?>> getTagsInUse(StateNamespace namespace) {
- Set<StateTag<?>> inUse = new HashSet<>();
- for (Map.Entry<StateTag<?>, State> entry :
+ public Set<StateTag> getTagsInUse(StateNamespace namespace) {
+ Set<StateTag> inUse = new HashSet<>();
+ for (Map.Entry<StateTag, State> entry :
inMemoryState.getTagsInUse(namespace).entrySet()) {
if (!isEmptyForTesting(entry.getValue())) {
inUse.add(entry.getKey());
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
index 13e4c43..8859bbb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
@@ -483,9 +483,9 @@
@Override
public OldAndNewHolds read() {
// Read both the element and extra holds.
- Instant elementHold = elementHoldState.read();
- Instant extraHold = extraHoldState.read();
- Instant oldHold;
+ @Nullable Instant elementHold = elementHoldState.read();
+ @Nullable Instant extraHold = extraHoldState.read();
+ @Nullable Instant oldHold;
// Find the minimum, accounting for null.
if (elementHold == null) {
oldHold = extraHold;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
new file mode 100644
index 0000000..811444c
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import io.grpc.stub.StreamObserver;
+import java.io.Closeable;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A client for the control plane of an SDK harness, which can issue requests to it over the Fn API.
+ *
+ * <p>This class presents a low-level Java API de-inverting the Fn API's gRPC layer.
+ *
+ * <p>The Fn API is inverted so the runner is the server and the SDK harness is the client, for
+ * firewalling reasons (the runner may execute in a more privileged environment forbidding outbound
+ * connections).
+ *
+ * <p>This low-level client is responsible only for correlating requests with responses.
+ *
+ * @deprecated Runners should depend on the beam-runners-java-fn-execution module for this
+ * functionality.
+ */
+@Deprecated
+class FnApiControlClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(FnApiControlClient.class);
+
+ // All writes to this StreamObserver need to be synchronized.
+ private final StreamObserver<BeamFnApi.InstructionRequest> requestReceiver;
+ private final ResponseStreamObserver responseObserver = new ResponseStreamObserver();
+ private final Map<String, SettableFuture<BeamFnApi.InstructionResponse>> outstandingRequests;
+ private volatile boolean isClosed;
+
+ private FnApiControlClient(StreamObserver<BeamFnApi.InstructionRequest> requestReceiver) {
+ this.requestReceiver = requestReceiver;
+ this.outstandingRequests = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Returns a {@link FnApiControlClient} which will submit its requests to the provided
+ * observer.
+ *
+ * <p>It is the responsibility of the caller to register this object as an observer of incoming
+ * responses (this will generally be done as part of fulfilling the contract of a gRPC service).
+ */
+ public static FnApiControlClient forRequestObserver(
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver) {
+ return new FnApiControlClient(requestObserver);
+ }
+
+ public synchronized ListenableFuture<BeamFnApi.InstructionResponse> handle(
+ BeamFnApi.InstructionRequest request) {
+ LOG.debug("Sending InstructionRequest {}", request);
+ SettableFuture<BeamFnApi.InstructionResponse> resultFuture = SettableFuture.create();
+ outstandingRequests.put(request.getInstructionId(), resultFuture);
+ requestReceiver.onNext(request);
+ return resultFuture;
+ }
+
+ StreamObserver<BeamFnApi.InstructionResponse> asResponseObserver() {
+ return responseObserver;
+ }
+
+ @Override
+ public void close() {
+ closeAndTerminateOutstandingRequests(new IllegalStateException("Runner closed connection"));
+ }
+
+ /** Closes this client and terminates any outstanding requests exceptionally. */
+ private synchronized void closeAndTerminateOutstandingRequests(Throwable cause) {
+ if (isClosed) {
+ return;
+ }
+
+ // Make a copy of the map to make the view of the outstanding requests consistent.
+ Map<String, SettableFuture<BeamFnApi.InstructionResponse>> outstandingRequestsCopy =
+ new ConcurrentHashMap<>(outstandingRequests);
+ outstandingRequests.clear();
+ isClosed = true;
+
+ if (outstandingRequestsCopy.isEmpty()) {
+ requestReceiver.onCompleted();
+ return;
+ }
+ requestReceiver.onError(
+ new StatusRuntimeException(Status.CANCELLED.withDescription(cause.getMessage())));
+
+ LOG.error(
+ "{} closed, clearing outstanding requests {}",
+ FnApiControlClient.class.getSimpleName(),
+ outstandingRequestsCopy);
+ for (SettableFuture<BeamFnApi.InstructionResponse> outstandingRequest :
+ outstandingRequestsCopy.values()) {
+ outstandingRequest.setException(cause);
+ }
+ }
+
+ /**
+ * A private view of this class as a {@link StreamObserver} for connecting as a gRPC listener.
+ */
+ private class ResponseStreamObserver implements StreamObserver<BeamFnApi.InstructionResponse> {
+ /**
+ * Processes an incoming {@link BeamFnApi.InstructionResponse} by correlating it with the
+ * corresponding {@link BeamFnApi.InstructionRequest} and completes the future that was returned
+ * by {@link #handle}.
+ */
+ @Override
+ public void onNext(BeamFnApi.InstructionResponse response) {
+ LOG.debug("Received InstructionResponse {}", response);
+ SettableFuture<BeamFnApi.InstructionResponse> completableFuture =
+ outstandingRequests.remove(response.getInstructionId());
+ if (completableFuture != null) {
+ completableFuture.set(response);
+ }
+ }
+
+ /** */
+ @Override
+ public void onCompleted() {
+ closeAndTerminateOutstandingRequests(
+ new IllegalStateException("SDK harness closed connection"));
+ }
+
+ @Override
+ public void onError(Throwable cause) {
+ LOG.error("{} received error {}", FnApiControlClient.class.getSimpleName(), cause);
+ closeAndTerminateOutstandingRequests(cause);
+ }
+ }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
new file mode 100644
index 0000000..21fc4f7
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.BlockingQueue;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A Fn API control service which adds incoming SDK harness connections to a pool.
+ *
+ * @deprecated Runners should depend on the beam-runners-java-fn-execution module for this
+ * functionality.
+ */
+@Deprecated
+public class FnApiControlClientPoolService extends BeamFnControlGrpc.BeamFnControlImplBase {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FnApiControlClientPoolService.class);
+
+ private final BlockingQueue<FnApiControlClient> clientPool;
+
+ private FnApiControlClientPoolService(BlockingQueue<FnApiControlClient> clientPool) {
+ this.clientPool = clientPool;
+ }
+
+ /**
+ * Creates a new {@link FnApiControlClientPoolService} which will enqueue and vend new SDK harness
+ * connections.
+ */
+ public static FnApiControlClientPoolService offeringClientsToPool(
+ BlockingQueue<FnApiControlClient> clientPool) {
+ return new FnApiControlClientPoolService(clientPool);
+ }
+
+ /**
+ * Called by gRPC for each incoming connection from an SDK harness, and enqueue an available SDK
+ * harness client.
+ *
+ * <p>Note: currently does not distinguish what sort of SDK it is, so a separate instance is
+ * required for each.
+ */
+ @Override
+ public StreamObserver<BeamFnApi.InstructionResponse> control(
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver) {
+ LOGGER.info("Beam Fn Control client connected.");
+ FnApiControlClient newClient = FnApiControlClient.forRequestObserver(requestObserver);
+ try {
+ clientPool.put(newClient);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ return newClient.asResponseObserver();
+ }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataReceiver.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataReceiver.java
new file mode 100644
index 0000000..639d678
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataReceiver.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import java.io.Closeable;
+
+/**
+ * A receiver of streamed data.
+ *
+ * <p>Provide a {@link FnDataReceiver} and target to a {@link FnDataService} to listen for incoming
+ * data.
+ *
+ * <p>Register a target with a {@link FnDataService} to gain a {@link FnDataReceiver} to which you
+ * may write outgoing data.
+ *
+ * @deprecated Runners should depend on the beam-runners-java-fn-execution module for this
+ * functionality.
+ */
+@Deprecated
+public interface FnDataReceiver<T> extends Closeable {
+ void accept(T input) throws Exception;
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
new file mode 100644
index 0000000..2a6777e
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.util.WindowedValue;
+
+/**
+ * The {@link FnDataService} is able to forward inbound elements to a consumer and is also a
+ * consumer of outbound elements. Callers can register themselves as consumers for inbound elements
+ * or can get a handle for a consumer for outbound elements.
+ *
+ * @deprecated Runners should depend on the beam-runners-java-fn-execution module for this
+ * functionality.
+ */
+@Deprecated
+public interface FnDataService {
+
+ /**
+ * A logical endpoint is a pair of an instruction ID corresponding to the {@link
+ * BeamFnApi.ProcessBundleRequest} and the {@link
+ * BeamFnApi.Target} within the processing graph. This enables the same
+ * {@link FnDataService} to be re-used across multiple bundles.
+ */
+ @AutoValue
+ abstract class LogicalEndpoint {
+
+ public abstract String getInstructionId();
+
+ public abstract BeamFnApi.Target getTarget();
+
+ public static LogicalEndpoint of(String instructionId, BeamFnApi.Target target) {
+ return new AutoValue_FnDataService_LogicalEndpoint(instructionId, target);
+ }
+ }
+
+ /**
+ * Registers a receiver to be notified upon any incoming elements.
+ *
+ * <p>The provided coder is used to decode inbound elements. The decoded elements are passed to
+ * the provided receiver.
+ *
+ * <p>Any failure during decoding or processing of the element will complete the returned future
+ * exceptionally. On successful termination of the stream, the returned future is completed
+ * successfully.
+ *
+ * <p>The provided receiver is not required to be thread safe.
+ */
+ <T> ListenableFuture<Void> listen(
+ LogicalEndpoint inputLocation,
+ Coder<WindowedValue<T>> coder,
+ FnDataReceiver<WindowedValue<T>> listener)
+ throws Exception;
+
+ /**
+ * Creates a receiver to which you can write data values and have them sent over this data plane
+ * service.
+ *
+ * <p>The provided coder is used to encode elements on the outbound stream.
+ *
+ * <p>Closing the returned receiver signals the end of the stream.
+ *
+ * <p>The returned receiver is not thread safe.
+ */
+ <T> FnDataReceiver<WindowedValue<T>> send(
+ LogicalEndpoint outputLocation, Coder<WindowedValue<T>> coder) throws Exception;
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
new file mode 100644
index 0000000..091dea1
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Function;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import java.io.IOException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+
+/**
+ * A high-level client for an SDK harness.
+ *
+ * <p>This provides a Java-friendly wrapper around {@link FnApiControlClient} and {@link
+ * FnDataReceiver}, which handle lower-level gRPC message wrangling.
+ *
+ * @deprecated Runners should depend on the beam-runners-java-fn-execution module for this
+ * functionality.
+ */
+@Deprecated
+public class SdkHarnessClient {
+
+ /**
+ * A supply of unique identifiers, used internally. These must be unique across all Fn API
+ * clients.
+ */
+ public interface IdGenerator {
+ String getId();
+ }
+
+ /** A supply of unique identifiers that are simply incrementing longs. */
+ private static class CountingIdGenerator implements IdGenerator {
+ private final AtomicLong nextId = new AtomicLong(0L);
+
+ @Override
+ public String getId() {
+ return String.valueOf(nextId.incrementAndGet());
+ }
+ }
+
+ /**
+ * An active bundle for a particular {@link
+ * BeamFnApi.ProcessBundleDescriptor}.
+ */
+ @AutoValue
+ public abstract static class ActiveBundle<InputT> {
+ public abstract String getBundleId();
+
+ public abstract Future<BeamFnApi.ProcessBundleResponse> getBundleResponse();
+
+ public abstract FnDataReceiver<InputT> getInputReceiver();
+
+ public static <InputT> ActiveBundle<InputT> create(
+ String bundleId,
+ Future<BeamFnApi.ProcessBundleResponse> response,
+ FnDataReceiver<InputT> dataReceiver) {
+ return new AutoValue_SdkHarnessClient_ActiveBundle(bundleId, response, dataReceiver);
+ }
+ }
+
+ private final IdGenerator idGenerator;
+ private final FnApiControlClient fnApiControlClient;
+
+ private SdkHarnessClient(
+ FnApiControlClient fnApiControlClient,
+ IdGenerator idGenerator) {
+ this.idGenerator = idGenerator;
+ this.fnApiControlClient = fnApiControlClient;
+ }
+
+ /**
+ * Creates a client for a particular SDK harness. It is the responsibility of the caller to ensure
+ * that these correspond to the same SDK harness, so control plane and data plane messages can be
+ * correctly associated.
+ */
+ public static SdkHarnessClient usingFnApiClient(FnApiControlClient fnApiControlClient) {
+ return new SdkHarnessClient(fnApiControlClient, new CountingIdGenerator());
+ }
+
+ public SdkHarnessClient withIdGenerator(IdGenerator idGenerator) {
+ return new SdkHarnessClient(fnApiControlClient, idGenerator);
+ }
+
+ /**
+ * Registers a {@link BeamFnApi.ProcessBundleDescriptor} for future
+ * processing.
+ *
+ * <p>A client may block on the result future, but may also proceed without blocking.
+ */
+ public Future<BeamFnApi.RegisterResponse> register(
+ Iterable<BeamFnApi.ProcessBundleDescriptor> processBundleDescriptors) {
+
+ // TODO: validate that all the necessary data endpoints are known
+
+ ListenableFuture<BeamFnApi.InstructionResponse> genericResponse =
+ fnApiControlClient.handle(
+ BeamFnApi.InstructionRequest.newBuilder()
+ .setInstructionId(idGenerator.getId())
+ .setRegister(
+ BeamFnApi.RegisterRequest.newBuilder()
+ .addAllProcessBundleDescriptor(processBundleDescriptors)
+ .build())
+ .build());
+
+ return Futures.transform(
+ genericResponse,
+ new Function<BeamFnApi.InstructionResponse, BeamFnApi.RegisterResponse>() {
+ @Override
+ public BeamFnApi.RegisterResponse apply(BeamFnApi.InstructionResponse input) {
+ return input.getRegister();
+ }
+ });
+ }
+
+ /**
+ * Start a new bundle for the given {@link
+ * BeamFnApi.ProcessBundleDescriptor} identifier.
+ *
+ * <p>The input channels for the returned {@link ActiveBundle} are derived from the
+ * instructions in the {@link BeamFnApi.ProcessBundleDescriptor}.
+ */
+ public ActiveBundle newBundle(String processBundleDescriptorId) {
+ String bundleId = idGenerator.getId();
+
+ // TODO: acquire an input receiver from appropriate FnDataService
+ FnDataReceiver dataReceiver = new FnDataReceiver() {
+ @Override
+ public void accept(Object input) throws Exception {
+ throw new UnsupportedOperationException("Placeholder FnDataReceiver cannot accept data.");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // noop
+ }
+ };
+
+ ListenableFuture<BeamFnApi.InstructionResponse> genericResponse =
+ fnApiControlClient.handle(
+ BeamFnApi.InstructionRequest.newBuilder()
+ .setProcessBundle(
+ BeamFnApi.ProcessBundleRequest.newBuilder()
+ .setProcessBundleDescriptorReference(processBundleDescriptorId))
+ .build());
+
+ ListenableFuture<BeamFnApi.ProcessBundleResponse> specificResponse =
+ Futures.transform(
+ genericResponse,
+ new Function<BeamFnApi.InstructionResponse, BeamFnApi.ProcessBundleResponse>() {
+ @Override
+ public BeamFnApi.ProcessBundleResponse apply(BeamFnApi.InstructionResponse input) {
+ return input.getProcessBundle();
+ }
+ });
+
+ return ActiveBundle.create(bundleId, specificResponse, dataReceiver);
+ }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
new file mode 100644
index 0000000..d27077f
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import java.util.concurrent.ExecutionException;
+import javax.annotation.Nullable;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor;
+import org.apache.beam.runners.core.DoFnRunner;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.joda.time.Instant;
+
+/**
+ * Processes a bundle by sending it to an SDK harness over the Fn API.
+ *
+ * @deprecated Runners should interact with the Control and Data plane directly, rather than through
+ * a {@link DoFnRunner}. Consider the beam-runners-java-fn-execution artifact instead.
+ */
+@Deprecated
+public class SdkHarnessDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
+
+ private final SdkHarnessClient sdkHarnessClient;
+ private final String processBundleDescriptorId;
+
+ /** {@code null} between bundles. */
+ @Nullable private SdkHarnessClient.ActiveBundle activeBundle;
+
+ private SdkHarnessDoFnRunner(
+ SdkHarnessClient sdkHarnessClient,
+ String processBundleDescriptorId) {
+ this.sdkHarnessClient = sdkHarnessClient;
+ this.processBundleDescriptorId = processBundleDescriptorId;
+ }
+
+ /**
+ * Returns a new {@link SdkHarnessDoFnRunner} suitable for just a particular {@link
+ * ProcessBundleDescriptor} (referenced by id here).
+ *
+ * <p>The {@link FnDataReceiver} must be the correct data plane service referenced
+ * in the primitive instructions in the
+ * {@link ProcessBundleDescriptor}.
+ *
+ * <p>Also outside of this class, the appropriate receivers must be registered with the
+ * output data plane channels of the descriptor.
+ */
+ public static <InputT, OutputT> SdkHarnessDoFnRunner<InputT, OutputT> create(
+ SdkHarnessClient sdkHarnessClient,
+ String processBundleDescriptorId) {
+ return new SdkHarnessDoFnRunner(sdkHarnessClient, processBundleDescriptorId);
+ }
+
+ @Override
+ public void startBundle() {
+ this.activeBundle =
+ sdkHarnessClient.newBundle(processBundleDescriptorId);
+ }
+
+ @Override
+ public void processElement(WindowedValue<InputT> elem) {
+ checkState(
+ activeBundle != null,
+ "%s attempted to process an element without an active bundle",
+ SdkHarnessDoFnRunner.class.getSimpleName());
+
+ try {
+ activeBundle.getInputReceiver().accept(elem);
+ } catch (Exception exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ @Override
+ public void onTimer(
+ String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) {
+ throw new UnsupportedOperationException("Timers are not supported over the Fn API");
+ }
+
+ @Override
+ public void finishBundle() {
+ try {
+ activeBundle.getBundleResponse().get();
+ } catch (InterruptedException interrupted) {
+ Thread.interrupted();
+ return;
+ } catch (ExecutionException exc) {
+ throw UserCodeException.wrap(exc);
+ }
+ }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/package-info.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/package-info.java
new file mode 100644
index 0000000..d24a597
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides utilities for a Beam runner to interact with a client using the Fn API.
+ */
+package org.apache.beam.runners.core.fn;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
index 4378bb9..886d681 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
@@ -21,8 +21,10 @@
import java.util.concurrent.atomic.AtomicLong;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricsContainer;
/**
* Tracks the current value (and delta) for a Counter metric for a specific context and bundle.
@@ -40,10 +42,12 @@
private final MetricName name;
/**
- * Package-visibility because all {@link CounterCell CounterCells} should be created by
- * {@link MetricsContainerImpl#getCounter(MetricName)}.
+ * Generally, runners should construct instances using the methods in
+ * {@link MetricsContainerImpl}, unless they need to define their own version of
+ * {@link MetricsContainer}. These constructors are *only* public so runners can instantiate.
*/
- CounterCell(MetricName name) {
+ @Internal
+ public CounterCell(MetricName name) {
this.name = name;
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
index 532fc2a..1976049 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
@@ -22,6 +22,7 @@
import java.util.concurrent.atomic.AtomicReference;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
/**
* Atomically tracks the dirty-state of a metric.
@@ -42,7 +43,8 @@
* completed.
*/
@Experimental(Kind.METRICS)
-class DirtyState implements Serializable {
+@Internal
+public class DirtyState implements Serializable {
private enum State {
/** Indicates that there have been changes to the MetricCell since last commit. */
DIRTY,
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
index 5a5099a..8713ec4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
@@ -21,8 +21,10 @@
import java.util.concurrent.atomic.AtomicReference;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.metrics.Distribution;
import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricsContainer;
/**
* Tracks the current value (and delta) for a Distribution metric.
@@ -41,10 +43,12 @@
private final MetricName name;
/**
- * Package-visibility because all {@link DistributionCell DistributionCells} should be created by
- * {@link MetricsContainerImpl#getDistribution(MetricName)}.
+ * Generally, runners should construct instances using the methods in
+ * {@link MetricsContainerImpl}, unless they need to define their own version of
+ * {@link MetricsContainer}. These constructors are *only* public so runners can instantiate.
*/
- DistributionCell(MetricName name) {
+ @Internal
+ public DistributionCell(MetricName name) {
this.name = name;
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
index 795e826..1c55021 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
@@ -20,8 +20,10 @@
import java.util.concurrent.atomic.AtomicReference;
import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.metrics.Gauge;
import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricsContainer;
/**
* Tracks the current value (and delta) for a {@link Gauge} metric.
@@ -39,10 +41,12 @@
private final MetricName name;
/**
- * Package-visibility because all {@link GaugeCell GaugeCells} should be created by
- * {@link MetricsContainerImpl#getGauge(MetricName)}.
+ * Generally, runners should construct instances using the methods in
+ * {@link MetricsContainerImpl}, unless they need to define their own version of
+ * {@link MetricsContainer}. These constructors are *only* public so runners can instantiate.
*/
- GaugeCell(MetricName name) {
+ @Internal
+ public GaugeCell(MetricName name) {
this.name = name;
}
@@ -70,7 +74,6 @@
return gaugeValue.get();
}
-
@Override
public MetricName getName() {
return name;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
index eae3305..ae91f71 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
@@ -22,6 +22,7 @@
import com.google.common.collect.Iterables;
import java.io.Serializable;
import java.util.Collections;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
index 6967bf0..1d5ad72 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
@@ -23,6 +23,8 @@
import com.google.common.collect.ImmutableList;
import java.io.Serializable;
import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -31,7 +33,9 @@
import org.apache.beam.sdk.metrics.MetricsContainer;
/**
- * Holds the metrics for a single step and unit-of-commit (bundle).
+ * Holds the metrics for a single step and uses metric cells that allow extracting the cumulative
+ * value. Generally, this implementation should be used for a specific unit of commitment (bundle)
+ * that wishes to report the values since the start of the bundle (eg., for committed metrics).
*
* <p>This class is thread-safe. It is intended to be used with 1 (or more) threads are updating
* metrics and at-most 1 thread is extracting updates by calling {@link #getUpdates} and
@@ -77,21 +81,60 @@
this.stepName = stepName;
}
+ /**
+ * Return a {@code CounterCell} named {@code metricName}. If it doesn't exist, create a
+ * {@code Metric} with the specified name.
+ */
@Override
public CounterCell getCounter(MetricName metricName) {
return counters.get(metricName);
}
+ /**
+ * Return a {@code CounterCell} named {@code metricName}. If it doesn't exist, return
+ * {@code null}.
+ */
+ @Nullable
+ public CounterCell tryGetCounter(MetricName metricName) {
+ return counters.tryGet(metricName);
+ }
+
+ /**
+ * Return a {@code DistributionCell} named {@code metricName}. If it doesn't exist, create a
+ * {@code Metric} with the specified name.
+ */
@Override
public DistributionCell getDistribution(MetricName metricName) {
return distributions.get(metricName);
}
+ /**
+ * Return a {@code DistributionCell} named {@code metricName}. If it doesn't exist, return
+ * {@code null}.
+ */
+ @Nullable
+ public DistributionCell tryGetDistribution(MetricName metricName) {
+ return distributions.tryGet(metricName);
+ }
+
+ /**
+ * Return a {@code GaugeCell} named {@code metricName}. If it doesn't exist, create a
+ * {@code Metric} with the specified name.
+ */
@Override
public GaugeCell getGauge(MetricName metricName) {
return gauges.get(metricName);
}
+ /**
+ * Return a {@code GaugeCell} named {@code metricName}. If it doesn't exist, return
+ * {@code null}.
+ */
+ @Nullable
+ public GaugeCell tryGetGauge(MetricName metricName) {
+ return gauges.tryGet(metricName);
+ }
+
private <UpdateT, CellT extends MetricCell<UpdateT>>
ImmutableList<MetricUpdate<UpdateT>> extractUpdates(MetricsMap<MetricName, CellT> cells) {
ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
index 68238e4..14b8ccb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
@@ -25,6 +25,8 @@
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.runners.core.construction.metrics.MetricFiltering;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.GaugeResult;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java
index 0f0c17c..3530ed1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java
@@ -23,7 +23,6 @@
import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.List;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.annotations.Experimental;
/**
@@ -31,7 +30,7 @@
* have fired.
*/
@Experimental(Experimental.Kind.TRIGGER)
-public class AfterAllStateMachine extends OnceTriggerStateMachine {
+public class AfterAllStateMachine extends TriggerStateMachine {
private AfterAllStateMachine(List<TriggerStateMachine> subTriggers) {
super(subTriggers);
@@ -42,11 +41,11 @@
* Returns an {@code AfterAll} {@code Trigger} with the given subtriggers.
*/
@SafeVarargs
- public static OnceTriggerStateMachine of(TriggerStateMachine... triggers) {
+ public static TriggerStateMachine of(TriggerStateMachine... triggers) {
return new AfterAllStateMachine(Arrays.<TriggerStateMachine>asList(triggers));
}
- public static OnceTriggerStateMachine of(Iterable<? extends TriggerStateMachine> triggers) {
+ public static TriggerStateMachine of(Iterable<? extends TriggerStateMachine> triggers) {
return new AfterAllStateMachine(ImmutableList.copyOf(triggers));
}
@@ -78,24 +77,21 @@
*/
@Override
public boolean shouldFire(TriggerContext context) throws Exception {
- for (ExecutableTriggerStateMachine subtrigger : context.trigger().subTriggers()) {
- if (!context.forTrigger(subtrigger).trigger().isFinished()
- && !subtrigger.invokeShouldFire(context)) {
+ for (ExecutableTriggerStateMachine subTrigger : context.trigger().subTriggers()) {
+ if (!context.forTrigger(subTrigger).trigger().isFinished()
+ && !subTrigger.invokeShouldFire(context)) {
return false;
}
}
return true;
}
- /**
- * Invokes {@link #onFire} for all subtriggers, eliding redundant calls to {@link #shouldFire}
- * because they all must be ready to fire.
- */
@Override
- public void onOnlyFiring(TriggerContext context) throws Exception {
- for (ExecutableTriggerStateMachine subtrigger : context.trigger().subTriggers()) {
- subtrigger.invokeOnFire(context);
+ public void onFire(TriggerContext context) throws Exception {
+ for (ExecutableTriggerStateMachine subTrigger : context.trigger().subTriggers()) {
+ subTrigger.invokeOnFire(context);
}
+ context.trigger().setFinished(true);
}
@Override
@@ -103,7 +99,6 @@
StringBuilder builder = new StringBuilder("AfterAll.of(");
Joiner.on(", ").appendTo(builder, subTriggers);
builder.append(")");
-
return builder.toString();
}
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
index 8d8d0de4..06c2066 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
@@ -27,7 +27,6 @@
import org.apache.beam.runners.core.StateMerging;
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTags;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.state.CombiningState;
@@ -50,7 +49,7 @@
// This class should be inlined to subclasses and deleted, simplifying them too
// https://issues.apache.org/jira/browse/BEAM-1486
@Experimental(Experimental.Kind.TRIGGER)
-public abstract class AfterDelayFromFirstElementStateMachine extends OnceTriggerStateMachine {
+public abstract class AfterDelayFromFirstElementStateMachine extends TriggerStateMachine {
protected static final List<SerializableFunction<Instant, Instant>> IDENTITY =
ImmutableList.<SerializableFunction<Instant, Instant>>of();
@@ -237,8 +236,9 @@
}
@Override
- protected void onOnlyFiring(TriggerStateMachine.TriggerContext context) throws Exception {
+ public final void onFire(TriggerContext context) throws Exception {
clear(context);
+ context.trigger().setFinished(true);
}
protected Instant computeTargetTimestamp(Instant time) {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java
index 840a65c..58c24c5 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java
@@ -23,7 +23,6 @@
import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.List;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.annotations.Experimental;
/**
@@ -31,7 +30,7 @@
* sub-triggers have fired.
*/
@Experimental(Experimental.Kind.TRIGGER)
-public class AfterFirstStateMachine extends OnceTriggerStateMachine {
+public class AfterFirstStateMachine extends TriggerStateMachine {
AfterFirstStateMachine(List<TriggerStateMachine> subTriggers) {
super(subTriggers);
@@ -42,12 +41,12 @@
* Returns an {@code AfterFirst} {@code Trigger} with the given subtriggers.
*/
@SafeVarargs
- public static OnceTriggerStateMachine of(
+ public static TriggerStateMachine of(
TriggerStateMachine... triggers) {
return new AfterFirstStateMachine(Arrays.<TriggerStateMachine>asList(triggers));
}
- public static OnceTriggerStateMachine of(
+ public static TriggerStateMachine of(
Iterable<? extends TriggerStateMachine> triggers) {
return new AfterFirstStateMachine(ImmutableList.copyOf(triggers));
}
@@ -79,18 +78,19 @@
}
@Override
- protected void onOnlyFiring(TriggerContext context) throws Exception {
- for (ExecutableTriggerStateMachine subtrigger : context.trigger().subTriggers()) {
- TriggerContext subContext = context.forTrigger(subtrigger);
- if (subtrigger.invokeShouldFire(subContext)) {
+ public void onFire(TriggerContext context) throws Exception {
+ for (ExecutableTriggerStateMachine subTrigger : context.trigger().subTriggers()) {
+ TriggerContext subContext = context.forTrigger(subTrigger);
+ if (subTrigger.invokeShouldFire(subContext)) {
// If the trigger is ready to fire, then do whatever it needs to do.
- subtrigger.invokeOnFire(subContext);
+ subTrigger.invokeOnFire(subContext);
} else {
// If the trigger is not ready to fire, it is nonetheless true that whatever
// pending pane it was tracking is now gone.
- subtrigger.invokeClear(subContext);
+ subTrigger.invokeClear(subContext);
}
}
+ context.trigger().setFinished(true);
}
@Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
index b9fbac3..1ce035a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
@@ -23,7 +23,6 @@
import org.apache.beam.runners.core.StateMerging;
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTags;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.state.CombiningState;
@@ -33,7 +32,7 @@
* {@link TriggerStateMachine}s that fire based on properties of the elements in the current pane.
*/
@Experimental(Experimental.Kind.TRIGGER)
-public class AfterPaneStateMachine extends OnceTriggerStateMachine {
+public class AfterPaneStateMachine extends TriggerStateMachine {
private static final StateTag<CombiningState<Long, long[], Long>>
ELEMENTS_IN_PANE_TAG =
@@ -130,7 +129,8 @@
}
@Override
- protected void onOnlyFiring(TriggerStateMachine.TriggerContext context) throws Exception {
+ public void onFire(TriggerStateMachine.TriggerContext context) throws Exception {
clear(context);
+ context.trigger().setFinished(true);
}
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
index c9eee15..2c99722 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
@@ -22,7 +22,6 @@
import com.google.common.collect.ImmutableList;
import java.util.Objects;
import javax.annotation.Nullable;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.state.TimeDomain;
@@ -101,6 +100,10 @@
@Override
public void onElement(OnElementContext c) throws Exception {
+ if (!endOfWindowReached(c)) {
+ c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME);
+ }
+
if (!c.trigger().isMerging()) {
// If merges can never happen, we just run the unfinished subtrigger
c.trigger().firstUnfinishedSubTrigger().invokeOnElement(c);
@@ -242,7 +245,7 @@
/**
* A watermark trigger targeted relative to the end of the window.
*/
- public static class FromEndOfWindow extends OnceTriggerStateMachine {
+ public static class FromEndOfWindow extends TriggerStateMachine {
private FromEndOfWindow() {
super(null);
@@ -271,7 +274,9 @@
// We're interested in knowing when the input watermark passes the end of the window.
// (It is possible this has already happened, in which case the timer will be fired
// almost immediately).
- c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME);
+ if (!endOfWindowReached(c)) {
+ c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME);
+ }
}
@Override
@@ -319,6 +324,8 @@
}
@Override
- protected void onOnlyFiring(TriggerStateMachine.TriggerContext context) throws Exception { }
+ public void onFire(TriggerStateMachine.TriggerContext context) throws Exception {
+ context.trigger().setFinished(true);
+ }
}
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java
index c4d89c2..cdcff64 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java
@@ -23,7 +23,6 @@
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
/**
@@ -46,17 +45,14 @@
private static <W extends BoundedWindow> ExecutableTriggerStateMachine create(
TriggerStateMachine trigger, int nextUnusedIndex) {
- if (trigger instanceof OnceTriggerStateMachine) {
- return new ExecutableOnceTriggerStateMachine(
- (OnceTriggerStateMachine) trigger, nextUnusedIndex);
- } else {
+
return new ExecutableTriggerStateMachine(trigger, nextUnusedIndex);
- }
+
}
public static <W extends BoundedWindow> ExecutableTriggerStateMachine createForOnceTrigger(
- OnceTriggerStateMachine trigger, int nextUnusedIndex) {
- return new ExecutableOnceTriggerStateMachine(trigger, nextUnusedIndex);
+ TriggerStateMachine trigger, int nextUnusedIndex) {
+ return new ExecutableTriggerStateMachine(trigger, nextUnusedIndex);
}
private ExecutableTriggerStateMachine(TriggerStateMachine trigger, int nextUnusedIndex) {
@@ -146,15 +142,4 @@
public void invokeClear(TriggerStateMachine.TriggerContext c) throws Exception {
trigger.clear(c.forTrigger(this));
}
-
- /**
- * {@link ExecutableTriggerStateMachine} that enforces the fact that the trigger should always
- * FIRE_AND_FINISH and never just FIRE.
- */
- private static class ExecutableOnceTriggerStateMachine extends ExecutableTriggerStateMachine {
-
- public ExecutableOnceTriggerStateMachine(OnceTriggerStateMachine trigger, int nextUnusedIndex) {
- super(trigger, nextUnusedIndex);
- }
- }
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/NeverStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/NeverStateMachine.java
index f32c7a8..f8c5e8b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/NeverStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/NeverStateMachine.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.runners.core.triggers;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -27,7 +26,7 @@
* <p>Using this trigger will only produce output when the watermark passes the end of the
* {@link BoundedWindow window} plus the allowed lateness.
*/
-public final class NeverStateMachine extends OnceTriggerStateMachine {
+public final class NeverStateMachine extends TriggerStateMachine {
/**
* Returns a trigger which never fires. Output will be produced from the using {@link GroupByKey}
* when the {@link BoundedWindow} closes.
@@ -53,7 +52,7 @@
}
@Override
- protected void onOnlyFiring(TriggerStateMachine.TriggerContext context) {
+ public void onFire(TriggerStateMachine.TriggerContext context) {
throw new UnsupportedOperationException(
String.format("%s should never fire", getClass().getSimpleName()));
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
index 6a2cf0c..880aa48 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
@@ -453,35 +453,8 @@
* }
* </pre>
*
- * <p>Note that if {@code t1} is {@link OnceTriggerStateMachine}, then {@code t1.orFinally(t2)} is
- * the same as {@code AfterFirst.of(t1, t2)}.
*/
public TriggerStateMachine orFinally(TriggerStateMachine until) {
return new OrFinallyStateMachine(this, until);
}
-
- /**
- * {@link TriggerStateMachine}s that are guaranteed to fire at most once should extend from this,
- * rather than the general {@link TriggerStateMachine} class to indicate that behavior.
- */
- public abstract static class OnceTriggerStateMachine extends TriggerStateMachine {
- protected OnceTriggerStateMachine(List<TriggerStateMachine> subTriggers) {
- super(subTriggers);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public final void onFire(TriggerContext context) throws Exception {
- onOnlyFiring(context);
- context.trigger().setFinished(true);
- }
-
- /**
- * Called exactly once by {@link #onFire} when the trigger is fired. By default,
- * invokes {@link #onFire} on all subtriggers for which {@link #shouldFire} is {@code true}.
- */
- protected abstract void onOnlyFiring(TriggerContext context) throws Exception;
- }
}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
index 88ea6ef..b643a7b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
@@ -24,6 +24,7 @@
import java.util.BitSet;
import java.util.Collection;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.beam.runners.core.MergingStateAccessor;
import org.apache.beam.runners.core.StateAccessor;
import org.apache.beam.runners.core.StateTag;
@@ -79,7 +80,7 @@
return FinishedTriggersBitSet.emptyWithCapacity(rootTrigger.getFirstIndexAfterSubtree());
}
- BitSet bitSet = state.read();
+ @Nullable BitSet bitSet = state.read();
return bitSet == null
? FinishedTriggersBitSet.emptyWithCapacity(rootTrigger.getFirstIndexAfterSubtree())
: FinishedTriggersBitSet.fromBitSet(bitSet);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
index a6b38ec..32c9153 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
@@ -19,7 +19,7 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.transforms.windowing.Trigger;
import org.joda.time.Duration;
import org.joda.time.Instant;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java
deleted file mode 100644
index f516046..0000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import org.apache.beam.sdk.coders.BigEndianLongCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.testing.CoderProperties;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-
-/**
- * Tests for {@link ElementAndRestrictionCoder}. Parroted from {@link
- * org.apache.beam.sdk.coders.KvCoderTest}.
- */
-@RunWith(Parameterized.class)
-public class ElementAndRestrictionCoderTest<K, V> {
- private static class CoderAndData<T> {
- Coder<T> coder;
- List<T> data;
- }
-
- private static class AnyCoderAndData {
- private CoderAndData<?> coderAndData;
- }
-
- private static <T> AnyCoderAndData coderAndData(Coder<T> coder, List<T> data) {
- CoderAndData<T> coderAndData = new CoderAndData<>();
- coderAndData.coder = coder;
- coderAndData.data = data;
- AnyCoderAndData res = new AnyCoderAndData();
- res.coderAndData = coderAndData;
- return res;
- }
-
- private static final List<AnyCoderAndData> TEST_DATA =
- Arrays.asList(
- coderAndData(
- VarIntCoder.of(), Arrays.asList(-1, 0, 1, 13, Integer.MAX_VALUE, Integer.MIN_VALUE)),
- coderAndData(
- BigEndianLongCoder.of(),
- Arrays.asList(-1L, 0L, 1L, 13L, Long.MAX_VALUE, Long.MIN_VALUE)),
- coderAndData(StringUtf8Coder.of(), Arrays.asList("", "hello", "goodbye", "1")),
- coderAndData(
- ElementAndRestrictionCoder.of(StringUtf8Coder.of(), VarIntCoder.of()),
- Arrays.asList(
- ElementAndRestriction.of("", -1),
- ElementAndRestriction.of("hello", 0),
- ElementAndRestriction.of("goodbye", Integer.MAX_VALUE))),
- coderAndData(
- ListCoder.of(VarLongCoder.of()),
- Arrays.asList(Arrays.asList(1L, 2L, 3L), Collections.<Long>emptyList())));
-
- @Parameterized.Parameters(name = "{index}: keyCoder={0} key={1} valueCoder={2} value={3}")
- public static Collection<Object[]> data() {
- List<Object[]> parameters = new ArrayList<>();
- for (AnyCoderAndData keyCoderAndData : TEST_DATA) {
- Coder keyCoder = keyCoderAndData.coderAndData.coder;
- for (Object key : keyCoderAndData.coderAndData.data) {
- for (AnyCoderAndData valueCoderAndData : TEST_DATA) {
- Coder valueCoder = valueCoderAndData.coderAndData.coder;
- for (Object value : valueCoderAndData.coderAndData.data) {
- parameters.add(new Object[] {keyCoder, key, valueCoder, value});
- }
- }
- }
- }
- return parameters;
- }
-
- @Parameter(0)
- public Coder<K> keyCoder;
- @Parameter(1)
- public K key;
- @Parameter(2)
- public Coder<V> valueCoder;
- @Parameter(3)
- public V value;
-
- @Test
- @SuppressWarnings("rawtypes")
- public void testDecodeEncodeEqual() throws Exception {
- CoderProperties.coderDecodeEncodeEqual(
- ElementAndRestrictionCoder.of(keyCoder, valueCoder),
- ElementAndRestriction.of(key, value));
- }
-
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void encodeNullThrowsCoderException() throws Exception {
- thrown.expect(CoderException.class);
- thrown.expectMessage("cannot encode a null ElementAndRestriction");
-
- CoderUtils.encodeToBase64(
- ElementAndRestrictionCoder.of(StringUtf8Coder.of(), VarIntCoder.of()), null);
- }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
index b526305..1c6cd30 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
@@ -17,545 +17,88 @@
*/
package org.apache.beam.runners.core;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItems;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
-import org.apache.beam.sdk.state.GroupingState;
import org.apache.beam.sdk.state.MapState;
-import org.apache.beam.sdk.state.ReadableState;
import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.ValueState;
import org.apache.beam.sdk.state.WatermarkHoldState;
import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.hamcrest.Matchers;
-import org.joda.time.Instant;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.junit.runners.Suite;
/**
- * Tests for {@link InMemoryStateInternals}.
+ * Tests for {@link InMemoryStateInternals}. This is based on {@link StateInternalsTest}.
*/
-@RunWith(JUnit4.class)
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ InMemoryStateInternalsTest.StandardStateInternalsTests.class,
+ InMemoryStateInternalsTest.OtherTests.class
+})
public class InMemoryStateInternalsTest {
- private static final BoundedWindow WINDOW_1 = new IntervalWindow(new Instant(0), new Instant(10));
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
- private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
- private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
- StateTags.value("stringValue", StringUtf8Coder.of());
- private static final StateTag<CombiningState<Integer, int[], Integer>>
- SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
- "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
- private static final StateTag<SetState<String>> STRING_SET_ADDR =
- StateTags.set("stringSet", StringUtf8Coder.of());
- private static final StateTag<MapState<String, Integer>> STRING_MAP_ADDR =
- StateTags.map("stringMap", StringUtf8Coder.of(), VarIntCoder.of());
- private static final StateTag<WatermarkHoldState> WATERMARK_EARLIEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_LATEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_EOW_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW);
-
- InMemoryStateInternals<String> underTest = InMemoryStateInternals.forKey("dummyKey");
-
- @Test
- public void testValue() throws Exception {
- ValueState<String> value = underTest.state(NAMESPACE_1, STRING_VALUE_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertThat(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), Matchers.sameInstance(value));
- assertThat(
- underTest.state(NAMESPACE_2, STRING_VALUE_ADDR),
- Matchers.not(Matchers.sameInstance(value)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.write("hello");
- assertThat(value.read(), equalTo("hello"));
- value.write("world");
- assertThat(value.read(), equalTo("world"));
-
- value.clear();
- assertThat(value.read(), Matchers.nullValue());
- assertThat(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_BAG_ADDR)));
- assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_BAG_ADDR))));
-
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), containsInAnyOrder("hello"));
-
- value.add("world");
- assertThat(value.read(), containsInAnyOrder("hello", "world"));
-
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertThat(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeBagIntoSource() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
-
- // Reading the merged bag gets both the contents
- assertThat(bag1.read(), containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeBagIntoNewNamespace() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
- BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
-
- // Reading the merged bag gets both the contents
- assertThat(bag3.read(), containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag1.read(), Matchers.emptyIterable());
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testSet() throws Exception {
- SetState<String> value = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_SET_ADDR)));
- assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_SET_ADDR))));
-
- // empty
- assertThat(value.read(), Matchers.emptyIterable());
- assertFalse(value.contains("A").read());
-
- // add
- value.add("A");
- value.add("B");
- value.add("A");
- assertFalse(value.addIfAbsent("B").read());
- assertThat(value.read(), containsInAnyOrder("A", "B"));
-
- // remove
- value.remove("A");
- assertThat(value.read(), containsInAnyOrder("B"));
- value.remove("C");
- assertThat(value.read(), containsInAnyOrder("B"));
-
- // contains
- assertFalse(value.contains("A").read());
- assertTrue(value.contains("B").read());
- value.add("C");
- value.add("D");
-
- // readLater
- assertThat(value.readLater().read(), containsInAnyOrder("B", "C", "D"));
- SetState<String> later = value.readLater();
- assertThat(later.read(), hasItems("C", "D"));
- assertFalse(later.contains("A").read());
-
- // clear
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertThat(underTest.state(NAMESPACE_1, STRING_SET_ADDR), Matchers.sameInstance(value));
-
- }
-
- @Test
- public void testSetIsEmpty() throws Exception {
- SetState<String> value = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeSetIntoSource() throws Exception {
- SetState<String> set1 = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
- SetState<String> set2 = underTest.state(NAMESPACE_2, STRING_SET_ADDR);
-
- set1.add("Hello");
- set2.add("Hello");
- set2.add("World");
- set1.add("!");
-
- StateMerging.mergeSets(Arrays.asList(set1, set2), set1);
-
- // Reading the merged set gets both the contents
- assertThat(set1.read(), containsInAnyOrder("Hello", "World", "!"));
- assertThat(set2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeSetIntoNewNamespace() throws Exception {
- SetState<String> set1 = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
- SetState<String> set2 = underTest.state(NAMESPACE_2, STRING_SET_ADDR);
- SetState<String> set3 = underTest.state(NAMESPACE_3, STRING_SET_ADDR);
-
- set1.add("Hello");
- set2.add("Hello");
- set2.add("World");
- set1.add("!");
-
- StateMerging.mergeSets(Arrays.asList(set1, set2, set3), set3);
-
- // Reading the merged set gets both the contents
- assertThat(set3.read(), containsInAnyOrder("Hello", "World", "!"));
- assertThat(set1.read(), Matchers.emptyIterable());
- assertThat(set2.read(), Matchers.emptyIterable());
- }
-
- // for testMap
- private static class MapEntry<K, V> implements Map.Entry<K, V> {
- private K key;
- private V value;
-
- private MapEntry(K key, V value) {
- this.key = key;
- this.value = value;
- }
-
- static <K, V> Map.Entry<K, V> of(K k, V v) {
- return new MapEntry<>(k, v);
- }
-
- public final K getKey() {
- return key;
- }
- public final V getValue() {
- return value;
- }
-
- public final String toString() {
- return key + "=" + value;
- }
-
- public final int hashCode() {
- return Objects.hashCode(key) ^ Objects.hashCode(value);
- }
-
- public final V setValue(V newValue) {
- V oldValue = value;
- value = newValue;
- return oldValue;
- }
-
- public final boolean equals(Object o) {
- if (o == this) {
- return true;
- }
- if (o instanceof Map.Entry) {
- Map.Entry<?, ?> e = (Map.Entry<?, ?>) o;
- if (Objects.equals(key, e.getKey())
- && Objects.equals(value, e.getValue())) {
- return true;
- }
- }
- return false;
+ /**
+ * A standard StateInternals test.
+ */
+ @RunWith(JUnit4.class)
+ public static class StandardStateInternalsTests extends StateInternalsTest {
+ @Override
+ protected StateInternals createStateInternals() {
+ return new InMemoryStateInternals<>("dummyKey");
}
}
- @Test
- public void testMap() throws Exception {
- MapState<String, Integer> value = underTest.state(NAMESPACE_1, STRING_MAP_ADDR);
+ /**
+ * A specific test of InMemoryStateInternals.
+ */
+ @RunWith(JUnit4.class)
+ public static class OtherTests {
- // State instances are cached, but depend on the namespace.
- assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_MAP_ADDR)));
- assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_MAP_ADDR))));
+ private static final StateNamespace NAMESPACE = new StateNamespaceForTest("ns");
- // put
- assertThat(value.entries().read(), Matchers.emptyIterable());
- value.put("A", 1);
- value.put("B", 2);
- value.put("A", 11);
- assertThat(value.putIfAbsent("B", 22).read(), equalTo(2));
- assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("A", 11),
- MapEntry.of("B", 2)));
+ private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
+ StateTags.value("stringValue", StringUtf8Coder.of());
+ private static final StateTag<CombiningState<Integer, int[], Integer>>
+ SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
+ "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
+ private static final StateTag<BagState<String>> STRING_BAG_ADDR =
+ StateTags.bag("stringBag", StringUtf8Coder.of());
+ private static final StateTag<SetState<String>> STRING_SET_ADDR =
+ StateTags.set("stringSet", StringUtf8Coder.of());
+ private static final StateTag<MapState<String, Integer>> STRING_MAP_ADDR =
+ StateTags.map("stringMap", StringUtf8Coder.of(), VarIntCoder.of());
+ private static final StateTag<WatermarkHoldState> WATERMARK_EARLIEST_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST);
+ private static final StateTag<WatermarkHoldState> WATERMARK_LATEST_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST);
+ private static final StateTag<WatermarkHoldState> WATERMARK_EOW_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW);
- // remove
- value.remove("A");
- assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("B", 2)));
- value.remove("C");
- assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("B", 2)));
+ StateInternals underTest = new InMemoryStateInternals<>("dummyKey");
- // get
- assertNull(value.get("A").read());
- assertThat(value.get("B").read(), equalTo(2));
- value.put("C", 3);
- value.put("D", 4);
- assertThat(value.get("C").read(), equalTo(3));
+ @Test
+ public void testSameInstance() {
+ assertSameInstance(STRING_VALUE_ADDR);
+ assertSameInstance(SUM_INTEGER_ADDR);
+ assertSameInstance(STRING_BAG_ADDR);
+ assertSameInstance(STRING_SET_ADDR);
+ assertSameInstance(STRING_MAP_ADDR);
+ assertSameInstance(WATERMARK_EARLIEST_ADDR);
+ }
- // iterate
- value.put("E", 5);
- value.remove("C");
- assertThat(value.keys().read(), containsInAnyOrder("B", "D", "E"));
- assertThat(value.values().read(), containsInAnyOrder(2, 4, 5));
- assertThat(
- value.entries().read(),
- containsInAnyOrder(MapEntry.of("B", 2), MapEntry.of("D", 4), MapEntry.of("E", 5)));
-
- // readLater
- assertThat(value.get("B").readLater().read(), equalTo(2));
- assertNull(value.get("A").readLater().read());
- assertThat(
- value.entries().readLater().read(),
- containsInAnyOrder(MapEntry.of("B", 2), MapEntry.of("D", 4), MapEntry.of("E", 5)));
-
- // clear
- value.clear();
- assertThat(value.entries().read(), Matchers.emptyIterable());
- assertThat(underTest.state(NAMESPACE_1, STRING_MAP_ADDR), Matchers.sameInstance(value));
+ private <T extends State> void assertSameInstance(StateTag<T> address) {
+ assertThat(underTest.state(NAMESPACE, address),
+ Matchers.sameInstance(underTest.state(NAMESPACE, address)));
+ }
}
- @Test
- public void testCombiningValue() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR)));
-
- assertThat(value.read(), equalTo(0));
- value.add(2);
- assertThat(value.read(), equalTo(2));
-
- value.add(3);
- assertThat(value.read(), equalTo(5));
-
- value.clear();
- assertThat(value.read(), equalTo(0));
- assertThat(underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testCombiningIsEmpty() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(5);
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeCombiningValueIntoSource() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- assertThat(value1.read(), equalTo(11));
- assertThat(value2.read(), equalTo(10));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1);
-
- assertThat(value1.read(), equalTo(21));
- assertThat(value2.read(), equalTo(0));
- }
-
- @Test
- public void testMergeCombiningValueIntoNewNamespace() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value3 =
- underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3);
-
- // Merging clears the old values and updates the result value.
- assertThat(value1.read(), equalTo(0));
- assertThat(value2.read(), equalTo(0));
- assertThat(value3.read(), equalTo(21));
- }
-
- @Test
- public void testWatermarkEarliestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), equalTo(new Instant(2000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), equalTo(new Instant(1000)));
-
- value.clear();
- assertThat(value.read(), equalTo(null));
- assertThat(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testWatermarkLatestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), equalTo(new Instant(3000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), equalTo(new Instant(3000)));
-
- value.clear();
- assertThat(value.read(), equalTo(null));
- assertThat(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testWatermarkEndOfWindowState() throws Exception {
- WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), equalTo(new Instant(2000)));
-
- value.clear();
- assertThat(value.read(), equalTo(null));
- assertThat(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), Matchers.sameInstance(value));
- }
-
- @Test
- public void testWatermarkStateIsEmpty() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(new Instant(1000));
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeEarliestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the merged value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value1, WINDOW_1);
-
- assertThat(value1.read(), equalTo(new Instant(2000)));
- assertThat(value2.read(), equalTo(null));
- }
-
- @Test
- public void testMergeLatestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value3 =
- underTest.state(NAMESPACE_3, WATERMARK_LATEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value3, WINDOW_1);
-
- // Merging clears the old values and updates the result value.
- assertThat(value3.read(), equalTo(new Instant(5000)));
- assertThat(value1.read(), equalTo(null));
- assertThat(value2.read(), equalTo(null));
- }
}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
index a2f6acc..959909e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
@@ -17,18 +17,22 @@
*/
package org.apache.beam.runners.core;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import java.util.Collection;
import java.util.concurrent.Executors;
+import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.transforms.splittabledofn.OffsetRange;
import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -42,19 +46,27 @@
/** Tests for {@link OutputAndTimeBoundedSplittableProcessElementInvoker}. */
public class OutputAndTimeBoundedSplittableProcessElementInvokerTest {
private static class SomeFn extends DoFn<Integer, String> {
+ private final int numOutputsPerProcessCall;
private final Duration sleepBeforeEachOutput;
- private SomeFn(Duration sleepBeforeEachOutput) {
+ private SomeFn(int numOutputsPerProcessCall, Duration sleepBeforeEachOutput) {
+ this.numOutputsPerProcessCall = numOutputsPerProcessCall;
this.sleepBeforeEachOutput = sleepBeforeEachOutput;
}
@ProcessElement
- public void process(ProcessContext context, OffsetRangeTracker tracker)
+ public ProcessContinuation process(ProcessContext context, OffsetRangeTracker tracker)
throws Exception {
- for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
+ for (long i = tracker.currentRestriction().getFrom(), numIterations = 1;
+ tracker.tryClaim(i);
+ ++i, ++numIterations) {
Thread.sleep(sleepBeforeEachOutput.getMillis());
context.output("" + i);
+ if (numIterations == numOutputsPerProcessCall) {
+ return resume();
+ }
}
+ return stop();
}
@GetInitialRestriction
@@ -64,8 +76,8 @@
}
private SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker>.Result
- runTest(int count, Duration sleepPerElement) {
- SomeFn fn = new SomeFn(sleepPerElement);
+ runTest(int totalNumOutputs, int numOutputsPerProcessCall, Duration sleepPerElement) {
+ SomeFn fn = new SomeFn(numOutputsPerProcessCall, sleepPerElement);
SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker> invoker =
new OutputAndTimeBoundedSplittableProcessElementInvoker<>(
fn,
@@ -93,14 +105,15 @@
return invoker.invokeProcessElement(
DoFnInvokers.invokerFor(fn),
- WindowedValue.of(count, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING),
- new OffsetRangeTracker(new OffsetRange(0, count)));
+ WindowedValue.of(totalNumOutputs, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING),
+ new OffsetRangeTracker(new OffsetRange(0, totalNumOutputs)));
}
@Test
public void testInvokeProcessElementOutputBounded() throws Exception {
SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker>.Result res =
- runTest(10000, Duration.ZERO);
+ runTest(10000, Integer.MAX_VALUE, Duration.ZERO);
+ assertFalse(res.getContinuation().shouldResume());
OffsetRange residualRange = res.getResidualRestriction();
// Should process the first 100 elements.
assertEquals(1000, residualRange.getFrom());
@@ -110,7 +123,8 @@
@Test
public void testInvokeProcessElementTimeBounded() throws Exception {
SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker>.Result res =
- runTest(10000, Duration.millis(100));
+ runTest(10000, Integer.MAX_VALUE, Duration.millis(100));
+ assertFalse(res.getContinuation().shouldResume());
OffsetRange residualRange = res.getResidualRestriction();
// Should process ideally around 30 elements - but due to timing flakiness, we can't enforce
// that precisely. Just test that it's not egregiously off.
@@ -120,9 +134,18 @@
}
@Test
- public void testInvokeProcessElementVoluntaryReturn() throws Exception {
+ public void testInvokeProcessElementVoluntaryReturnStop() throws Exception {
SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker>.Result res =
- runTest(5, Duration.millis(100));
+ runTest(5, Integer.MAX_VALUE, Duration.millis(100));
+ assertFalse(res.getContinuation().shouldResume());
assertNull(res.getResidualRestriction());
}
+
+ @Test
+ public void testInvokeProcessElementVoluntaryReturnResume() throws Exception {
+ SplittableProcessElementInvoker<Integer, String, OffsetRange, OffsetRangeTracker>.Result res =
+ runTest(10, 5, Duration.millis(100));
+ assertTrue(res.getContinuation().shouldResume());
+ assertEquals(new OffsetRange(5, 10), res.getResidualRestriction());
+ }
}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
index 9e71300..2341502 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
@@ -39,6 +39,7 @@
import com.google.common.collect.Iterables;
import java.util.List;
import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
+import org.apache.beam.runners.core.triggers.DefaultTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachine;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.metrics.MetricName;
@@ -55,6 +56,7 @@
import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
@@ -67,6 +69,7 @@
import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
@@ -140,7 +143,40 @@
}
})
.when(mockTrigger).onFire(anyTriggerContext());
- }
+ }
+
+ /**
+ * Tests that a processing time timer does not cause window GC.
+ */
+ @Test
+ public void testProcessingTimeTimerDoesNotGc() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.ZERO)
+ .withTrigger(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceProcessingTime(new Instant(5000));
+ injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
+ injectElement(tester, 5);
+
+ tester.advanceProcessingTime(new Instant(10000));
+
+ tester.assertHasOnlyGlobalAndStateFor(
+ new IntervalWindow(new Instant(0), new Instant(100)));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0))));
+ }
@Test
public void testOnElementBufferingDiscarding() throws Exception {
@@ -211,6 +247,134 @@
tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
}
+ /**
+ * When the watermark passes the end-of-window and window expiration time
+ * in a single update, this tests that it does not crash.
+ */
+ @Test
+ public void testSessionEowAndGcTogether() throws Exception {
+ ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
+ ReduceFnTester.nonCombining(
+ Sessions.withGapDuration(Duration.millis(10)),
+ DefaultTriggerStateMachine.<IntervalWindow>of(),
+ AccumulationMode.ACCUMULATING_FIRED_PANES,
+ Duration.millis(50),
+ ClosingBehavior.FIRE_ALWAYS);
+
+ tester.setAutoAdvanceOutputWatermark(true);
+
+ tester.advanceInputWatermark(new Instant(0));
+ injectElement(tester, 1);
+ tester.advanceInputWatermark(new Instant(100));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ contains(1), 1, 1, 11, PaneInfo.createPane(true, true, Timing.ON_TIME))));
+ }
+
+ /**
+ * When the watermark passes the end-of-window and window expiration time
+ * in a single update, this tests that it does not crash.
+ */
+ @Test
+ public void testFixedWindowsEowAndGcTogether() throws Exception {
+ ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
+ ReduceFnTester.nonCombining(
+ FixedWindows.of(Duration.millis(10)),
+ DefaultTriggerStateMachine.<IntervalWindow>of(),
+ AccumulationMode.ACCUMULATING_FIRED_PANES,
+ Duration.millis(50),
+ ClosingBehavior.FIRE_ALWAYS);
+
+ tester.setAutoAdvanceOutputWatermark(true);
+
+ tester.advanceInputWatermark(new Instant(0));
+ injectElement(tester, 1);
+ tester.advanceInputWatermark(new Instant(100));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME))));
+ }
+
+ /**
+ * When the watermark passes the end-of-window and window expiration time
+ * in a single update, this tests that it does not crash.
+ */
+ @Test
+ public void testFixedWindowsEowAndGcTogetherFireIfNonEmpty() throws Exception {
+ ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
+ ReduceFnTester.nonCombining(
+ FixedWindows.of(Duration.millis(10)),
+ DefaultTriggerStateMachine.<IntervalWindow>of(),
+ AccumulationMode.ACCUMULATING_FIRED_PANES,
+ Duration.millis(50),
+ ClosingBehavior.FIRE_IF_NON_EMPTY);
+
+ tester.setAutoAdvanceOutputWatermark(true);
+
+ tester.advanceInputWatermark(new Instant(0));
+ injectElement(tester, 1);
+ tester.advanceInputWatermark(new Instant(100));
+
+ List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
+ assertThat(
+ output,
+ contains(
+ isSingleWindowedValue(
+ contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME))));
+ }
+
+ /**
+ * Tests that with the default trigger we will not produce two ON_TIME panes, even
+ * if there are two outputs that are both candidates.
+ */
+ @Test
+ public void testOnlyOneOnTimePane() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
+ .withTrigger(DefaultTrigger.of())
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.millis(100));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceInputWatermark(new Instant(0));
+
+ int value1 = 1;
+ int value2 = 3;
+
+ // A single element that should be in the ON_TIME output
+ tester.injectElements(
+ TimestampedValue.of(value1, new Instant(1)));
+
+ // Should fire ON_TIME
+ tester.advanceInputWatermark(new Instant(10));
+
+ // The DefaultTrigger should cause output labeled LATE, even though it does not have to be
+ // labeled as such.
+ tester.injectElements(
+ TimestampedValue.of(value2, new Instant(3)));
+
+ List<WindowedValue<Integer>> output = tester.extractOutput();
+ assertEquals(2, output.size());
+
+ assertThat(output.get(0), WindowMatchers.isWindowedValue(equalTo(value1)));
+ assertThat(output.get(1), WindowMatchers.isWindowedValue(equalTo(value1 + value2)));
+
+ assertThat(
+ output.get(0),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0)));
+ assertThat(
+ output.get(1),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 1, 1)));
+ }
+
@Test
public void testOnElementCombiningDiscarding() throws Exception {
// Test basic execution of a trigger using a non-combining window set and discarding mode.
@@ -250,6 +414,76 @@
}
/**
+ * Tests that when a processing time timer comes in after a window is expired
+ * it is just ignored.
+ */
+ @Test
+ public void testLateProcessingTimeTimer() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.ZERO)
+ .withTrigger(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceProcessingTime(new Instant(5000));
+ injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
+ injectElement(tester, 5);
+
+ // After this advancement, the window is expired and only the GC process
+ // should be allowed to touch it
+ tester.advanceInputWatermarkNoTimers(new Instant(100));
+
+ // This should not output
+ tester.advanceProcessingTime(new Instant(6000));
+
+ assertThat(tester.extractOutput(), emptyIterable());
+ }
+
+ /**
+ * Tests that when a processing time timer comes in after a window is expired
+ * but in the same bundle it does not cause a spurious output.
+ */
+ @Test
+ public void testCombiningAccumulatingProcessingTime() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.ZERO)
+ .withTrigger(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceProcessingTime(new Instant(5000));
+ injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
+ injectElement(tester, 5);
+
+ tester.advanceInputWatermarkNoTimers(new Instant(100));
+ tester.advanceProcessingTimeNoTimers(new Instant(5010));
+
+ // Fires the GC/EOW timer at the same time as the processing time timer.
+ tester.fireTimers(
+ new IntervalWindow(new Instant(0), new Instant(100)),
+ TimestampedValue.of(TimeDomain.EVENT_TIME, new Instant(100)),
+ TimestampedValue.of(TimeDomain.PROCESSING_TIME, new Instant(5010)));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
+ }
+
+ /**
* Tests that the garbage collection time for a fixed window does not overflow the end of time.
*/
@Test
@@ -316,6 +550,67 @@
assertThat(tester.extractOutput(), contains(isWindowedValue(equalTo(55))));
}
+ /**
+ * Tests that when a processing time timers comes in after a window is expired
+ * and GC'd it does not cause a spurious output.
+ */
+ @Test
+ public void testCombiningAccumulatingProcessingTimeSeparateBundles() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.ZERO)
+ .withTrigger(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceProcessingTime(new Instant(5000));
+ injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
+ injectElement(tester, 5);
+
+ tester.advanceInputWatermark(new Instant(100));
+ tester.advanceProcessingTime(new Instant(5011));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
+ }
+
+ /**
+ * Tests that if end-of-window and GC timers come in together, that the pane is correctly
+ * marked as final.
+ */
+ @Test
+ public void testCombiningAccumulatingEventTime() throws Exception {
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.millis(1))
+ .withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow()));
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
+ injectElement(tester, 5);
+
+ tester.advanceInputWatermark(new Instant(1000));
+
+ assertThat(
+ tester.extractOutput(),
+ contains(
+ isSingleWindowedValue(
+ equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
+ }
+
+
@Test
public void testOnElementCombiningAccumulating() throws Exception {
// Test basic execution of a trigger using a non-combining window set and accumulating mode.
@@ -1289,6 +1584,166 @@
}
/**
+ * Test that it won't fire an empty on-time pane when OnTimeBehavior is FIRE_IF_NON_EMPTY.
+ */
+ @Test
+ public void testEmptyOnTimeWithOnTimeBehaviorFireIfNonEmpty() throws Exception {
+
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withTrigger(
+ AfterEach.<IntervalWindow>inOrder(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(new Duration(5)))
+ .orFinally(AfterWatermark.pastEndOfWindow()),
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(new Duration(25)))))
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.millis(100))
+ .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)
+ .withOnTimeBehavior(Window.OnTimeBehavior.FIRE_IF_NON_EMPTY);
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceInputWatermark(new Instant(0));
+ tester.advanceProcessingTime(new Instant(0));
+
+ // Processing time timer for 5
+ tester.injectElements(
+ TimestampedValue.of(1, new Instant(1)),
+ TimestampedValue.of(1, new Instant(3)),
+ TimestampedValue.of(1, new Instant(7)),
+ TimestampedValue.of(1, new Instant(5)));
+
+ // Should fire early pane
+ tester.advanceProcessingTime(new Instant(6));
+
+ // Should not fire empty on time pane
+ tester.advanceInputWatermark(new Instant(11));
+
+ // Should fire final GC pane
+ tester.advanceInputWatermark(new Instant(10 + 100));
+ List<WindowedValue<Integer>> output = tester.extractOutput();
+ assertEquals(2, output.size());
+
+ assertThat(output.get(0), WindowMatchers.isSingleWindowedValue(4, 1, 0, 10));
+ assertThat(output.get(1), WindowMatchers.isSingleWindowedValue(4, 9, 0, 10));
+
+ assertThat(
+ output.get(0),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
+ assertThat(
+ output.get(1),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 1, 0)));
+ }
+
+ /**
+ * Test that it fires an empty on-time isFinished pane when OnTimeBehavior is FIRE_ALWAYS
+ * and ClosingBehavior is FIRE_IF_NON_EMPTY.
+ *
+ * <p>This is a test just for backward compatibility.
+ */
+ @Test
+ public void testEmptyOnTimeWithOnTimeBehaviorBackwardCompatibility() throws Exception {
+
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withTrigger(AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(AfterPane.elementCountAtLeast(1)))
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.millis(0))
+ .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY);
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceInputWatermark(new Instant(0));
+ tester.advanceProcessingTime(new Instant(0));
+
+ tester.injectElements(
+ TimestampedValue.of(1, new Instant(1)));
+
+ // Should fire empty on time isFinished pane
+ tester.advanceInputWatermark(new Instant(11));
+
+ List<WindowedValue<Integer>> output = tester.extractOutput();
+ assertEquals(2, output.size());
+
+ assertThat(
+ output.get(0),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
+ assertThat(
+ output.get(1),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0)));
+ }
+
+ /**
+ * Test that it won't fire an empty on-time pane when OnTimeBehavior is FIRE_IF_NON_EMPTY
+ * and when receiving late data.
+ */
+ @Test
+ public void testEmptyOnTimeWithOnTimeBehaviorFireIfNonEmptyAndLateData() throws Exception {
+
+ WindowingStrategy<?, IntervalWindow> strategy =
+ WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .withTrigger(
+ AfterEach.<IntervalWindow>inOrder(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(new Duration(5)))
+ .orFinally(AfterWatermark.pastEndOfWindow()),
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(new Duration(25)))))
+ .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
+ .withAllowedLateness(Duration.millis(100))
+ .withOnTimeBehavior(Window.OnTimeBehavior.FIRE_IF_NON_EMPTY);
+
+ ReduceFnTester<Integer, Integer, IntervalWindow> tester =
+ ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
+
+ tester.advanceInputWatermark(new Instant(0));
+ tester.advanceProcessingTime(new Instant(0));
+
+ // Processing time timer for 5
+ tester.injectElements(
+ TimestampedValue.of(1, new Instant(1)),
+ TimestampedValue.of(1, new Instant(3)),
+ TimestampedValue.of(1, new Instant(7)),
+ TimestampedValue.of(1, new Instant(5)));
+
+ // Should fire early pane
+ tester.advanceProcessingTime(new Instant(6));
+
+ // Should not fire empty on time pane
+ tester.advanceInputWatermark(new Instant(11));
+
+ // Processing late data, and should fire late pane
+ tester.injectElements(
+ TimestampedValue.of(1, new Instant(9)));
+ tester.advanceProcessingTime(new Instant(6 + 25 + 1));
+
+ List<WindowedValue<Integer>> output = tester.extractOutput();
+ assertEquals(2, output.size());
+
+ assertThat(output.get(0), WindowMatchers.isSingleWindowedValue(4, 1, 0, 10));
+ assertThat(output.get(1), WindowMatchers.isSingleWindowedValue(5, 9, 0, 10));
+
+ assertThat(
+ output.get(0),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
+ assertThat(
+ output.get(1),
+ WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 1, 0)));
+ }
+
+ /**
* Tests for processing time firings after the watermark passes the end of the window.
* Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
* when the on-time pane is non-empty.
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index 7de8f3b..6f7a4f4 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -21,6 +21,7 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Equivalence;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
@@ -38,7 +39,7 @@
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.TimerInternals.TimerData;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachineRunner;
@@ -116,7 +117,7 @@
return new ReduceFnTester<Integer, Iterable<Integer>, W>(
windowingStrategy,
TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger())),
+ TriggerTranslation.toProto(windowingStrategy.getTrigger())),
SystemReduceFn.<String, Integer, W>buffering(VarIntCoder.of()),
IterableCoder.of(VarIntCoder.of()),
PipelineOptionsFactory.create(),
@@ -179,7 +180,8 @@
return combining(
strategy,
- TriggerStateMachines.stateMachineForTrigger(Triggers.toProto(strategy.getTrigger())),
+ TriggerStateMachines.stateMachineForTrigger(
+ TriggerTranslation.toProto(strategy.getTrigger())),
combineFn,
outputCoder);
}
@@ -227,7 +229,8 @@
return combining(
strategy,
- TriggerStateMachines.stateMachineForTrigger(Triggers.toProto(strategy.getTrigger())),
+ TriggerStateMachines.stateMachineForTrigger(
+ TriggerTranslation.toProto(strategy.getTrigger())),
combineFn,
outputCoder,
options,
@@ -316,6 +319,19 @@
}
@SafeVarargs
+ public final void assertHasOnlyGlobalAndStateFor(W... expectedWindows) {
+ assertHasOnlyGlobalAndAllowedTags(
+ ImmutableSet.copyOf(expectedWindows),
+ ImmutableSet.<StateTag<?>>of(
+ ((SystemReduceFn<?, ?, ?, ?, ?>) reduceFn).getBufferTag(),
+ TriggerStateMachineRunner.FINISHED_BITS_TAG,
+ PaneInfoTracker.PANE_INFO_TAG,
+ WatermarkHold.watermarkHoldTagForTimestampCombiner(
+ objectStrategy.getTimestampCombiner()),
+ WatermarkHold.EXTRA_HOLD_TAG));
+ }
+
+ @SafeVarargs
public final void assertHasOnlyGlobalAndFinishedSetsAndPaneInfoFor(W... expectedWindows) {
assertHasOnlyGlobalAndAllowedTags(
ImmutableSet.copyOf(expectedWindows),
@@ -350,28 +366,41 @@
private void assertHasOnlyGlobalAndAllowedTags(
Set<W> expectedWindows, Set<StateTag<?>> allowedTags) {
Set<StateNamespace> expectedWindowsSet = new HashSet<>();
+
+ Set<Equivalence.Wrapper<StateTag>> allowedEquivalentTags = new HashSet<>();
+ for (StateTag tag : allowedTags) {
+ allowedEquivalentTags.add(StateTags.ID_EQUIVALENCE.wrap(tag));
+ }
+
for (W expectedWindow : expectedWindows) {
expectedWindowsSet.add(windowNamespace(expectedWindow));
}
- Map<StateNamespace, Set<StateTag<?>>> actualWindows = new HashMap<>();
+ Map<StateNamespace, Set<Equivalence.Wrapper<StateTag>>> actualWindows = new HashMap<>();
for (StateNamespace namespace : stateInternals.getNamespacesInUse()) {
if (namespace instanceof StateNamespaces.GlobalNamespace) {
continue;
} else if (namespace instanceof StateNamespaces.WindowNamespace) {
- Set<StateTag<?>> tagsInUse = stateInternals.getTagsInUse(namespace);
+ Set<Equivalence.Wrapper<StateTag>> tagsInUse = new HashSet<>();
+ for (StateTag tag : stateInternals.getTagsInUse(namespace)) {
+ tagsInUse.add(StateTags.ID_EQUIVALENCE.wrap(tag));
+ }
if (tagsInUse.isEmpty()) {
continue;
}
actualWindows.put(namespace, tagsInUse);
- Set<StateTag<?>> unexpected = Sets.difference(tagsInUse, allowedTags);
+ Set<Equivalence.Wrapper<StateTag>> unexpected =
+ Sets.difference(tagsInUse, allowedEquivalentTags);
if (unexpected.isEmpty()) {
continue;
} else {
fail(namespace + " has unexpected states: " + tagsInUse);
}
} else if (namespace instanceof StateNamespaces.WindowAndTriggerNamespace) {
- Set<StateTag<?>> tagsInUse = stateInternals.getTagsInUse(namespace);
+ Set<Equivalence.Wrapper<StateTag>> tagsInUse = new HashSet<>();
+ for (StateTag tag : stateInternals.getTagsInUse(namespace)) {
+ tagsInUse.add(StateTags.ID_EQUIVALENCE.wrap(tag));
+ }
assertTrue(namespace + " contains " + tagsInUse, tagsInUse.isEmpty());
} else {
fail("Unrecognized namespace " + namespace);
@@ -418,6 +447,10 @@
return result;
}
+ public void advanceInputWatermarkNoTimers(Instant newInputWatermark) throws Exception {
+ timerInternals.advanceInputWatermark(newInputWatermark);
+ }
+
/**
* Advance the input watermark to the specified time, firing any timers that should
* fire. Then advance the output watermark as far as possible.
@@ -449,6 +482,10 @@
runner.persist();
}
+ public void advanceProcessingTimeNoTimers(Instant newProcessingTime) throws Exception {
+ timerInternals.advanceProcessingTime(newProcessingTime);
+ }
+
/**
* If {@link #autoAdvanceOutputWatermark} is {@literal false}, advance the output watermark
* to the given value. Otherwise throw.
@@ -506,8 +543,8 @@
for (TimestampedValue<InputT> value : values) {
WindowTracing.trace("TriggerTester.injectElements: {}", value);
}
- ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
- runner.processElements(
+
+ Iterable<WindowedValue<InputT>> inputs =
Iterables.transform(
Arrays.asList(values),
new Function<TimestampedValue<InputT>, WindowedValue<InputT>>() {
@@ -525,7 +562,12 @@
throw new RuntimeException(e);
}
}
- }));
+ });
+
+ ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
+ runner.processElements(
+ new LateDataDroppingDoFnRunner.LateDataFilter(objectStrategy, timerInternals)
+ .filter(KEY, inputs));
// Persist after each bundle.
runner.persist();
@@ -533,13 +575,27 @@
public void fireTimer(W window, Instant timestamp, TimeDomain domain) throws Exception {
ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
- ArrayList timers = new ArrayList(1);
+ ArrayList<TimerData> timers = new ArrayList<>(1);
timers.add(
TimerData.of(StateNamespaces.window(windowFn.windowCoder(), window), timestamp, domain));
runner.onTimers(timers);
runner.persist();
}
+ public void fireTimers(W window, TimestampedValue<TimeDomain>... timers) throws Exception {
+ ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
+ ArrayList<TimerData> timerData = new ArrayList<>(timers.length);
+ for (TimestampedValue<TimeDomain> timer : timers) {
+ timerData.add(
+ TimerData.of(
+ StateNamespaces.window(windowFn.windowCoder(), window),
+ timer.getTimestamp(),
+ timer.getValue()));
+ }
+ runner.onTimers(timerData);
+ runner.persist();
+ }
+
/**
* Convey the simulated state and implement {@link #outputWindowedValue} to capture all output
* elements.
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index abefd1c..f331b65 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -29,7 +29,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
import org.apache.beam.runners.core.DoFnRunners.OutputManager;
import org.apache.beam.runners.core.TimerInternals.TimerData;
import org.apache.beam.sdk.coders.Coder;
@@ -63,7 +62,8 @@
public class SimpleDoFnRunnerTest {
@Rule public ExpectedException thrown = ExpectedException.none();
- @Mock StepContext mockStepContext;
+ @Mock
+ StepContext mockStepContext;
@Mock TimerInternals mockTimerInternals;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
new file mode 100644
index 0000000..7449af3
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.Executors;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SplittableParDoViaKeyedWorkItems.ProcessFn}. */
+@RunWith(JUnit4.class)
+public class SplittableParDoProcessFnTest {
+ private static final int MAX_OUTPUTS_PER_BUNDLE = 10000;
+ private static final Duration MAX_BUNDLE_DURATION = Duration.standardSeconds(5);
+
+ // ----------------- Tests for whether the transform sets boundedness correctly --------------
+ private static class SomeRestriction
+ implements Serializable, HasDefaultTracker<SomeRestriction, SomeRestrictionTracker> {
+ @Override
+ public SomeRestrictionTracker newTracker() {
+ return new SomeRestrictionTracker(this);
+ }
+ }
+
+ private static class SomeRestrictionTracker implements RestrictionTracker<SomeRestriction> {
+ private final SomeRestriction someRestriction;
+
+ public SomeRestrictionTracker(SomeRestriction someRestriction) {
+ this.someRestriction = someRestriction;
+ }
+
+ @Override
+ public SomeRestriction currentRestriction() {
+ return someRestriction;
+ }
+
+ @Override
+ public SomeRestriction checkpoint() {
+ return someRestriction;
+ }
+
+ @Override
+ public void checkDone() {}
+ }
+
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ /**
+ * A helper for testing {@link ProcessFn} on 1 element (but
+ * possibly over multiple {@link DoFn.ProcessElement} calls).
+ */
+ private static class ProcessFnTester<
+ InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+ implements AutoCloseable {
+ private final DoFnTester<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> tester;
+ private Instant currentProcessingTime;
+
+ private InMemoryTimerInternals timerInternals;
+ private TestInMemoryStateInternals<String> stateInternals;
+
+ ProcessFnTester(
+ Instant currentProcessingTime,
+ final DoFn<InputT, OutputT> fn,
+ Coder<InputT> inputCoder,
+ Coder<RestrictionT> restrictionCoder,
+ int maxOutputsPerBundle,
+ Duration maxBundleDuration)
+ throws Exception {
+ // The exact windowing strategy doesn't matter in this test, but it should be able to
+ // encode IntervalWindow's because that's what all tests here use.
+ WindowingStrategy<InputT, BoundedWindow> windowingStrategy =
+ (WindowingStrategy) WindowingStrategy.of(FixedWindows.of(Duration.standardSeconds(1)));
+ final ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
+ new ProcessFn<>(
+ fn, inputCoder, restrictionCoder, windowingStrategy);
+ this.tester = DoFnTester.of(processFn);
+ this.timerInternals = new InMemoryTimerInternals();
+ this.stateInternals = new TestInMemoryStateInternals<>("dummy");
+ processFn.setStateInternalsFactory(
+ new StateInternalsFactory<String>() {
+ @Override
+ public StateInternals stateInternalsForKey(String key) {
+ return stateInternals;
+ }
+ });
+ processFn.setTimerInternalsFactory(
+ new TimerInternalsFactory<String>() {
+ @Override
+ public TimerInternals timerInternalsForKey(String key) {
+ return timerInternals;
+ }
+ });
+ processFn.setProcessElementInvoker(
+ new OutputAndTimeBoundedSplittableProcessElementInvoker<
+ InputT, OutputT, RestrictionT, TrackerT>(
+ fn,
+ tester.getPipelineOptions(),
+ new OutputWindowedValueToDoFnTester<>(tester),
+ new SideInputReader() {
+ @Override
+ public <T> T get(PCollectionView<T> view, BoundedWindow window) {
+ throw new NoSuchElementException();
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return true;
+ }
+ },
+ Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory()),
+ maxOutputsPerBundle,
+ maxBundleDuration));
+ // Do not clone since ProcessFn references non-serializable DoFnTester itself
+ // through the state/timer/output callbacks.
+ this.tester.setCloningBehavior(DoFnTester.CloningBehavior.DO_NOT_CLONE);
+ this.tester.startBundle();
+ timerInternals.advanceProcessingTime(currentProcessingTime);
+
+ this.currentProcessingTime = currentProcessingTime;
+ }
+
+ @Override
+ public void close() throws Exception {
+ tester.close();
+ }
+
+ /** Performs a seed {@link DoFn.ProcessElement} call feeding the element and restriction. */
+ void startElement(InputT element, RestrictionT restriction) throws Exception {
+ startElement(
+ WindowedValue.of(
+ KV.of(element, restriction),
+ currentProcessingTime,
+ GlobalWindow.INSTANCE,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING));
+ }
+
+ void startElement(WindowedValue<KV<InputT, RestrictionT>> windowedValue) throws Exception {
+ tester.processElement(
+ KeyedWorkItems.elementsWorkItem("key", Collections.singletonList(windowedValue)));
+ }
+
+ /**
+ * Advances processing time by a given duration and, if any timers fired, performs a non-seed
+ * {@link DoFn.ProcessElement} call, feeding it the timers.
+ */
+ boolean advanceProcessingTimeBy(Duration duration) throws Exception {
+ currentProcessingTime = currentProcessingTime.plus(duration);
+ timerInternals.advanceProcessingTime(currentProcessingTime);
+
+ List<TimerInternals.TimerData> timers = new ArrayList<>();
+ TimerInternals.TimerData nextTimer;
+ while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) {
+ timers.add(nextTimer);
+ }
+ if (timers.isEmpty()) {
+ return false;
+ }
+ tester.processElement(
+ KeyedWorkItems.<String, KV<InputT, RestrictionT>>timersWorkItem("key", timers));
+ return true;
+ }
+
+ List<TimestampedValue<OutputT>> peekOutputElementsInWindow(BoundedWindow window) {
+ return tester.peekOutputElementsInWindow(window);
+ }
+
+ List<OutputT> takeOutputElements() {
+ return tester.takeOutputElements();
+ }
+
+ public Instant getWatermarkHold() {
+ return stateInternals.earliestWatermarkHold();
+ }
+ }
+
+ private static class OutputWindowedValueToDoFnTester<OutputT>
+ implements OutputWindowedValue<OutputT> {
+ private final DoFnTester<?, OutputT> tester;
+
+ private OutputWindowedValueToDoFnTester(DoFnTester<?, OutputT> tester) {
+ this.tester = tester;
+ }
+
+ @Override
+ public void outputWindowedValue(
+ OutputT output,
+ Instant timestamp,
+ Collection<? extends BoundedWindow> windows,
+ PaneInfo pane) {
+ outputWindowedValue(tester.getMainOutputTag(), output, timestamp, windows, pane);
+ }
+
+ @Override
+ public <AdditionalOutputT> void outputWindowedValue(
+ TupleTag<AdditionalOutputT> tag,
+ AdditionalOutputT output,
+ Instant timestamp,
+ Collection<? extends BoundedWindow> windows,
+ PaneInfo pane) {
+ for (BoundedWindow window : windows) {
+ tester.getMutableOutput(tag).add(ValueInSingleWindow.of(output, timestamp, window, pane));
+ }
+ }
+ }
+
+ /** A simple splittable {@link DoFn} that's actually monolithic. */
+ private static class ToStringFn extends DoFn<Integer, String> {
+ @ProcessElement
+ public void process(ProcessContext c, SomeRestrictionTracker tracker) {
+ c.output(c.element().toString() + "a");
+ c.output(c.element().toString() + "b");
+ c.output(c.element().toString() + "c");
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer elem) {
+ return new SomeRestriction();
+ }
+ }
+
+ @Test
+ public void testTrivialProcessFnPropagatesOutputWindowAndTimestamp() throws Exception {
+ // Tests that ProcessFn correctly propagates the window and timestamp of the element
+ // inside the KeyedWorkItem.
+ // The underlying DoFn is actually monolithic, so this doesn't test splitting.
+ DoFn<Integer, String> fn = new ToStringFn();
+
+ Instant base = Instant.now();
+
+ IntervalWindow w =
+ new IntervalWindow(
+ base.minus(Duration.standardMinutes(1)), base.plus(Duration.standardMinutes(1)));
+
+ ProcessFnTester<Integer, String, SomeRestriction, SomeRestrictionTracker> tester =
+ new ProcessFnTester<>(
+ base,
+ fn,
+ BigEndianIntegerCoder.of(),
+ SerializableCoder.of(SomeRestriction.class),
+ MAX_OUTPUTS_PER_BUNDLE,
+ MAX_BUNDLE_DURATION);
+ tester.startElement(
+ WindowedValue.of(
+ KV.of(42, new SomeRestriction()),
+ base,
+ Collections.singletonList(w),
+ PaneInfo.ON_TIME_AND_ONLY_FIRING));
+
+ assertEquals(
+ Arrays.asList(
+ TimestampedValue.of("42a", base),
+ TimestampedValue.of("42b", base),
+ TimestampedValue.of("42c", base)),
+ tester.peekOutputElementsInWindow(w));
+ }
+
+ private static class WatermarkUpdateFn extends DoFn<Instant, String> {
+ @ProcessElement
+ public void process(ProcessContext c, OffsetRangeTracker tracker) {
+ for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
+ c.updateWatermark(c.element().plus(Duration.standardSeconds(i)));
+ c.output(String.valueOf(i));
+ }
+ }
+
+ @GetInitialRestriction
+ public OffsetRange getInitialRestriction(Instant elem) {
+ throw new IllegalStateException("Expected to be supplied explicitly in this test");
+ }
+
+ @NewTracker
+ public OffsetRangeTracker newTracker(OffsetRange range) {
+ return new OffsetRangeTracker(range);
+ }
+ }
+
+ @Test
+ public void testUpdatesWatermark() throws Exception {
+ DoFn<Instant, String> fn = new WatermarkUpdateFn();
+ Instant base = Instant.now();
+
+ ProcessFnTester<Instant, String, OffsetRange, OffsetRangeTracker> tester =
+ new ProcessFnTester<>(
+ base,
+ fn,
+ InstantCoder.of(),
+ SerializableCoder.of(OffsetRange.class),
+ 3,
+ MAX_BUNDLE_DURATION);
+
+ tester.startElement(base, new OffsetRange(0, 8));
+ assertThat(tester.takeOutputElements(), hasItems("0", "1", "2"));
+ assertEquals(base.plus(Duration.standardSeconds(2)), tester.getWatermarkHold());
+
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ assertThat(tester.takeOutputElements(), hasItems("3", "4", "5"));
+ assertEquals(base.plus(Duration.standardSeconds(5)), tester.getWatermarkHold());
+
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ assertThat(tester.takeOutputElements(), hasItems("6", "7"));
+ assertEquals(null, tester.getWatermarkHold());
+ }
+
+ /** A simple splittable {@link DoFn} that outputs the given element every 5 seconds forever. */
+ private static class SelfInitiatedResumeFn extends DoFn<Integer, String> {
+ @ProcessElement
+ public ProcessContinuation process(ProcessContext c, SomeRestrictionTracker tracker) {
+ c.output(c.element().toString());
+ return resume().withResumeDelay(Duration.standardSeconds(5));
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer elem) {
+ return new SomeRestriction();
+ }
+ }
+
+ @Test
+ public void testResumeSetsTimer() throws Exception {
+ DoFn<Integer, String> fn = new SelfInitiatedResumeFn();
+ Instant base = Instant.now();
+ ProcessFnTester<Integer, String, SomeRestriction, SomeRestrictionTracker> tester =
+ new ProcessFnTester<>(
+ base,
+ fn,
+ BigEndianIntegerCoder.of(),
+ SerializableCoder.of(SomeRestriction.class),
+ MAX_OUTPUTS_PER_BUNDLE,
+ MAX_BUNDLE_DURATION);
+
+ tester.startElement(42, new SomeRestriction());
+ assertThat(tester.takeOutputElements(), contains("42"));
+
+ // Should resume after 5 seconds: advancing by 3 seconds should have no effect.
+ assertFalse(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
+ assertTrue(tester.takeOutputElements().isEmpty());
+
+ // 6 seconds should be enough should invoke the fn again.
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
+ assertThat(tester.takeOutputElements(), contains("42"));
+
+ // Should again resume after 5 seconds: advancing by 3 seconds should again have no effect.
+ assertFalse(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
+ assertTrue(tester.takeOutputElements().isEmpty());
+
+ // 6 seconds should again be enough.
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(3)));
+ assertThat(tester.takeOutputElements(), contains("42"));
+ }
+
+ /** A splittable {@link DoFn} that generates the sequence [init, init + total). */
+ private static class CounterFn extends DoFn<Integer, String> {
+ private final int numOutputsPerCall;
+
+ public CounterFn(int numOutputsPerCall) {
+ this.numOutputsPerCall = numOutputsPerCall;
+ }
+
+ @ProcessElement
+ public ProcessContinuation process(ProcessContext c, OffsetRangeTracker tracker) {
+ for (long i = tracker.currentRestriction().getFrom(), numIterations = 0;
+ tracker.tryClaim(i); ++i, ++numIterations) {
+ c.output(String.valueOf(c.element() + i));
+ if (numIterations == numOutputsPerCall) {
+ return resume();
+ }
+ }
+ return stop();
+ }
+
+ @GetInitialRestriction
+ public OffsetRange getInitialRestriction(Integer elem) {
+ throw new UnsupportedOperationException("Expected to be supplied explicitly in this test");
+ }
+ }
+
+ public void testResumeCarriesOverState() throws Exception {
+ DoFn<Integer, String> fn = new CounterFn(1);
+ Instant base = Instant.now();
+ ProcessFnTester<Integer, String, OffsetRange, OffsetRangeTracker> tester =
+ new ProcessFnTester<>(
+ base,
+ fn,
+ BigEndianIntegerCoder.of(),
+ SerializableCoder.of(OffsetRange.class),
+ MAX_OUTPUTS_PER_BUNDLE,
+ MAX_BUNDLE_DURATION);
+
+ tester.startElement(42, new OffsetRange(0, 3));
+ assertThat(tester.takeOutputElements(), contains("42"));
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ assertThat(tester.takeOutputElements(), contains("43"));
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ assertThat(tester.takeOutputElements(), contains("44"));
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ // After outputting all 3 items, should not output anything more.
+ assertEquals(0, tester.takeOutputElements().size());
+ // Should also not ask to resume.
+ assertFalse(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ }
+
+ @Test
+ public void testCheckpointsAfterNumOutputs() throws Exception {
+ int max = 100;
+ DoFn<Integer, String> fn = new CounterFn(Integer.MAX_VALUE);
+ Instant base = Instant.now();
+ int baseIndex = 42;
+
+ ProcessFnTester<Integer, String, OffsetRange, OffsetRangeTracker> tester =
+ new ProcessFnTester<>(
+ base, fn, BigEndianIntegerCoder.of(), SerializableCoder.of(OffsetRange.class),
+ max, MAX_BUNDLE_DURATION);
+
+ List<String> elements;
+
+ // Create an fn that attempts to 2x output more than checkpointing allows.
+ tester.startElement(baseIndex, new OffsetRange(0, 2 * max + max / 2));
+ elements = tester.takeOutputElements();
+ assertEquals(max, elements.size());
+ // Should output the range [0, max)
+ assertThat(elements, hasItem(String.valueOf(baseIndex)));
+ assertThat(elements, hasItem(String.valueOf(baseIndex + max - 1)));
+
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ elements = tester.takeOutputElements();
+ assertEquals(max, elements.size());
+ // Should output the range [max, 2*max)
+ assertThat(elements, hasItem(String.valueOf(baseIndex + max)));
+ assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max - 1)));
+
+ assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
+ elements = tester.takeOutputElements();
+ assertEquals(max / 2, elements.size());
+ // Should output the range [2*max, 2*max + max/2)
+ assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max)));
+ assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max + max / 2 - 1)));
+ assertThat(elements, not(hasItem((String.valueOf(baseIndex + 2 * max + max / 2)))));
+ }
+
+ @Test
+ public void testCheckpointsAfterDuration() throws Exception {
+ // Don't bound number of outputs.
+ int max = Integer.MAX_VALUE;
+ // But bound bundle duration - the bundle should terminate.
+ Duration maxBundleDuration = Duration.standardSeconds(1);
+ // Create an fn that attempts to 2x output more than checkpointing allows.
+ DoFn<Integer, String> fn = new CounterFn(Integer.MAX_VALUE);
+ Instant base = Instant.now();
+ int baseIndex = 42;
+
+ ProcessFnTester<Integer, String, OffsetRange, OffsetRangeTracker> tester =
+ new ProcessFnTester<>(
+ base, fn, BigEndianIntegerCoder.of(), SerializableCoder.of(OffsetRange.class),
+ max, maxBundleDuration);
+
+ List<String> elements;
+
+ tester.startElement(baseIndex, new OffsetRange(0, Long.MAX_VALUE));
+ // Bundle should terminate, and should do at least some processing.
+ elements = tester.takeOutputElements();
+ assertFalse(elements.isEmpty());
+ // Bundle should have run for at least the requested duration.
+ assertThat(
+ Instant.now().getMillis() - base.getMillis(),
+ greaterThanOrEqualTo(maxBundleDuration.getMillis()));
+ }
+
+ private static class LifecycleVerifyingFn extends DoFn<Integer, String> {
+ private enum State {
+ BEFORE_SETUP,
+ OUTSIDE_BUNDLE,
+ INSIDE_BUNDLE,
+ TORN_DOWN
+ }
+
+ private State state = State.BEFORE_SETUP;
+
+ @ProcessElement
+ public void process(ProcessContext c, SomeRestrictionTracker tracker) {
+ assertEquals(State.INSIDE_BUNDLE, state);
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return new SomeRestriction();
+ }
+
+ @Setup
+ public void setup() {
+ assertEquals(State.BEFORE_SETUP, state);
+ state = State.OUTSIDE_BUNDLE;
+ }
+
+ @Teardown
+ public void tearDown() {
+ assertEquals(State.OUTSIDE_BUNDLE, state);
+ state = State.TORN_DOWN;
+ }
+
+ @StartBundle
+ public void startBundle() {
+ assertEquals(State.OUTSIDE_BUNDLE, state);
+ state = State.INSIDE_BUNDLE;
+ }
+
+ @FinishBundle
+ public void finishBundle() {
+ assertEquals(State.INSIDE_BUNDLE, state);
+ state = State.OUTSIDE_BUNDLE;
+ }
+ }
+
+ @Test
+ public void testInvokesLifecycleMethods() throws Exception {
+ DoFn<Integer, String> fn = new LifecycleVerifyingFn();
+ try (ProcessFnTester<Integer, String, SomeRestriction, SomeRestrictionTracker> tester =
+ new ProcessFnTester<>(
+ Instant.now(),
+ fn,
+ BigEndianIntegerCoder.of(),
+ SerializableCoder.of(SomeRestriction.class),
+ MAX_OUTPUTS_PER_BUNDLE,
+ MAX_BUNDLE_DURATION)) {
+ tester.startElement(42, new SomeRestriction());
+ }
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
deleted file mode 100644
index be4cf08..0000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasItems;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.concurrent.Executors;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.InstantCoder;
-import org.apache.beam.sdk.coders.SerializableCoder;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
-import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
-import org.apache.beam.sdk.transforms.DoFnTester;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
-import org.apache.beam.sdk.transforms.splittabledofn.OffsetRange;
-import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TimestampedValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.sdk.values.ValueInSingleWindow;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link SplittableParDo}. */
-@RunWith(JUnit4.class)
-public class SplittableParDoTest {
- private static final int MAX_OUTPUTS_PER_BUNDLE = 10000;
- private static final Duration MAX_BUNDLE_DURATION = Duration.standardSeconds(5);
-
- // ----------------- Tests for whether the transform sets boundedness correctly --------------
- private static class SomeRestriction
- implements Serializable, HasDefaultTracker<SomeRestriction, SomeRestrictionTracker> {
- @Override
- public SomeRestrictionTracker newTracker() {
- return new SomeRestrictionTracker(this);
- }
- }
-
- private static class SomeRestrictionTracker implements RestrictionTracker<SomeRestriction> {
- private final SomeRestriction someRestriction;
-
- public SomeRestrictionTracker(SomeRestriction someRestriction) {
- this.someRestriction = someRestriction;
- }
-
- @Override
- public SomeRestriction currentRestriction() {
- return someRestriction;
- }
-
- @Override
- public SomeRestriction checkpoint() {
- return someRestriction;
- }
-
- @Override
- public void checkDone() {}
- }
-
- @BoundedPerElement
- private static class BoundedFakeFn extends DoFn<Integer, String> {
- @ProcessElement
- public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
-
- @GetInitialRestriction
- public SomeRestriction getInitialRestriction(Integer element) {
- return null;
- }
- }
-
- @UnboundedPerElement
- private static class UnboundedFakeFn extends DoFn<Integer, String> {
- @ProcessElement
- public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
-
- @GetInitialRestriction
- public SomeRestriction getInitialRestriction(Integer element) {
- return null;
- }
- }
-
- private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) {
- return pipeline
- .apply("unbounded", Create.of(1, 2, 3))
- .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
- }
-
- private static PCollection<Integer> makeBoundedCollection(Pipeline pipeline) {
- return pipeline
- .apply("bounded", Create.of(1, 2, 3))
- .setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
- }
-
- private static final TupleTag<String> MAIN_OUTPUT_TAG = new TupleTag<String>() {};
-
- private ParDo.MultiOutput<Integer, String> makeParDo(DoFn<Integer, String> fn) {
- return ParDo.of(fn).withOutputTags(MAIN_OUTPUT_TAG, TupleTagList.empty());
- }
-
- @Rule
- public TestPipeline pipeline = TestPipeline.create();
-
- @Test
- public void testBoundednessForBoundedFn() {
- pipeline.enableAbandonedNodeEnforcement(false);
-
- DoFn<Integer, String> boundedFn = new BoundedFakeFn();
- assertEquals(
- "Applying a bounded SDF to a bounded collection produces a bounded collection",
- PCollection.IsBounded.BOUNDED,
- makeBoundedCollection(pipeline)
- .apply("bounded to bounded", new SplittableParDo<>(makeParDo(boundedFn)))
- .get(MAIN_OUTPUT_TAG)
- .isBounded());
- assertEquals(
- "Applying a bounded SDF to an unbounded collection produces an unbounded collection",
- PCollection.IsBounded.UNBOUNDED,
- makeUnboundedCollection(pipeline)
- .apply("bounded to unbounded", new SplittableParDo<>(makeParDo(boundedFn)))
- .get(MAIN_OUTPUT_TAG)
- .isBounded());
- }
-
- @Test
- public void testBoundednessForUnboundedFn() {
- pipeline.enableAbandonedNodeEnforcement(false);
-
- DoFn<Integer, String> unboundedFn = new UnboundedFakeFn();
- assertEquals(
- "Applying an unbounded SDF to a bounded collection produces a bounded collection",
- PCollection.IsBounded.UNBOUNDED,
- makeBoundedCollection(pipeline)
- .apply("unbounded to bounded", new SplittableParDo<>(makeParDo(unboundedFn)))
- .get(MAIN_OUTPUT_TAG)
- .isBounded());
- assertEquals(
- "Applying an unbounded SDF to an unbounded collection produces an unbounded collection",
- PCollection.IsBounded.UNBOUNDED,
- makeUnboundedCollection(pipeline)
- .apply("unbounded to unbounded", new SplittableParDo<>(makeParDo(unboundedFn)))
- .get(MAIN_OUTPUT_TAG)
- .isBounded());
- }
-
- // ------------------------------- Tests for ProcessFn ---------------------------------
-
- /**
- * A helper for testing {@link SplittableParDo.ProcessFn} on 1 element (but possibly over multiple
- * {@link DoFn.ProcessElement} calls).
- */
- private static class ProcessFnTester<
- InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
- implements AutoCloseable {
- private final DoFnTester<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
- tester;
- private Instant currentProcessingTime;
-
- private InMemoryTimerInternals timerInternals;
- private TestInMemoryStateInternals<String> stateInternals;
-
- ProcessFnTester(
- Instant currentProcessingTime,
- final DoFn<InputT, OutputT> fn,
- Coder<InputT> inputCoder,
- Coder<RestrictionT> restrictionCoder,
- int maxOutputsPerBundle,
- Duration maxBundleDuration)
- throws Exception {
- // The exact windowing strategy doesn't matter in this test, but it should be able to
- // encode IntervalWindow's because that's what all tests here use.
- WindowingStrategy<InputT, BoundedWindow> windowingStrategy =
- (WindowingStrategy) WindowingStrategy.of(FixedWindows.of(Duration.standardSeconds(1)));
- final SplittableParDo.ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
- new SplittableParDo.ProcessFn<>(
- fn, inputCoder, restrictionCoder, windowingStrategy);
- this.tester = DoFnTester.of(processFn);
- this.timerInternals = new InMemoryTimerInternals();
- this.stateInternals = new TestInMemoryStateInternals<>("dummy");
- processFn.setStateInternalsFactory(
- new StateInternalsFactory<String>() {
- @Override
- public StateInternals stateInternalsForKey(String key) {
- return stateInternals;
- }
- });
- processFn.setTimerInternalsFactory(
- new TimerInternalsFactory<String>() {
- @Override
- public TimerInternals timerInternalsForKey(String key) {
- return timerInternals;
- }
- });
- processFn.setProcessElementInvoker(
- new OutputAndTimeBoundedSplittableProcessElementInvoker<
- InputT, OutputT, RestrictionT, TrackerT>(
- fn,
- tester.getPipelineOptions(),
- new OutputWindowedValueToDoFnTester<>(tester),
- new SideInputReader() {
- @Nullable
- @Override
- public <T> T get(PCollectionView<T> view, BoundedWindow window) {
- throw new NoSuchElementException();
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- return false;
- }
-
- @Override
- public boolean isEmpty() {
- return true;
- }
- },
- Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory()),
- maxOutputsPerBundle,
- maxBundleDuration));
- // Do not clone since ProcessFn references non-serializable DoFnTester itself
- // through the state/timer/output callbacks.
- this.tester.setCloningBehavior(DoFnTester.CloningBehavior.DO_NOT_CLONE);
- this.tester.startBundle();
- timerInternals.advanceProcessingTime(currentProcessingTime);
-
- this.currentProcessingTime = currentProcessingTime;
- }
-
- @Override
- public void close() throws Exception {
- tester.close();
- }
-
- /** Performs a seed {@link DoFn.ProcessElement} call feeding the element and restriction. */
- void startElement(InputT element, RestrictionT restriction) throws Exception {
- startElement(
- WindowedValue.of(
- ElementAndRestriction.of(element, restriction),
- currentProcessingTime,
- GlobalWindow.INSTANCE,
- PaneInfo.ON_TIME_AND_ONLY_FIRING));
- }
-
- void startElement(WindowedValue<ElementAndRestriction<InputT, RestrictionT>> windowedValue)
- throws Exception {
- tester.processElement(
- KeyedWorkItems.elementsWorkItem("key", Collections.singletonList(windowedValue)));
- }
-
- /**
- * Advances processing time by a given duration and, if any timers fired, performs a non-seed
- * {@link DoFn.ProcessElement} call, feeding it the timers.
- */
- boolean advanceProcessingTimeBy(Duration duration) throws Exception {
- currentProcessingTime = currentProcessingTime.plus(duration);
- timerInternals.advanceProcessingTime(currentProcessingTime);
-
- List<TimerInternals.TimerData> timers = new ArrayList<>();
- TimerInternals.TimerData nextTimer;
- while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) {
- timers.add(nextTimer);
- }
- if (timers.isEmpty()) {
- return false;
- }
- tester.processElement(
- KeyedWorkItems.<String, ElementAndRestriction<InputT, RestrictionT>>timersWorkItem(
- "key", timers));
- return true;
- }
-
- List<TimestampedValue<OutputT>> peekOutputElementsInWindow(BoundedWindow window) {
- return tester.peekOutputElementsInWindow(window);
- }
-
- List<OutputT> takeOutputElements() {
- return tester.takeOutputElements();
- }
-
- public Instant getWatermarkHold() {
- return stateInternals.earliestWatermarkHold();
- }
- }
-
- private static class OutputWindowedValueToDoFnTester<OutputT>
- implements OutputWindowedValue<OutputT> {
- private final DoFnTester<?, OutputT> tester;
-
- private OutputWindowedValueToDoFnTester(DoFnTester<?, OutputT> tester) {
- this.tester = tester;
- }
-
- @Override
- public void outputWindowedValue(
- OutputT output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- outputWindowedValue(tester.getMainOutputTag(), output, timestamp, windows, pane);
- }
-
- @Override
- public <AdditionalOutputT> void outputWindowedValue(
- TupleTag<AdditionalOutputT> tag,
- AdditionalOutputT output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- for (BoundedWindow window : windows) {
- tester.getMutableOutput(tag).add(ValueInSingleWindow.of(output, timestamp, window, pane));
- }
- }
- }
-
- /** A simple splittable {@link DoFn} that's actually monolithic. */
- private static class ToStringFn extends DoFn<Integer, String> {
- @ProcessElement
- public void process(ProcessContext c, SomeRestrictionTracker tracker) {
- c.output(c.element().toString() + "a");
- c.output(c.element().toString() + "b");
- c.output(c.element().toString() + "c");
- }
-
- @GetInitialRestriction
- public SomeRestriction getInitialRestriction(Integer elem) {
- return new SomeRestriction();
- }
- }
-
- @Test
- public void testTrivialProcessFnPropagatesOutputWindowAndTimestamp() throws Exception {
- // Tests that ProcessFn correctly propagates the window and timestamp of the element
- // inside the KeyedWorkItem.
- // The underlying DoFn is actually monolithic, so this doesn't test splitting.
- DoFn<Integer, String> fn = new ToStringFn();
-
- Instant base = Instant.now();
-
- IntervalWindow w =
- new IntervalWindow(
- base.minus(Duration.standardMinutes(1)), base.plus(Duration.standardMinutes(1)));
-
- ProcessFnTester<Integer, String, SomeRestriction, SomeRestrictionTracker> tester =
- new ProcessFnTester<>(
- base,
- fn,
- BigEndianIntegerCoder.of(),
- SerializableCoder.of(SomeRestriction.class),
- MAX_OUTPUTS_PER_BUNDLE,
- MAX_BUNDLE_DURATION);
- tester.startElement(
- WindowedValue.of(
- ElementAndRestriction.of(42, new SomeRestriction()),
- base,
- Collections.singletonList(w),
- PaneInfo.ON_TIME_AND_ONLY_FIRING));
-
- assertEquals(
- Arrays.asList(
- TimestampedValue.of("42a", base),
- TimestampedValue.of("42b", base),
- TimestampedValue.of("42c", base)),
- tester.peekOutputElementsInWindow(w));
- }
-
- private static class WatermarkUpdateFn extends DoFn<Instant, String> {
- @ProcessElement
- public void process(ProcessContext c, OffsetRangeTracker tracker) {
- for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
- c.updateWatermark(c.element().plus(Duration.standardSeconds(i)));
- c.output(String.valueOf(i));
- }
- }
-
- @GetInitialRestriction
- public OffsetRange getInitialRestriction(Instant elem) {
- throw new IllegalStateException("Expected to be supplied explicitly in this test");
- }
-
- @NewTracker
- public OffsetRangeTracker newTracker(OffsetRange range) {
- return new OffsetRangeTracker(range);
- }
- }
-
- @Test
- public void testUpdatesWatermark() throws Exception {
- DoFn<Instant, String> fn = new WatermarkUpdateFn();
- Instant base = Instant.now();
-
- ProcessFnTester<Instant, String, OffsetRange, OffsetRangeTracker> tester =
- new ProcessFnTester<>(
- base,
- fn,
- InstantCoder.of(),
- SerializableCoder.of(OffsetRange.class),
- 3,
- MAX_BUNDLE_DURATION);
-
- tester.startElement(base, new OffsetRange(0, 8));
- assertThat(tester.takeOutputElements(), hasItems("0", "1", "2"));
- assertEquals(base.plus(Duration.standardSeconds(2)), tester.getWatermarkHold());
-
- assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
- assertThat(tester.takeOutputElements(), hasItems("3", "4", "5"));
- assertEquals(base.plus(Duration.standardSeconds(5)), tester.getWatermarkHold());
-
- assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
- assertThat(tester.takeOutputElements(), hasItems("6", "7"));
- assertEquals(null, tester.getWatermarkHold());
- }
-
- /**
- * A splittable {@link DoFn} that generates the sequence [init, init + total).
- */
- private static class CounterFn extends DoFn<Integer, String> {
- @ProcessElement
- public void process(ProcessContext c, OffsetRangeTracker tracker) {
- for (long i = tracker.currentRestriction().getFrom();
- tracker.tryClaim(i); ++i) {
- c.output(String.valueOf(c.element() + i));
- }
- }
-
- @GetInitialRestriction
- public OffsetRange getInitialRestriction(Integer elem) {
- throw new UnsupportedOperationException("Expected to be supplied explicitly in this test");
- }
- }
-
- @Test
- public void testCheckpointsAfterNumOutputs() throws Exception {
- int max = 100;
- DoFn<Integer, String> fn = new CounterFn();
- Instant base = Instant.now();
- int baseIndex = 42;
-
- ProcessFnTester<Integer, String, OffsetRange, OffsetRangeTracker> tester =
- new ProcessFnTester<>(
- base, fn, BigEndianIntegerCoder.of(), SerializableCoder.of(OffsetRange.class),
- max, MAX_BUNDLE_DURATION);
-
- List<String> elements;
-
- // Create an fn that attempts to 2x output more than checkpointing allows.
- tester.startElement(baseIndex, new OffsetRange(0, 2 * max + max / 2));
- elements = tester.takeOutputElements();
- assertEquals(max, elements.size());
- // Should output the range [0, max)
- assertThat(elements, hasItem(String.valueOf(baseIndex)));
- assertThat(elements, hasItem(String.valueOf(baseIndex + max - 1)));
-
- assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
- elements = tester.takeOutputElements();
- assertEquals(max, elements.size());
- // Should output the range [max, 2*max)
- assertThat(elements, hasItem(String.valueOf(baseIndex + max)));
- assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max - 1)));
-
- assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1)));
- elements = tester.takeOutputElements();
- assertEquals(max / 2, elements.size());
- // Should output the range [2*max, 2*max + max/2)
- assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max)));
- assertThat(elements, hasItem(String.valueOf(baseIndex + 2 * max + max / 2 - 1)));
- assertThat(elements, not(hasItem((String.valueOf(baseIndex + 2 * max + max / 2)))));
- }
-
- @Test
- public void testCheckpointsAfterDuration() throws Exception {
- // Don't bound number of outputs.
- int max = Integer.MAX_VALUE;
- // But bound bundle duration - the bundle should terminate.
- Duration maxBundleDuration = Duration.standardSeconds(1);
- // Create an fn that attempts to 2x output more than checkpointing allows.
- DoFn<Integer, String> fn = new CounterFn();
- Instant base = Instant.now();
- int baseIndex = 42;
-
- ProcessFnTester<Integer, String, OffsetRange, OffsetRangeTracker> tester =
- new ProcessFnTester<>(
- base, fn, BigEndianIntegerCoder.of(), SerializableCoder.of(OffsetRange.class),
- max, maxBundleDuration);
-
- List<String> elements;
-
- tester.startElement(baseIndex, new OffsetRange(0, Long.MAX_VALUE));
- // Bundle should terminate, and should do at least some processing.
- elements = tester.takeOutputElements();
- assertFalse(elements.isEmpty());
- // Bundle should have run for at least the requested duration.
- assertThat(
- Instant.now().getMillis() - base.getMillis(),
- greaterThanOrEqualTo(maxBundleDuration.getMillis()));
- }
-
- private static class LifecycleVerifyingFn extends DoFn<Integer, String> {
- private enum State {
- BEFORE_SETUP,
- OUTSIDE_BUNDLE,
- INSIDE_BUNDLE,
- TORN_DOWN
- }
-
- private State state = State.BEFORE_SETUP;
-
- @ProcessElement
- public void process(ProcessContext c, SomeRestrictionTracker tracker) {
- assertEquals(State.INSIDE_BUNDLE, state);
- }
-
- @GetInitialRestriction
- public SomeRestriction getInitialRestriction(Integer element) {
- return new SomeRestriction();
- }
-
- @Setup
- public void setup() {
- assertEquals(State.BEFORE_SETUP, state);
- state = State.OUTSIDE_BUNDLE;
- }
-
- @Teardown
- public void tearDown() {
- assertEquals(State.OUTSIDE_BUNDLE, state);
- state = State.TORN_DOWN;
- }
-
- @StartBundle
- public void startBundle() {
- assertEquals(State.OUTSIDE_BUNDLE, state);
- state = State.INSIDE_BUNDLE;
- }
-
- @FinishBundle
- public void finishBundle() {
- assertEquals(State.INSIDE_BUNDLE, state);
- state = State.OUTSIDE_BUNDLE;
- }
- }
-
- @Test
- public void testInvokesLifecycleMethods() throws Exception {
- DoFn<Integer, String> fn = new LifecycleVerifyingFn();
- try (ProcessFnTester<Integer, String, SomeRestriction, SomeRestrictionTracker> tester =
- new ProcessFnTester<>(
- Instant.now(),
- fn,
- BigEndianIntegerCoder.of(),
- SerializableCoder.of(SomeRestriction.class),
- MAX_OUTPUTS_PER_BUNDLE,
- MAX_BUNDLE_DURATION)) {
- tester.startElement(42, new SomeRestriction());
- }
- }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java
new file mode 100644
index 0000000..eb438ba
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java
@@ -0,0 +1,672 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.hamcrest.Matchers;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for {@link StateInternals}.
+ */
+public abstract class StateInternalsTest {
+
+ private static final BoundedWindow WINDOW_1 = new IntervalWindow(new Instant(0), new Instant(10));
+ private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
+ private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
+ private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
+
+ private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
+ StateTags.value("stringValue", StringUtf8Coder.of());
+ private static final StateTag<CombiningState<Integer, int[], Integer>>
+ SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
+ "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
+ private static final StateTag<BagState<String>> STRING_BAG_ADDR =
+ StateTags.bag("stringBag", StringUtf8Coder.of());
+ private static final StateTag<SetState<String>> STRING_SET_ADDR =
+ StateTags.set("stringSet", StringUtf8Coder.of());
+ private static final StateTag<MapState<String, Integer>> STRING_MAP_ADDR =
+ StateTags.map("stringMap", StringUtf8Coder.of(), VarIntCoder.of());
+ private static final StateTag<WatermarkHoldState> WATERMARK_EARLIEST_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST);
+ private static final StateTag<WatermarkHoldState> WATERMARK_LATEST_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST);
+ private static final StateTag<WatermarkHoldState> WATERMARK_EOW_ADDR =
+ StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW);
+
+ // Two distinct tags because they have non-equals() coders
+ private static final StateTag<BagState<String>> STRING_BAG_ADDR1 =
+ StateTags.bag("badStringBag", new StringCoderWithIdentityEquality());
+
+ private static final StateTag<BagState<String>> STRING_BAG_ADDR2 =
+ StateTags.bag("badStringBag", new StringCoderWithIdentityEquality());
+
+ private StateInternals underTest;
+
+ @Before
+ public void setUp() {
+ this.underTest = createStateInternals();
+ }
+
+ protected abstract StateInternals createStateInternals();
+
+ @Test
+ public void testValue() throws Exception {
+ ValueState<String> value = underTest.state(NAMESPACE_1, STRING_VALUE_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertThat(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), equalTo(value));
+ assertThat(
+ underTest.state(NAMESPACE_2, STRING_VALUE_ADDR),
+ Matchers.not(equalTo(value)));
+
+ assertThat(value.read(), Matchers.nullValue());
+ value.write("hello");
+ assertThat(value.read(), equalTo("hello"));
+ value.write("world");
+ assertThat(value.read(), equalTo("world"));
+
+ value.clear();
+ assertThat(value.read(), Matchers.nullValue());
+ assertThat(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testBag() throws Exception {
+ BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_BAG_ADDR)));
+ assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_BAG_ADDR))));
+
+ assertThat(value.read(), Matchers.emptyIterable());
+ value.add("hello");
+ assertThat(value.read(), containsInAnyOrder("hello"));
+
+ value.add("world");
+ assertThat(value.read(), containsInAnyOrder("hello", "world"));
+
+ value.clear();
+ assertThat(value.read(), Matchers.emptyIterable());
+ assertThat(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testBagIsEmpty() throws Exception {
+ BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+
+ assertThat(value.isEmpty().read(), Matchers.is(true));
+ ReadableState<Boolean> readFuture = value.isEmpty();
+ value.add("hello");
+ assertThat(readFuture.read(), Matchers.is(false));
+
+ value.clear();
+ assertThat(readFuture.read(), Matchers.is(true));
+ }
+
+ @Test
+ public void testMergeBagIntoSource() throws Exception {
+ BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
+
+ bag1.add("Hello");
+ bag2.add("World");
+ bag1.add("!");
+
+ StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
+
+ // Reading the merged bag gets both the contents
+ assertThat(bag1.read(), containsInAnyOrder("Hello", "World", "!"));
+ assertThat(bag2.read(), Matchers.emptyIterable());
+ }
+
+ @Test
+ public void testMergeBagIntoNewNamespace() throws Exception {
+ BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
+ BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
+
+ bag1.add("Hello");
+ bag2.add("World");
+ bag1.add("!");
+
+ StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
+
+ // Reading the merged bag gets both the contents
+ assertThat(bag3.read(), containsInAnyOrder("Hello", "World", "!"));
+ assertThat(bag1.read(), Matchers.emptyIterable());
+ assertThat(bag2.read(), Matchers.emptyIterable());
+ }
+
+ @Test
+ public void testSet() throws Exception {
+
+ SetState<String> value = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_SET_ADDR)));
+ assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_SET_ADDR))));
+
+ // empty
+ assertThat(value.read(), Matchers.emptyIterable());
+ assertFalse(value.contains("A").read());
+
+ // add
+ value.add("A");
+ value.add("B");
+ value.add("A");
+ assertFalse(value.addIfAbsent("B").read());
+ assertThat(value.read(), containsInAnyOrder("A", "B"));
+
+ // remove
+ value.remove("A");
+ assertThat(value.read(), containsInAnyOrder("B"));
+ value.remove("C");
+ assertThat(value.read(), containsInAnyOrder("B"));
+
+ // contains
+ assertFalse(value.contains("A").read());
+ assertTrue(value.contains("B").read());
+ value.add("C");
+ value.add("D");
+
+ // readLater
+ assertThat(value.readLater().read(), containsInAnyOrder("B", "C", "D"));
+ SetState<String> later = value.readLater();
+ assertThat(later.read(), hasItems("C", "D"));
+ assertFalse(later.contains("A").read());
+
+ // clear
+ value.clear();
+ assertThat(value.read(), Matchers.emptyIterable());
+ assertThat(underTest.state(NAMESPACE_1, STRING_SET_ADDR), equalTo(value));
+
+ }
+
+ @Test
+ public void testSetIsEmpty() throws Exception {
+
+ SetState<String> value = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
+
+ assertThat(value.isEmpty().read(), Matchers.is(true));
+ ReadableState<Boolean> readFuture = value.isEmpty();
+ value.add("hello");
+ assertThat(readFuture.read(), Matchers.is(false));
+
+ value.clear();
+ assertThat(readFuture.read(), Matchers.is(true));
+ }
+
+ @Test
+ public void testMergeSetIntoSource() throws Exception {
+
+ SetState<String> set1 = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
+ SetState<String> set2 = underTest.state(NAMESPACE_2, STRING_SET_ADDR);
+
+ set1.add("Hello");
+ set2.add("Hello");
+ set2.add("World");
+ set1.add("!");
+
+ StateMerging.mergeSets(Arrays.asList(set1, set2), set1);
+
+ // Reading the merged set gets both the contents
+ assertThat(set1.read(), containsInAnyOrder("Hello", "World", "!"));
+ assertThat(set2.read(), Matchers.emptyIterable());
+ }
+
+ @Test
+ public void testMergeSetIntoNewNamespace() throws Exception {
+
+ SetState<String> set1 = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
+ SetState<String> set2 = underTest.state(NAMESPACE_2, STRING_SET_ADDR);
+ SetState<String> set3 = underTest.state(NAMESPACE_3, STRING_SET_ADDR);
+
+ set1.add("Hello");
+ set2.add("Hello");
+ set2.add("World");
+ set1.add("!");
+
+ StateMerging.mergeSets(Arrays.asList(set1, set2, set3), set3);
+
+ // Reading the merged set gets both the contents
+ assertThat(set3.read(), containsInAnyOrder("Hello", "World", "!"));
+ assertThat(set1.read(), Matchers.emptyIterable());
+ assertThat(set2.read(), Matchers.emptyIterable());
+ }
+
+ // for testMap
+ private static class MapEntry<K, V> implements Map.Entry<K, V> {
+ private K key;
+ private V value;
+
+ private MapEntry(K key, V value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ static <K, V> Map.Entry<K, V> of(K k, V v) {
+ return new MapEntry<>(k, v);
+ }
+
+ public final K getKey() {
+ return key;
+ }
+ public final V getValue() {
+ return value;
+ }
+
+ public final String toString() {
+ return key + "=" + value;
+ }
+
+ public final int hashCode() {
+ return Objects.hashCode(key) ^ Objects.hashCode(value);
+ }
+
+ public final V setValue(V newValue) {
+ V oldValue = value;
+ value = newValue;
+ return oldValue;
+ }
+
+ public final boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o instanceof Map.Entry) {
+ Map.Entry<?, ?> e = (Map.Entry<?, ?>) o;
+ if (Objects.equals(key, e.getKey())
+ && Objects.equals(value, e.getValue())) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+
+ MapState<String, Integer> value = underTest.state(NAMESPACE_1, STRING_MAP_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertThat(value, equalTo(underTest.state(NAMESPACE_1, STRING_MAP_ADDR)));
+ assertThat(value, not(equalTo(underTest.state(NAMESPACE_2, STRING_MAP_ADDR))));
+
+ // put
+ assertThat(value.entries().read(), Matchers.emptyIterable());
+ value.put("A", 1);
+ value.put("B", 2);
+ value.put("A", 11);
+ assertThat(value.putIfAbsent("B", 22).read(), equalTo(2));
+ assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("A", 11),
+ MapEntry.of("B", 2)));
+
+ // remove
+ value.remove("A");
+ assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("B", 2)));
+ value.remove("C");
+ assertThat(value.entries().read(), containsInAnyOrder(MapEntry.of("B", 2)));
+
+ // get
+ assertNull(value.get("A").read());
+ assertThat(value.get("B").read(), equalTo(2));
+ value.put("C", 3);
+ value.put("D", 4);
+ assertThat(value.get("C").read(), equalTo(3));
+
+ // iterate
+ value.put("E", 5);
+ value.remove("C");
+ assertThat(value.keys().read(), containsInAnyOrder("B", "D", "E"));
+ assertThat(value.values().read(), containsInAnyOrder(2, 4, 5));
+ assertThat(
+ value.entries().read(),
+ containsInAnyOrder(MapEntry.of("B", 2), MapEntry.of("D", 4), MapEntry.of("E", 5)));
+
+ // readLater
+ assertThat(value.get("B").readLater().read(), equalTo(2));
+ assertNull(value.get("A").readLater().read());
+ assertThat(
+ value.entries().readLater().read(),
+ containsInAnyOrder(MapEntry.of("B", 2), MapEntry.of("D", 4), MapEntry.of("E", 5)));
+
+ // clear
+ value.clear();
+ assertThat(value.entries().read(), Matchers.emptyIterable());
+ assertThat(underTest.state(NAMESPACE_1, STRING_MAP_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testCombiningValue() throws Exception {
+
+ GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertEquals(value, underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR));
+ assertFalse(value.equals(underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR)));
+
+ assertThat(value.read(), equalTo(0));
+ value.add(2);
+ assertThat(value.read(), equalTo(2));
+
+ value.add(3);
+ assertThat(value.read(), equalTo(5));
+
+ value.clear();
+ assertThat(value.read(), equalTo(0));
+ assertThat(underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testCombiningIsEmpty() throws Exception {
+ GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
+
+ assertThat(value.isEmpty().read(), Matchers.is(true));
+ ReadableState<Boolean> readFuture = value.isEmpty();
+ value.add(5);
+ assertThat(readFuture.read(), Matchers.is(false));
+
+ value.clear();
+ assertThat(readFuture.read(), Matchers.is(true));
+ }
+
+ @Test
+ public void testMergeCombiningValueIntoSource() throws Exception {
+ CombiningState<Integer, int[], Integer> value1 =
+ underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
+ CombiningState<Integer, int[], Integer> value2 =
+ underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
+
+ value1.add(5);
+ value2.add(10);
+ value1.add(6);
+
+ assertThat(value1.read(), equalTo(11));
+ assertThat(value2.read(), equalTo(10));
+
+ // Merging clears the old values and updates the result value.
+ StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1);
+
+ assertThat(value1.read(), equalTo(21));
+ assertThat(value2.read(), equalTo(0));
+ }
+
+ @Test
+ public void testMergeCombiningValueIntoNewNamespace() throws Exception {
+ CombiningState<Integer, int[], Integer> value1 =
+ underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
+ CombiningState<Integer, int[], Integer> value2 =
+ underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
+ CombiningState<Integer, int[], Integer> value3 =
+ underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR);
+
+ value1.add(5);
+ value2.add(10);
+ value1.add(6);
+
+ StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3);
+
+ // Merging clears the old values and updates the result value.
+ assertThat(value1.read(), equalTo(0));
+ assertThat(value2.read(), equalTo(0));
+ assertThat(value3.read(), equalTo(21));
+ }
+
+ @Test
+ public void testWatermarkEarliestState() throws Exception {
+ WatermarkHoldState value =
+ underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR));
+ assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR)));
+
+ assertThat(value.read(), Matchers.nullValue());
+ value.add(new Instant(2000));
+ assertThat(value.read(), equalTo(new Instant(2000)));
+
+ value.add(new Instant(3000));
+ assertThat(value.read(), equalTo(new Instant(2000)));
+
+ value.add(new Instant(1000));
+ assertThat(value.read(), equalTo(new Instant(1000)));
+
+ value.clear();
+ assertThat(value.read(), equalTo(null));
+ assertThat(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testWatermarkLatestState() throws Exception {
+ WatermarkHoldState value =
+ underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR));
+ assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR)));
+
+ assertThat(value.read(), Matchers.nullValue());
+ value.add(new Instant(2000));
+ assertThat(value.read(), equalTo(new Instant(2000)));
+
+ value.add(new Instant(3000));
+ assertThat(value.read(), equalTo(new Instant(3000)));
+
+ value.add(new Instant(1000));
+ assertThat(value.read(), equalTo(new Instant(3000)));
+
+ value.clear();
+ assertThat(value.read(), equalTo(null));
+ assertThat(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testWatermarkEndOfWindowState() throws Exception {
+ WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR);
+
+ // State instances are cached, but depend on the namespace.
+ assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR));
+ assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR)));
+
+ assertThat(value.read(), Matchers.nullValue());
+ value.add(new Instant(2000));
+ assertThat(value.read(), equalTo(new Instant(2000)));
+
+ value.clear();
+ assertThat(value.read(), equalTo(null));
+ assertThat(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), equalTo(value));
+ }
+
+ @Test
+ public void testWatermarkStateIsEmpty() throws Exception {
+ WatermarkHoldState value =
+ underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
+
+ assertThat(value.isEmpty().read(), Matchers.is(true));
+ ReadableState<Boolean> readFuture = value.isEmpty();
+ value.add(new Instant(1000));
+ assertThat(readFuture.read(), Matchers.is(false));
+
+ value.clear();
+ assertThat(readFuture.read(), Matchers.is(true));
+ }
+
+ @Test
+ public void testMergeEarliestWatermarkIntoSource() throws Exception {
+ WatermarkHoldState value1 =
+ underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
+ WatermarkHoldState value2 =
+ underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR);
+
+ value1.add(new Instant(3000));
+ value2.add(new Instant(5000));
+ value1.add(new Instant(4000));
+ value2.add(new Instant(2000));
+
+ // Merging clears the old values and updates the merged value.
+ StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value1, WINDOW_1);
+
+ assertThat(value1.read(), equalTo(new Instant(2000)));
+ assertThat(value2.read(), equalTo(null));
+ }
+
+ @Test
+ public void testMergeLatestWatermarkIntoSource() throws Exception {
+ WatermarkHoldState value1 =
+ underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
+ WatermarkHoldState value2 =
+ underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR);
+ WatermarkHoldState value3 =
+ underTest.state(NAMESPACE_3, WATERMARK_LATEST_ADDR);
+
+ value1.add(new Instant(3000));
+ value2.add(new Instant(5000));
+ value1.add(new Instant(4000));
+ value2.add(new Instant(2000));
+
+ // Merging clears the old values and updates the result value.
+ StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value3, WINDOW_1);
+
+ // Merging clears the old values and updates the result value.
+ assertThat(value3.read(), equalTo(new Instant(5000)));
+ assertThat(value1.read(), equalTo(null));
+ assertThat(value2.read(), equalTo(null));
+ }
+
+ @Test
+ public void testSetReadable() throws Exception {
+ SetState<String> value = underTest.state(NAMESPACE_1, STRING_SET_ADDR);
+
+ // test contains
+ ReadableState<Boolean> readable = value.contains("A");
+ value.add("A");
+ assertFalse(readable.read());
+
+ // test addIfAbsent
+ value.addIfAbsent("B");
+ assertTrue(value.contains("B").read());
+ }
+
+ @Test
+ public void testMapReadable() throws Exception {
+ MapState<String, Integer> value = underTest.state(NAMESPACE_1, STRING_MAP_ADDR);
+
+ // test iterable, should just return a iterable view of the values contained in this map.
+ // The iterable is backed by the map, so changes to the map are reflected in the iterable.
+ ReadableState<Iterable<String>> keys = value.keys();
+ ReadableState<Iterable<Integer>> values = value.values();
+ ReadableState<Iterable<Map.Entry<String, Integer>>> entries = value.entries();
+ value.put("A", 1);
+ assertFalse(Iterables.isEmpty(keys.read()));
+ assertFalse(Iterables.isEmpty(values.read()));
+ assertFalse(Iterables.isEmpty(entries.read()));
+
+ // test get
+ ReadableState<Integer> get = value.get("B");
+ value.put("B", 2);
+ assertNull(get.read());
+
+ // test addIfAbsent
+ value.putIfAbsent("C", 3);
+ assertThat(value.get("C").read(), equalTo(3));
+ }
+
+ @Test
+ public void testBagWithBadCoderEquality() throws Exception {
+ // Ensure two instances of the bad coder are distinct; models user who fails to
+ // override equals() or inherit from CustomCoder for StructuredCoder
+ assertThat(
+ new StringCoderWithIdentityEquality(), not(equalTo(new StringCoderWithIdentityEquality())));
+
+ BagState<String> state1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR1);
+ state1.add("hello");
+
+ BagState<String> state2 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR2);
+ assertThat(state2.read(), containsInAnyOrder("hello"));
+ }
+
+ private static class StringCoderWithIdentityEquality extends Coder<String> {
+
+ private final StringUtf8Coder realCoder = StringUtf8Coder.of();
+
+ @Override
+ public void encode(String value, OutputStream outStream) throws CoderException, IOException {
+ realCoder.encode(value, outStream);
+ }
+
+ @Override
+ public String decode(InputStream inStream) throws CoderException, IOException {
+ return realCoder.decode(inStream);
+ }
+
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return null;
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {}
+
+ @Override
+ public boolean equals(Object other) {
+ return other == this;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index 5172f43..4f155dc 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -24,7 +24,6 @@
import com.google.common.base.MoreObjects;
import java.util.Collections;
-import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
@@ -69,7 +68,8 @@
private static final IntervalWindow WINDOW_2 =
new IntervalWindow(new Instant(10), new Instant(20));
- @Mock StepContext mockStepContext;
+ @Mock
+ StepContext mockStepContext;
private InMemoryStateInternals<String> stateInternals;
private InMemoryTimerInternals timerInternals;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java
index 9769d10..26cbfee 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java
@@ -116,6 +116,21 @@
}
public static <T> Matcher<WindowedValue<? extends T>> isSingleWindowedValue(
+ Matcher<T> valueMatcher,
+ long timestamp,
+ long windowStart,
+ long windowEnd,
+ PaneInfo paneInfo) {
+ IntervalWindow intervalWindow =
+ new IntervalWindow(new Instant(windowStart), new Instant(windowEnd));
+ return WindowMatchers.<T>isSingleWindowedValue(
+ valueMatcher,
+ Matchers.describedAs("%0", Matchers.equalTo(new Instant(timestamp)), timestamp),
+ Matchers.<BoundedWindow>equalTo(intervalWindow),
+ Matchers.equalTo(paneInfo));
+ }
+
+ public static <T> Matcher<WindowedValue<? extends T>> isSingleWindowedValue(
Matcher<? super T> valueMatcher,
Matcher<? super Instant> timestampMatcher,
Matcher<? super BoundedWindow> windowMatcher) {
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
new file mode 100644
index 0000000..da02d92
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link FnApiControlClientPoolService}. */
+@RunWith(JUnit4.class)
+public class FnApiControlClientPoolServiceTest {
+
+ // For ease of straight-line testing, we use a LinkedBlockingQueue; in practice a SynchronousQueue
+ // for matching incoming connections and server threads is likely.
+ private final BlockingQueue<FnApiControlClient> pool = new LinkedBlockingQueue<>();
+ private FnApiControlClientPoolService controlService =
+ FnApiControlClientPoolService.offeringClientsToPool(pool);
+
+ @Test
+ public void testIncomingConnection() throws Exception {
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver = mock(StreamObserver.class);
+ StreamObserver<BeamFnApi.InstructionResponse> responseObserver =
+ controlService.control(requestObserver);
+
+ FnApiControlClient client = pool.take();
+
+ // Check that the client is wired up to the request channel
+ String id = "fakeInstruction";
+ ListenableFuture<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+ verify(requestObserver).onNext(any(BeamFnApi.InstructionRequest.class));
+ assertThat(responseFuture.isDone(), is(false));
+
+ // Check that the response channel really came from the client
+ responseObserver.onNext(
+ BeamFnApi.InstructionResponse.newBuilder().setInstructionId(id).build());
+ responseFuture.get();
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
new file mode 100644
index 0000000..279e974
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isA;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Unit tests for {@link FnApiControlClient}. */
+@RunWith(JUnit4.class)
+public class FnApiControlClientTest {
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Mock public StreamObserver<BeamFnApi.InstructionRequest> mockObserver;
+ private FnApiControlClient client;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ client = FnApiControlClient.forRequestObserver(mockObserver);
+ }
+
+ @Test
+ public void testRequestSent() {
+ String id = "instructionId";
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ verify(mockObserver).onNext(any(BeamFnApi.InstructionRequest.class));
+ }
+
+ @Test
+ public void testRequestSuccess() throws Exception {
+ String id = "successfulInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+ client
+ .asResponseObserver()
+ .onNext(BeamFnApi.InstructionResponse.newBuilder().setInstructionId(id).build());
+
+ BeamFnApi.InstructionResponse response = responseFuture.get();
+
+ assertThat(response.getInstructionId(), equalTo(id));
+ }
+
+ @Test
+ public void testUnknownResponseIgnored() throws Exception {
+ String id = "actualInstruction";
+ String unknownId = "unknownInstruction";
+
+ ListenableFuture<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client
+ .asResponseObserver()
+ .onNext(BeamFnApi.InstructionResponse.newBuilder().setInstructionId(unknownId).build());
+
+ assertThat(responseFuture.isDone(), is(false));
+ assertThat(responseFuture.isCancelled(), is(false));
+ }
+
+ @Test
+ public void testOnCompletedCancelsOutstanding() throws Exception {
+ String id = "clientHangUpInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client.asResponseObserver().onCompleted();
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(IllegalStateException.class));
+ thrown.expectMessage("closed");
+ responseFuture.get();
+ }
+
+ @Test
+ public void testOnErrorCancelsOutstanding() throws Exception {
+ String id = "errorInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ class FrazzleException extends Exception {}
+ client.asResponseObserver().onError(new FrazzleException());
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(FrazzleException.class));
+ responseFuture.get();
+ }
+
+ @Test
+ public void testCloseCancelsOutstanding() throws Exception {
+ String id = "serverCloseInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client.close();
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(IllegalStateException.class));
+ thrown.expectMessage("closed");
+ responseFuture.get();
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
new file mode 100644
index 0000000..7783b2f
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.SettableFuture;
+import java.util.concurrent.Future;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Unit tests for {@link SdkHarnessClient}. */
+@RunWith(JUnit4.class)
+public class SdkHarnessClientTest {
+
+ @Mock public FnApiControlClient fnApiControlClient;
+
+ private SdkHarnessClient sdkHarnessClient;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ sdkHarnessClient = SdkHarnessClient.usingFnApiClient(fnApiControlClient);
+ }
+
+ @Test
+ public void testRegisterDoesNotCrash() throws Exception {
+ String descriptorId1 = "descriptor1";
+ String descriptorId2 = "descriptor2";
+
+ SettableFuture<BeamFnApi.InstructionResponse> registerResponseFuture = SettableFuture.create();
+ when(fnApiControlClient.handle(any(BeamFnApi.InstructionRequest.class)))
+ .thenReturn(registerResponseFuture);
+
+ Future<BeamFnApi.RegisterResponse> responseFuture = sdkHarnessClient.register(
+ ImmutableList.of(
+ BeamFnApi.ProcessBundleDescriptor.newBuilder().setId(descriptorId1).build(),
+ BeamFnApi.ProcessBundleDescriptor.newBuilder().setId(descriptorId2).build()));
+
+ // Correlating the RegisterRequest and RegisterResponse is owned by the underlying
+ // FnApiControlClient. The SdkHarnessClient owns just wrapping the request and unwrapping
+ // the response.
+ //
+ // Currently there are no fields so there's nothing to check. This test is formulated
+ // to match the pattern it should have if/when the response is meaningful.
+ BeamFnApi.RegisterResponse response = BeamFnApi.RegisterResponse.getDefaultInstance();
+ registerResponseFuture.set(
+ BeamFnApi.InstructionResponse.newBuilder().setRegister(response).build());
+ responseFuture.get();
+ }
+
+ @Test
+ public void testNewBundleNoDataDoesNotCrash() throws Exception {
+ String descriptorId1 = "descriptor1";
+
+ SettableFuture<BeamFnApi.InstructionResponse> processBundleResponseFuture =
+ SettableFuture.create();
+ when(fnApiControlClient.handle(any(BeamFnApi.InstructionRequest.class)))
+ .thenReturn(processBundleResponseFuture);
+
+ SdkHarnessClient.ActiveBundle activeBundle = sdkHarnessClient.newBundle(descriptorId1);
+
+ // Correlating the ProcessBundleRequest and ProcessBundleReponse is owned by the underlying
+ // FnApiControlClient. The SdkHarnessClient owns just wrapping the request and unwrapping
+ // the response.
+ //
+ // Currently there are no fields so there's nothing to check. This test is formulated
+ // to match the pattern it should have if/when the response is meaningful.
+ BeamFnApi.ProcessBundleResponse response = BeamFnApi.ProcessBundleResponse.getDefaultInstance();
+ processBundleResponseFuture.set(
+ BeamFnApi.InstructionResponse.newBuilder().setProcessBundle(response).build());
+ activeBundle.getBundleResponse().get();
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
new file mode 100644
index 0000000..8f16004
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.fn;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+import com.google.common.util.concurrent.SettableFuture;
+import java.io.IOException;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Unit tests for {@link SdkHarnessDoFnRunner}. */
+@RunWith(JUnit4.class)
+public class SdkHarnessDoFnRunnerTest {
+ @Mock private SdkHarnessClient mockClient;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @Test
+ public void testStartAndFinishBundleDoesNotCrash() {
+ String processBundleDescriptorId = "testDescriptor";
+ String bundleId = "testBundle";
+ SdkHarnessDoFnRunner<Void, Void> underTest =
+ SdkHarnessDoFnRunner.<Void, Void>create(mockClient, processBundleDescriptorId);
+
+ SettableFuture<BeamFnApi.ProcessBundleResponse> processBundleResponseFuture =
+ SettableFuture.create();
+ FnDataReceiver dummyInputReceiver = new FnDataReceiver() {
+ @Override
+ public void accept(Object input) throws Exception {
+ fail("Dummy input receiver should not have received data");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // noop
+ }
+ };
+ SdkHarnessClient.ActiveBundle activeBundle =
+ SdkHarnessClient.ActiveBundle.create(
+ bundleId, processBundleResponseFuture, dummyInputReceiver);
+
+ when(mockClient.newBundle(anyString())).thenReturn(activeBundle);
+ underTest.startBundle();
+ processBundleResponseFuture.set(BeamFnApi.ProcessBundleResponse.getDefaultInstance());
+ underTest.finishBundle();
+ }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
index b304d3b..ab4b709 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
@@ -22,6 +22,7 @@
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import org.apache.beam.sdk.metrics.MetricName;
@@ -67,6 +68,9 @@
c1.inc(8L);
assertThat(container.getUpdates().counterUpdates(), contains(
metricUpdate("name1", 13L)));
+
+ CounterCell dne = container.tryGetCounter(MetricName.named("ns", "dne"));
+ assertEquals(dne, null);
}
@Test
@@ -89,6 +93,9 @@
assertThat(container.getCumulative().counterUpdates(), containsInAnyOrder(
metricUpdate("name1", 13L),
metricUpdate("name2", 4L)));
+
+ CounterCell readC1 = container.tryGetCounter(MetricName.named("ns", "name1"));
+ assertEquals((long) readC1.getCumulative(), 13L);
}
@Test
@@ -126,5 +133,8 @@
assertThat(container.getUpdates().distributionUpdates(), contains(
metricUpdate("name1", DistributionData.create(17, 3, 4, 8))));
container.commitUpdates();
+
+ DistributionCell dne = container.tryGetDistribution(MetricName.named("ns", "dne"));
+ assertEquals(dne, null);
}
}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachineTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachineTest.java
index 453c8ff..2be90de 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachineTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachineTest.java
@@ -21,7 +21,6 @@
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachineTester.SimpleTriggerStateMachineTester;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -42,8 +41,8 @@
@RunWith(JUnit4.class)
public class AfterFirstStateMachineTest {
- @Mock private OnceTriggerStateMachine mockTrigger1;
- @Mock private OnceTriggerStateMachine mockTrigger2;
+ @Mock private TriggerStateMachine mockTrigger1;
+ @Mock private TriggerStateMachine mockTrigger2;
private SimpleTriggerStateMachineTester<IntervalWindow> tester;
private static TriggerStateMachine.TriggerContext anyTriggerContext() {
return Mockito.<TriggerStateMachine.TriggerContext>any();
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachineTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachineTest.java
index e4d10a0..65c8be3 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachineTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachineTest.java
@@ -17,16 +17,19 @@
*/
package org.apache.beam.runners.core.triggers;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnMergeContext;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachineTester.SimpleTriggerStateMachineTester;
+import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.Sessions;
@@ -46,8 +49,8 @@
@RunWith(JUnit4.class)
public class AfterWatermarkStateMachineTest {
- @Mock private OnceTriggerStateMachine mockEarly;
- @Mock private OnceTriggerStateMachine mockLate;
+ @Mock private TriggerStateMachine mockEarly;
+ @Mock private TriggerStateMachine mockLate;
private SimpleTriggerStateMachineTester<IntervalWindow> tester;
private static TriggerStateMachine.TriggerContext anyTriggerContext() {
@@ -70,7 +73,7 @@
MockitoAnnotations.initMocks(this);
}
- public void testRunningAsTrigger(OnceTriggerStateMachine mockTrigger, IntervalWindow window)
+ public void testRunningAsTrigger(TriggerStateMachine mockTrigger, IntervalWindow window)
throws Exception {
// Don't fire due to mock saying no
@@ -105,6 +108,31 @@
}
@Test
+ public void testTimerForEndOfWindow() throws Exception {
+ tester = TriggerStateMachineTester.forTrigger(
+ AfterWatermarkStateMachine.pastEndOfWindow(),
+ FixedWindows.of(Duration.millis(100)));
+
+ assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), nullValue());
+ injectElements(1);
+ IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
+ assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), equalTo(window.maxTimestamp()));
+ }
+
+ @Test
+ public void testTimerForEndOfWindowCompound() throws Exception {
+ tester =
+ TriggerStateMachineTester.forTrigger(
+ AfterWatermarkStateMachine.pastEndOfWindow().withEarlyFirings(NeverStateMachine.ever()),
+ FixedWindows.of(Duration.millis(100)));
+
+ assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), nullValue());
+ injectElements(1);
+ IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
+ assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), equalTo(window.maxTimestamp()));
+ }
+
+ @Test
public void testAtWatermarkAndLate() throws Exception {
tester = TriggerStateMachineTester.forTrigger(
AfterWatermarkStateMachine.pastEndOfWindow()
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java
index 4512848..1bc757e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java
@@ -18,12 +18,11 @@
package org.apache.beam.runners.core.triggers;
import com.google.common.collect.Lists;
-import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
/**
- * No-op {@link OnceTriggerStateMachine} implementation for testing.
+ * No-op {@link TriggerStateMachine} implementation for testing.
*/
-abstract class StubTriggerStateMachine extends OnceTriggerStateMachine {
+abstract class StubTriggerStateMachine extends TriggerStateMachine {
/**
* Create a stub {@link TriggerStateMachine} instance which returns the specified name on {@link
* #toString()}.
@@ -42,7 +41,7 @@
}
@Override
- protected void onOnlyFiring(TriggerContext context) throws Exception {
+ public void onFire(TriggerContext context) throws Exception {
}
@Override
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
index 9a10f53..0f38be01 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
@@ -198,6 +198,12 @@
}
}
+ /** Retrieves the next timer for this time domain, if any, for use in assertions. */
+ @Nullable
+ public Instant getNextTimer(TimeDomain domain) {
+ return timerInternals.getNextTimer(domain);
+ }
+
/**
* Returns {@code true} if the {@link TriggerStateMachine} under test is finished for the given
* window.
@@ -263,11 +269,6 @@
for (W window : assignedWindows) {
activeWindows.addActiveForTesting(window);
-
- // Today, triggers assume onTimer firing at the watermark time, whether or not they
- // explicitly set the timer themselves. So this tester must set it.
- timerInternals.setTimer(
- TimerData.of(windowNamespace(window), window.maxTimestamp(), TimeDomain.EVENT_TIME));
}
windowedValues.add(WindowedValue.of(value, timestamp, assignedWindows, PaneInfo.NO_FIRING));
@@ -351,8 +352,6 @@
executableTrigger.invokeOnMerge(contextFactory.createOnMergeContext(mergeResult,
new TestTimers(windowNamespace(mergeResult)), executableTrigger,
getFinishedSet(mergeResult), mergingFinishedSets));
- timerInternals.setTimer(TimerData.of(
- windowNamespace(mergeResult), mergeResult.maxTimestamp(), TimeDomain.EVENT_TIME));
}
});
}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
index 5158f50..754110e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
@@ -22,7 +22,7 @@
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.state.TimeDomain;
import org.joda.time.Duration;
import org.junit.Test;
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index c581113..752af44 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -40,6 +40,92 @@
</resource>
</resources>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>org.apache.beam:beam-model-pipeline</include>
+ <include>org.apache.beam:beam-runners-core-construction-java</include>
+ <include>org.apache.beam:beam-runners-core-java</include>
+ <include>com.google.code.findbugs:jsr305</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.beam.runners.core</pattern>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.runners.core
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.beam.model</pattern>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.model
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <excludes>
+ <!-- com.google.common is too generic, need to exclude guava-testlib -->
+ <exclude>com.google.common.**.testing.*</exclude>
+ </excludes>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.com.google.common
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.com.google.protobuf
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.com.google.thirdparty
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax.annotation</pattern>
+ <shadedPattern>
+ org.apache.beam.runners.direct.repackaged.javax.annotation
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -58,16 +144,19 @@
</goals>
<configuration>
<groups>org.apache.beam.sdk.testing.NeedsRunner</groups>
+ <!-- 100MB keys work on the direct runner, but make the test too slow. -->
+ <excludedGroups>org.apache.beam.sdk.testing.LargeKeys$Above100MB</excludedGroups>
<parallel>none</parallel>
<failIfNoTests>true</failIfNoTests>
<dependenciesToScan>
<dependency>org.apache.beam:beam-sdks-java-core</dependency>
- <dependency>org.apache.beam:beam-runners-core-java</dependency>
+ <dependency>org.apache.beam:beam-runners-java-core</dependency>
</dependenciesToScan>
<systemPropertyVariables>
<beamTestPipelineOptions>
[
- "--runner=DirectRunner"
+ "--runner=DirectRunner",
+ "--runnerDeterminedSharding=false"
]
</beamTestPipelineOptions>
</systemPropertyVariables>
@@ -77,95 +166,6 @@
</executions>
</plugin>
- <!-- Ensure that the Maven jar plugin runs before the Maven
- shade plugin by listing the plugin higher within the file. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-and-repackage</id>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadeTestJar>true</shadeTestJar>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>org.apache.beam:beam-runners-core-construction-java</include>
- <include>org.apache.beam:beam-runners-core-java</include>
- <include>org.apache.beam:beam-sdks-common-runner-api</include>
- <include>com.google.code.findbugs:jsr305</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>org.apache.beam.runners.core</pattern>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.runners.core
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.beam.sdk.common</pattern>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.sdk.common
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <excludes>
- <!-- com.google.common is too generic, need to exclude guava-testlib -->
- <exclude>com.google.common.**.testing.*</exclude>
- </excludes>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.com.google.common
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.protobuf</pattern>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.com.google.protobuf
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.thirdparty</pattern>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.com.google.thirdparty
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>javax.annotation</pattern>
- <shadedPattern>
- org.apache.beam.runners.direct.repackaged.javax.annotation
- </shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
@@ -177,12 +177,12 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-sdks-java-core</artifactId>
</dependency>
<dependency>
@@ -313,7 +313,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 76db861..fcaaa84 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -33,10 +33,10 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
import org.apache.beam.runners.direct.StepTransformResult.Builder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
-import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.Read.Bounded;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -180,16 +180,17 @@
}
@AutoValue
- abstract static class BoundedSourceShard<T> {
+ abstract static class BoundedSourceShard<T> implements SourceShard<T> {
static <T> BoundedSourceShard<T> of(BoundedSource<T> source) {
return new AutoValue_BoundedReadEvaluatorFactory_BoundedSourceShard<>(source);
}
- abstract BoundedSource<T> getSource();
+ @Override
+ public abstract BoundedSource<T> getSource();
}
static class InputProvider<T>
- implements RootInputProvider<T, BoundedSourceShard<T>, PBegin, Read.Bounded<T>> {
+ implements RootInputProvider<T, BoundedSourceShard<T>, PBegin> {
private final EvaluationContext evaluationContext;
InputProvider(EvaluationContext evaluationContext) {
@@ -198,9 +199,10 @@
@Override
public Collection<CommittedBundle<BoundedSourceShard<T>>> getInitialInputs(
- AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform, int targetParallelism)
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform,
+ int targetParallelism)
throws Exception {
- BoundedSource<T> source = transform.getTransform().getSource();
+ BoundedSource<T> source = ReadTranslation.boundedSourceFromTransform(transform);
PipelineOptions options = evaluationContext.getPipelineOptions();
long estimatedBytes = source.getEstimatedSizeBytes(options);
long bytesPerBundle = estimatedBytes / targetParallelism;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
index 8c45449..70e3ac3 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
@@ -19,8 +19,8 @@
package org.apache.beam.runners.direct;
import com.google.auto.value.AutoValue;
+import com.google.common.base.Optional;
import java.util.Set;
-import javax.annotation.Nullable;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
@@ -36,12 +36,10 @@
/**
* Returns the {@link CommittedBundle} that contains the input elements that could not be
- * processed by the evaluation.
- *
- * <p>{@code null} if the input bundle was null.
+ * processed by the evaluation. The returned optional is present if there were any unprocessed
+ * input elements, and absent otherwise.
*/
- @Nullable
- public abstract CommittedBundle<?> getUnprocessedInputs();
+ public abstract Optional<? extends CommittedBundle<?>> getUnprocessedInputs();
/**
* Returns the outputs produced by the transform.
@@ -59,7 +57,7 @@
public static CommittedResult create(
TransformResult<?> original,
- CommittedBundle<?> unprocessedElements,
+ Optional<? extends CommittedBundle<?>> unprocessedElements,
Iterable<? extends CommittedBundle<?>> outputs,
Set<OutputType> producedOutputs) {
return new AutoValue_CommittedResult(original.getTransform(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
index 3c701c7..848bf71 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
@@ -264,8 +264,12 @@
}
private boolean containedInUnderlying(StateNamespace namespace, StateTag<?> tag) {
- return underlying.isPresent() && underlying.get().isNamespaceInUse(namespace)
- && underlying.get().getTagsInUse(namespace).containsKey(tag);
+ return underlying.isPresent()
+ && underlying.get().isNamespaceInUse(namespace)
+ && underlying
+ .get()
+ .getTagsInUse(namespace)
+ .containsKey(tag);
}
@Override
@@ -388,7 +392,7 @@
public Instant readThroughAndGetEarliestHold(StateTable readTo) {
Instant earliestHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
for (StateNamespace namespace : underlying.getNamespacesInUse()) {
- for (Map.Entry<StateTag<?>, ? extends State> existingState :
+ for (Map.Entry<StateTag, State> existingState :
underlying.getTagsInUse(namespace).entrySet()) {
if (!((InMemoryState<?>) existingState.getValue()).isCleared()) {
// Only read through non-cleared values to ensure that completed windows are
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 107f39a..e8ad8d7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -17,10 +17,10 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.core.BaseExecutionContext;
-import org.apache.beam.runners.core.ExecutionContext;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
@@ -30,12 +30,12 @@
* <p>This implementation is not thread safe. A new {@link DirectExecutionContext} must be created
* for each thread that requires it.
*/
-class DirectExecutionContext
- extends BaseExecutionContext<DirectStepContext> {
+class DirectExecutionContext {
private final Clock clock;
private final StructuralKey<?> key;
private final CopyOnAccessInMemoryStateInternals existingState;
private final TransformWatermarks watermarks;
+ private Map<String, DirectStepContext> cachedStepContexts = new LinkedHashMap<>();
public DirectExecutionContext(
Clock clock,
@@ -48,23 +48,30 @@
this.watermarks = watermarks;
}
- @Override
- protected DirectStepContext createStepContext(String stepName, String transformName) {
- return new DirectStepContext(this, stepName, transformName);
+ private DirectStepContext createStepContext() {
+ return new DirectStepContext();
+ }
+
+ /**
+ * Returns the {@link StepContext} associated with the given step.
+ */
+ public DirectStepContext getStepContext(String stepName) {
+ DirectStepContext context = cachedStepContexts.get(stepName);
+ if (context == null) {
+ context = createStepContext();
+ cachedStepContexts.put(stepName, context);
+ }
+ return context;
}
/**
* Step Context for the {@link DirectRunner}.
*/
- public class DirectStepContext
- extends BaseExecutionContext.StepContext {
+ public class DirectStepContext implements StepContext {
private CopyOnAccessInMemoryStateInternals<?> stateInternals;
private DirectTimerInternals timerInternals;
- public DirectStepContext(
- ExecutionContext executionContext, String stepName, String transformName) {
- super(executionContext, stepName, transformName);
- }
+ public DirectStepContext() { }
@Override
public CopyOnAccessInMemoryStateInternals<?> stateInternals() {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
index 64eecc8..3fefe20 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
@@ -18,17 +18,14 @@
package org.apache.beam.runners.direct;
import org.apache.beam.runners.core.KeyedWorkItem;
-import org.apache.beam.runners.core.SplittableParDo.GBKIntoKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-/**
- * Provides an implementation of {@link SplittableParDo.GBKIntoKeyedWorkItems} for the Direct
- * Runner.
- */
+/** Provides an implementation of {@link GBKIntoKeyedWorkItems} for the Direct Runner. */
class DirectGBKIntoKeyedWorkItemsOverrideFactory<KeyT, InputT>
extends SingleInputOutputOverrideFactory<
PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>,
@@ -39,7 +36,7 @@
getReplacementTransform(
AppliedPTransform<
PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>,
- GBKIntoKeyedWorkItems<KeyT, InputT>>
+ GBKIntoKeyedWorkItems<KeyT, InputT>>
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
index 83b214a..ad17b2b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.runners.direct;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.common.collect.ListMultimap;
import java.util.Collection;
import java.util.List;
@@ -24,9 +26,9 @@
import java.util.Set;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
/**
@@ -34,49 +36,75 @@
* executed with the {@link DirectRunner}.
*/
class DirectGraph {
- private final Map<POutput, AppliedPTransform<?, ?, ?>> producers;
- private final ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers;
- private final Set<PCollectionView<?>> views;
+ private final Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers;
+ private final Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters;
+ private final ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers;
+ private final ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers;
private final Set<AppliedPTransform<?, ?, ?>> rootTransforms;
private final Map<AppliedPTransform<?, ?, ?>, String> stepNames;
public static DirectGraph create(
- Map<POutput, AppliedPTransform<?, ?, ?>> producers,
- ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers,
- Set<PCollectionView<?>> views,
+ Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers,
+ Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters,
+ ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers,
+ ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers,
Set<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<AppliedPTransform<?, ?, ?>, String> stepNames) {
- return new DirectGraph(producers, primitiveConsumers, views, rootTransforms, stepNames);
+ return new DirectGraph(
+ producers, viewWriters, perElementConsumers, allConsumers, rootTransforms, stepNames);
}
private DirectGraph(
- Map<POutput, AppliedPTransform<?, ?, ?>> producers,
- ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers,
- Set<PCollectionView<?>> views,
+ Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers,
+ Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters,
+ ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers,
+ ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers,
Set<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<AppliedPTransform<?, ?, ?>, String> stepNames) {
this.producers = producers;
- this.primitiveConsumers = primitiveConsumers;
- this.views = views;
+ this.viewWriters = viewWriters;
+ this.perElementConsumers = perElementConsumers;
+ this.allConsumers = allConsumers;
this.rootTransforms = rootTransforms;
this.stepNames = stepNames;
+ for (AppliedPTransform<?, ?, ?> step : stepNames.keySet()) {
+ for (PValue input : step.getInputs().values()) {
+ checkArgument(
+ allConsumers.get(input).contains(step),
+ "Step %s lists value %s as input, but it is not in the graph of consumers",
+ step.getFullName(),
+ input);
+ }
+ }
}
- AppliedPTransform<?, ?, ?> getProducer(PValue produced) {
+ AppliedPTransform<?, ?, ?> getProducer(PCollection<?> produced) {
return producers.get(produced);
}
- List<AppliedPTransform<?, ?, ?>> getPrimitiveConsumers(PValue consumed) {
- return primitiveConsumers.get(consumed);
+ AppliedPTransform<?, ?, ?> getWriter(PCollectionView<?> view) {
+ return viewWriters.get(view);
+ }
+
+ List<AppliedPTransform<?, ?, ?>> getPerElementConsumers(PValue consumed) {
+ return perElementConsumers.get(consumed);
+ }
+
+ List<AppliedPTransform<?, ?, ?>> getAllConsumers(PValue consumed) {
+ return allConsumers.get(consumed);
}
Set<AppliedPTransform<?, ?, ?>> getRootTransforms() {
return rootTransforms;
}
+ Set<PCollection<?>> getPCollections() {
+ return producers.keySet();
+ }
+
Set<PCollectionView<?>> getViews() {
- return views;
+ return viewWriters.keySet();
}
String getStepName(AppliedPTransform<?, ?, ?> step) {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index 1ee8ceb..675de2c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,19 +21,26 @@
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Sets;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
+import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -41,12 +48,17 @@
* input after the upstream transform has produced and committed output.
*/
class DirectGraphVisitor extends PipelineVisitor.Defaults {
- private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
+ private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
- private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers =
+ private Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
+ private Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters = new HashMap<>();
+ private Set<PCollectionView<?>> consumedViews = new HashSet<>();
+
+ private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers =
+ ArrayListMultimap.create();
+ private ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers =
ArrayListMultimap.create();
- private Set<PCollectionView<?>> views = new HashSet<>();
private Set<AppliedPTransform<?, ?, ?>> rootTransforms = new HashSet<>();
private Map<AppliedPTransform<?, ?, ?>, String> stepNames = new HashMap<>();
private int numTransforms = 0;
@@ -72,6 +84,13 @@
getClass().getSimpleName());
if (node.isRootNode()) {
finalized = true;
+ checkState(
+ viewWriters.keySet().containsAll(consumedViews),
+ "All %ss that are consumed must be written by some %s %s: Missing %s",
+ PCollectionView.class.getSimpleName(),
+ WriteView.class.getSimpleName(),
+ PTransform.class.getSimpleName(),
+ Sets.difference(consumedViews, viewWriters.keySet()));
}
}
@@ -82,26 +101,40 @@
if (node.getInputs().isEmpty()) {
rootTransforms.add(appliedTransform);
} else {
- for (PValue value : node.getInputs().values()) {
- primitiveConsumers.put(value, appliedTransform);
+ Collection<PValue> mainInputs =
+ TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
+ if (!mainInputs.containsAll(node.getInputs().values())) {
+ LOG.debug(
+ "Inputs reduced to {} from {} by removing additional inputs",
+ mainInputs,
+ node.getInputs().values());
}
+ for (PValue value : mainInputs) {
+ perElementConsumers.put(value, appliedTransform);
+ }
+ for (PValue value : node.getInputs().values()) {
+ allConsumers.put(value, appliedTransform);
+ }
+ }
+ if (node.getTransform() instanceof ParDo.MultiOutput) {
+ consumedViews.addAll(((ParDo.MultiOutput<?, ?>) node.getTransform()).getSideInputs());
+ } else if (node.getTransform() instanceof ViewOverrideFactory.WriteView) {
+ viewWriters.put(
+ ((WriteView) node.getTransform()).getView(), node.toAppliedPTransform(getPipeline()));
}
}
- @Override
+ @Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {
AppliedPTransform<?, ?, ?> appliedTransform = getAppliedTransform(producer);
- if (value instanceof PCollectionView) {
- views.add((PCollectionView<?>) value);
- }
- if (!producers.containsKey(value)) {
- producers.put(value, appliedTransform);
+ if (value instanceof PCollection && !producers.containsKey(value)) {
+ producers.put((PCollection<?>) value, appliedTransform);
}
}
private AppliedPTransform<?, ?, ?> getAppliedTransform(TransformHierarchy.Node node) {
@SuppressWarnings({"rawtypes", "unchecked"})
- AppliedPTransform<?, ?, ?> application = node.toAppliedPTransform();
+ AppliedPTransform<?, ?, ?> application = node.toAppliedPTransform(getPipeline());
return application;
}
@@ -110,11 +143,12 @@
}
/**
- * Get the graph constructed by this {@link DirectGraphVisitor}, which provides
- * lookups for producers and consumers of {@link PValue PValues}.
+ * Get the graph constructed by this {@link DirectGraphVisitor}, which provides lookups for
+ * producers and consumers of {@link PValue PValues}.
*/
public DirectGraph getGraph() {
checkState(finalized, "Can't get a graph before the Pipeline has been completely traversed");
- return DirectGraph.create(producers, primitiveConsumers, views, rootTransforms, stepNames);
+ return DirectGraph.create(
+ producers, viewWriters, perElementConsumers, allConsumers, rootTransforms, stepNames);
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
index 791615a..0053360 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -23,7 +23,6 @@
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.KeyedWorkItemCoder;
import org.apache.beam.runners.core.construction.ForwardingPTransform;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -35,10 +34,17 @@
class DirectGroupByKey<K, V>
extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
- private final GroupByKey<K, V> original;
+ private final PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> original;
- DirectGroupByKey(GroupByKey<K, V> from) {
- this.original = from;
+ static final String DIRECT_GBKO_URN = "urn:beam:directrunner:transforms:gbko:v1";
+ static final String DIRECT_GABW_URN = "urn:beam:directrunner:transforms:gabw:v1";
+ private final WindowingStrategy<?, ?> outputWindowingStrategy;
+
+ DirectGroupByKey(
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> original,
+ WindowingStrategy<?, ?> outputWindowingStrategy) {
+ this.original = original;
+ this.outputWindowingStrategy = outputWindowingStrategy;
}
@Override
@@ -53,9 +59,6 @@
// key/value input elements and the window merge operation of the
// window function associated with the input PCollection.
WindowingStrategy<?, ?> inputWindowingStrategy = input.getWindowingStrategy();
- // Update the windowing strategy as appropriate.
- WindowingStrategy<?, ?> outputWindowingStrategy =
- original.updateWindowingStrategy(inputWindowingStrategy);
// By default, implement GroupByKey via a series of lower-level operations.
return input
@@ -72,24 +75,21 @@
@Override
public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded());
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ input.isBounded(),
+ KeyedWorkItemCoder.of(
+ GroupByKey.getKeyCoder(input.getCoder()),
+ GroupByKey.getInputValueCoder(input.getCoder()),
+ input.getWindowingStrategy().getWindowFn().windowCoder()));
}
DirectGroupByKeyOnly() {}
-
- @Override
- protected Coder<?> getDefaultOutputCoder(
- @SuppressWarnings("unused") PCollection<KV<K, V>> input)
- throws CannotProvideCoderException {
- return KeyedWorkItemCoder.of(
- GroupByKey.getKeyCoder(input.getCoder()),
- GroupByKey.getInputValueCoder(input.getCoder()),
- input.getWindowingStrategy().getWindowFn().windowCoder());
- }
}
static final class DirectGroupAlsoByWindow<K, V>
- extends PTransform<PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>> {
+ extends PTransform<
+ PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>> {
private final WindowingStrategy<?, ?> inputWindowingStrategy;
private final WindowingStrategy<?, ?> outputWindowingStrategy;
@@ -123,17 +123,11 @@
}
@Override
- protected Coder<?> getDefaultOutputCoder(
- @SuppressWarnings("unused") PCollection<KeyedWorkItem<K, V>> input)
- throws CannotProvideCoderException {
- KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder());
- return KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder()));
- }
-
- @Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) {
+ KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder());
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), outputWindowingStrategy, input.isBounded());
+ input.getPipeline(), outputWindowingStrategy, input.isBounded(),
+ KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder())));
}
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
index c2eb5e7..9c2de3d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
@@ -17,26 +17,34 @@
*/
package org.apache.beam.runners.direct;
+import com.google.common.collect.Iterables;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
/** A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms. */
final class DirectGroupByKeyOverrideFactory<K, V>
extends SingleInputOutputOverrideFactory<
- PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>, GroupByKey<K, V>> {
+ PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>,
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>> {
@Override
public PTransformReplacement<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>
getReplacementTransform(
AppliedPTransform<
- PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>, GroupByKey<K, V>>
+ PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>,
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>>
transform) {
+
+ PCollection<KV<K, Iterable<V>>> output =
+ (PCollection<KV<K, Iterable<V>>>) Iterables.getOnlyElement(transform.getOutputs().values());
+
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new DirectGroupByKey<>(transform.getTransform()));
+ new DirectGroupByKey<>(transform.getTransform(), output.getWindowingStrategy()));
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
index 8169332..48b0113 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
@@ -32,10 +32,10 @@
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.concurrent.GuardedBy;
+import org.apache.beam.runners.core.construction.metrics.MetricFiltering;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.DistributionData;
import org.apache.beam.runners.core.metrics.GaugeData;
-import org.apache.beam.runners.core.metrics.MetricFiltering;
-import org.apache.beam.runners.core.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.MetricUpdates;
import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
import org.apache.beam.runners.core.metrics.MetricsMap;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
index 0e6fbab..53fb2f2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
@@ -50,7 +50,7 @@
@Override
public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
return ImmutableList.<Class<? extends PipelineOptions>>of(
- DirectOptions.class);
+ DirectOptions.class, DirectTestOptions.class);
}
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 181896f..d041a5a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -17,35 +17,35 @@
*/
package org.apache.beam.runners.direct;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.beam.runners.core.SplittableParDo.GBKIntoKeyedWorkItems;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.PipelineTranslation;
+import org.apache.beam.runners.core.construction.SplittableParDo;
import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.PipelineRunner;
-import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.metrics.MetricResults;
import org.apache.beam.sdk.metrics.MetricsEnvironment;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.testing.TestStream;
-import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.PCollection;
import org.joda.time.Duration;
@@ -72,16 +72,17 @@
IMMUTABILITY {
@Override
public boolean appliesTo(PCollection<?> collection, DirectGraph graph) {
- return CONTAINS_UDF.contains(graph.getProducer(collection).getTransform().getClass());
+ return CONTAINS_UDF.contains(
+ PTransformTranslation.urnForTransform(graph.getProducer(collection).getTransform()));
}
};
/**
* The set of {@link PTransform PTransforms} that execute a UDF. Useful for some enforcements.
*/
- private static final Set<Class<? extends PTransform>> CONTAINS_UDF =
+ private static final Set<String> CONTAINS_UDF =
ImmutableSet.of(
- Read.Bounded.class, Read.Unbounded.class, ParDo.SingleOutput.class, MultiOutput.class);
+ PTransformTranslation.READ_TRANSFORM_URN, PTransformTranslation.PAR_DO_TRANSFORM_URN);
public abstract boolean appliesTo(PCollection<?> collection, DirectGraph graph);
@@ -110,22 +111,19 @@
return bundleFactory;
}
- @SuppressWarnings("rawtypes")
- private static Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
+ private static Map<String, Collection<ModelEnforcementFactory>>
defaultModelEnforcements(Set<Enforcement> enabledEnforcements) {
- ImmutableMap.Builder<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
- enforcements = ImmutableMap.builder();
+ ImmutableMap.Builder<String, Collection<ModelEnforcementFactory>> enforcements =
+ ImmutableMap.builder();
ImmutableList.Builder<ModelEnforcementFactory> enabledParDoEnforcements =
ImmutableList.builder();
if (enabledEnforcements.contains(Enforcement.IMMUTABILITY)) {
enabledParDoEnforcements.add(ImmutabilityEnforcementFactory.create());
}
Collection<ModelEnforcementFactory> parDoEnforcements = enabledParDoEnforcements.build();
- enforcements.put(ParDo.SingleOutput.class, parDoEnforcements);
- enforcements.put(MultiOutput.class, parDoEnforcements);
+ enforcements.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, parDoEnforcements);
return enforcements.build();
}
-
}
////////////////////////////////////////////////////////////////////////////////////////////////
@@ -161,7 +159,14 @@
}
@Override
- public DirectPipelineResult run(Pipeline pipeline) {
+ public DirectPipelineResult run(Pipeline originalPipeline) {
+ Pipeline pipeline;
+ try {
+ RunnerApi.Pipeline protoPipeline = PipelineTranslation.toProto(originalPipeline);
+ pipeline = PipelineTranslation.fromProto(protoPipeline);
+ } catch (IOException exception) {
+ throw new RuntimeException("Error preparing pipeline for direct execution.", exception);
+ }
pipeline.replaceAll(defaultTransformOverrides());
MetricsEnvironment.setMetricsSupported(true);
DirectGraphVisitor graphVisitor = new DirectGraphVisitor();
@@ -219,38 +224,52 @@
* iteration order based on the order at which elements are added to it.
*/
@SuppressWarnings("rawtypes")
- private List<PTransformOverride> defaultTransformOverrides() {
- return ImmutableList.<PTransformOverride>builder()
- .add(
- PTransformOverride.of(
- PTransformMatchers.writeWithRunnerDeterminedSharding(),
- new WriteWithShardingFactory())) /* Uses a view internally. */
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(CreatePCollectionView.class),
- new ViewOverrideFactory())) /* Uses pardos and GBKs */
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(TestStream.class),
- new DirectTestStreamFactory(this))) /* primitive */
- // SplittableParMultiDo is implemented in terms of nonsplittable simple ParDos and extra
- // primitives
- .add(
- PTransformOverride.of(
- PTransformMatchers.splittableParDoMulti(), new ParDoMultiOverrideFactory()))
- // state and timer pardos are implemented in terms of simple ParDos and extra primitives
- .add(
- PTransformOverride.of(
- PTransformMatchers.stateOrTimerParDoMulti(), new ParDoMultiOverrideFactory()))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(GBKIntoKeyedWorkItems.class),
- new DirectGBKIntoKeyedWorkItemsOverrideFactory())) /* Returns a GBKO */
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(GroupByKey.class),
- new DirectGroupByKeyOverrideFactory())) /* returns two chained primitives. */
- .build();
+ @VisibleForTesting
+ List<PTransformOverride> defaultTransformOverrides() {
+ DirectTestOptions testOptions = options.as(DirectTestOptions.class);
+ ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder();
+ if (testOptions.isRunnerDeterminedSharding()) {
+ builder.add(
+ PTransformOverride.of(
+ PTransformMatchers.writeWithRunnerDeterminedSharding(),
+ new WriteWithShardingFactory())); /* Uses a view internally. */
+ }
+ builder =
+ builder
+ .add(
+ PTransformOverride.of(
+ MultiStepCombine.matcher(), MultiStepCombine.Factory.create()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN),
+ new ViewOverrideFactory())) /* Uses pardos and GBKs */
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(PTransformTranslation.TEST_STREAM_TRANSFORM_URN),
+ new DirectTestStreamFactory(this))) /* primitive */
+ // SplittableParMultiDo is implemented in terms of nonsplittable simple ParDos and extra
+ // primitives
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.splittableParDo(), new ParDoMultiOverrideFactory()))
+ // state and timer pardos are implemented in terms of simple ParDos and extra primitives
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.stateOrTimerParDo(), new ParDoMultiOverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(
+ SplittableParDo.SPLITTABLE_PROCESS_KEYED_ELEMENTS_URN),
+ new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(SplittableParDo.SPLITTABLE_GBKIKWI_URN),
+ new DirectGBKIntoKeyedWorkItemsOverrideFactory())) /* Returns a GBKO */
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN),
+ new DirectGroupByKeyOverrideFactory())); /* returns two chained primitives. */
+ return builder.build();
}
/**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTestOptions.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTestOptions.java
new file mode 100644
index 0000000..a426443
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTestOptions.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Hidden;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+/**
+ * Internal-only options for tweaking the behavior of the {@link DirectRunner} in ways that users
+ * should never do.
+ *
+ * <p>Currently, the only use is to disable user-friendly overrides that prevent fully testing
+ * certain composite transforms.
+ */
+@Internal
+@Hidden
+public interface DirectTestOptions extends PipelineOptions, ApplicationNameOptions {
+ @Default.Boolean(true)
+ @Description(
+ "Indicates whether this is an automatically-run unit test.")
+ boolean isRunnerDeterminedSharding();
+ void setRunnerDeterminedSharding(boolean goAheadAndDetermineSharding);
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
index a099368..7db12a4 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
@@ -52,6 +52,9 @@
timerUpdateBuilder.setTimer(TimerData.of(timerId, namespace, target, timeDomain));
}
+ /**
+ * @deprecated use {@link #setTimer(StateNamespace, String, Instant, TimeDomain)}.
+ */
@Deprecated
@Override
public void setTimer(TimerData timerData) {
@@ -63,12 +66,18 @@
throw new UnsupportedOperationException("Canceling of timer by ID is not yet supported.");
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(StateNamespace namespace, String timerId) {
throw new UnsupportedOperationException("Canceling of timer by ID is not yet supported.");
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(TimerData timerKey) {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
index c36879a..a5a53bc 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
@@ -20,13 +20,12 @@
import java.util.Collection;
import java.util.Collections;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
/** A {@link RootInputProvider} that provides a singleton empty bundle. */
-class EmptyInputProvider<T>
- implements RootInputProvider<T, Void, PCollectionList<T>, Flatten.PCollections<T>> {
+class EmptyInputProvider<T> implements RootInputProvider<T, Void, PCollectionList<T>> {
EmptyInputProvider() {}
/**
@@ -36,7 +35,8 @@
*/
@Override
public Collection<CommittedBundle<Void>> getInitialInputs(
- AppliedPTransform<PCollectionList<T>, PCollection<T>, Flatten.PCollections<T>>
+ AppliedPTransform<
+ PCollectionList<T>, PCollection<T>, PTransform<PCollectionList<T>, PCollection<T>>>
transform,
int targetParallelism) {
return Collections.emptyList();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index c627119..d192785 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.MoreExecutors;
@@ -31,7 +32,6 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.annotation.Nullable;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
import org.apache.beam.runners.core.SideInputReader;
import org.apache.beam.runners.core.TimerInternals.TimerData;
@@ -52,22 +52,20 @@
import org.joda.time.Instant;
/**
- * The evaluation context for a specific pipeline being executed by the
- * {@link DirectRunner}. Contains state shared within the execution across all
- * transforms.
+ * The evaluation context for a specific pipeline being executed by the {@link DirectRunner}.
+ * Contains state shared within the execution across all transforms.
*
- * <p>{@link EvaluationContext} contains shared state for an execution of the
- * {@link DirectRunner} that can be used while evaluating a {@link PTransform}. This
- * consists of views into underlying state and watermark implementations, access to read and write
- * {@link PCollectionView PCollectionViews}, and managing the
- * {@link ExecutionContext ExecutionContexts}. This includes executing callbacks asynchronously when
- * state changes to the appropriate point (e.g. when a {@link PCollectionView} is requested and
- * known to be empty).
+ * <p>{@link EvaluationContext} contains shared state for an execution of the {@link DirectRunner}
+ * that can be used while evaluating a {@link PTransform}. This consists of views into underlying
+ * state and watermark implementations, access to read and write {@link PCollectionView
+ * PCollectionViews}, and managing the {@link DirectExecutionContext ExecutionContexts}. This
+ * includes executing callbacks asynchronously when state changes to the appropriate point (e.g.
+ * when a {@link PCollectionView} is requested and known to be empty).
*
- * <p>{@link EvaluationContext} also handles results by committing finalizing bundles based
- * on the current global state and updating the global state appropriately. This includes updating
- * the per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that
- * can be executed.
+ * <p>{@link EvaluationContext} also handles results by committing finalizing bundles based on the
+ * current global state and updating the global state appropriately. This includes updating the
+ * per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that can be
+ * executed.
*/
class EvaluationContext {
/**
@@ -161,12 +159,9 @@
} else {
outputTypes.add(OutputType.BUNDLE);
}
- CommittedResult committedResult = CommittedResult.create(result,
- completedBundle == null
- ? null
- : completedBundle.withElements((Iterable) result.getUnprocessedElements()),
- committedBundles,
- outputTypes);
+ CommittedResult committedResult =
+ CommittedResult.create(
+ result, getUnprocessedInput(completedBundle, result), committedBundles, outputTypes);
// Update state internals
CopyOnAccessInMemoryStateInternals theirState = result.getState();
if (theirState != null) {
@@ -190,6 +185,22 @@
return committedResult;
}
+ /**
+ * Returns an {@link Optional} containing a bundle which contains all of the unprocessed elements
+ * that were not processed from the {@code completedBundle}. If all of the elements of the {@code
+ * completedBundle} were processed, or if {@code completedBundle} is null, returns an absent
+ * {@link Optional}.
+ */
+ private Optional<? extends CommittedBundle<?>> getUnprocessedInput(
+ @Nullable CommittedBundle<?> completedBundle, TransformResult<?> result) {
+ if (completedBundle == null || Iterables.isEmpty(result.getUnprocessedElements())) {
+ return Optional.absent();
+ }
+ CommittedBundle<?> residual =
+ completedBundle.withElements((Iterable) result.getUnprocessedElements());
+ return Optional.of(residual);
+ }
+
private Iterable<? extends CommittedBundle<?>> commitBundles(
Iterable<? extends UncommittedBundle<?>> bundles) {
ImmutableList.Builder<CommittedBundle<?>> completed = ImmutableList.builder();
@@ -279,7 +290,7 @@
* callback will be executed regardless of whether values have been produced.
*/
public void scheduleAfterOutputWouldBeProduced(
- PValue value,
+ PCollection<?> value,
BoundedWindow window,
WindowingStrategy<?, ?> windowingStrategy,
Runnable runnable) {
@@ -290,6 +301,21 @@
}
/**
+ * Schedule a callback to be executed after output would be produced for the given window if there
+ * had been input.
+ */
+ public void scheduleAfterOutputWouldBeProduced(
+ PCollectionView<?> view,
+ BoundedWindow window,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Runnable runnable) {
+ AppliedPTransform<?, ?, ?> producing = graph.getWriter(view);
+ callbackExecutor.callOnGuaranteedFiring(producing, window, windowingStrategy, runnable);
+
+ fireAvailableCallbacks(producing);
+ }
+
+ /**
* Schedule a callback to be executed after the given window is expired.
*
* <p>For example, upstream state associated with the window may be cleared.
@@ -312,7 +338,7 @@
}
/**
- * Get an {@link ExecutionContext} for the provided {@link AppliedPTransform} and key.
+ * Get a {@link DirectExecutionContext} for the provided {@link AppliedPTransform} and key.
*/
public DirectExecutionContext getExecutionContext(
AppliedPTransform<?, ?, ?> application, StructuralKey<?> key) {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 71ab4cc..75e2562 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -49,11 +49,11 @@
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.KeyedWorkItems;
import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
@@ -77,9 +77,7 @@
private final DirectGraph graph;
private final RootProviderRegistry rootProviderRegistry;
private final TransformEvaluatorRegistry registry;
- @SuppressWarnings("rawtypes")
- private final Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
- transformEnforcements;
+ private final Map<String, Collection<ModelEnforcementFactory>> transformEnforcements;
private final EvaluationContext evaluationContext;
@@ -112,9 +110,7 @@
DirectGraph graph,
RootProviderRegistry rootProviderRegistry,
TransformEvaluatorRegistry registry,
- @SuppressWarnings("rawtypes")
- Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
- transformEnforcements,
+ Map<String, Collection<ModelEnforcementFactory>> transformEnforcements,
EvaluationContext context) {
return new ExecutorServiceParallelExecutor(
targetParallelism,
@@ -130,8 +126,7 @@
DirectGraph graph,
RootProviderRegistry rootProviderRegistry,
TransformEvaluatorRegistry registry,
- @SuppressWarnings("rawtypes")
- Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>> transformEnforcements,
+ Map<String, Collection<ModelEnforcementFactory>> transformEnforcements,
EvaluationContext context) {
this.targetParallelism = targetParallelism;
// Don't use Daemon threads for workers. The Pipeline should continue to execute even if there
@@ -237,7 +232,8 @@
Collection<ModelEnforcementFactory> enforcements =
MoreObjects.firstNonNull(
- transformEnforcements.get(transform.getTransform().getClass()),
+ transformEnforcements.get(
+ PTransformTranslation.urnForTransform(transform.getTransform())),
Collections.<ModelEnforcementFactory>emptyList());
TransformExecutor<T> callable =
@@ -355,17 +351,18 @@
for (CommittedBundle<?> outputBundle : committedResult.getOutputs()) {
allUpdates.offer(
ExecutorUpdate.fromBundle(
- outputBundle, graph.getPrimitiveConsumers(outputBundle.getPCollection())));
+ outputBundle, graph.getPerElementConsumers(outputBundle.getPCollection())));
}
- CommittedBundle<?> unprocessedInputs = committedResult.getUnprocessedInputs();
- if (unprocessedInputs != null && !Iterables.isEmpty(unprocessedInputs.getElements())) {
+ Optional<? extends CommittedBundle<?>> unprocessedInputs =
+ committedResult.getUnprocessedInputs();
+ if (unprocessedInputs.isPresent()) {
if (inputBundle.getPCollection() == null) {
// TODO: Split this logic out of an if statement
- pendingRootBundles.get(result.getTransform()).offer(unprocessedInputs);
+ pendingRootBundles.get(result.getTransform()).offer(unprocessedInputs.get());
} else {
allUpdates.offer(
ExecutorUpdate.fromBundle(
- unprocessedInputs,
+ unprocessedInputs.get(),
Collections.<AppliedPTransform<?, ?, ?>>singleton(
committedResult.getTransform())));
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 84be15d..d80e4ff 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -24,23 +24,21 @@
import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.Collection;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly;
-import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
-import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.OutputWindowedValue;
import org.apache.beam.runners.core.ReduceFnRunner;
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -57,7 +55,7 @@
/**
* The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
- * {@link GroupByKeyOnly} {@link PTransform}.
+ * {@link DirectGroupAlsoByWindow} {@link PTransform}.
*/
class GroupAlsoByWindowEvaluatorFactory implements TransformEvaluatorFactory {
private final EvaluationContext evaluationContext;
@@ -92,8 +90,9 @@
}
/**
- * A transform evaluator for the pseudo-primitive {@link GroupAlsoByWindow}. Windowing is ignored;
- * all input should be in the global window since all output will be as well.
+ * A transform evaluator for the pseudo-primitive {@link DirectGroupAlsoByWindow}. The window of
+ * the input {@link KeyedWorkItem} is ignored; it should be in the global window, as element
+ * windows are reified in the {@link KeyedWorkItem#elementsIterable()}.
*
* @see GroupByKeyViaGroupByKeyOnly
*/
@@ -130,8 +129,8 @@
structuralKey = inputBundle.getKey();
stepContext = evaluationContext
.getExecutionContext(application, inputBundle.getKey())
- .getOrCreateStepContext(
- evaluationContext.getStepName(application), application.getTransform().getName());
+ .getStepContext(
+ evaluationContext.getStepName(application));
windowingStrategy =
(WindowingStrategy<?, BoundedWindow>)
application.getTransform().getInputWindowingStrategy();
@@ -163,7 +162,7 @@
(CopyOnAccessInMemoryStateInternals) stepContext.stateInternals();
DirectTimerInternals timerInternals = stepContext.timerInternals();
RunnerApi.Trigger runnerApiTrigger =
- Triggers.toProto(windowingStrategy.getTrigger());
+ TriggerTranslation.toProto(windowingStrategy.getTrigger());
ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner =
new ReduceFnRunner<>(
key,
@@ -173,7 +172,7 @@
stateInternals,
timerInternals,
new OutputWindowedValueToBundle<>(bundle),
- new UnsupportedSideInputReader("GroupAlsoByWindow"),
+ new UnsupportedSideInputReader(DirectGroupAlsoByWindow.class.getSimpleName()),
reduceFn,
evaluationContext.getPipelineOptions());
@@ -226,8 +225,9 @@
// The element is too late for this window.
droppedDueToLateness.inc();
WindowTracing.debug(
- "GroupAlsoByWindow: Dropping element at {} for key: {}; "
+ "{}: Dropping element at {} for key: {}; "
+ "window: {} since it is too far behind inputWatermark: {}",
+ DirectGroupAlsoByWindow.class.getSimpleName(),
input.getTimestamp(),
key,
window,
@@ -264,7 +264,9 @@
Instant timestamp,
Collection<? extends BoundedWindow> windows,
PaneInfo pane) {
- throw new UnsupportedOperationException("GroupAlsoByWindow should not use tagged outputs");
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s should not use tagged outputs", DirectGroupAlsoByWindow.class.getSimpleName()));
}
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
index f9b6eba..6eadaba 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
@@ -23,7 +23,7 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
@@ -44,11 +44,11 @@
*/
// TODO: Handle Key-preserving transforms when appropriate and more aggressively make PTransforms
// unkeyed
-class KeyedPValueTrackingVisitor implements PipelineVisitor {
+class KeyedPValueTrackingVisitor extends PipelineVisitor.Defaults {
private static final Set<Class<? extends PTransform>> PRODUCES_KEYED_OUTPUTS =
- ImmutableSet.of(
- SplittableParDo.GBKIntoKeyedWorkItems.class,
+ ImmutableSet.<Class<? extends PTransform>>of(
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class,
DirectGroupByKeyOnly.class,
DirectGroupAlsoByWindow.class);
@@ -91,9 +91,6 @@
}
@Override
- public void visitPrimitiveTransform(TransformHierarchy.Node node) {}
-
- @Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {
boolean inputsAreKeyed = true;
for (PValue input : producer.getInputs().values()) {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java
new file mode 100644
index 0000000..5253ef5
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.direct;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.CombineTranslation;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
+import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformMatcher;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Combine.PerKey;
+import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Instant;
+
+/** A {@link Combine} that performs the combine in multiple steps. */
+class MultiStepCombine<K, InputT, AccumT, OutputT>
+ extends RawPTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> {
+ public static PTransformMatcher matcher() {
+ return new PTransformMatcher() {
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ if (PTransformTranslation.COMBINE_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+ try {
+ GlobalCombineFn fn = CombineTranslation.getCombineFn(application);
+ return isApplicable(application.getInputs(), fn);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return false;
+ }
+
+ private <K, InputT> boolean isApplicable(
+ Map<TupleTag<?>, PValue> inputs, GlobalCombineFn<InputT, ?, ?> fn) {
+ if (!(fn instanceof CombineFn)) {
+ return false;
+ }
+ if (inputs.size() == 1) {
+ PCollection<KV<K, InputT>> input =
+ (PCollection<KV<K, InputT>>) Iterables.getOnlyElement(inputs.values());
+ WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
+ boolean windowFnApplicable = windowingStrategy.getWindowFn().isNonMerging();
+ // Triggering with count based triggers is not appropriately handled here. Disabling
+ // most triggers is safe, though more broad than is technically required.
+ boolean triggerApplicable = DefaultTrigger.of().equals(windowingStrategy.getTrigger());
+ boolean accumulatorCoderAvailable;
+ try {
+ if (input.getCoder() instanceof KvCoder) {
+ KvCoder<K, InputT> kvCoder = (KvCoder<K, InputT>) input.getCoder();
+ Coder<?> accumulatorCoder =
+ fn.getAccumulatorCoder(
+ input.getPipeline().getCoderRegistry(), kvCoder.getValueCoder());
+ accumulatorCoderAvailable = accumulatorCoder != null;
+ } else {
+ accumulatorCoderAvailable = false;
+ }
+ } catch (CannotProvideCoderException e) {
+ throw new RuntimeException(
+ String.format(
+ "Could not construct an accumulator %s for %s. Accumulator %s for a %s may be"
+ + " null, but may not throw an exception",
+ Coder.class.getSimpleName(),
+ fn,
+ Coder.class.getSimpleName(),
+ Combine.class.getSimpleName()),
+ e);
+ }
+ return windowFnApplicable && triggerApplicable && accumulatorCoderAvailable;
+ }
+ return false;
+ }
+ };
+ }
+
+ static class Factory<K, InputT, AccumT, OutputT>
+ extends SingleInputOutputOverrideFactory<
+ PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>,
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
+ public static PTransformOverrideFactory create() {
+ return new Factory<>();
+ }
+
+ private Factory() {}
+
+ @Override
+ public PTransformReplacement<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>
+ getReplacementTransform(
+ AppliedPTransform<
+ PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>,
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>>
+ transform) {
+ try {
+ GlobalCombineFn<?, ?, ?> globalFn = CombineTranslation.getCombineFn(transform);
+ checkState(
+ globalFn instanceof CombineFn,
+ "%s.matcher() should only match %s instances using %s, got %s",
+ MultiStepCombine.class.getSimpleName(),
+ PerKey.class.getSimpleName(),
+ CombineFn.class.getSimpleName(),
+ globalFn.getClass().getName());
+ @SuppressWarnings("unchecked")
+ CombineFn<InputT, AccumT, OutputT> fn = (CombineFn<InputT, AccumT, OutputT>) globalFn;
+ @SuppressWarnings("unchecked")
+ PCollection<KV<K, InputT>> input =
+ (PCollection<KV<K, InputT>>) Iterables.getOnlyElement(transform.getInputs().values());
+ @SuppressWarnings("unchecked")
+ PCollection<KV<K, OutputT>> output =
+ (PCollection<KV<K, OutputT>>) Iterables.getOnlyElement(transform.getOutputs().values());
+ return PTransformReplacement.of(input, new MultiStepCombine<>(fn, output.getCoder()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ // ===========================================================================================
+
+ private final CombineFn<InputT, AccumT, OutputT> combineFn;
+ private final Coder<KV<K, OutputT>> outputCoder;
+
+ public static <K, InputT, AccumT, OutputT> MultiStepCombine<K, InputT, AccumT, OutputT> of(
+ CombineFn<InputT, AccumT, OutputT> combineFn, Coder<KV<K, OutputT>> outputCoder) {
+ return new MultiStepCombine<>(combineFn, outputCoder);
+ }
+
+ private MultiStepCombine(
+ CombineFn<InputT, AccumT, OutputT> combineFn, Coder<KV<K, OutputT>> outputCoder) {
+ this.combineFn = combineFn;
+ this.outputCoder = outputCoder;
+ }
+
+ @Nonnull
+ @Override
+ public String getUrn() {
+ return "urn:beam:directrunner:transforms:multistepcombine:v1";
+ }
+
+ @Nullable
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ return null;
+ }
+
+ @Override
+ public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, InputT>> input) {
+ checkArgument(
+ input.getCoder() instanceof KvCoder,
+ "Expected input to have a %s of type %s, got %s",
+ Coder.class.getSimpleName(),
+ KvCoder.class.getSimpleName(),
+ input.getCoder());
+ KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) input.getCoder();
+ Coder<InputT> inputValueCoder = inputCoder.getValueCoder();
+ Coder<AccumT> accumulatorCoder;
+ try {
+ accumulatorCoder =
+ combineFn.getAccumulatorCoder(input.getPipeline().getCoderRegistry(), inputValueCoder);
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalStateException(
+ String.format(
+ "Could not construct an Accumulator Coder with the provided %s %s",
+ CombineFn.class.getSimpleName(), combineFn),
+ e);
+ }
+ return input
+ .apply(
+ ParDo.of(
+ new CombineInputs<>(
+ combineFn,
+ input.getWindowingStrategy().getTimestampCombiner(),
+ inputCoder.getKeyCoder())))
+ .setCoder(KvCoder.of(inputCoder.getKeyCoder(), accumulatorCoder))
+ .apply(GroupByKey.<K, AccumT>create())
+ .apply(new MergeAndExtractAccumulatorOutput<>(combineFn, outputCoder));
+ }
+
+ private static class CombineInputs<K, InputT, AccumT> extends DoFn<KV<K, InputT>, KV<K, AccumT>> {
+ private final CombineFn<InputT, AccumT, ?> combineFn;
+ private final TimestampCombiner timestampCombiner;
+ private final Coder<K> keyCoder;
+
+ /**
+ * Per-bundle state. Accumulators and output timestamps should only be tracked while a bundle
+ * is being processed, and must be cleared when a bundle is completed.
+ */
+ private transient Map<WindowedStructuralKey<K>, AccumT> accumulators;
+ private transient Map<WindowedStructuralKey<K>, Instant> timestamps;
+
+ private CombineInputs(
+ CombineFn<InputT, AccumT, ?> combineFn,
+ TimestampCombiner timestampCombiner,
+ Coder<K> keyCoder) {
+ this.combineFn = combineFn;
+ this.timestampCombiner = timestampCombiner;
+ this.keyCoder = keyCoder;
+ }
+
+ @StartBundle
+ public void startBundle() {
+ accumulators = new LinkedHashMap<>();
+ timestamps = new LinkedHashMap<>();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context, BoundedWindow window) {
+ WindowedStructuralKey<K>
+ key = WindowedStructuralKey.create(keyCoder, context.element().getKey(), window);
+ AccumT accumulator = accumulators.get(key);
+ Instant assignedTs = timestampCombiner.assign(window, context.timestamp());
+ if (accumulator == null) {
+ accumulator = combineFn.createAccumulator();
+ accumulators.put(key, accumulator);
+ timestamps.put(key, assignedTs);
+ }
+ accumulators.put(key, combineFn.addInput(accumulator, context.element().getValue()));
+ timestamps.put(key, timestampCombiner.combine(assignedTs, timestamps.get(key)));
+ }
+
+ @FinishBundle
+ public void outputAccumulators(FinishBundleContext context) {
+ for (Map.Entry<WindowedStructuralKey<K>, AccumT> preCombineEntry : accumulators.entrySet()) {
+ context.output(
+ KV.of(preCombineEntry.getKey().getKey(), combineFn.compact(preCombineEntry.getValue())),
+ timestamps.get(preCombineEntry.getKey()),
+ preCombineEntry.getKey().getWindow());
+ }
+ accumulators = null;
+ timestamps = null;
+ }
+ }
+
+ static class WindowedStructuralKey<K> {
+ public static <K> WindowedStructuralKey<K> create(
+ Coder<K> keyCoder, K key, BoundedWindow window) {
+ return new WindowedStructuralKey<>(StructuralKey.of(key, keyCoder), window);
+ }
+
+ private final StructuralKey<K> key;
+ private final BoundedWindow window;
+
+ private WindowedStructuralKey(StructuralKey<K> key, BoundedWindow window) {
+ this.key = checkNotNull(key, "key cannot be null");
+ this.window = checkNotNull(window, "Window cannot be null");
+ }
+
+ public K getKey() {
+ return key.getKey();
+ }
+
+ public BoundedWindow getWindow() {
+ return window;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof MultiStepCombine.WindowedStructuralKey)) {
+ return false;
+ }
+ WindowedStructuralKey that = (WindowedStructuralKey<?>) other;
+ return this.window.equals(that.window) && this.key.equals(that.key);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(window, key);
+ }
+ }
+
+ static final String DIRECT_MERGE_ACCUMULATORS_EXTRACT_OUTPUT_URN =
+ "urn:beam:directrunner:transforms:merge_accumulators_extract_output:v1";
+ /**
+ * A primitive {@link PTransform} that merges iterables of accumulators and extracts the output.
+ *
+ * <p>Required to ensure that Immutability Enforcement is not applied. Accumulators
+ * are explicitly mutable.
+ */
+ static class MergeAndExtractAccumulatorOutput<K, AccumT, OutputT>
+ extends RawPTransform<PCollection<KV<K, Iterable<AccumT>>>, PCollection<KV<K, OutputT>>> {
+ private final CombineFn<?, AccumT, OutputT> combineFn;
+ private final Coder<KV<K, OutputT>> outputCoder;
+
+ private MergeAndExtractAccumulatorOutput(
+ CombineFn<?, AccumT, OutputT> combineFn, Coder<KV<K, OutputT>> outputCoder) {
+ this.combineFn = combineFn;
+ this.outputCoder = outputCoder;
+ }
+
+ CombineFn<?, AccumT, OutputT> getCombineFn() {
+ return combineFn;
+ }
+
+ @Override
+ public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, Iterable<AccumT>>> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder);
+ }
+
+ @Nonnull
+ @Override
+ public String getUrn() {
+ return DIRECT_MERGE_ACCUMULATORS_EXTRACT_OUTPUT_URN;
+ }
+
+ @Nullable
+ @Override
+ public RunnerApi.FunctionSpec getSpec() {
+ return null;
+ }
+ }
+
+ static class MergeAndExtractAccumulatorOutputEvaluatorFactory
+ implements TransformEvaluatorFactory {
+ private final EvaluationContext ctxt;
+
+ public MergeAndExtractAccumulatorOutputEvaluatorFactory(EvaluationContext ctxt) {
+ this.ctxt = ctxt;
+ }
+
+ @Nullable
+ @Override
+ public <InputT> TransformEvaluator<InputT> forApplication(
+ AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception {
+ return createEvaluator((AppliedPTransform) application, (CommittedBundle) inputBundle);
+ }
+
+ private <K, AccumT, OutputT> TransformEvaluator<KV<K, Iterable<AccumT>>> createEvaluator(
+ AppliedPTransform<
+ PCollection<KV<K, Iterable<AccumT>>>, PCollection<KV<K, OutputT>>,
+ MergeAndExtractAccumulatorOutput<K, AccumT, OutputT>>
+ application,
+ CommittedBundle<KV<K, Iterable<AccumT>>> inputBundle) {
+ return new MergeAccumulatorsAndExtractOutputEvaluator<>(ctxt, application);
+ }
+
+ @Override
+ public void cleanup() throws Exception {}
+ }
+
+ private static class MergeAccumulatorsAndExtractOutputEvaluator<K, AccumT, OutputT>
+ implements TransformEvaluator<KV<K, Iterable<AccumT>>> {
+ private final AppliedPTransform<
+ PCollection<KV<K, Iterable<AccumT>>>, PCollection<KV<K, OutputT>>,
+ MergeAndExtractAccumulatorOutput<K, AccumT, OutputT>>
+ application;
+ private final CombineFn<?, AccumT, OutputT> combineFn;
+ private final UncommittedBundle<KV<K, OutputT>> output;
+
+ public MergeAccumulatorsAndExtractOutputEvaluator(
+ EvaluationContext ctxt,
+ AppliedPTransform<
+ PCollection<KV<K, Iterable<AccumT>>>, PCollection<KV<K, OutputT>>,
+ MergeAndExtractAccumulatorOutput<K, AccumT, OutputT>>
+ application) {
+ this.application = application;
+ this.combineFn = application.getTransform().getCombineFn();
+ this.output =
+ ctxt.createBundle(
+ (PCollection<KV<K, OutputT>>)
+ Iterables.getOnlyElement(application.getOutputs().values()));
+ }
+
+ @Override
+ public void processElement(WindowedValue<KV<K, Iterable<AccumT>>> element) throws Exception {
+ checkState(
+ element.getWindows().size() == 1,
+ "Expected inputs to %s to be in exactly one window. Got %s",
+ MergeAccumulatorsAndExtractOutputEvaluator.class.getSimpleName(),
+ element.getWindows().size());
+ Iterable<AccumT> inputAccumulators = element.getValue().getValue();
+ try {
+ AccumT first = combineFn.createAccumulator();
+ AccumT merged = combineFn.mergeAccumulators(Iterables.concat(Collections.singleton(first),
+ inputAccumulators,
+ Collections.singleton(combineFn.createAccumulator())));
+ OutputT extracted = combineFn.extractOutput(merged);
+ output.add(element.withValue(KV.of(element.getValue().getKey(), extracted)));
+ } catch (Exception e) {
+ throw UserCodeException.wrap(e);
+ }
+ }
+
+ @Override
+ public TransformResult<KV<K, Iterable<AccumT>>> finishBundle() throws Exception {
+ return StepTransformResult.<KV<K, Iterable<AccumT>>>withoutHold(application)
+ .addOutput(output)
+ .build();
+ }
+ }
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
index 28fc68d..26da6c6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -17,8 +17,9 @@
*/
package org.apache.beam.runners.direct;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.common.collect.ImmutableList;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -218,7 +219,6 @@
static class BundleOutputManager implements OutputManager {
private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
- private final Map<TupleTag<?>, List<?>> undeclaredOutputs;
public static BundleOutputManager create(Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
return new BundleOutputManager(outputBundles);
@@ -226,23 +226,13 @@
private BundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
this.bundles = bundles;
- undeclaredOutputs = new HashMap<>();
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- UncommittedBundle bundle = bundles.get(tag);
- if (bundle == null) {
- List<WindowedValue<T>> undeclaredContents = (List) undeclaredOutputs.get(tag);
- if (undeclaredContents == null) {
- undeclaredContents = new ArrayList<>();
- undeclaredOutputs.put(tag, undeclaredContents);
- }
- undeclaredContents.add(output);
- } else {
- bundle.add(output);
- }
+ checkArgument(bundles.containsKey(tag), "Unknown output tag %s", tag);
+ ((UncommittedBundle) bundles.get(tag)).add(output);
}
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 74470bf..47df0d4 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,10 +20,10 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
@@ -40,50 +40,43 @@
final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluatorFactory {
private static final Logger LOG = LoggerFactory.getLogger(ParDoEvaluatorFactory.class);
- private final LoadingCache<DoFn<?, ?>, DoFnLifecycleManager> fnClones;
+ private final LoadingCache<AppliedPTransform<?, ?, ?>, DoFnLifecycleManager> fnClones;
private final EvaluationContext evaluationContext;
private final ParDoEvaluator.DoFnRunnerFactory<InputT, OutputT> runnerFactory;
ParDoEvaluatorFactory(
EvaluationContext evaluationContext,
- ParDoEvaluator.DoFnRunnerFactory<InputT, OutputT> runnerFactory) {
+ ParDoEvaluator.DoFnRunnerFactory<InputT, OutputT> runnerFactory,
+ CacheLoader<AppliedPTransform<?, ?, ?>, DoFnLifecycleManager> doFnCacheLoader) {
this.evaluationContext = evaluationContext;
this.runnerFactory = runnerFactory;
fnClones =
- CacheBuilder.newBuilder()
- .build(
- new CacheLoader<DoFn<?, ?>, DoFnLifecycleManager>() {
- @Override
- public DoFnLifecycleManager load(DoFn<?, ?> key) throws Exception {
- return DoFnLifecycleManager.of(key);
- }
- });
+ CacheBuilder.newBuilder().build(doFnCacheLoader);
+ }
+
+ static CacheLoader<AppliedPTransform<?, ?, ?>, DoFnLifecycleManager> basicDoFnCacheLoader() {
+ return new CacheLoader<AppliedPTransform<?, ?, ?>, DoFnLifecycleManager>() {
+ @Override
+ public DoFnLifecycleManager load(AppliedPTransform<?, ?, ?> application) throws Exception {
+ return DoFnLifecycleManager.of(ParDoTranslation.getDoFn(application));
+ }
+ };
}
@Override
public <T> TransformEvaluator<T> forApplication(
AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception {
- @SuppressWarnings("unchecked")
- AppliedPTransform<PCollection<InputT>, PCollectionTuple, ParDo.MultiOutput<InputT, OutputT>>
- parDoApplication =
- (AppliedPTransform<
- PCollection<InputT>, PCollectionTuple, ParDo.MultiOutput<InputT, OutputT>>)
- application;
-
- ParDo.MultiOutput<InputT, OutputT> transform = parDoApplication.getTransform();
- final DoFn<InputT, OutputT> doFn = transform.getFn();
-
@SuppressWarnings({"unchecked", "rawtypes"})
TransformEvaluator<T> evaluator =
(TransformEvaluator<T>)
createEvaluator(
(AppliedPTransform) application,
+ (PCollection<InputT>) inputBundle.getPCollection(),
inputBundle.getKey(),
- doFn,
- transform.getSideInputs(),
- transform.getMainOutputTag(),
- transform.getAdditionalOutputTags().getAll());
+ ParDoTranslation.getSideInputs(application),
+ (TupleTag<OutputT>) ParDoTranslation.getMainOutputTag(application),
+ ParDoTranslation.getAdditionalOutputTags(application).getAll());
return evaluator;
}
@@ -102,8 +95,8 @@
@SuppressWarnings({"unchecked", "rawtypes"})
DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
+ PCollection<InputT> mainInput,
StructuralKey<?> inputBundleKey,
- DoFn<InputT, OutputT> doFn,
List<PCollectionView<?>> sideInputs,
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> additionalOutputTags)
@@ -112,14 +105,15 @@
DirectStepContext stepContext =
evaluationContext
.getExecutionContext(application, inputBundleKey)
- .getOrCreateStepContext(stepName, stepName);
+ .getStepContext(stepName);
- DoFnLifecycleManager fnManager = fnClones.getUnchecked(doFn);
+ DoFnLifecycleManager fnManager = fnClones.getUnchecked(application);
return DoFnLifecycleManagerRemovingTransformEvaluator.wrapping(
createParDoEvaluator(
application,
inputBundleKey,
+ mainInput,
sideInputs,
mainOutputTag,
additionalOutputTags,
@@ -132,6 +126,7 @@
ParDoEvaluator<InputT> createParDoEvaluator(
AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
StructuralKey<?> key,
+ PCollection<InputT> mainInput,
List<PCollectionView<?>> sideInputs,
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> additionalOutputTags,
@@ -144,8 +139,7 @@
evaluationContext,
stepContext,
application,
- ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
- .getWindowingStrategy(),
+ mainInput.getWindowingStrategy(),
fn,
key,
sideInputs,
@@ -173,5 +167,4 @@
}
return pcs;
}
-
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
index 89903da..e8a9c83 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
@@ -19,14 +19,17 @@
import static com.google.common.base.Preconditions.checkState;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.KeyedWorkItemCoder;
import org.apache.beam.runners.core.KeyedWorkItems;
-import org.apache.beam.runners.core.SplittableParDo;
import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.runners.core.construction.SplittableParDo;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -35,7 +38,6 @@
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.windowing.AfterPane;
@@ -47,6 +49,8 @@
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
@@ -59,36 +63,48 @@
*/
class ParDoMultiOverrideFactory<InputT, OutputT>
implements PTransformOverrideFactory<
- PCollection<? extends InputT>, PCollectionTuple, MultiOutput<InputT, OutputT>> {
+ PCollection<? extends InputT>, PCollectionTuple,
+ PTransform<PCollection<? extends InputT>, PCollectionTuple>> {
@Override
public PTransformReplacement<PCollection<? extends InputT>, PCollectionTuple>
getReplacementTransform(
AppliedPTransform<
- PCollection<? extends InputT>, PCollectionTuple, MultiOutput<InputT, OutputT>>
- transform) {
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- getReplacementTransform(transform.getTransform()));
+ PCollection<? extends InputT>, PCollectionTuple,
+ PTransform<PCollection<? extends InputT>, PCollectionTuple>>
+ application) {
+
+ try {
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(application),
+ getReplacementForApplication(application));
+ } catch (IOException exc) {
+ throw new RuntimeException(exc);
+ }
}
@SuppressWarnings("unchecked")
- private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementTransform(
- MultiOutput<InputT, OutputT> transform) {
+ private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication(
+ AppliedPTransform<
+ PCollection<? extends InputT>, PCollectionTuple,
+ PTransform<PCollection<? extends InputT>, PCollectionTuple>>
+ application)
+ throws IOException {
- DoFn<InputT, OutputT> fn = transform.getFn();
+ DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application);
+
DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
+
if (signature.processElement().isSplittable()) {
- return new SplittableParDo(transform);
+ return SplittableParDo.forAppliedParDo((AppliedPTransform) application);
} else if (signature.stateDeclarations().size() > 0
|| signature.timerDeclarations().size() > 0) {
- // Based on the fact that the signature is stateful, DoFnSignatures ensures
- // that it is also keyed
- MultiOutput<KV<?, ?>, OutputT> keyedTransform =
- (MultiOutput<KV<?, ?>, OutputT>) transform;
-
- return new GbkThenStatefulParDo(keyedTransform);
+ return new GbkThenStatefulParDo(
+ fn,
+ ParDoTranslation.getMainOutputTag(application),
+ ParDoTranslation.getAdditionalOutputTags(application),
+ ParDoTranslation.getSideInputs(application));
} else {
- return transform;
+ return application.getTransform();
}
}
@@ -100,10 +116,25 @@
static class GbkThenStatefulParDo<K, InputT, OutputT>
extends PTransform<PCollection<KV<K, InputT>>, PCollectionTuple> {
- private final MultiOutput<KV<K, InputT>, OutputT> underlyingParDo;
+ private final transient DoFn<KV<K, InputT>, OutputT> doFn;
+ private final TupleTagList additionalOutputTags;
+ private final TupleTag<OutputT> mainOutputTag;
+ private final List<PCollectionView<?>> sideInputs;
- public GbkThenStatefulParDo(MultiOutput<KV<K, InputT>, OutputT> underlyingParDo) {
- this.underlyingParDo = underlyingParDo;
+ public GbkThenStatefulParDo(
+ DoFn<KV<K, InputT>, OutputT> doFn,
+ TupleTag<OutputT> mainOutputTag,
+ TupleTagList additionalOutputTags,
+ List<PCollectionView<?>> sideInputs) {
+ this.doFn = doFn;
+ this.additionalOutputTags = additionalOutputTags;
+ this.mainOutputTag = mainOutputTag;
+ this.sideInputs = sideInputs;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
@Override
@@ -159,33 +190,54 @@
adjustedInput
// Explode the resulting iterable into elements that are exactly the ones from
// the input
- .apply("Stateful ParDo", new StatefulParDo<>(underlyingParDo, input));
+ .apply(
+ "Stateful ParDo",
+ new StatefulParDo<>(doFn, mainOutputTag, additionalOutputTags, sideInputs));
return outputs;
}
}
+ static final String DIRECT_STATEFUL_PAR_DO_URN =
+ "urn:beam:directrunner:transforms:stateful_pardo:v1";
+
static class StatefulParDo<K, InputT, OutputT>
extends PTransform<PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>>, PCollectionTuple> {
- private final transient MultiOutput<KV<K, InputT>, OutputT> underlyingParDo;
- private final transient PCollection<KV<K, InputT>> originalInput;
+ private final transient DoFn<KV<K, InputT>, OutputT> doFn;
+ private final TupleTagList additionalOutputTags;
+ private final TupleTag<OutputT> mainOutputTag;
+ private final List<PCollectionView<?>> sideInputs;
public StatefulParDo(
- MultiOutput<KV<K, InputT>, OutputT> underlyingParDo,
- PCollection<KV<K, InputT>> originalInput) {
- this.underlyingParDo = underlyingParDo;
- this.originalInput = originalInput;
+ DoFn<KV<K, InputT>, OutputT> doFn,
+ TupleTag<OutputT> mainOutputTag,
+ TupleTagList additionalOutputTags,
+ List<PCollectionView<?>> sideInputs) {
+ this.doFn = doFn;
+ this.mainOutputTag = mainOutputTag;
+ this.additionalOutputTags = additionalOutputTags;
+ this.sideInputs = sideInputs;
}
- public MultiOutput<KV<K, InputT>, OutputT> getUnderlyingParDo() {
- return underlyingParDo;
+ public DoFn<KV<K, InputT>, OutputT> getDoFn() {
+ return doFn;
+ }
+
+ public TupleTag<OutputT> getMainOutputTag() {
+ return mainOutputTag;
+ }
+
+ public List<PCollectionView<?>> getSideInputs() {
+ return sideInputs;
+ }
+
+ public TupleTagList getAdditionalOutputTags() {
+ return additionalOutputTags;
}
@Override
- public <T> Coder<T> getDefaultOutputCoder(
- PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>> input, PCollection<T> output)
- throws CannotProvideCoderException {
- return underlyingParDo.getDefaultOutputCoder(originalInput, output);
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
@Override
@@ -194,8 +246,9 @@
PCollectionTuple outputs =
PCollectionTuple.ofPrimitiveOutputsInternal(
input.getPipeline(),
- TupleTagList.of(underlyingParDo.getMainOutputTag())
- .and(underlyingParDo.getAdditionalOutputTags().getAll()),
+ TupleTagList.of(getMainOutputTag()).and(getAdditionalOutputTags().getAll()),
+ // TODO
+ Collections.<TupleTag<?>, Coder<?>>emptyMap(),
input.getWindowingStrategy(),
input.isBounded());
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
new file mode 100644
index 0000000..8521706
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * A {@link TransformEvaluatorFactory} that produces {@link TransformEvaluator TransformEvaluators}
+ * for the {@link Read Read} primitives, whether bounded or unbounded.
+ */
+final class ReadEvaluatorFactory implements TransformEvaluatorFactory {
+
+ final BoundedReadEvaluatorFactory boundedFactory;
+ final UnboundedReadEvaluatorFactory unboundedFactory;
+
+ public ReadEvaluatorFactory(EvaluationContext context) {
+ boundedFactory = new BoundedReadEvaluatorFactory(context);
+ unboundedFactory = new UnboundedReadEvaluatorFactory(context);
+ }
+
+ @Nullable
+ @Override
+ public <InputT> TransformEvaluator<InputT> forApplication(
+ AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception {
+ switch (ReadTranslation.sourceIsBounded(application)) {
+ case BOUNDED:
+ return boundedFactory.forApplication(application, inputBundle);
+ case UNBOUNDED:
+ return unboundedFactory.forApplication(application, inputBundle);
+ default:
+ throw new IllegalArgumentException("PCollection is neither bounded nor unbounded?!?");
+ }
+ }
+
+ @Override
+ public void cleanup() throws Exception {
+ boundedFactory.cleanup();
+ unboundedFactory.cleanup();
+ }
+
+ static <T> InputProvider<T> inputProvider(EvaluationContext context) {
+ return new InputProvider(context);
+ }
+
+ private static class InputProvider<T> implements RootInputProvider<T, SourceShard<T>, PBegin> {
+
+ private final UnboundedReadEvaluatorFactory.InputProvider<T> unboundedInputProvider;
+ private final BoundedReadEvaluatorFactory.InputProvider<T> boundedInputProvider;
+
+ InputProvider(EvaluationContext context) {
+ this.unboundedInputProvider = new UnboundedReadEvaluatorFactory.InputProvider<T>(context);
+ this.boundedInputProvider = new BoundedReadEvaluatorFactory.InputProvider<T>(context);
+ }
+
+ @Override
+ public Collection<CommittedBundle<SourceShard<T>>> getInitialInputs(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>>
+ appliedTransform,
+ int targetParallelism)
+ throws Exception {
+ switch (ReadTranslation.sourceIsBounded(appliedTransform)) {
+ case BOUNDED:
+ // This cast could be made unnecessary, but too much bounded polymorphism
+ return (Collection)
+ boundedInputProvider.getInitialInputs(appliedTransform, targetParallelism);
+ case UNBOUNDED:
+ // This cast could be made unnecessary, but too much bounded polymorphism
+ return (Collection)
+ unboundedInputProvider.getInitialInputs(appliedTransform, targetParallelism);
+ default:
+ throw new IllegalArgumentException("PCollection is neither bounded nor unbounded?!?");
+ }
+ }
+ }
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
index ce69518..0b3de32 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
@@ -29,8 +29,7 @@
* Provides {@link CommittedBundle bundles} that will be provided to the {@link PTransform
* PTransforms} that are at the root of a {@link Pipeline}.
*/
-interface RootInputProvider<
- T, ShardT, InputT extends PInput, TransformT extends PTransform<InputT, PCollection<T>>> {
+interface RootInputProvider<T, ShardT, InputT extends PInput> {
/**
* Get the initial inputs for the {@link AppliedPTransform}. The {@link AppliedPTransform} will be
* provided with these {@link CommittedBundle bundles} as input when the {@link Pipeline} runs.
@@ -44,6 +43,8 @@
* greater than or equal to 1.
*/
Collection<CommittedBundle<ShardT>> getInitialInputs(
- AppliedPTransform<InputT, PCollection<T>, TransformT> transform, int targetParallelism)
+ AppliedPTransform<InputT, PCollection<T>, PTransform<InputT, PCollection<T>>>
+ transform,
+ int targetParallelism)
throws Exception;
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
index 4b0c06d..5cbeab7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
@@ -18,13 +18,14 @@
package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.FLATTEN_TRANSFORM_URN;
+import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN;
import com.google.common.collect.ImmutableMap;
import java.util.Collection;
import java.util.Map;
-import org.apache.beam.sdk.io.Read;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten.PCollections;
import org.apache.beam.sdk.transforms.PTransform;
/**
@@ -33,34 +34,31 @@
*/
class RootProviderRegistry {
public static RootProviderRegistry defaultRegistry(EvaluationContext context) {
- ImmutableMap.Builder<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>>
+ ImmutableMap.Builder<String, RootInputProvider<?, ?, ?>>
defaultProviders = ImmutableMap.builder();
defaultProviders
- .put(Read.Bounded.class, new BoundedReadEvaluatorFactory.InputProvider(context))
- .put(Read.Unbounded.class, new UnboundedReadEvaluatorFactory.InputProvider(context))
- .put(
- TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream.class,
- new TestStreamEvaluatorFactory.InputProvider(context))
- .put(PCollections.class, new EmptyInputProvider());
+ .put(PTransformTranslation.READ_TRANSFORM_URN, ReadEvaluatorFactory.inputProvider(context))
+ .put(DIRECT_TEST_STREAM_URN, new TestStreamEvaluatorFactory.InputProvider(context))
+ .put(FLATTEN_TRANSFORM_URN, new EmptyInputProvider());
return new RootProviderRegistry(defaultProviders.build());
}
- private final Map<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>> providers;
+ private final Map<String, RootInputProvider<?, ?, ?>> providers;
private RootProviderRegistry(
- Map<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>> providers) {
+ Map<String, RootInputProvider<?, ?, ?>> providers) {
this.providers = providers;
}
public Collection<CommittedBundle<?>> getInitialInputs(
AppliedPTransform<?, ?, ?> transform, int targetParallelism) throws Exception {
- Class<? extends PTransform> transformClass = transform.getTransform().getClass();
+ String transformUrn = PTransformTranslation.urnForTransform(transform.getTransform());
RootInputProvider provider =
checkNotNull(
- providers.get(transformClass),
- "Tried to get a %s for a Transform of type %s, but there is no such provider",
+ providers.get(transformUrn),
+ "Tried to get a %s for a transform \"%s\", but there is no such provider",
RootInputProvider.class.getSimpleName(),
- transformClass.getSimpleName());
+ transformUrn);
return provider.getInitialInputs(transform, targetParallelism);
}
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java
new file mode 100644
index 0000000..a054333
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+
+/**
+ * A shard for a source in the {@link Read} transform.
+ *
+ * <p>Since {@link UnboundedSource} and {@link BoundedSource} have radically different needs, this
+ * is a mostly-empty interface.
+ */
+interface SourceShard<T> {
+ Source<T> getSource();
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index f490b0b..852ad2f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -24,14 +24,13 @@
import java.util.concurrent.Executors;
import org.apache.beam.runners.core.DoFnRunners;
import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ElementAndRestriction;
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
import org.apache.beam.runners.core.OutputWindowedValue;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
-import org.apache.beam.runners.core.SplittableParDo;
-import org.apache.beam.runners.core.SplittableParDo.ProcessFn;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateInternalsFactory;
import org.apache.beam.runners.core.TimerInternals;
@@ -43,6 +42,7 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
@@ -54,8 +54,7 @@
class SplittableProcessElementsEvaluatorFactory<
InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
implements TransformEvaluatorFactory {
- private final ParDoEvaluatorFactory<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
+ private final ParDoEvaluatorFactory<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>
delegateFactory;
private final EvaluationContext evaluationContext;
@@ -65,7 +64,8 @@
new ParDoEvaluatorFactory<>(
evaluationContext,
SplittableProcessElementsEvaluatorFactory
- .<InputT, OutputT, RestrictionT>processFnRunnerFactory());
+ .<InputT, OutputT, RestrictionT>processFnRunnerFactory(),
+ ParDoEvaluatorFactory.basicDoFnCacheLoader());
}
@Override
@@ -84,16 +84,14 @@
}
@SuppressWarnings({"unchecked", "rawtypes"})
- private TransformEvaluator<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
- createEvaluator(
- AppliedPTransform<
- PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>,
- PCollectionTuple,
- SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>>
- application,
- CommittedBundle<InputT> inputBundle)
- throws Exception {
- final SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform =
+ private TransformEvaluator<KeyedWorkItem<String, KV<InputT, RestrictionT>>> createEvaluator(
+ AppliedPTransform<
+ PCollection<KeyedWorkItem<String, KV<InputT, RestrictionT>>>, PCollectionTuple,
+ ProcessElements<InputT, OutputT, RestrictionT, TrackerT>>
+ application,
+ CommittedBundle<InputT> inputBundle)
+ throws Exception {
+ final ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform =
application.getTransform();
ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
@@ -102,21 +100,21 @@
DoFnLifecycleManager fnManager = DoFnLifecycleManager.of(processFn);
processFn =
((ProcessFn<InputT, OutputT, RestrictionT, TrackerT>)
- fnManager
- .<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
- get());
+ fnManager.<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>get());
String stepName = evaluationContext.getStepName(application);
final DirectExecutionContext.DirectStepContext stepContext =
evaluationContext
.getExecutionContext(application, inputBundle.getKey())
- .getOrCreateStepContext(stepName, stepName);
+ .getStepContext(stepName);
- final ParDoEvaluator<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
+ final ParDoEvaluator<KeyedWorkItem<String, KV<InputT, RestrictionT>>>
parDoEvaluator =
delegateFactory.createParDoEvaluator(
application,
inputBundle.getKey(),
+ (PCollection<KeyedWorkItem<String, KV<InputT, RestrictionT>>>)
+ inputBundle.getPCollection(),
transform.getSideInputs(),
transform.getMainOutputTag(),
transform.getAdditionalOutputTags().getAll(),
@@ -181,24 +179,25 @@
.setDaemon(true)
.setNameFormat("direct-splittable-process-element-checkpoint-executor")
.build()),
- 10000,
- Duration.standardSeconds(10)));
+ // Setting small values here to stimulate frequent checkpointing and better exercise
+ // splittable DoFn's in that respect.
+ 100,
+ Duration.standardSeconds(1)));
return DoFnLifecycleManagerRemovingTransformEvaluator.wrapping(parDoEvaluator, fnManager);
}
private static <InputT, OutputT, RestrictionT>
- ParDoEvaluator.DoFnRunnerFactory<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
+ ParDoEvaluator.DoFnRunnerFactory<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>
processFnRunnerFactory() {
return new ParDoEvaluator.DoFnRunnerFactory<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>() {
+ KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>() {
@Override
public PushbackSideInputDoFnRunner<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>
+ KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>
createRunner(
PipelineOptions options,
- DoFn<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT> fn,
+ DoFn<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> fn,
List<PCollectionView<?>> sideInputs,
ReadyCheckingSideInputReader sideInputReader,
OutputManager outputManager,
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 985c3be..42bfe0b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -66,7 +66,19 @@
StatefulParDoEvaluatorFactory(EvaluationContext evaluationContext) {
this.delegateFactory =
new ParDoEvaluatorFactory<>(
- evaluationContext, ParDoEvaluator.<KV<K, InputT>, OutputT>defaultRunnerFactory());
+ evaluationContext,
+ ParDoEvaluator.<KV<K, InputT>, OutputT>defaultRunnerFactory(),
+ new CacheLoader<AppliedPTransform<?, ?, ?>, DoFnLifecycleManager>() {
+ @Override
+ public DoFnLifecycleManager load(AppliedPTransform<?, ?, ?> appliedStatefulParDo)
+ throws Exception {
+ // StatefulParDo is overridden after the portable pipeline is received, so we
+ // do not go through the portability translation layers
+ StatefulParDo<?, ?, ?> statefulParDo =
+ (StatefulParDo<?, ?, ?>) appliedStatefulParDo.getTransform();
+ return DoFnLifecycleManager.of(statefulParDo.getDoFn());
+ }
+ });
this.cleanupRegistry =
CacheBuilder.newBuilder()
.weakValues()
@@ -98,7 +110,7 @@
throws Exception {
final DoFn<KV<K, InputT>, OutputT> doFn =
- application.getTransform().getUnderlyingParDo().getFn();
+ application.getTransform().getDoFn();
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
// If the DoFn is stateful, schedule state clearing.
@@ -117,11 +129,11 @@
DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
delegateFactory.createEvaluator(
(AppliedPTransform) application,
+ (PCollection) inputBundle.getPCollection(),
inputBundle.getKey(),
- doFn,
- application.getTransform().getUnderlyingParDo().getSideInputs(),
- application.getTransform().getUnderlyingParDo().getMainOutputTag(),
- application.getTransform().getUnderlyingParDo().getAdditionalOutputTags().getAll());
+ application.getTransform().getSideInputs(),
+ application.getTransform().getMainOutputTag(),
+ application.getTransform().getAdditionalOutputTags().getAll());
return new StatefulParDoEvaluator<>(delegateEvaluator);
}
@@ -151,19 +163,18 @@
transformOutputWindow
.getTransform()
.getTransform()
- .getUnderlyingParDo()
.getMainOutputTag());
WindowingStrategy<?, ?> windowingStrategy = pc.getWindowingStrategy();
BoundedWindow window = transformOutputWindow.getWindow();
final DoFn<?, ?> doFn =
- transformOutputWindow.getTransform().getTransform().getUnderlyingParDo().getFn();
+ transformOutputWindow.getTransform().getTransform().getDoFn();
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
final DirectStepContext stepContext =
evaluationContext
.getExecutionContext(
transformOutputWindow.getTransform(), transformOutputWindow.getKey())
- .getOrCreateStepContext(stepName, stepName);
+ .getStepContext(stepName);
final StateNamespace namespace =
StateNamespaces.window(
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index 8b21d5a..e42b5fe 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -22,6 +22,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -29,6 +30,7 @@
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.TestStreamTranslation;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.testing.TestStream;
@@ -159,7 +161,8 @@
}
static class DirectTestStreamFactory<T>
- implements PTransformOverrideFactory<PBegin, PCollection<T>, TestStream<T>> {
+ implements PTransformOverrideFactory<
+ PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> {
private final DirectRunner runner;
DirectTestStreamFactory(DirectRunner runner) {
@@ -168,10 +171,17 @@
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
- AppliedPTransform<PBegin, PCollection<T>, TestStream<T>> transform) {
- return PTransformReplacement.of(
- transform.getPipeline().begin(),
- new DirectTestStream<T>(runner, transform.getTransform()));
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) {
+ try {
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(),
+ new DirectTestStream<T>(runner, TestStreamTranslation.getTestStream(transform)));
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Transform could not be converted to %s", TestStream.class.getSimpleName()),
+ exc);
+ }
}
@Override
@@ -180,6 +190,8 @@
return ReplacementOutputs.singleton(outputs, newOutput);
}
+ static final String DIRECT_TEST_STREAM_URN = "urn:beam:directrunner:transforms:test_stream:v1";
+
static class DirectTestStream<T> extends PTransform<PBegin, PCollection<T>> {
private final transient DirectRunner runner;
private final TestStream<T> original;
@@ -193,16 +205,16 @@
@Override
public PCollection<T> expand(PBegin input) {
runner.setClockSupplier(new TestClockSupplier());
- return PCollection.<T>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
- .setCoder(original.getValueCoder());
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ IsBounded.UNBOUNDED,
+ original.getValueCoder());
}
}
}
- static class InputProvider<T>
- implements RootInputProvider<
- T, TestStreamIndex<T>, PBegin, DirectTestStreamFactory.DirectTestStream<T>> {
+ static class InputProvider<T> implements RootInputProvider<T, TestStreamIndex<T>, PBegin> {
private final EvaluationContext evaluationContext;
InputProvider(EvaluationContext evaluationContext) {
@@ -211,15 +223,17 @@
@Override
public Collection<CommittedBundle<TestStreamIndex<T>>> getInitialInputs(
- AppliedPTransform<PBegin, PCollection<T>, DirectTestStreamFactory.DirectTestStream<T>>
- transform,
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform,
int targetParallelism) {
+
+ // This will always be run on an execution-time transform, so it can be downcast
+ DirectTestStreamFactory.DirectTestStream<T> testStream =
+ (DirectTestStreamFactory.DirectTestStream<T>) transform.getTransform();
+
CommittedBundle<TestStreamIndex<T>> initialBundle =
evaluationContext
.<TestStreamIndex<T>>createRootBundle()
- .add(
- WindowedValue.valueInGlobalWindow(
- TestStreamIndex.of(transform.getTransform().original)))
+ .add(WindowedValue.valueInGlobalWindow(TestStreamIndex.of(testStream.original)))
.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
return Collections.singleton(initialBundle);
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index d0e622d..708a931 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -19,23 +19,33 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.FLATTEN_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.PAR_DO_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.READ_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.WINDOW_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.SplittableParDo.SPLITTABLE_PROCESS_URN;
+import static org.apache.beam.runners.direct.DirectGroupByKey.DIRECT_GABW_URN;
+import static org.apache.beam.runners.direct.DirectGroupByKey.DIRECT_GBKO_URN;
+import static org.apache.beam.runners.direct.MultiStepCombine.DIRECT_MERGE_ACCUMULATORS_EXTRACT_OUTPUT_URN;
+import static org.apache.beam.runners.direct.ParDoMultiOverrideFactory.DIRECT_STATEFUL_PAR_DO_URN;
+import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN;
+import static org.apache.beam.runners.direct.ViewOverrideFactory.DIRECT_WRITE_VIEW_URN;
+import com.google.auto.service.AutoService;
import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.beam.runners.core.SplittableParDo;
-import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
-import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
-import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
-import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
-import org.apache.beam.sdk.io.Read;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar;
+import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten.PCollections;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.Window;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,43 +55,98 @@
*/
class TransformEvaluatorRegistry implements TransformEvaluatorFactory {
private static final Logger LOG = LoggerFactory.getLogger(TransformEvaluatorRegistry.class);
+
public static TransformEvaluatorRegistry defaultRegistry(EvaluationContext ctxt) {
- @SuppressWarnings({"rawtypes"})
- ImmutableMap<Class<? extends PTransform>, TransformEvaluatorFactory> primitives =
- ImmutableMap.<Class<? extends PTransform>, TransformEvaluatorFactory>builder()
- .put(Read.Bounded.class, new BoundedReadEvaluatorFactory(ctxt))
- .put(Read.Unbounded.class, new UnboundedReadEvaluatorFactory(ctxt))
+ ImmutableMap<String, TransformEvaluatorFactory> primitives =
+ ImmutableMap.<String, TransformEvaluatorFactory>builder()
+ // Beam primitives
+ .put(READ_TRANSFORM_URN, new ReadEvaluatorFactory(ctxt))
.put(
- ParDo.MultiOutput.class,
- new ParDoEvaluatorFactory<>(ctxt, ParDoEvaluator.defaultRunnerFactory()))
- .put(StatefulParDo.class, new StatefulParDoEvaluatorFactory<>(ctxt))
- .put(PCollections.class, new FlattenEvaluatorFactory(ctxt))
- .put(WriteView.class, new ViewEvaluatorFactory(ctxt))
- .put(Window.Assign.class, new WindowEvaluatorFactory(ctxt))
- // Runner-specific primitives used in expansion of GroupByKey
- .put(DirectGroupByKeyOnly.class, new GroupByKeyOnlyEvaluatorFactory(ctxt))
- .put(DirectGroupAlsoByWindow.class, new GroupAlsoByWindowEvaluatorFactory(ctxt))
+ PAR_DO_TRANSFORM_URN,
+ new ParDoEvaluatorFactory<>(
+ ctxt,
+ ParDoEvaluator.defaultRunnerFactory(),
+ ParDoEvaluatorFactory.basicDoFnCacheLoader()))
+ .put(FLATTEN_TRANSFORM_URN, new FlattenEvaluatorFactory(ctxt))
+ .put(WINDOW_TRANSFORM_URN, new WindowEvaluatorFactory(ctxt))
+
+ // Runner-specific primitives
+ .put(DIRECT_WRITE_VIEW_URN, new ViewEvaluatorFactory(ctxt))
+ .put(DIRECT_STATEFUL_PAR_DO_URN, new StatefulParDoEvaluatorFactory<>(ctxt))
+ .put(DIRECT_GBKO_URN, new GroupByKeyOnlyEvaluatorFactory(ctxt))
+ .put(DIRECT_GABW_URN, new GroupAlsoByWindowEvaluatorFactory(ctxt))
+ .put(DIRECT_TEST_STREAM_URN, new TestStreamEvaluatorFactory(ctxt))
.put(
- TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream.class,
- new TestStreamEvaluatorFactory(ctxt))
- // Runner-specific primitive used in expansion of SplittableParDo
- .put(
- SplittableParDo.ProcessElements.class,
- new SplittableProcessElementsEvaluatorFactory<>(ctxt))
+ DIRECT_MERGE_ACCUMULATORS_EXTRACT_OUTPUT_URN,
+ new MultiStepCombine.MergeAndExtractAccumulatorOutputEvaluatorFactory(ctxt))
+
+ // Runners-core primitives
+ .put(SPLITTABLE_PROCESS_URN, new SplittableProcessElementsEvaluatorFactory<>(ctxt))
.build();
return new TransformEvaluatorRegistry(primitives);
}
+ /** Registers classes specialized to the direct runner. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class DirectTransformsRegistrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<
+ ? extends Class<? extends PTransform>,
+ ? extends PTransformTranslation.TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return ImmutableMap
+ .<Class<? extends PTransform>, PTransformTranslation.TransformPayloadTranslator>builder()
+ .put(
+ DirectGroupByKey.DirectGroupByKeyOnly.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(DIRECT_GBKO_URN))
+ .put(
+ DirectGroupByKey.DirectGroupAlsoByWindow.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(DIRECT_GABW_URN))
+ .put(
+ ParDoMultiOverrideFactory.StatefulParDo.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(DIRECT_STATEFUL_PAR_DO_URN))
+ .put(
+ ViewOverrideFactory.WriteView.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(DIRECT_WRITE_VIEW_URN))
+ .put(
+ DirectTestStream.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(DIRECT_TEST_STREAM_URN))
+ .put(
+ SplittableParDoViaKeyedWorkItems.ProcessElements.class,
+ TransformPayloadTranslator.NotSerializable.forUrn(SPLITTABLE_PROCESS_URN))
+ .build();
+ }
+
+ @Override
+ public Map<String, TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+
+ /**
+ * A translator just to vend the URN. This will need to be moved to runners-core-construction-java
+ * once SDF is reorganized appropriately.
+ */
+ private static class SplittableParDoProcessElementsTranslator
+ extends TransformPayloadTranslator.NotSerializable<ProcessElements<?, ?, ?, ?>> {
+
+ private SplittableParDoProcessElementsTranslator() {}
+
+ @Override
+ public String getUrn(ProcessElements<?, ?, ?, ?> transform) {
+ return SPLITTABLE_PROCESS_URN;
+ }
+ }
+
// the TransformEvaluatorFactories can construct instances of all generic types of transform,
// so all instances of a primitive can be handled with the same evaluator factory.
- @SuppressWarnings("rawtypes")
- private final Map<Class<? extends PTransform>, TransformEvaluatorFactory> factories;
+ private final Map<String, TransformEvaluatorFactory> factories;
private final AtomicBoolean finished = new AtomicBoolean(false);
private TransformEvaluatorRegistry(
@SuppressWarnings("rawtypes")
- Map<Class<? extends PTransform>, TransformEvaluatorFactory> factories) {
+ Map<String, TransformEvaluatorFactory> factories) {
this.factories = factories;
}
@@ -91,10 +156,12 @@
throws Exception {
checkState(
!finished.get(), "Tried to get an evaluator for a finished TransformEvaluatorRegistry");
- Class<? extends PTransform> transformClass = application.getTransform().getClass();
+
+ String urn = PTransformTranslation.urnForTransform(application.getTransform());
+
TransformEvaluatorFactory factory =
checkNotNull(
- factories.get(transformClass), "No evaluator for PTransform type %s", transformClass);
+ factories.get(urn), "No evaluator for PTransform \"%s\"", urn);
return factory.forApplication(application, inputBundle);
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index cba826cc..7d4bba1 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -29,6 +29,7 @@
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
import org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.Read.Unbounded;
@@ -253,7 +254,8 @@
}
@AutoValue
- abstract static class UnboundedSourceShard<T, CheckpointT extends CheckpointMark> {
+ abstract static class UnboundedSourceShard<T, CheckpointT extends CheckpointMark>
+ implements SourceShard<T> {
static <T, CheckpointT extends CheckpointMark> UnboundedSourceShard<T, CheckpointT> unstarted(
UnboundedSource<T, CheckpointT> source, UnboundedReadDeduplicator deduplicator) {
return of(source, deduplicator, null, null);
@@ -268,7 +270,8 @@
source, deduplicator, reader, checkpoint);
}
- abstract UnboundedSource<T, CheckpointT> getSource();
+ @Override
+ public abstract UnboundedSource<T, CheckpointT> getSource();
abstract UnboundedReadDeduplicator getDeduplicator();
@@ -283,9 +286,8 @@
}
}
- static class InputProvider<OutputT>
- implements RootInputProvider<
- OutputT, UnboundedSourceShard<OutputT, ?>, PBegin, Unbounded<OutputT>> {
+ static class InputProvider<T>
+ implements RootInputProvider<T, UnboundedSourceShard<T, ?>, PBegin> {
private final EvaluationContext evaluationContext;
InputProvider(EvaluationContext evaluationContext) {
@@ -293,27 +295,28 @@
}
@Override
- public Collection<CommittedBundle<UnboundedSourceShard<OutputT, ?>>> getInitialInputs(
- AppliedPTransform<PBegin, PCollection<OutputT>, Unbounded<OutputT>> transform,
+ public Collection<CommittedBundle<UnboundedSourceShard<T, ?>>> getInitialInputs(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>>
+ transform,
int targetParallelism)
throws Exception {
- UnboundedSource<OutputT, ?> source = transform.getTransform().getSource();
- List<? extends UnboundedSource<OutputT, ?>> splits =
+ UnboundedSource<T, ?> source = ReadTranslation.unboundedSourceFromTransform(transform);
+ List<? extends UnboundedSource<T, ?>> splits =
source.split(targetParallelism, evaluationContext.getPipelineOptions());
UnboundedReadDeduplicator deduplicator =
source.requiresDeduping()
? UnboundedReadDeduplicator.CachedIdDeduplicator.create()
: NeverDeduplicator.create();
- ImmutableList.Builder<CommittedBundle<UnboundedSourceShard<OutputT, ?>>> initialShards =
+ ImmutableList.Builder<CommittedBundle<UnboundedSourceShard<T, ?>>> initialShards =
ImmutableList.builder();
- for (UnboundedSource<OutputT, ?> split : splits) {
- UnboundedSourceShard<OutputT, ?> shard =
+ for (UnboundedSource<T, ?> split : splits) {
+ UnboundedSourceShard<T, ?> shard =
UnboundedSourceShard.unstarted(split, deduplicator);
initialShards.add(
evaluationContext
- .<UnboundedSourceShard<OutputT, ?>>createRootBundle()
- .add(WindowedValue.<UnboundedSourceShard<OutputT, ?>>valueInGlobalWindow(shard))
+ .<UnboundedSourceShard<T, ?>>createRootBundle()
+ .add(WindowedValue.<UnboundedSourceShard<T, ?>>valueInGlobalWindow(shard))
.commit(BoundedWindow.TIMESTAMP_MAX_VALUE));
}
return initialShards.build();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index 057f4a1..8a281a7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -28,7 +28,6 @@
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
/**
* The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link CreatePCollectionView}
@@ -60,12 +59,13 @@
public void cleanup() throws Exception {}
private <InT, OuT> TransformEvaluator<Iterable<InT>> createEvaluator(
- final AppliedPTransform<PCollection<Iterable<InT>>, PCollectionView<OuT>, WriteView<InT, OuT>>
+ final AppliedPTransform<
+ PCollection<Iterable<InT>>, PCollection<Iterable<InT>>, WriteView<InT, OuT>>
application) {
PCollection<Iterable<InT>> input =
(PCollection<Iterable<InT>>) Iterables.getOnlyElement(application.getInputs().values());
- final PCollectionViewWriter<InT, OuT> writer = context.createPCollectionViewWriter(input,
- (PCollectionView<OuT>) Iterables.getOnlyElement(application.getOutputs().values()));
+ final PCollectionViewWriter<InT, OuT> writer =
+ context.createPCollectionViewWriter(input, application.getTransform().getView());
return new TransformEvaluator<Iterable<InT>>() {
private final List<WindowedValue<InT>> elements = new ArrayList<>();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index b3bbac8..0079f98 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -18,10 +18,11 @@
package org.apache.beam.runners.direct;
-import java.util.Collections;
+import java.io.IOException;
import java.util.Map;
-import org.apache.beam.runners.core.construction.ForwardingPTransform;
+import org.apache.beam.runners.core.construction.CreatePCollectionViewTranslation;
import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -42,46 +43,56 @@
*/
class ViewOverrideFactory<ElemT, ViewT>
implements PTransformOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>> {
@Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>> getReplacementTransform(
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>>
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>>
transform) {
- return PTransformReplacement.of(
+
+ PCollectionView<ViewT> view;
+ try {
+ view = CreatePCollectionViewTranslation.getView(transform);
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Could not extract %s from transform %s",
+ PCollectionView.class.getSimpleName(), transform),
+ exc);
+ }
+
+ return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new GroupAndWriteView<>(transform.getTransform()));
+ new GroupAndWriteView<ElemT, ViewT>(view));
}
@Override
public Map<PValue, ReplacementOutput> mapOutputs(
- Map<TupleTag<?>, PValue> outputs, PCollectionView<ViewT> newOutput) {
- return Collections.emptyMap();
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
}
/** The {@link DirectRunner} composite override for {@link CreatePCollectionView}. */
static class GroupAndWriteView<ElemT, ViewT>
- extends ForwardingPTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
- private final CreatePCollectionView<ElemT, ViewT> og;
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
- private GroupAndWriteView(CreatePCollectionView<ElemT, ViewT> og) {
- this.og = og;
+ private GroupAndWriteView(PCollectionView<ViewT> view) {
+ this.view = view;
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return input
+ public PCollection<ElemT> expand(final PCollection<ElemT> input) {
+ input
.apply(WithKeys.<Void, ElemT>of((Void) null))
.setCoder(KvCoder.of(VoidCoder.of(), input.getCoder()))
.apply(GroupByKey.<Void, ElemT>create())
.apply(Values.<Iterable<ElemT>>create())
- .apply(new WriteView<ElemT, ViewT>(og));
- }
-
- @Override
- protected PTransform<PCollection<ElemT>, PCollectionView<ViewT>> delegate() {
- return og;
+ .apply(new WriteView<ElemT, ViewT>(view));
+ return input;
}
}
@@ -93,22 +104,26 @@
* to {@link ViewT}.
*/
static final class WriteView<ElemT, ViewT>
- extends PTransform<PCollection<Iterable<ElemT>>, PCollectionView<ViewT>> {
- private final CreatePCollectionView<ElemT, ViewT> og;
+ extends PTransform<PCollection<Iterable<ElemT>>, PCollection<Iterable<ElemT>>> {
+ private final PCollectionView<ViewT> view;
- WriteView(CreatePCollectionView<ElemT, ViewT> og) {
- this.og = og;
+ WriteView(PCollectionView<ViewT> view) {
+ this.view = view;
}
@Override
@SuppressWarnings("deprecation")
- public PCollectionView<ViewT> expand(PCollection<Iterable<ElemT>> input) {
- return og.getView();
+ public PCollection<Iterable<ElemT>> expand(PCollection<Iterable<ElemT>> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
}
@SuppressWarnings("deprecation")
public PCollectionView<ViewT> getView() {
- return og.getView();
+ return view;
}
}
+
+ public static final String DIRECT_WRITE_VIEW_URN =
+ "urn:beam:directrunner:transforms:write_view:v1";
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 4f1b831..599b74f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -54,14 +54,15 @@
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
import org.joda.time.Instant;
/**
@@ -790,6 +791,18 @@
}
}
+ private TransformWatermarks getValueWatermark(PValue pvalue) {
+ if (pvalue instanceof PCollection) {
+ return getTransformWatermark(graph.getProducer((PCollection<?>) pvalue));
+ } else if (pvalue instanceof PCollectionView<?>) {
+ return getTransformWatermark(graph.getWriter((PCollectionView<?>) pvalue));
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Unknown type of %s %s", PValue.class.getSimpleName(), pvalue.getClass()));
+ }
+ }
+
private TransformWatermarks getTransformWatermark(AppliedPTransform<?, ?, ?> transform) {
TransformWatermarks wms = transformToWatermarks.get(transform);
if (wms == null) {
@@ -818,14 +831,13 @@
private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?, ?, ?> transform) {
ImmutableList.Builder<Watermark> inputWmsBuilder = ImmutableList.builder();
- Map<TupleTag<?>, PValue> inputs = transform.getInputs();
+ Collection<PValue> inputs = TransformInputs.nonAdditionalInputs(transform);
if (inputs.isEmpty()) {
inputWmsBuilder.add(THE_END_OF_TIME);
}
- for (PValue pvalue : inputs.values()) {
+ for (PValue pvalue : inputs) {
Watermark producerOutputWatermark =
- getTransformWatermark(graph.getProducer(pvalue))
- .synchronizedProcessingOutputWatermark;
+ getValueWatermark(pvalue).synchronizedProcessingOutputWatermark;
inputWmsBuilder.add(producerOutputWatermark);
}
return inputWmsBuilder.build();
@@ -833,13 +845,12 @@
private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform) {
ImmutableList.Builder<Watermark> inputWatermarksBuilder = ImmutableList.builder();
- Map<TupleTag<?>, PValue> inputs = transform.getInputs();
+ Collection< PValue> inputs = TransformInputs.nonAdditionalInputs(transform);
if (inputs.isEmpty()) {
inputWatermarksBuilder.add(THE_END_OF_TIME);
}
- for (PValue pvalue : inputs.values()) {
- Watermark producerOutputWatermark =
- getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
+ for (PValue pvalue : inputs) {
+ Watermark producerOutputWatermark = getValueWatermark(pvalue).outputWatermark;
inputWatermarksBuilder.add(producerOutputWatermark);
}
List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
@@ -976,16 +987,16 @@
// refresh.
for (CommittedBundle<?> bundle : result.getOutputs()) {
for (AppliedPTransform<?, ?, ?> consumer :
- graph.getPrimitiveConsumers(bundle.getPCollection())) {
+ graph.getPerElementConsumers(bundle.getPCollection())) {
TransformWatermarks watermarks = transformToWatermarks.get(consumer);
watermarks.addPending(bundle);
}
}
TransformWatermarks completedTransform = transformToWatermarks.get(result.getTransform());
- if (input != null) {
+ if (result.getUnprocessedInputs().isPresent()) {
// Add the unprocessed inputs
- completedTransform.addPending(result.getUnprocessedInputs());
+ completedTransform.addPending(result.getUnprocessedInputs().get());
}
completedTransform.updateTimers(timerUpdate);
if (input != null) {
@@ -1024,7 +1035,7 @@
if (updateResult.isAdvanced()) {
Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
for (PValue outputPValue : toRefresh.getOutputs().values()) {
- additionalRefreshes.addAll(graph.getPrimitiveConsumers(outputPValue));
+ additionalRefreshes.addAll(graph.getPerElementConsumers(outputPValue));
}
return additionalRefreshes;
}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
index f4228d9..2d37b27 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
@@ -20,6 +20,7 @@
import com.google.common.collect.Iterables;
import java.util.Collection;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.WindowIntoTranslation;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -52,7 +53,9 @@
private <InputT> TransformEvaluator<InputT> createTransformEvaluator(
AppliedPTransform<PCollection<InputT>, PCollection<InputT>, Window.Assign<InputT>>
transform) {
- WindowFn<? super InputT, ?> fn = transform.getTransform().getWindowFn();
+
+ WindowFn<? super InputT, ?> fn = (WindowFn) WindowIntoTranslation.getWindowFn(transform);
+
UncommittedBundle<InputT> outputBundle =
evaluationContext.createBundle(
(PCollection<InputT>) Iterables.getOnlyElement(transform.getOutputs().values()));
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
index 65a5a19..3f17f4d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
@@ -21,12 +21,15 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
+import java.io.IOException;
import java.io.Serializable;
-import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.WriteFilesTranslation;
import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.Count;
@@ -38,34 +41,50 @@
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
/**
- * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles}
- * {@link PTransform PTransforms} with an unspecified number of shards with a write with a
- * specified number of shards. The number of shards is the log base 10 of the number of input
- * records, with up to 2 additional shards.
+ * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles} {@link PTransform
+ * PTransforms} with an unspecified number of shards with a write with a specified number of shards.
+ * The number of shards is the log base 10 of the number of input records, with up to 2 additional
+ * shards.
*/
-class WriteWithShardingFactory<InputT>
- implements PTransformOverrideFactory<PCollection<InputT>, PDone, WriteFiles<InputT>> {
+class WriteWithShardingFactory<InputT, DestinationT>
+ implements PTransformOverrideFactory<
+ PCollection<InputT>, WriteFilesResult<DestinationT>,
+ PTransform<PCollection<InputT>, WriteFilesResult<DestinationT>>> {
static final int MAX_RANDOM_EXTRA_SHARDS = 3;
@VisibleForTesting static final int MIN_SHARDS_FOR_LOG = 3;
@Override
- public PTransformReplacement<PCollection<InputT>, PDone> getReplacementTransform(
- AppliedPTransform<PCollection<InputT>, PDone, WriteFiles<InputT>> transform) {
-
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- transform.getTransform().withSharding(new LogElementShardsWithDrift<InputT>()));
+ public PTransformReplacement<PCollection<InputT>, WriteFilesResult<DestinationT>>
+ getReplacementTransform(
+ AppliedPTransform<
+ PCollection<InputT>, WriteFilesResult<DestinationT>,
+ PTransform<PCollection<InputT>, WriteFilesResult<DestinationT>>>
+ transform) {
+ try {
+ WriteFiles<InputT, DestinationT, ?> replacement =
+ WriteFiles.to(WriteFilesTranslation.getSink(transform))
+ .withSideInputs(WriteFilesTranslation.getDynamicDestinationSideInputs(transform))
+ .withSharding(new LogElementShardsWithDrift<InputT>());
+ if (WriteFilesTranslation.isWindowedWrites(transform)) {
+ replacement = replacement.withWindowedWrites();
+ }
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform), replacement);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
public Map<PValue, ReplacementOutput> mapOutputs(
- Map<TupleTag<?>, PValue> outputs, PDone newOutput) {
- return Collections.emptyMap();
+ Map<TupleTag<?>, PValue> outputs, WriteFilesResult<DestinationT> newOutput) {
+ // We must connect the new output from WriteFilesResult to the outputs provided by the original
+ // transform.
+ return ReplacementOutputs.tagged(outputs, newOutput);
}
private static class LogElementShardsWithDrift<T>
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
index 6180d29..cb8168c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
@@ -380,10 +380,6 @@
}
@Override
- public void validate() {
- }
-
- @Override
public long getMaxEndOffset(PipelineOptions options) throws Exception {
return elems.length;
}
@@ -395,7 +391,7 @@
}
@Override
- public Coder<T> getDefaultOutputCoder() {
+ public Coder<T> getOutputCoder() {
return coder;
}
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
index cf19dc2..29ed55d 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
@@ -18,15 +18,16 @@
package org.apache.beam.runners.direct;
-import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
+import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import java.io.Serializable;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import org.apache.beam.runners.direct.CommittedResult.OutputType;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
@@ -72,7 +73,7 @@
CommittedResult result =
CommittedResult.create(
StepTransformResult.withoutHold(transform).build(),
- bundleFactory.createBundle(created).commit(Instant.now()),
+ Optional.<CommittedBundle<?>>absent(),
Collections.<CommittedBundle<?>>emptyList(),
EnumSet.noneOf(OutputType.class));
@@ -88,11 +89,11 @@
CommittedResult result =
CommittedResult.create(
StepTransformResult.withoutHold(transform).build(),
- bundle,
+ Optional.of(bundle),
Collections.<CommittedBundle<?>>emptyList(),
EnumSet.noneOf(OutputType.class));
- assertThat(result.getUnprocessedInputs(),
+ assertThat(result.getUnprocessedInputs().get(),
Matchers.<CommittedBundle<?>>equalTo(bundle));
}
@@ -101,26 +102,40 @@
CommittedResult result =
CommittedResult.create(
StepTransformResult.withoutHold(transform).build(),
- null,
+ Optional.<CommittedBundle<?>>absent(),
Collections.<CommittedBundle<?>>emptyList(),
EnumSet.noneOf(OutputType.class));
- assertThat(result.getUnprocessedInputs(), nullValue());
+ assertThat(
+ result.getUnprocessedInputs(),
+ Matchers.<Optional<? extends CommittedBundle<?>>>equalTo(
+ Optional.<CommittedBundle<?>>absent()));
}
@Test
public void getOutputsEqualInput() {
- List<? extends CommittedBundle<?>> outputs =
- ImmutableList.of(bundleFactory.createBundle(PCollection.createPrimitiveOutputInternal(p,
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.BOUNDED)).commit(Instant.now()),
- bundleFactory.createBundle(PCollection.createPrimitiveOutputInternal(p,
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.UNBOUNDED)).commit(Instant.now()));
+ List<? extends CommittedBundle<Integer>> outputs =
+ ImmutableList.of(
+ bundleFactory
+ .createBundle(
+ PCollection.createPrimitiveOutputInternal(
+ p,
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.BOUNDED,
+ VarIntCoder.of()))
+ .commit(Instant.now()),
+ bundleFactory
+ .createBundle(
+ PCollection.createPrimitiveOutputInternal(
+ p,
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ VarIntCoder.of()))
+ .commit(Instant.now()));
CommittedResult result =
CommittedResult.create(
StepTransformResult.withoutHold(transform).build(),
- bundleFactory.createBundle(created).commit(Instant.now()),
+ Optional.<CommittedBundle<?>>absent(),
outputs,
EnumSet.of(OutputType.BUNDLE, OutputType.PCOLLECTION_VIEW));
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
index 1e60ca3..657bb7f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
@@ -29,6 +29,7 @@
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
+import com.google.common.collect.Lists;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateNamespaceForTest;
import org.apache.beam.runners.core.StateNamespaces;
@@ -63,8 +64,10 @@
@RunWith(JUnit4.class)
public class CopyOnAccessInMemoryStateInternalsTest {
- @Rule public final TestPipeline pipeline = TestPipeline.create();
- @Rule public ExpectedException thrown = ExpectedException.none();
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
private String key = "foo";
@Test
@@ -114,7 +117,7 @@
*/
@Test
public void testGetWithPresentInUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -125,7 +128,7 @@
underlyingValue.write("bar");
assertThat(underlyingValue.read(), equalTo("bar"));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
ValueState<String> copyOnAccessState = internals.state(namespace, valueTag);
assertThat(copyOnAccessState.read(), equalTo("bar"));
@@ -140,7 +143,7 @@
@Test
public void testBagStateWithUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -151,7 +154,7 @@
underlyingValue.add(1);
assertThat(underlyingValue.read(), containsInAnyOrder(1));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
BagState<Integer> copyOnAccessState = internals.state(namespace, valueTag);
assertThat(copyOnAccessState.read(), containsInAnyOrder(1));
@@ -161,12 +164,13 @@
assertThat(underlyingValue.read(), containsInAnyOrder(1));
BagState<Integer> reReadUnderlyingValue = underlying.state(namespace, valueTag);
- assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read()));
+ assertThat(Lists.newArrayList(underlyingValue.read()),
+ equalTo(Lists.newArrayList(reReadUnderlyingValue.read())));
}
@Test
public void testSetStateWithUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -177,7 +181,7 @@
underlyingValue.add(1);
assertThat(underlyingValue.read(), containsInAnyOrder(1));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
SetState<Integer> copyOnAccessState = internals.state(namespace, valueTag);
assertThat(copyOnAccessState.read(), containsInAnyOrder(1));
@@ -192,7 +196,7 @@
@Test
public void testMapStateWithUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -204,7 +208,7 @@
underlyingValue.put("hello", 1);
assertThat(underlyingValue.get("hello").read(), equalTo(1));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
MapState<String, Integer> copyOnAccessState = internals.state(namespace, valueTag);
assertThat(copyOnAccessState.get("hello").read(), equalTo(1));
@@ -221,7 +225,7 @@
@Test
public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs();
@@ -236,7 +240,7 @@
underlyingValue.add(1L);
assertThat(underlyingValue.read(), equalTo(1L));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag);
assertThat(copyOnAccessState.read(), equalTo(1L));
@@ -251,7 +255,7 @@
@Test
public void testWatermarkHoldStateWithUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
TimestampCombiner timestampCombiner = TimestampCombiner.EARLIEST;
@@ -265,7 +269,7 @@
underlyingValue.add(new Instant(250L));
assertThat(underlyingValue.read(), equalTo(new Instant(250L)));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
WatermarkHoldState copyOnAccessState = internals.state(namespace, stateTag);
assertThat(copyOnAccessState.read(), equalTo(new Instant(250L)));
@@ -284,7 +288,7 @@
@Test
public void testCommitWithoutUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
@@ -304,9 +308,9 @@
@Test
public void testCommitWithUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -331,11 +335,11 @@
@Test
public void testCommitWithClearedInUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
- CopyOnAccessInMemoryStateInternals<String>secondUnderlying =
+ CopyOnAccessInMemoryStateInternals<String> secondUnderlying =
spy(CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, secondUnderlying);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -361,9 +365,9 @@
@Test
public void testCommitWithOverwrittenUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -392,9 +396,9 @@
@Test
public void testCommitWithAddedUnderlying() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
internals.commit();
@@ -416,7 +420,7 @@
@Test
public void testCommitWithEmptyTableIsEmpty() {
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
internals.commit();
@@ -426,7 +430,7 @@
@Test
public void testCommitWithOnlyClearedValuesIsEmpty() {
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -444,9 +448,9 @@
@Test
public void testCommitWithEmptyNewAndFullUnderlyingIsNotEmpty() {
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
StateNamespace namespace = new StateNamespaceForTest("foo");
@@ -475,7 +479,7 @@
return new Instant(689743L);
}
};
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying("foo", null);
StateTag<WatermarkHoldState> firstHoldAddress =
@@ -508,7 +512,7 @@
return new Instant(689743L);
}
};
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying("foo", null);
StateTag<WatermarkHoldState> firstHoldAddress =
StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST);
@@ -516,7 +520,7 @@
underlying.state(StateNamespaces.window(null, first), firstHoldAddress);
firstHold.add(new Instant(22L));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying("foo", underlying.commit());
StateTag<WatermarkHoldState> secondHoldAddress =
@@ -545,7 +549,7 @@
return new Instant(689743L);
}
};
- CopyOnAccessInMemoryStateInternals<String>underlying =
+ CopyOnAccessInMemoryStateInternals<String> underlying =
CopyOnAccessInMemoryStateInternals.withUnderlying("foo", null);
StateTag<WatermarkHoldState> firstHoldAddress =
StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST);
@@ -553,7 +557,7 @@
underlying.state(StateNamespaces.window(null, first), firstHoldAddress);
firstHold.add(new Instant(224L));
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying("foo", underlying.commit());
StateTag<WatermarkHoldState> secondHoldAddress =
@@ -568,7 +572,7 @@
@Test
public void testGetEarliestHoldBeforeCommit() {
- CopyOnAccessInMemoryStateInternals<String>internals =
+ CopyOnAccessInMemoryStateInternals<String> internals =
CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
internals
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
index 7f46a0e..bf3e83e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
@@ -78,6 +78,9 @@
.apply(View.<String>asList());
PCollectionView<Object> singletonView =
p.apply("singletonCreate", Create.<Object>of(1, 2, 3)).apply(View.<Object>asSingleton());
+ p.replaceAll(
+ DirectRunner.fromOptions(TestPipeline.testingPipelineOptions())
+ .defaultTransformOverrides());
p.traverseTopologically(visitor);
assertThat(
visitor.getGraph().getViews(),
@@ -148,13 +151,13 @@
graph.getProducer(flattened);
assertThat(
- graph.getPrimitiveConsumers(created),
+ graph.getPerElementConsumers(created),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(
transformedProducer, flattenedProducer));
assertThat(
- graph.getPrimitiveConsumers(transformed),
+ graph.getPerElementConsumers(transformed),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(flattenedProducer));
- assertThat(graph.getPrimitiveConsumers(flattened), emptyIterable());
+ assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
@Test
@@ -170,10 +173,10 @@
AppliedPTransform<?, ?, ?> flattenedProducer = graph.getProducer(flattened);
assertThat(
- graph.getPrimitiveConsumers(created),
+ graph.getPerElementConsumers(created),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(flattenedProducer,
flattenedProducer));
- assertThat(graph.getPrimitiveConsumers(flattened), emptyIterable());
+ assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
@Test
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
index 2f048fa..7707f7f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
@@ -18,11 +18,20 @@
package org.apache.beam.runners.direct;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
/** Test utilities for the {@link DirectRunner}. */
final class DirectGraphs {
+ public static void performDirectOverrides(Pipeline p) {
+ p.replaceAll(
+ DirectRunner.fromOptions(PipelineOptionsFactory.create().as(DirectOptions.class))
+ .defaultTransformOverrides());
+ }
+
public static DirectGraph getGraph(Pipeline p) {
DirectGraphVisitor visitor = new DirectGraphVisitor();
p.traverseTopologically(visitor);
@@ -30,6 +39,12 @@
}
public static AppliedPTransform<?, ?, ?> getProducer(PValue value) {
- return getGraph(value.getPipeline()).getProducer(value);
+ if (value instanceof PCollection) {
+ return getGraph(value.getPipeline()).getProducer((PCollection<?>) value);
+ } else if (value instanceof PCollectionView) {
+ return getGraph(value.getPipeline()).getWriter((PCollectionView<?>) value);
+ }
+ throw new IllegalArgumentException(
+ String.format("Unexpected type of %s %s", PValue.class.getSimpleName(), value.getClass()));
}
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
index 8cdf323..4ce5342 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
@@ -26,9 +26,9 @@
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.DistributionData;
import org.apache.beam.runners.core.metrics.GaugeData;
-import org.apache.beam.runners.core.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.MetricUpdates;
import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
import org.apache.beam.sdk.metrics.DistributionResult;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
index 603e43e..4b909bc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
@@ -37,7 +37,7 @@
@Test
public void testCorrectOptionsAreReturned() {
assertEquals(
- ImmutableList.of(DirectOptions.class),
+ ImmutableList.of(DirectOptions.class, DirectTestOptions.class),
new Options().getPipelineOptions());
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 943d27c..d3f407a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -573,8 +573,8 @@
}
@Override
- public Coder<T> getDefaultOutputCoder() {
- return underlying.getDefaultOutputCoder();
+ public Coder<T> getOutputCoder() {
+ return underlying.getOutputCoder();
}
}
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 72b1bbc..cc9ce60 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -40,6 +40,7 @@
import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.io.GenerateSequence;
@@ -101,10 +102,13 @@
view = created.apply(View.<Integer>asIterable());
unbounded = p.apply(GenerateSequence.from(0));
+ p.replaceAll(runner.defaultTransformOverrides());
+
KeyedPValueTrackingVisitor keyedPValueTrackingVisitor = KeyedPValueTrackingVisitor.create();
p.traverseTopologically(keyedPValueTrackingVisitor);
BundleFactory bundleFactory = ImmutableListBundleFactory.create();
+ DirectGraphs.performDirectOverrides(p);
graph = DirectGraphs.getGraph(p);
context =
EvaluationContext.create(
@@ -116,7 +120,7 @@
createdProducer = graph.getProducer(created);
downstreamProducer = graph.getProducer(downstream);
- viewProducer = graph.getProducer(view);
+ viewProducer = graph.getWriter(view);
unboundedProducer = graph.getProducer(unbounded);
}
@@ -124,8 +128,11 @@
public void writeToViewWriterThenReadReads() {
PCollectionViewWriter<Integer, Iterable<Integer>> viewWriter =
context.createPCollectionViewWriter(
- PCollection.<Iterable<Integer>>createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED),
+ PCollection.createPrimitiveOutputInternal(
+ p,
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ IterableCoder.of(VarIntCoder.of())),
view);
BoundedWindow window = new TestBoundedWindow(new Instant(1024L));
BoundedWindow second = new TestBoundedWindow(new Instant(899999L));
@@ -160,7 +167,7 @@
StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
- DirectStepContext stepContext = fooContext.getOrCreateStepContext("s1", "s1");
+ DirectStepContext stepContext = fooContext.getStepContext("s1");
stepContext.stateInternals().state(StateNamespaces.global(), intBag).add(1);
context.handleResult(
@@ -177,7 +184,7 @@
StructuralKey.of("foo", StringUtf8Coder.of()));
assertThat(
secondFooContext
- .getOrCreateStepContext("s1", "s1")
+ .getStepContext("s1")
.stateInternals()
.state(StateNamespaces.global(), intBag)
.read(),
@@ -194,7 +201,7 @@
StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
fooContext
- .getOrCreateStepContext("s1", "s1")
+ .getStepContext("s1")
.stateInternals()
.state(StateNamespaces.global(), intBag)
.add(1);
@@ -205,7 +212,7 @@
assertThat(barContext, not(equalTo(fooContext)));
assertThat(
barContext
- .getOrCreateStepContext("s1", "s1")
+ .getStepContext("s1")
.stateInternals()
.state(StateNamespaces.global(), intBag)
.read(),
@@ -221,7 +228,7 @@
StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
fooContext
- .getOrCreateStepContext("s1", "s1")
+ .getStepContext("s1")
.stateInternals()
.state(StateNamespaces.global(), intBag)
.add(1);
@@ -230,7 +237,7 @@
context.getExecutionContext(downstreamProducer, myKey);
assertThat(
barContext
- .getOrCreateStepContext("s1", "s1")
+ .getStepContext("s1")
.stateInternals()
.state(StateNamespaces.global(), intBag)
.read(),
@@ -246,7 +253,7 @@
StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
CopyOnAccessInMemoryStateInternals<?> state =
- fooContext.getOrCreateStepContext("s1", "s1").stateInternals();
+ fooContext.getStepContext("s1").stateInternals();
BagState<Integer> bag = state.state(StateNamespaces.global(), intBag);
bag.add(1);
bag.add(2);
@@ -266,7 +273,7 @@
context.getExecutionContext(downstreamProducer, myKey);
CopyOnAccessInMemoryStateInternals<?> afterResultState =
- afterResultContext.getOrCreateStepContext("s1", "s1").stateInternals();
+ afterResultContext.getStepContext("s1").stateInternals();
assertThat(afterResultState.state(StateNamespaces.global(), intBag).read(), contains(1, 2, 4));
}
@@ -411,7 +418,7 @@
StepTransformResult.withoutHold(unboundedProducer).build());
assertThat(context.isDone(), is(false));
- for (AppliedPTransform<?, ?, ?> consumers : graph.getPrimitiveConsumers(created)) {
+ for (AppliedPTransform<?, ?, ?> consumers : graph.getPerElementConsumers(created)) {
context.handleResult(
committedBundle,
ImmutableList.<TimerData>of(),
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
index c0919b9..365b6c4 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
@@ -64,7 +64,9 @@
c.element()[0] = 'b';
}
}));
- consumer = DirectGraphs.getProducer(pcollection.apply(Count.<byte[]>globally()));
+ PCollection<Long> consumer = pcollection.apply(Count.<byte[]>globally());
+ DirectGraphs.performDirectOverrides(p);
+ this.consumer = DirectGraphs.getProducer(consumer);
}
@Test
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MultiStepCombineTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MultiStepCombineTest.java
new file mode 100644
index 0000000..0c11a8a
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MultiStepCombineTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.auto.value.AutoValue;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.VarInt;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link MultiStepCombine}.
+ */
+@RunWith(JUnit4.class)
+public class MultiStepCombineTest implements Serializable {
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+
+ private transient KvCoder<String, Long> combinedCoder =
+ KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of());
+
+ @Test
+ public void testMultiStepCombine() {
+ PCollection<KV<String, Long>> combined =
+ pipeline
+ .apply(
+ Create.of(
+ KV.of("foo", 1L),
+ KV.of("bar", 2L),
+ KV.of("bizzle", 3L),
+ KV.of("bar", 4L),
+ KV.of("bizzle", 11L)))
+ .apply(Combine.<String, Long, Long>perKey(new MultiStepCombineFn()));
+
+ PAssert.that(combined)
+ .containsInAnyOrder(KV.of("foo", 1L), KV.of("bar", 6L), KV.of("bizzle", 14L));
+ pipeline.run();
+ }
+
+ @Test
+ public void testMultiStepCombineWindowed() {
+ SlidingWindows windowFn = SlidingWindows.of(Duration.millis(6L)).every(Duration.millis(3L));
+ PCollection<KV<String, Long>> combined =
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("foo", 1L), new Instant(1L)),
+ TimestampedValue.of(KV.of("bar", 2L), new Instant(2L)),
+ TimestampedValue.of(KV.of("bizzle", 3L), new Instant(3L)),
+ TimestampedValue.of(KV.of("bar", 4L), new Instant(4L)),
+ TimestampedValue.of(KV.of("bizzle", 11L), new Instant(11L))))
+ .apply(Window.<KV<String, Long>>into(windowFn))
+ .apply(Combine.<String, Long, Long>perKey(new MultiStepCombineFn()));
+
+ PAssert.that("Windows should combine only elements in their windows", combined)
+ .inWindow(new IntervalWindow(new Instant(0L), Duration.millis(6L)))
+ .containsInAnyOrder(KV.of("foo", 1L), KV.of("bar", 6L), KV.of("bizzle", 3L));
+ PAssert.that("Elements should appear in all the windows they are assigned to", combined)
+ .inWindow(new IntervalWindow(new Instant(-3L), Duration.millis(6L)))
+ .containsInAnyOrder(KV.of("foo", 1L), KV.of("bar", 2L));
+ PAssert.that(combined)
+ .inWindow(new IntervalWindow(new Instant(6L), Duration.millis(6L)))
+ .containsInAnyOrder(KV.of("bizzle", 11L));
+ PAssert.that(combined)
+ .containsInAnyOrder(
+ KV.of("foo", 1L),
+ KV.of("foo", 1L),
+ KV.of("bar", 6L),
+ KV.of("bar", 2L),
+ KV.of("bar", 4L),
+ KV.of("bizzle", 11L),
+ KV.of("bizzle", 11L),
+ KV.of("bizzle", 3L),
+ KV.of("bizzle", 3L));
+ pipeline.run();
+ }
+
+ @Test
+ public void testMultiStepCombineTimestampCombiner() {
+ TimestampCombiner combiner = TimestampCombiner.LATEST;
+ combinedCoder = KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of());
+ PCollection<KV<String, Long>> combined =
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("foo", 4L), new Instant(1L)),
+ TimestampedValue.of(KV.of("foo", 1L), new Instant(4L)),
+ TimestampedValue.of(KV.of("bazzle", 4L), new Instant(4L)),
+ TimestampedValue.of(KV.of("foo", 12L), new Instant(12L))))
+ .apply(
+ Window.<KV<String, Long>>into(FixedWindows.of(Duration.millis(5L)))
+ .withTimestampCombiner(combiner))
+ .apply(Combine.<String, Long, Long>perKey(new MultiStepCombineFn()));
+ PCollection<KV<String, TimestampedValue<Long>>> reified =
+ combined.apply(
+ ParDo.of(
+ new DoFn<KV<String, Long>, KV<String, TimestampedValue<Long>>>() {
+ @ProcessElement
+ public void reifyTimestamp(ProcessContext context) {
+ context.output(
+ KV.of(
+ context.element().getKey(),
+ TimestampedValue.of(
+ context.element().getValue(), context.timestamp())));
+ }
+ }));
+
+ PAssert.that(reified)
+ .containsInAnyOrder(
+ KV.of("foo", TimestampedValue.of(5L, new Instant(4L))),
+ KV.of("bazzle", TimestampedValue.of(4L, new Instant(4L))),
+ KV.of("foo", TimestampedValue.of(12L, new Instant(12L))));
+ pipeline.run();
+ }
+
+ private static class MultiStepCombineFn extends CombineFn<Long, MultiStepAccumulator, Long> {
+ @Override
+ public Coder<MultiStepAccumulator> getAccumulatorCoder(
+ CoderRegistry registry, Coder<Long> inputCoder) throws CannotProvideCoderException {
+ return new MultiStepAccumulatorCoder();
+ }
+
+ @Override
+ public MultiStepAccumulator createAccumulator() {
+ return MultiStepAccumulator.of(0L, false);
+ }
+
+ @Override
+ public MultiStepAccumulator addInput(MultiStepAccumulator accumulator, Long input) {
+ return MultiStepAccumulator.of(accumulator.getValue() + input, accumulator.isDeserialized());
+ }
+
+ @Override
+ public MultiStepAccumulator mergeAccumulators(Iterable<MultiStepAccumulator> accumulators) {
+ MultiStepAccumulator result = MultiStepAccumulator.of(0L, false);
+ for (MultiStepAccumulator accumulator : accumulators) {
+ result = result.merge(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public Long extractOutput(MultiStepAccumulator accumulator) {
+ assertThat(
+ "Accumulators should have been serialized and deserialized within the Pipeline",
+ accumulator.isDeserialized(),
+ is(true));
+ return accumulator.getValue();
+ }
+ }
+
+ @AutoValue
+ abstract static class MultiStepAccumulator {
+ private static MultiStepAccumulator of(long value, boolean deserialized) {
+ return new AutoValue_MultiStepCombineTest_MultiStepAccumulator(value, deserialized);
+ }
+
+ MultiStepAccumulator merge(MultiStepAccumulator other) {
+ return MultiStepAccumulator.of(
+ this.getValue() + other.getValue(), this.isDeserialized() || other.isDeserialized());
+ }
+
+ abstract long getValue();
+
+ abstract boolean isDeserialized();
+ }
+
+ private static class MultiStepAccumulatorCoder extends CustomCoder<MultiStepAccumulator> {
+ @Override
+ public void encode(MultiStepAccumulator value, OutputStream outStream)
+ throws CoderException, IOException {
+ VarInt.encode(value.getValue(), outStream);
+ }
+
+ @Override
+ public MultiStepAccumulator decode(InputStream inStream) throws CoderException, IOException {
+ return MultiStepAccumulator.of(VarInt.decodeLong(inStream), true);
+ }
+ }
+}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 286e44d..7912538 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@
when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
ParDoEvaluator<Integer> evaluator =
- createEvaluator(singletonView, fn, output);
+ createEvaluator(singletonView, fn, inputPc, output);
IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,6 +132,7 @@
private ParDoEvaluator<Integer> createEvaluator(
PCollectionView<Integer> singletonView,
RecorderFn fn,
+ PCollection<Integer> input,
PCollection<Integer> output) {
when(
evaluationContext.createSideInputReader(
@@ -140,8 +141,8 @@
DirectExecutionContext executionContext = mock(DirectExecutionContext.class);
DirectStepContext stepContext = mock(DirectStepContext.class);
when(
- executionContext.getOrCreateStepContext(
- Mockito.any(String.class), Mockito.any(String.class)))
+ executionContext.getStepContext(
+ Mockito.any(String.class)))
.thenReturn(stepContext);
when(stepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
when(
@@ -149,6 +150,7 @@
Mockito.any(AppliedPTransform.class), Mockito.any(StructuralKey.class)))
.thenReturn(executionContext);
+ DirectGraphs.performDirectOverrides(p);
@SuppressWarnings("unchecked")
AppliedPTransform<PCollection<Integer>, ?, ?> transform =
(AppliedPTransform<PCollection<Integer>, ?, ?>) DirectGraphs.getProducer(output);
@@ -156,8 +158,7 @@
evaluationContext,
stepContext,
transform,
- ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
- .getWindowingStrategy(),
+ input.getWindowingStrategy(),
fn,
null /* key */,
ImmutableList.<PCollectionView<?>>of(singletonView),
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index eb54d5c..fe0b743 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -41,6 +41,7 @@
import org.apache.beam.runners.core.StateNamespaces;
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTags;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -52,7 +53,6 @@
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
@@ -128,16 +128,17 @@
input
.apply(
new ParDoMultiOverrideFactory.GbkThenStatefulParDo<>(
- ParDo.of(
- new DoFn<KV<String, Integer>, Integer>() {
- @StateId(stateId)
- private final StateSpec<ValueState<String>> spec =
- StateSpecs.value(StringUtf8Coder.of());
+ new DoFn<KV<String, Integer>, Integer>() {
+ @StateId(stateId)
+ private final StateSpec<ValueState<String>> spec =
+ StateSpecs.value(StringUtf8Coder.of());
- @ProcessElement
- public void process(ProcessContext c) {}
- })
- .withOutputTags(mainOutput, TupleTagList.empty())))
+ @ProcessElement
+ public void process(ProcessContext c) {}
+ },
+ mainOutput,
+ TupleTagList.empty(),
+ Collections.<PCollectionView<?>>emptyList()))
.get(mainOutput)
.setCoder(VarIntCoder.of());
@@ -153,8 +154,7 @@
when(mockEvaluationContext.getExecutionContext(
eq(producingTransform), Mockito.<StructuralKey>any()))
.thenReturn(mockExecutionContext);
- when(mockExecutionContext.getOrCreateStepContext(anyString(), anyString()))
- .thenReturn(mockStepContext);
+ when(mockExecutionContext.getStepContext(anyString())).thenReturn(mockStepContext);
IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(9));
IntervalWindow secondWindow = new IntervalWindow(new Instant(10), new Instant(19));
@@ -241,18 +241,17 @@
mainInput
.apply(
new ParDoMultiOverrideFactory.GbkThenStatefulParDo<>(
- ParDo
- .of(
- new DoFn<KV<String, Integer>, Integer>() {
- @StateId(stateId)
- private final StateSpec<ValueState<String>> spec =
- StateSpecs.value(StringUtf8Coder.of());
+ new DoFn<KV<String, Integer>, Integer>() {
+ @StateId(stateId)
+ private final StateSpec<ValueState<String>> spec =
+ StateSpecs.value(StringUtf8Coder.of());
- @ProcessElement
- public void process(ProcessContext c) {}
- })
- .withSideInputs(sideInput)
- .withOutputTags(mainOutput, TupleTagList.empty())))
+ @ProcessElement
+ public void process(ProcessContext c) {}
+ },
+ mainOutput,
+ TupleTagList.empty(),
+ Collections.<PCollectionView<?>>singletonList(sideInput)))
.get(mainOutput)
.setCoder(VarIntCoder.of());
@@ -269,8 +268,7 @@
when(mockEvaluationContext.getExecutionContext(
eq(producingTransform), Mockito.<StructuralKey>any()))
.thenReturn(mockExecutionContext);
- when(mockExecutionContext.getOrCreateStepContext(anyString(), anyString()))
- .thenReturn(mockStepContext);
+ when(mockExecutionContext.getStepContext(anyString())).thenReturn(mockStepContext);
when(mockEvaluationContext.createBundle(Matchers.<PCollection<Integer>>any()))
.thenReturn(mockUncommittedBundle);
when(mockStepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
@@ -287,11 +285,8 @@
// global window state merely by having the evaluator created. The cleanup logic does not
// depend on the window.
String key = "hello";
- WindowedValue<KV<String, Integer>> firstKv = WindowedValue.of(
- KV.of(key, 1),
- new Instant(3),
- firstWindow,
- PaneInfo.NO_FIRING);
+ WindowedValue<KV<String, Integer>> firstKv =
+ WindowedValue.of(KV.of(key, 1), new Instant(3), firstWindow, PaneInfo.NO_FIRING);
WindowedValue<KeyedWorkItem<String, KV<String, Integer>>> gbkOutputElement =
firstKv.withValue(
@@ -306,7 +301,8 @@
BUNDLE_FACTORY
.createBundle(
(PCollection<KeyedWorkItem<String, KV<String, Integer>>>)
- Iterables.getOnlyElement(producingTransform.getInputs().values()))
+ Iterables.getOnlyElement(
+ TransformInputs.nonAdditionalInputs(producingTransform)))
.add(gbkOutputElement)
.commit(Instant.now());
TransformEvaluator<KeyedWorkItem<String, KV<String, Integer>>> evaluator =
@@ -316,8 +312,7 @@
// This should push back every element as a KV<String, Iterable<Integer>>
// in the appropriate window. Since the keys are equal they are single-threaded
- TransformResult<KeyedWorkItem<String, KV<String, Integer>>> result =
- evaluator.finishBundle();
+ TransformResult<KeyedWorkItem<String, KV<String, Integer>>> result = evaluator.finishBundle();
List<Integer> pushedBackInts = new ArrayList<>();
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index 86412a0..b7f5a7c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -25,6 +25,8 @@
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.when;
+import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.ArrayList;
import java.util.Collection;
@@ -90,6 +92,7 @@
created = p.apply(Create.of("foo", "spam", "third"));
PCollection<KV<Integer, String>> downstream = created.apply(WithKeys.<Integer, String>of(3));
+ DirectGraphs.performDirectOverrides(p);
DirectGraph graph = DirectGraphs.getGraph(p);
createdProducer = graph.getProducer(created);
downstreamProducer = graph.getProducer(downstream);
@@ -414,8 +417,13 @@
? Collections.emptyList()
: result.getUnprocessedElements();
- CommittedBundle<?> unprocessedBundle =
- inputBundle == null ? null : inputBundle.withElements(unprocessedElements);
+ Optional<? extends CommittedBundle<?>> unprocessedBundle;
+ if (inputBundle == null || Iterables.isEmpty(unprocessedElements)) {
+ unprocessedBundle = Optional.absent();
+ } else {
+ unprocessedBundle =
+ Optional.<CommittedBundle<?>>of(inputBundle.withElements(unprocessedElements));
+ }
return CommittedResult.create(
result,
unprocessedBundle,
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index 2a01db5..12c10a7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -474,10 +474,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<T> getDefaultOutputCoder() {
+ public Coder<T> getOutputCoder() {
return coder;
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index 419698e..5bc48b7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -36,7 +36,6 @@
import org.apache.beam.sdk.transforms.WithKeys;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
import org.joda.time.Instant;
import org.junit.Rule;
@@ -66,12 +65,13 @@
.setCoder(KvCoder.of(VoidCoder.of(), StringUtf8Coder.of()))
.apply(GroupByKey.<Void, String>create())
.apply(Values.<Iterable<String>>create());
- PCollectionView<Iterable<String>> view =
- concat.apply(new ViewOverrideFactory.WriteView<>(createView));
+ PCollection<Iterable<String>> view =
+ concat.apply(
+ new ViewOverrideFactory.WriteView<String, Iterable<String>>(createView.getView()));
EvaluationContext context = mock(EvaluationContext.class);
TestViewWriter<String, Iterable<String>> viewWriter = new TestViewWriter<>();
- when(context.createPCollectionViewWriter(concat, view)).thenReturn(viewWriter);
+ when(context.createPCollectionViewWriter(concat, createView.getView())).thenReturn(viewWriter);
CommittedBundle<String> inputBundle = bundleFactory.createBundle(input).commit(Instant.now());
AppliedPTransform<?, ?, ?> producer = DirectGraphs.getProducer(view);
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index 024e15c..94d8d70 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -23,22 +23,20 @@
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
-import com.google.common.collect.ImmutableSet;
import java.io.Serializable;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
-import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
@@ -59,56 +57,15 @@
new ViewOverrideFactory<>();
@Test
- public void replacementSucceeds() {
- PCollection<Integer> ints = p.apply("CreateContents", Create.of(1, 2, 3));
- final PCollectionView<List<Integer>> view =
- PCollectionViews.listView(ints, WindowingStrategy.globalDefault(), ints.getCoder());
- PTransformReplacement<PCollection<Integer>, PCollectionView<List<Integer>>>
- replacementTransform =
- factory.getReplacementTransform(
- AppliedPTransform
- .<PCollection<Integer>, PCollectionView<List<Integer>>,
- CreatePCollectionView<Integer, List<Integer>>>
- of(
- "foo",
- ints.expand(),
- view.expand(),
- CreatePCollectionView.<Integer, List<Integer>>of(view),
- p));
- PCollectionView<List<Integer>> afterReplacement =
- ints.apply(replacementTransform.getTransform());
- assertThat(
- "The CreatePCollectionView replacement should return the same View",
- afterReplacement,
- equalTo(view));
-
- PCollection<Set<Integer>> outputViewContents =
- p.apply("CreateSingleton", Create.of(0))
- .apply(
- "OutputContents",
- ParDo.of(
- new DoFn<Integer, Set<Integer>>() {
- @ProcessElement
- public void outputSideInput(ProcessContext context) {
- context.output(ImmutableSet.copyOf(context.sideInput(view)));
- }
- })
- .withSideInputs(view));
- PAssert.thatSingleton(outputViewContents).isEqualTo(ImmutableSet.of(1, 2, 3));
-
- p.run();
- }
-
- @Test
public void replacementGetViewReturnsOriginal() {
final PCollection<Integer> ints = p.apply("CreateContents", Create.of(1, 2, 3));
final PCollectionView<List<Integer>> view =
PCollectionViews.listView(ints, WindowingStrategy.globalDefault(), ints.getCoder());
- PTransformReplacement<PCollection<Integer>, PCollectionView<List<Integer>>> replacement =
+ PTransformReplacement<PCollection<Integer>, PCollection<Integer>> replacement =
factory.getReplacementTransform(
AppliedPTransform
- .<PCollection<Integer>, PCollectionView<List<Integer>>,
- CreatePCollectionView<Integer, List<Integer>>>
+ .<PCollection<Integer>, PCollection<Integer>,
+ PTransform<PCollection<Integer>, PCollection<Integer>>>
of(
"foo",
ints.expand(),
@@ -126,8 +83,19 @@
"There should only be one WriteView primitive in the graph",
writeViewVisited.getAndSet(true),
is(false));
- PCollectionView replacementView = ((WriteView) node.getTransform()).getView();
- assertThat(replacementView, Matchers.<PCollectionView>theInstance(view));
+ PCollectionView<?> replacementView = ((WriteView) node.getTransform()).getView();
+
+ // replacementView.getPCollection() is null, but that is not a requirement
+ // so not asserted one way or the other
+ assertThat(
+ replacementView.getTagInternal(),
+ equalTo(view.getTagInternal()));
+ assertThat(
+ replacementView.getViewFn(),
+ Matchers.<ViewFn<?, ?>>equalTo(view.getViewFn()));
+ assertThat(
+ replacementView.getWindowMappingFn(),
+ Matchers.<WindowMappingFn<?>>equalTo(view.getWindowMappingFn()));
assertThat(node.getInputs().entrySet(), hasSize(1));
}
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
index b667346..1d8aac1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
@@ -59,6 +59,7 @@
public void setup() {
PCollection<Integer> created = p.apply(Create.of(1, 2, 3));
PCollection<Integer> summed = created.apply(Sum.integersGlobally());
+ DirectGraphs.performDirectOverrides(p);
DirectGraph graph = DirectGraphs.getGraph(p);
create = graph.getProducer(created);
sum = graph.getProducer(summed);
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
index 9528ac9..e3f6215 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -24,6 +24,7 @@
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
+import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -121,6 +122,7 @@
flattened = preFlatten.apply("flattened", Flatten.<Integer>pCollections());
clock = MockClock.fromInstant(new Instant(1000));
+ DirectGraphs.performDirectOverrides(p);
graph = DirectGraphs.getGraph(p);
manager = WatermarkManager.create(clock, graph);
@@ -317,7 +319,7 @@
TimerUpdate.empty(),
CommittedResult.create(
StepTransformResult.withoutHold(graph.getProducer(created)).build(),
- root.withElements(Collections.<WindowedValue<Void>>emptyList()),
+ Optional.<CommittedBundle<?>>absent(),
Collections.singleton(createBundle),
EnumSet.allOf(OutputType.class)),
BoundedWindow.TIMESTAMP_MAX_VALUE);
@@ -331,7 +333,7 @@
TimerUpdate.empty(),
CommittedResult.create(
StepTransformResult.withoutHold(theFlatten).build(),
- createBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Optional.<CommittedBundle<?>>absent(),
Collections.<CommittedBundle<?>>emptyList(),
EnumSet.allOf(OutputType.class)),
BoundedWindow.TIMESTAMP_MAX_VALUE);
@@ -344,7 +346,7 @@
TimerUpdate.empty(),
CommittedResult.create(
StepTransformResult.withoutHold(theFlatten).build(),
- createBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Optional.<CommittedBundle<?>>absent(),
Collections.<CommittedBundle<?>>emptyList(),
EnumSet.allOf(OutputType.class)),
BoundedWindow.TIMESTAMP_MAX_VALUE);
@@ -1500,9 +1502,15 @@
AppliedPTransform<?, ?, ?> transform,
@Nullable CommittedBundle<?> unprocessedBundle,
Iterable<? extends CommittedBundle<?>> bundles) {
+ Optional<? extends CommittedBundle<?>> unprocessedElements;
+ if (unprocessedBundle == null || Iterables.isEmpty(unprocessedBundle.getElements())) {
+ unprocessedElements = Optional.absent();
+ } else {
+ unprocessedElements = Optional.of(unprocessedBundle);
+ }
return CommittedResult.create(
StepTransformResult.withoutHold(transform).build(),
- unprocessedBundle,
+ unprocessedElements,
bundles,
Iterables.isEmpty(bundles)
? EnumSet.noneOf(OutputType.class)
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
index a91bab5..96fdfab 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
@@ -35,6 +35,7 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
@@ -304,6 +305,15 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is not compatible with any other %s.",
+ EvaluatorTestWindowFn.class.getSimpleName(), WindowFn.class.getSimpleName()));
+ }
+
+ @Override
public Coder<BoundedWindow> windowCoder() {
@SuppressWarnings({"unchecked", "rawtypes"}) Coder coder =
(Coder) GlobalWindow.Coder.INSTANCE;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index 5c4fea1..79a23cc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -30,21 +30,23 @@
import java.io.File;
import java.io.FileReader;
import java.io.Reader;
+import java.io.Serializable;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import javax.annotation.Nullable;
import org.apache.beam.runners.direct.WriteWithShardingFactory.CalculateShardsFn;
import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.DefaultFilenamePolicy;
+import org.apache.beam.sdk.io.DynamicFileDestinations;
import org.apache.beam.sdk.io.FileBasedSink;
-import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.LocalResources;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
@@ -53,11 +55,13 @@
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
-import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -71,11 +75,18 @@
* Tests for {@link WriteWithShardingFactory}.
*/
@RunWith(JUnit4.class)
-public class WriteWithShardingFactoryTest {
+public class WriteWithShardingFactoryTest implements Serializable {
+
private static final int INPUT_SIZE = 10000;
- @Rule public TemporaryFolder tmp = new TemporaryFolder();
- private WriteWithShardingFactory<Object> factory = new WriteWithShardingFactory<>();
- @Rule public final TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ @Rule public transient TemporaryFolder tmp = new TemporaryFolder();
+
+ private transient WriteWithShardingFactory<Object, Void> factory =
+ new WriteWithShardingFactory<>();
+
+ @Rule
+ public final transient TestPipeline p =
+ TestPipeline.create().enableAbandonedNodeEnforcement(false);
@Test
public void dynamicallyReshardedWrite() throws Exception {
@@ -129,21 +140,26 @@
@Test
public void withNoShardingSpecifiedReturnsNewTransform() {
ResourceId outputDirectory = LocalResources.fromString("/foo", true /* isDirectory */);
- FilenamePolicy policy = DefaultFilenamePolicy.constructUsingStandardParameters(
- StaticValueProvider.of(outputDirectory), DefaultFilenamePolicy.DEFAULT_SHARD_TEMPLATE, "");
- WriteFiles<Object> original = WriteFiles.to(
- new FileBasedSink<Object>(StaticValueProvider.of(outputDirectory), policy) {
- @Override
- public WriteOperation<Object> createWriteOperation() {
- throw new IllegalArgumentException("Should not be used");
- }
- });
+
+ PTransform<PCollection<Object>, WriteFilesResult<Void>> original =
+ WriteFiles.to(
+ new FileBasedSink<Object, Void, Object>(
+ StaticValueProvider.of(outputDirectory),
+ DynamicFileDestinations.constant(new FakeFilenamePolicy())) {
+ @Override
+ public WriteOperation<Void, Object> createWriteOperation() {
+ throw new IllegalArgumentException("Should not be used");
+ }
+ });
@SuppressWarnings("unchecked")
PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of()));
- AppliedPTransform<PCollection<Object>, PDone, WriteFiles<Object>> originalApplication =
- AppliedPTransform.of(
- "write", objs.expand(), Collections.<TupleTag<?>, PValue>emptyMap(), original, p);
+ AppliedPTransform<
+ PCollection<Object>, WriteFilesResult<Void>,
+ PTransform<PCollection<Object>, WriteFilesResult<Void>>>
+ originalApplication =
+ AppliedPTransform.of(
+ "write", objs.expand(), Collections.<TupleTag<?>, PValue>emptyMap(), original, p);
assertThat(
factory.getReplacementTransform(originalApplication).getTransform(),
@@ -223,4 +239,25 @@
List<Integer> shards = fnTester.processBundle((long) count);
assertThat(shards, containsInAnyOrder(13));
}
+
+ private static class FakeFilenamePolicy extends FileBasedSink.FilenamePolicy {
+ @Override
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ FileBasedSink.OutputFileHints outputFileHints) {
+ throw new IllegalArgumentException("Should not be used");
+ }
+
+ @Nullable
+ @Override
+ public ResourceId unwindowedFilename(
+ int shardNumber,
+ int numShards,
+ FileBasedSink.OutputFileHints outputFileHints) {
+ throw new IllegalArgumentException("Should not be used");
+ }
+ }
}
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index ff73ec1..7840c32 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -31,7 +31,7 @@
<packaging>jar</packaging>
<properties>
- <flink.version>1.2.1</flink.version>
+ <flink.version>1.3.0</flink.version>
</properties>
<profiles>
@@ -57,6 +57,7 @@
<groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
<excludedGroups>
org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
+ org.apache.beam.sdk.testing.LargeKeys$Above100MB,
org.apache.beam.sdk.testing.UsesSplittableParDo,
org.apache.beam.sdk.testing.UsesCommittedMetrics,
org.apache.beam.sdk.testing.UsesTestStream
@@ -89,8 +90,7 @@
<groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
<excludedGroups>
org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
- org.apache.beam.sdk.testing.UsesSetState,
- org.apache.beam.sdk.testing.UsesMapState,
+ org.apache.beam.sdk.testing.LargeKeys$Above100MB,
org.apache.beam.sdk.testing.UsesCommittedMetrics,
org.apache.beam.sdk.testing.UsesTestStream,
org.apache.beam.sdk.testing.UsesSplittableParDo
@@ -256,16 +256,6 @@
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -376,9 +366,17 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
new file mode 100644
index 0000000..ceecc1f
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink;
+
+import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Flink streaming overrides for various view (side input) transforms. */
+class CreateStreamingFlinkView<ElemT, ViewT>
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
+
+ public static final String CREATE_STREAMING_FLINK_VIEW_URN =
+ "beam:transform:flink:create-streaming-flink-view:v1";
+
+ public CreateStreamingFlinkView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ @Override
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ input
+ .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
+ .apply(CreateFlinkPCollectionView.<ElemT, ViewT>of(view));
+ return input;
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link CreateStreamingFlinkView}. This combiner requires that the input
+ * {@link PCollection} fits in memory. For a large {@link PCollection} this is expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<T>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
+
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateFlinkPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollection<List<ElemT>>> {
+ private PCollectionView<ViewT> view;
+
+ private CreateFlinkPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateFlinkPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateFlinkPCollectionView<>(view);
+ }
+
+ @Override
+ public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+ }
+
+ public static class Factory<ElemT, ViewT>
+ implements PTransformOverrideFactory<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
+ public Factory() {}
+
+ @Override
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
+ AppliedPTransform<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
+ transform) {
+ return PTransformReplacement.of(
+ (PCollection<ElemT>) Iterables.getOnlyElement(transform.getInputs().values()),
+ new CreateStreamingFlinkView<ElemT, ViewT>(transform.getTransform().getView()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
index 854b674..d22a5da 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.flink;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.TransformHierarchy;
@@ -112,7 +113,7 @@
BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator;
// create the applied PTransform on the batchContext
- batchContext.setCurrentTransform(node.toAppliedPTransform());
+ batchContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
typedTranslator.translateNode(typedTransform, batchContext);
}
@@ -127,7 +128,7 @@
* Returns a translator for the given node, if it is possible, otherwise null.
*/
private static BatchTransformTranslator<?> getTranslator(TransformHierarchy.Node node) {
- PTransform<?, ?> transform = node.getTransform();
+ @Nullable PTransform<?, ?> transform = node.getTransform();
// Root of the graph is null
if (transform == null) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 0439119..6e70198 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,6 +20,7 @@
import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -143,7 +144,7 @@
@SuppressWarnings("unchecked")
<T extends PValue> T getInput(PTransform<T, ?> transform) {
- return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+ return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
}
Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java
index 7765a00..d2a2016 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java
@@ -84,6 +84,8 @@
this.flinkBatchEnv = null;
this.flinkStreamEnv = null;
+ pipeline.replaceAll(FlinkTransformOverrides.getDefaultOverrides(options.isStreaming()));
+
PipelineTranslationOptimizer optimizer =
new PipelineTranslationOptimizer(TranslationMode.BATCH, options);
@@ -227,7 +229,9 @@
if (checkpointInterval < 1) {
throw new IllegalArgumentException("The checkpoint interval must be positive");
}
- flinkStreamEnv.enableCheckpointing(checkpointInterval);
+ flinkStreamEnv.enableCheckpointing(checkpointInterval, options.getCheckpointingMode());
+ flinkStreamEnv.getCheckpointConfig().setCheckpointTimeout(
+ options.getCheckpointTimeoutMillis());
boolean externalizedCheckpoint = options.isExternalizedCheckpointsEnabled();
boolean retainOnCancellation = options.getRetainExternalizedCheckpointsOnCancellation();
if (externalizedCheckpoint) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
index 764fa5f..2432394 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
@@ -26,6 +26,7 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.streaming.api.CheckpointingMode;
/**
* Options which can be used to configure a Flink PipelineRunner.
@@ -70,6 +71,16 @@
Long getCheckpointingInterval();
void setCheckpointingInterval(Long interval);
+ @Description("The checkpointing mode that defines consistency guarantee.")
+ @Default.Enum("AT_LEAST_ONCE")
+ CheckpointingMode getCheckpointingMode();
+ void setCheckpointingMode(CheckpointingMode mode);
+
+ @Description("The maximum time that a checkpoint may take before being discarded.")
+ @Default.Long(20 * 60 * 1000)
+ Long getCheckpointTimeoutMillis();
+ void setCheckpointTimeoutMillis(Long checkpointTimeoutMillis);
+
@Description("Sets the number of times that failed tasks are re-executed. "
+ "A value of zero effectively disables fault tolerance. A value of -1 indicates "
+ "that the system default value (as defined in the configuration) should be used.")
@@ -116,4 +127,15 @@
@Default.Boolean(false)
Boolean getRetainExternalizedCheckpointsOnCancellation();
void setRetainExternalizedCheckpointsOnCancellation(Boolean retainOnCancellation);
+
+ @Description("The maximum number of elements in a bundle.")
+ @Default.Long(1000)
+ Long getMaxBundleSize();
+ void setMaxBundleSize(Long size);
+
+ @Description("The maximum time to wait before finalising a bundle (in milliseconds).")
+ @Default.Long(1000)
+ Long getMaxBundleTimeMills();
+ void setMaxBundleTimeMills(Long time);
+
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
index 80ef7bb..ca12615 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
@@ -38,7 +38,6 @@
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.PValue;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.client.program.DetachedEnvironment;
import org.slf4j.Logger;
@@ -199,10 +198,7 @@
// have just recorded the full names during apply time.
if (!ptransformViewsWithNonDeterministicKeyCoders.isEmpty()) {
final SortedSet<String> ptransformViewNamesWithNonDeterministicKeyCoders = new TreeSet<>();
- pipeline.traverseTopologically(new Pipeline.PipelineVisitor() {
- @Override
- public void visitValue(PValue value, TransformHierarchy.Node producer) {
- }
+ pipeline.traverseTopologically(new Pipeline.PipelineVisitor.Defaults() {
@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
@@ -218,10 +214,6 @@
}
return CompositeBehavior.ENTER_TRANSFORM;
}
-
- @Override
- public void leaveCompositeTransform(TransformHierarchy.Node node) {
- }
});
LOG.warn("Unable to use indexed implementation for View.AsMap and View.AsMultimap for {} "
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
index 35d1bcd..f733e2e 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
@@ -17,26 +17,18 @@
*/
package org.apache.beam.runners.flink;
-import com.google.common.collect.ImmutableList;
-import java.util.List;
import java.util.Map;
-import org.apache.beam.runners.core.SplittableParDo;
-import org.apache.beam.runners.core.construction.PTransformMatchers;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
-import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.runners.core.construction.SplittableParDo;
import org.apache.beam.runners.core.construction.UnconsumedReads;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.util.InstanceBuilder;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PValue;
@@ -72,50 +64,8 @@
@Override
public void translate(Pipeline pipeline) {
- List<PTransformOverride> transformOverrides =
- ImmutableList.<PTransformOverride>builder()
- .add(
- PTransformOverride.of(
- PTransformMatchers.splittableParDoMulti(),
- new SplittableParDoOverrideFactory()))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsIterable.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsList.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsList.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMap.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsMap.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMultimap.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsMultimap.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsSingleton.class, flinkRunner)))
- // this has to be last since the ViewAsSingleton override
- // can expand to a Combine.GloballyAsSingletonView
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingCombineGloballyAsSingletonView.class,
- flinkRunner)))
- .build();
-
// Ensure all outputs of all reads are consumed.
UnconsumedReads.ensureAllReadsConsumed(pipeline);
- pipeline.replaceAll(transformOverrides);
super.translate(pipeline);
}
@@ -183,7 +133,7 @@
StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
// create the applied PTransform on the streamingContext
- streamingContext.setCurrentTransform(node.toAppliedPTransform());
+ streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
typedTranslator.translateNode(typedTransform, streamingContext);
}
@@ -198,7 +148,7 @@
@SuppressWarnings("unchecked")
StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
- streamingContext.setCurrentTransform(node.toAppliedPTransform());
+ streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
return typedTranslator.canTranslate(typedTransform, streamingContext);
}
@@ -223,35 +173,6 @@
}
}
- private static class ReflectiveOneToOneOverrideFactory<
- InputT, OutputT, TransformT extends PTransform<PCollection<InputT>, PCollection<OutputT>>>
- extends SingleInputOutputOverrideFactory<
- PCollection<InputT>, PCollection<OutputT>, TransformT> {
- private final Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement;
- private final FlinkRunner runner;
-
- private ReflectiveOneToOneOverrideFactory(
- Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement,
- FlinkRunner runner) {
- this.replacement = replacement;
- this.runner = runner;
- }
-
- @Override
- public PTransformReplacement<PCollection<InputT>, PCollection<OutputT>> getReplacementTransform(
- AppliedPTransform<PCollection<InputT>, PCollection<OutputT>, TransformT> transform) {
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- InstanceBuilder.ofType(replacement)
- .withArg(FlinkRunner.class, runner)
- .withArg(
- (Class<PTransform<PCollection<InputT>, PCollection<OutputT>>>)
- transform.getTransform().getClass(),
- transform.getTransform())
- .build());
- }
- }
-
/**
* A {@link PTransformOverrideFactory} that overrides a <a
* href="https://s.apache.org/splittable-do-fn">Splittable DoFn</a> with {@link SplittableParDo}.
@@ -267,7 +188,7 @@
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new SplittableParDo<>(transform.getTransform()));
+ (SplittableParDo<InputT, OutputT, ?>) SplittableParDo.forAppliedParDo(transform));
}
@Override
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 9a93205..cec01f8 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -18,9 +18,10 @@
package org.apache.beam.runners.flink;
-import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.runners.core.construction.SplittableParDo.SPLITTABLE_PROCESS_URN;
-import com.google.common.collect.Lists;
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -29,11 +30,13 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.beam.runners.core.ElementAndRestriction;
+import javax.annotation.Nullable;
import org.apache.beam.runners.core.KeyedWorkItem;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.SystemReduceFn;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar;
import org.apache.beam.runners.flink.translation.functions.FlinkAssignWindows;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.runners.flink.translation.wrappers.streaming.DoFnOperator;
@@ -70,7 +73,9 @@
import org.apache.beam.sdk.util.AppliedCombineFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
@@ -84,16 +89,15 @@
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.datastream.SplitStream;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
import org.apache.flink.streaming.api.transformations.TwoInputTransformation;
import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
/**
* This class contains all the mappings between Beam and Flink
@@ -107,37 +111,38 @@
// Transform Translator Registry
// --------------------------------------------------------------------------------------------
+ /**
+ * A map from a Transform URN to the translator.
+ */
@SuppressWarnings("rawtypes")
- private static final Map<
- Class<? extends PTransform>,
- FlinkStreamingPipelineTranslator.StreamTransformTranslator> TRANSLATORS = new HashMap<>();
+ private static final Map<String, FlinkStreamingPipelineTranslator.StreamTransformTranslator>
+ TRANSLATORS = new HashMap<>();
// here you can find all the available translators.
static {
- TRANSLATORS.put(Read.Bounded.class, new BoundedReadSourceTranslator());
- TRANSLATORS.put(Read.Unbounded.class, new UnboundedReadSourceTranslator());
+ TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslator());
- TRANSLATORS.put(ParDo.MultiOutput.class, new ParDoStreamingTranslator());
+ TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoStreamingTranslator());
TRANSLATORS.put(
- SplittableParDo.ProcessElements.class, new SplittableProcessElementsStreamingTranslator());
- TRANSLATORS.put(
- SplittableParDo.GBKIntoKeyedWorkItems.class, new GBKIntoKeyedWorkItemsTranslator());
+ SPLITTABLE_PROCESS_URN, new SplittableProcessElementsStreamingTranslator());
+ TRANSLATORS.put(SplittableParDo.SPLITTABLE_GBKIKWI_URN, new GBKIntoKeyedWorkItemsTranslator());
-
- TRANSLATORS.put(Window.Assign.class, new WindowAssignTranslator());
- TRANSLATORS.put(Flatten.PCollections.class, new FlattenPCollectionTranslator());
+ TRANSLATORS.put(PTransformTranslation.WINDOW_TRANSFORM_URN, new WindowAssignTranslator());
TRANSLATORS.put(
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView.class,
+ PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslator());
+ TRANSLATORS.put(
+ CreateStreamingFlinkView.CREATE_STREAMING_FLINK_VIEW_URN,
new CreateViewStreamingTranslator());
- TRANSLATORS.put(Reshuffle.class, new ReshuffleTranslatorStreaming());
- TRANSLATORS.put(GroupByKey.class, new GroupByKeyTranslator());
- TRANSLATORS.put(Combine.PerKey.class, new CombinePerKeyTranslator());
+ TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorStreaming());
+ TRANSLATORS.put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslator());
+ TRANSLATORS.put(PTransformTranslation.COMBINE_TRANSFORM_URN, new CombinePerKeyTranslator());
}
public static FlinkStreamingPipelineTranslator.StreamTransformTranslator<?> getTranslator(
PTransform<?, ?> transform) {
- return TRANSLATORS.get(transform.getClass());
+ @Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform);
+ return urn == null ? null : TRANSLATORS.get(urn);
}
// --------------------------------------------------------------------------------------------
@@ -180,9 +185,9 @@
if (transform.getSource().requiresDeduping()) {
source = nonDedupSource.keyBy(
new ValueWithRecordIdKeySelector<T>())
- .transform("debuping", outputTypeInfo, new DedupingOperator<T>());
+ .transform("deduping", outputTypeInfo, new DedupingOperator<T>());
} else {
- source = nonDedupSource.flatMap(new StripIdsMap<T>());
+ source = nonDedupSource.flatMap(new StripIdsMap<T>()).returns(outputTypeInfo);
}
} catch (Exception e) {
throw new RuntimeException(
@@ -219,6 +224,26 @@
}
+ private static class ReadSourceTranslator<T>
+ extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
+ PTransform<PBegin, PCollection<T>>> {
+
+ private final BoundedReadSourceTranslator<T> boundedTranslator =
+ new BoundedReadSourceTranslator<>();
+ private final UnboundedReadSourceTranslator<T> unboundedTranslator =
+ new UnboundedReadSourceTranslator<>();
+
+ @Override
+ void translateNode(
+ PTransform<PBegin, PCollection<T>> transform, FlinkStreamingTranslationContext context) {
+ if (context.getOutput(transform).isBounded().equals(PCollection.IsBounded.BOUNDED)) {
+ boundedTranslator.translateNode((Read.Bounded<T>) transform, context);
+ } else {
+ unboundedTranslator.translateNode((Read.Unbounded<T>) transform, context);
+ }
+ }
+ }
+
private static class BoundedReadSourceTranslator<T>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<Read.Bounded<T>> {
@@ -329,12 +354,13 @@
}
/**
- * Helper for translating {@link ParDo.MultiOutput} and {@link SplittableParDo.ProcessElements}.
+ * Helper for translating {@link ParDo.MultiOutput} and {@link
+ * SplittableParDoViaKeyedWorkItems.ProcessElements}.
*/
static class ParDoTranslationHelper {
interface DoFnOperatorFactory<InputT, OutputT> {
- DoFnOperator<InputT, OutputT, RawUnionValue> createDoFnOperator(
+ DoFnOperator<InputT, OutputT> createDoFnOperator(
DoFn<InputT, OutputT> doFn,
String stepName,
List<PCollectionView<?>> sideInputs,
@@ -342,7 +368,9 @@
List<TupleTag<?>> additionalOutputTags,
FlinkStreamingTranslationContext context,
WindowingStrategy<?, ?> windowingStrategy,
- Map<TupleTag<?>, Integer> tagsToLabels,
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders,
+ Map<TupleTag<?>, Integer> tagsToIds,
Coder<WindowedValue<InputT>> inputCoder,
Coder keyCoder,
Map<Integer, PCollectionView<?>> transformedSideInputs);
@@ -351,7 +379,6 @@
static <InputT, OutputT> void translateParDo(
String transformName,
DoFn<InputT, OutputT> doFn,
- String stepName,
PCollection<InputT> input,
List<PCollectionView<?>> sideInputs,
Map<TupleTag<?>, PValue> outputs,
@@ -363,10 +390,32 @@
// we assume that the transformation does not change the windowing strategy.
WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- Map<TupleTag<?>, Integer> tagsToLabels =
- transformTupleTagsToLabels(mainOutputTag, outputs);
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags = Maps.newHashMap();
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders = Maps.newHashMap();
- SingleOutputStreamOperator<RawUnionValue> unionOutputStream;
+ // We associate output tags with ids, the Integer is easier to serialize than TupleTag.
+ // The return map of AppliedPTransform.getOutputs() is an ImmutableMap, its implementation is
+ // RegularImmutableMap, its entrySet order is the same with the order of insertion.
+ // So we can use the original AppliedPTransform.getOutputs() to produce deterministic ids.
+ Map<TupleTag<?>, Integer> tagsToIds = Maps.newHashMap();
+ int idCount = 0;
+ tagsToIds.put(mainOutputTag, idCount++);
+ for (Map.Entry<TupleTag<?>, PValue> entry : outputs.entrySet()) {
+ if (!tagsToOutputTags.containsKey(entry.getKey())) {
+ tagsToOutputTags.put(
+ entry.getKey(),
+ new OutputTag<>(
+ entry.getKey().getId(),
+ (TypeInformation) context.getTypeInfo((PCollection<?>) entry.getValue())
+ )
+ );
+ tagsToCoders.put(entry.getKey(),
+ (Coder) context.getCoder((PCollection<OutputT>) entry.getValue()));
+ tagsToIds.put(entry.getKey(), idCount++);
+ }
+ }
+
+ SingleOutputStreamOperator<WindowedValue<OutputT>> outputStream;
Coder<WindowedValue<InputT>> inputCoder = context.getCoder(input);
@@ -382,14 +431,18 @@
keyCoder = ((KvCoder) input.getCoder()).getKeyCoder();
inputDataStream = inputDataStream.keyBy(new KvToByteBufferKeySelector(keyCoder));
stateful = true;
- } else if (doFn instanceof SplittableParDo.ProcessFn) {
+ } else if (doFn instanceof SplittableParDoViaKeyedWorkItems.ProcessFn) {
// we know that it is keyed on String
keyCoder = StringUtf8Coder.of();
stateful = true;
}
+ CoderTypeInformation<WindowedValue<OutputT>> outputTypeInformation =
+ new CoderTypeInformation<>(
+ context.getCoder((PCollection<OutputT>) outputs.get(mainOutputTag)));
+
if (sideInputs.isEmpty()) {
- DoFnOperator<InputT, OutputT, RawUnionValue> doFnOperator =
+ DoFnOperator<InputT, OutputT> doFnOperator =
doFnOperatorFactory.createDoFnOperator(
doFn,
context.getCurrentTransform().getFullName(),
@@ -398,24 +451,21 @@
additionalOutputTags,
context,
windowingStrategy,
- tagsToLabels,
+ tagsToOutputTags,
+ tagsToCoders,
+ tagsToIds,
inputCoder,
keyCoder,
new HashMap<Integer, PCollectionView<?>>() /* side-input mapping */);
- UnionCoder outputUnionCoder = createUnionCoder(outputs);
-
- CoderTypeInformation<RawUnionValue> outputUnionTypeInformation =
- new CoderTypeInformation<>(outputUnionCoder);
-
- unionOutputStream = inputDataStream
- .transform(transformName, outputUnionTypeInformation, doFnOperator);
+ outputStream = inputDataStream
+ .transform(transformName, outputTypeInformation, doFnOperator);
} else {
Tuple2<Map<Integer, PCollectionView<?>>, DataStream<RawUnionValue>> transformedSideInputs =
transformSideInputs(sideInputs, context);
- DoFnOperator<InputT, OutputT, RawUnionValue> doFnOperator =
+ DoFnOperator<InputT, OutputT> doFnOperator =
doFnOperatorFactory.createDoFnOperator(
doFn,
context.getCurrentTransform().getFullName(),
@@ -424,16 +474,13 @@
additionalOutputTags,
context,
windowingStrategy,
- tagsToLabels,
+ tagsToOutputTags,
+ tagsToCoders,
+ tagsToIds,
inputCoder,
keyCoder,
transformedSideInputs.f0);
- UnionCoder outputUnionCoder = createUnionCoder(outputs);
-
- CoderTypeInformation<RawUnionValue> outputUnionTypeInformation =
- new CoderTypeInformation<>(outputUnionCoder);
-
if (stateful) {
// we have to manually contruct the two-input transform because we're not
// allowed to have only one input keyed, normally.
@@ -445,99 +492,52 @@
keyedStream.getTransformation(),
transformedSideInputs.f1.broadcast().getTransformation(),
transformName,
- (TwoInputStreamOperator) doFnOperator,
- outputUnionTypeInformation,
+ doFnOperator,
+ outputTypeInformation,
keyedStream.getParallelism());
rawFlinkTransform.setStateKeyType(keyedStream.getKeyType());
rawFlinkTransform.setStateKeySelectors(keyedStream.getKeySelector(), null);
- unionOutputStream = new SingleOutputStreamOperator(
- keyedStream.getExecutionEnvironment(),
- rawFlinkTransform) {}; // we have to cheat around the ctor being protected
+ outputStream = new SingleOutputStreamOperator(
+ keyedStream.getExecutionEnvironment(),
+ rawFlinkTransform) {
+ }; // we have to cheat around the ctor being protected
keyedStream.getExecutionEnvironment().addOperator(rawFlinkTransform);
} else {
- unionOutputStream = inputDataStream
+ outputStream = inputDataStream
.connect(transformedSideInputs.f1.broadcast())
- .transform(transformName, outputUnionTypeInformation, doFnOperator);
+ .transform(transformName, outputTypeInformation, doFnOperator);
}
}
- SplitStream<RawUnionValue> splitStream = unionOutputStream
- .split(new OutputSelector<RawUnionValue>() {
- @Override
- public Iterable<String> select(RawUnionValue value) {
- return Collections.singletonList(Integer.toString(value.getUnionTag()));
- }
- });
+ context.setOutputDataStream(outputs.get(mainOutputTag), outputStream);
- for (Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
- final int outputTag = tagsToLabels.get(output.getKey());
-
- TypeInformation outputTypeInfo = context.getTypeInfo((PCollection<?>) output.getValue());
-
- @SuppressWarnings("unchecked")
- DataStream unwrapped = splitStream.select(String.valueOf(outputTag))
- .flatMap(new FlatMapFunction<RawUnionValue, Object>() {
- @Override
- public void flatMap(RawUnionValue value, Collector<Object> out) throws Exception {
- out.collect(value.getValue());
- }
- }).returns(outputTypeInfo);
-
- context.setOutputDataStream(output.getValue(), unwrapped);
- }
- }
-
- private static Map<TupleTag<?>, Integer> transformTupleTagsToLabels(
- TupleTag<?> mainTag,
- Map<TupleTag<?>, PValue> allTaggedValues) {
-
- Map<TupleTag<?>, Integer> tagToLabelMap = Maps.newHashMap();
- int count = 0;
- tagToLabelMap.put(mainTag, count++);
- for (TupleTag<?> key : allTaggedValues.keySet()) {
- if (!tagToLabelMap.containsKey(key)) {
- tagToLabelMap.put(key, count++);
+ for (Map.Entry<TupleTag<?>, PValue> entry : outputs.entrySet()) {
+ if (!entry.getKey().equals(mainOutputTag)) {
+ context.setOutputDataStream(entry.getValue(),
+ outputStream.getSideOutput(tagsToOutputTags.get(entry.getKey())));
}
}
- return tagToLabelMap;
- }
-
- private static UnionCoder createUnionCoder(Map<TupleTag<?>, PValue> taggedCollections) {
- List<Coder<?>> outputCoders = Lists.newArrayList();
- for (PValue taggedColl : taggedCollections.values()) {
- checkArgument(
- taggedColl instanceof PCollection,
- "A Union Coder can only be created for a Collection of Tagged %s. Got %s",
- PCollection.class.getSimpleName(),
- taggedColl.getClass().getSimpleName());
- PCollection<?> coll = (PCollection<?>) taggedColl;
- WindowedValue.FullWindowedValueCoder<?> windowedValueCoder =
- WindowedValue.getFullCoder(
- coll.getCoder(),
- coll.getWindowingStrategy().getWindowFn().windowCoder());
- outputCoders.add(windowedValueCoder);
- }
- return UnionCoder.of(outputCoders);
}
}
private static class ParDoStreamingTranslator<InputT, OutputT>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
- ParDo.MultiOutput<InputT, OutputT>> {
+ PTransform<PCollection<InputT>, PCollectionTuple>> {
@Override
public void translateNode(
- ParDo.MultiOutput<InputT, OutputT> transform,
+ PTransform<PCollection<InputT>, PCollectionTuple> rawTransform,
FlinkStreamingTranslationContext context) {
+ ParDo.MultiOutput<InputT, OutputT> transform = (ParDo.MultiOutput) rawTransform;
+
ParDoTranslationHelper.translateParDo(
transform.getName(),
transform.getFn(),
- context.getCurrentTransform().getFullName(),
(PCollection<InputT>) context.getInput(transform),
transform.getSideInputs(),
context.getOutputs(transform),
@@ -546,7 +546,7 @@
context,
new ParDoTranslationHelper.DoFnOperatorFactory<InputT, OutputT>() {
@Override
- public DoFnOperator<InputT, OutputT, RawUnionValue> createDoFnOperator(
+ public DoFnOperator<InputT, OutputT> createDoFnOperator(
DoFn<InputT, OutputT> doFn,
String stepName,
List<PCollectionView<?>> sideInputs,
@@ -554,7 +554,9 @@
List<TupleTag<?>> additionalOutputTags,
FlinkStreamingTranslationContext context,
WindowingStrategy<?, ?> windowingStrategy,
- Map<TupleTag<?>, Integer> tagsToLabels,
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders,
+ Map<TupleTag<?>, Integer> tagsToIds,
Coder<WindowedValue<InputT>> inputCoder,
Coder keyCoder,
Map<Integer, PCollectionView<?>> transformedSideInputs) {
@@ -564,7 +566,8 @@
inputCoder,
mainOutputTag,
additionalOutputTags,
- new DoFnOperator.MultiOutputOutputManagerFactory(tagsToLabels),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(
+ mainOutputTag, tagsToOutputTags, tagsToCoders, tagsToIds),
windowingStrategy,
transformedSideInputs,
sideInputs,
@@ -578,55 +581,50 @@
private static class SplittableProcessElementsStreamingTranslator<
InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
- SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
+ SplittableParDoViaKeyedWorkItems.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
@Override
public void translateNode(
- SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
+ SplittableParDoViaKeyedWorkItems.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>
+ transform,
FlinkStreamingTranslationContext context) {
ParDoTranslationHelper.translateParDo(
transform.getName(),
transform.newProcessFn(transform.getFn()),
- context.getCurrentTransform().getFullName(),
- (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
- context.getInput(transform),
+ context.getInput(transform),
transform.getSideInputs(),
context.getOutputs(transform),
transform.getMainOutputTag(),
transform.getAdditionalOutputTags().getAll(),
context,
new ParDoTranslationHelper.DoFnOperatorFactory<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT>() {
+ KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>() {
@Override
- public DoFnOperator<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>,
- OutputT,
- RawUnionValue> createDoFnOperator(
- DoFn<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>,
- OutputT> doFn,
- String stepName,
- List<PCollectionView<?>> sideInputs,
- TupleTag<OutputT> mainOutputTag,
- List<TupleTag<?>> additionalOutputTags,
- FlinkStreamingTranslationContext context,
- WindowingStrategy<?, ?> windowingStrategy,
- Map<TupleTag<?>, Integer> tagsToLabels,
- Coder<
- WindowedValue<
- KeyedWorkItem<
- String,
- ElementAndRestriction<InputT, RestrictionT>>>> inputCoder,
- Coder keyCoder,
- Map<Integer, PCollectionView<?>> transformedSideInputs) {
+ public DoFnOperator<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT>
+ createDoFnOperator(
+ DoFn<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> doFn,
+ String stepName,
+ List<PCollectionView<?>> sideInputs,
+ TupleTag<OutputT> mainOutputTag,
+ List<TupleTag<?>> additionalOutputTags,
+ FlinkStreamingTranslationContext context,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders,
+ Map<TupleTag<?>, Integer> tagsToIds,
+ Coder<WindowedValue<KeyedWorkItem<String, KV<InputT, RestrictionT>>>>
+ inputCoder,
+ Coder keyCoder,
+ Map<Integer, PCollectionView<?>> transformedSideInputs) {
return new SplittableDoFnOperator<>(
doFn,
stepName,
inputCoder,
mainOutputTag,
additionalOutputTags,
- new DoFnOperator.MultiOutputOutputManagerFactory(tagsToLabels),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(
+ mainOutputTag, tagsToOutputTags, tagsToCoders, tagsToIds),
windowingStrategy,
transformedSideInputs,
sideInputs,
@@ -639,17 +637,17 @@
private static class CreateViewStreamingTranslator<ElemT, ViewT>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView<ElemT, ViewT>> {
+ CreateStreamingFlinkView.CreateFlinkPCollectionView<ElemT, ViewT>> {
@Override
public void translateNode(
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView<ElemT, ViewT> transform,
+ CreateStreamingFlinkView.CreateFlinkPCollectionView<ElemT, ViewT> transform,
FlinkStreamingTranslationContext context) {
// just forward
DataStream<WindowedValue<List<ElemT>>> inputDataSet =
context.getInputDataStream(context.getInput(transform));
- PCollectionView<ViewT> view = context.getOutput(transform);
+ PCollectionView<ViewT> view = transform.getView();
context.setOutputDataStream(view, inputDataSet);
}
@@ -750,21 +748,21 @@
SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, BoundedWindow> reduceFn =
SystemReduceFn.buffering(inputKvCoder.getValueCoder());
+ Coder<WindowedValue<KV<K, Iterable<InputT>>>> outputCoder =
+ context.getCoder(context.getOutput(transform));
TypeInformation<WindowedValue<KV<K, Iterable<InputT>>>> outputTypeInfo =
context.getTypeInfo(context.getOutput(transform));
- DoFnOperator.DefaultOutputManagerFactory<
- WindowedValue<KV<K, Iterable<InputT>>>> outputManagerFactory =
- new DoFnOperator.DefaultOutputManagerFactory<>();
+ TupleTag<KV<K, Iterable<InputT>>> mainTag = new TupleTag<>("main output");
WindowDoFnOperator<K, InputT, Iterable<InputT>> doFnOperator =
new WindowDoFnOperator<>(
reduceFn,
context.getCurrentTransform().getFullName(),
(Coder) windowedWorkItemCoder,
- new TupleTag<KV<K, Iterable<InputT>>>("main output"),
+ mainTag,
Collections.<TupleTag<?>>emptyList(),
- outputManagerFactory,
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, outputCoder),
windowingStrategy,
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
@@ -851,6 +849,8 @@
AppliedCombineFn.withInputCoder(
transform.getFn(), input.getPipeline().getCoderRegistry(), inputKvCoder));
+ Coder<WindowedValue<KV<K, OutputT>>> outputCoder =
+ context.getCoder(context.getOutput(transform));
TypeInformation<WindowedValue<KV<K, OutputT>>> outputTypeInfo =
context.getTypeInfo(context.getOutput(transform));
@@ -858,14 +858,15 @@
if (sideInputs.isEmpty()) {
+ TupleTag<KV<K, OutputT>> mainTag = new TupleTag<>("main output");
WindowDoFnOperator<K, InputT, OutputT> doFnOperator =
new WindowDoFnOperator<>(
reduceFn,
context.getCurrentTransform().getFullName(),
(Coder) windowedWorkItemCoder,
- new TupleTag<KV<K, OutputT>>("main output"),
+ mainTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<WindowedValue<KV<K, OutputT>>>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, outputCoder),
windowingStrategy,
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
@@ -884,14 +885,15 @@
Tuple2<Map<Integer, PCollectionView<?>>, DataStream<RawUnionValue>> transformSideInputs =
transformSideInputs(sideInputs, context);
+ TupleTag<KV<K, OutputT>> mainTag = new TupleTag<>("main output");
WindowDoFnOperator<K, InputT, OutputT> doFnOperator =
new WindowDoFnOperator<>(
reduceFn,
context.getCurrentTransform().getFullName(),
(Coder) windowedWorkItemCoder,
- new TupleTag<KV<K, OutputT>>("main output"),
+ mainTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<WindowedValue<KV<K, OutputT>>>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, outputCoder),
windowingStrategy,
transformSideInputs.f0,
sideInputs,
@@ -930,18 +932,18 @@
private static class GBKIntoKeyedWorkItemsTranslator<K, InputT>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
- SplittableParDo.GBKIntoKeyedWorkItems<K, InputT>> {
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT>> {
@Override
boolean canTranslate(
- SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform,
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT> transform,
FlinkStreamingTranslationContext context) {
return true;
}
@Override
public void translateNode(
- SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform,
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT> transform,
FlinkStreamingTranslationContext context) {
PCollection<KV<K, InputT>> input = context.getInput(transform);
@@ -1075,4 +1077,94 @@
}
}
+ /**
+ * A translator just to vend the URN. This will need to be moved to runners-core-construction-java
+ * once SDF is reorganized appropriately.
+ */
+ private static class SplittableParDoProcessElementsTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.NotSerializable<
+ SplittableParDoViaKeyedWorkItems.ProcessElements<?, ?, ?, ?>> {
+
+ private SplittableParDoProcessElementsTranslator() {}
+
+ @Override
+ public String getUrn(SplittableParDoViaKeyedWorkItems.ProcessElements<?, ?, ?, ?> transform) {
+ return SPLITTABLE_PROCESS_URN;
+ }
+ }
+
+ /** Registers classes specialized to the Flink runner. */
+ @AutoService(TransformPayloadTranslatorRegistrar.class)
+ public static class FlinkTransformsRegistrar implements TransformPayloadTranslatorRegistrar {
+ @Override
+ public Map<
+ ? extends Class<? extends PTransform>,
+ ? extends PTransformTranslation.TransformPayloadTranslator>
+ getTransformPayloadTranslators() {
+ return ImmutableMap
+ .<Class<? extends PTransform>, PTransformTranslation.TransformPayloadTranslator>builder()
+ .put(
+ CreateStreamingFlinkView.CreateFlinkPCollectionView.class,
+ new CreateStreamingFlinkViewPayloadTranslator())
+ .put(
+ SplittableParDoViaKeyedWorkItems.ProcessElements.class,
+ new SplittableParDoProcessElementsTranslator())
+ .put(
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class,
+ new SplittableParDoGbkIntoKeyedWorkItemsPayloadTranslator())
+ .build();
+ }
+
+ @Override
+ public Map<String, PTransformTranslation.TransformPayloadTranslator> getTransformRehydrators() {
+ return Collections.emptyMap();
+ }
+ }
+
+ /**
+ * A translator just to vend the URN. This will need to be moved to runners-core-construction-java
+ * once SDF is reorganized appropriately.
+ */
+ private static class SplittableParDoProcessElementsPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.NotSerializable<
+ SplittableParDoViaKeyedWorkItems.ProcessElements<?, ?, ?, ?>> {
+
+ private SplittableParDoProcessElementsPayloadTranslator() {}
+
+ @Override
+ public String getUrn(SplittableParDoViaKeyedWorkItems.ProcessElements<?, ?, ?, ?> transform) {
+ return SplittableParDo.SPLITTABLE_PROCESS_URN;
+ }
+ }
+
+ /**
+ * A translator just to vend the URN. This will need to be moved to runners-core-construction-java
+ * once SDF is reorganized appropriately.
+ */
+ private static class SplittableParDoGbkIntoKeyedWorkItemsPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.NotSerializable<
+ SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<?, ?>> {
+
+ private SplittableParDoGbkIntoKeyedWorkItemsPayloadTranslator() {}
+
+ @Override
+ public String getUrn(SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<?, ?> transform) {
+ return SplittableParDo.SPLITTABLE_GBKIKWI_URN;
+ }
+ }
+
+ /**
+ * A translator just to vend the URN.
+ */
+ private static class CreateStreamingFlinkViewPayloadTranslator
+ extends PTransformTranslation.TransformPayloadTranslator.NotSerializable<
+ CreateStreamingFlinkView.CreateFlinkPCollectionView<?, ?>> {
+
+ private CreateStreamingFlinkViewPayloadTranslator() {}
+
+ @Override
+ public String getUrn(CreateStreamingFlinkView.CreateFlinkPCollectionView<?, ?> transform) {
+ return CreateStreamingFlinkView.CREATE_STREAMING_FLINK_VIEW_URN;
+ }
+ }
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index ea5f6b3..74a5fb9 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,6 +22,7 @@
import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -113,7 +114,7 @@
@SuppressWarnings("unchecked")
public <T extends PValue> T getInput(PTransform<T, ?> transform) {
- return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+ return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
}
public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
deleted file mode 100644
index ce1c895..0000000
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.flink;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
-
-/**
- * Flink streaming overrides for various view (side input) transforms.
- */
-class FlinkStreamingViewOverrides {
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
- * for the Flink runner in streaming mode.
- */
- static class StreamingViewAsMap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
-
- private final transient FlinkRunner runner;
-
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsMap(FlinkRunner runner, View.AsMap<K, V> transform) {
- this.runner = runner;
- }
-
- @Override
- public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, V>> view =
- PCollectionViews.mapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- runner.recordViewUsesNonDeterministicKeyCoder(this);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<KV<K, V>, Map<K, V>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMap";
- }
- }
-
- /**
- * Specialized expansion for {@link
- * View.AsMultimap View.AsMultimap} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsMultimap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
-
- private final transient FlinkRunner runner;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsMultimap(FlinkRunner runner, View.AsMultimap<K, V> transform) {
- this.runner = runner;
- }
-
- @Override
- public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, Iterable<V>>> view =
- PCollectionViews.multimapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- runner.recordViewUsesNonDeterministicKeyCoder(this);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMultimap";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link View.AsList View.AsList} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsList<T>
- extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsList(FlinkRunner runner, View.AsList<T> transform) {}
-
- @Override
- public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view =
- PCollectionViews.listView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<T, List<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsList";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link View.AsIterable View.AsIterable} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsIterable(FlinkRunner runner, View.AsIterable<T> transform) { }
-
- @Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<T, Iterable<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsIterable";
- }
- }
-
- /**
- * Specialized expansion for
- * {@link View.AsSingleton View.AsSingleton} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsSingleton<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
- private View.AsSingleton<T> transform;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsSingleton(FlinkRunner runner, View.AsSingleton<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<T> expand(PCollection<T> input) {
- Combine.Globally<T, T> combine = Combine.globally(
- new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
- if (!transform.hasDefaultValue()) {
- combine = combine.withoutDefaults();
- }
- return input.apply(combine.asSingletonView());
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsSingleton";
- }
-
- private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
- private boolean hasDefaultValue;
- private T defaultValue;
-
- SingletonCombine(boolean hasDefaultValue, T defaultValue) {
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- }
-
- @Override
- public T apply(T left, T right) {
- throw new IllegalArgumentException("PCollection with more than one element "
- + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
- + "combine the PCollection into a single value");
- }
-
- @Override
- public T identity() {
- if (hasDefaultValue) {
- return defaultValue;
- } else {
- throw new IllegalArgumentException(
- "Empty PCollection accessed as a singleton view. "
- + "Consider setting withDefault to provide a default value");
- }
- }
- }
- }
-
- static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
- extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
- Combine.GloballyAsSingletonView<InputT, OutputT> transform;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingCombineGloballyAsSingletonView(
- FlinkRunner runner,
- Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<OutputT> expand(PCollection<InputT> input) {
- PCollection<OutputT> combined =
- input.apply(Combine.globally(transform.getCombineFn())
- .withoutDefaults()
- .withFanout(transform.getFanout()));
-
- PCollectionView<OutputT> view = PCollectionViews.singletonView(
- combined,
- combined.getWindowingStrategy(),
- transform.getInsertDefault(),
- transform.getInsertDefault()
- ? transform.getCombineFn().defaultValue() : null,
- combined.getCoder());
- return combined
- .apply(ParDo.of(new WrapAsList<OutputT>()))
- .apply(CreateFlinkPCollectionView.<OutputT, OutputT>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingCombineGloballyAsSingletonView";
- }
- }
-
- private static class WrapAsList<T> extends DoFn<T, List<T>> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(Collections.singletonList(c.element()));
- }
- }
-
- /**
- * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
- *
- * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
- * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
- * They require the input {@link PCollection} fits in memory.
- * For a large {@link PCollection} this is expected to crash!
- *
- * @param <T> the type of elements to concatenate.
- */
- private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
- @Override
- public List<T> createAccumulator() {
- return new ArrayList<T>();
- }
-
- @Override
- public List<T> addInput(List<T> accumulator, T input) {
- accumulator.add(input);
- return accumulator;
- }
-
- @Override
- public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
- List<T> result = createAccumulator();
- for (List<T> accumulator : accumulators) {
- result.addAll(accumulator);
- }
- return result;
- }
-
- @Override
- public List<T> extractOutput(List<T> accumulator) {
- return accumulator;
- }
-
- @Override
- public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
-
- @Override
- public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
- }
-
- /**
- * Creates a primitive {@link PCollectionView}.
- *
- * <p>For internal use only by runner implementors.
- *
- * @param <ElemT> The type of the elements of the input PCollection
- * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
- */
- public static class CreateFlinkPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
- private PCollectionView<ViewT> view;
-
- private CreateFlinkPCollectionView(PCollectionView<ViewT> view) {
- this.view = view;
- }
-
- public static <ElemT, ViewT> CreateFlinkPCollectionView<ElemT, ViewT> of(
- PCollectionView<ViewT> view) {
- return new CreateFlinkPCollectionView<>(view);
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
- return view;
- }
- }
-}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
new file mode 100644
index 0000000..1dc8de9
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
+import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.sdk.runners.PTransformOverride;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.View;
+
+/**
+ * {@link PTransform} overrides for Flink runner.
+ */
+public class FlinkTransformOverrides {
+ public static List<PTransformOverride> getDefaultOverrides(boolean streaming) {
+ if (streaming) {
+ return ImmutableList.<PTransformOverride>builder()
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.splittableParDoMulti(),
+ new FlinkStreamingPipelineTranslator.SplittableParDoOverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+ new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
+ new CreateStreamingFlinkView.Factory()))
+ .build();
+ } else {
+ return ImmutableList.of();
+ }
+ }
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java
index 447b1e5..26d6721 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.runners.flink.translation.functions;
-import static com.google.common.base.Preconditions.checkArgument;
-
import com.google.common.collect.Iterables;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -35,13 +33,14 @@
FlinkAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) {
fn.super();
- checkArgument(
- Iterables.size(value.getWindows()) == 1,
- String.format(
- "%s passed to window assignment must be in a single window, but it was in %s: %s",
- WindowedValue.class.getSimpleName(),
- Iterables.size(value.getWindows()),
- value.getWindows()));
+ if (Iterables.size(value.getWindows()) != 1) {
+ throw new IllegalArgumentException(
+ String.format(
+ "%s passed to window assignment must be in a single window, but it was in %s: %s",
+ WindowedValue.class.getSimpleName(),
+ Iterables.size(value.getWindows()),
+ value.getWindows()));
+ }
this.value = value;
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
index 9205bce..3048168 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
@@ -17,13 +17,14 @@
*/
package org.apache.beam.runners.flink.translation.functions;
-import java.util.Collections;
+import com.google.common.collect.Lists;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.FlinkPipelineOptions;
import org.apache.beam.runners.flink.metrics.DoFnRunnerWithMetricsUpdate;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.join.RawUnionValue;
@@ -49,7 +50,7 @@
public class FlinkDoFnFunction<InputT, OutputT>
extends RichMapPartitionFunction<WindowedValue<InputT>, WindowedValue<OutputT>> {
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
private final DoFn<InputT, OutputT> doFn;
private final String stepName;
@@ -74,7 +75,7 @@
this.doFn = doFn;
this.stepName = stepName;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(options);
+ this.serializedOptions = new SerializablePipelineOptions(options);
this.windowingStrategy = windowingStrategy;
this.outputMap = outputMap;
this.mainOutputTag = mainOutputTag;
@@ -89,7 +90,7 @@
RuntimeContext runtimeContext = getRuntimeContext();
DoFnRunners.OutputManager outputManager;
- if (outputMap == null) {
+ if (outputMap.size() == 1) {
outputManager = new FlinkDoFnFunction.DoFnOutputManager(out);
} else {
// it has some additional outputs
@@ -97,17 +98,18 @@
new FlinkDoFnFunction.MultiDoFnOutputManager((Collector) out, outputMap);
}
+ List<TupleTag<?>> additionalOutputTags = Lists.newArrayList(outputMap.keySet());
+
DoFnRunner<InputT, OutputT> doFnRunner = DoFnRunners.simpleRunner(
- serializedOptions.getPipelineOptions(), doFn,
+ serializedOptions.get(), doFn,
new FlinkSideInputReader(sideInputs, runtimeContext),
outputManager,
mainOutputTag,
- // see SimpleDoFnRunner, just use it to limit number of additional outputs
- Collections.<TupleTag<?>>emptyList(),
+ additionalOutputTags,
new FlinkNoOpStepContext(),
windowingStrategy);
- if ((serializedOptions.getPipelineOptions().as(FlinkPipelineOptions.class))
+ if ((serializedOptions.get().as(FlinkPipelineOptions.class))
.getEnableMetrics()) {
doFnRunner = new DoFnRunnerWithMetricsUpdate<>(stepName, doFnRunner, getRuntimeContext());
}
@@ -144,7 +146,9 @@
@Override
@SuppressWarnings("unchecked")
public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- collector.collect(output);
+ collector.collect(
+ WindowedValue.of(new RawUnionValue(0 /* single output */, output.getValue()),
+ output.getTimestamp(), output.getWindows(), output.getPane()));
}
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
index 13be913..c73dade 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.flink.translation.functions;
import java.util.Map;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.CombineFnBase;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -47,7 +47,7 @@
private final Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs;
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
public FlinkMergingNonShuffleReduceFunction(
CombineFnBase.GlobalCombineFn<InputT, AccumT, OutputT> combineFn,
@@ -60,7 +60,7 @@
this.windowingStrategy = windowingStrategy;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
}
@@ -69,7 +69,7 @@
Iterable<WindowedValue<KV<K, InputT>>> elements,
Collector<WindowedValue<KV<K, OutputT>>> out) throws Exception {
- PipelineOptions options = serializedOptions.getPipelineOptions();
+ PipelineOptions options = serializedOptions.get();
FlinkSideInputReader sideInputReader =
new FlinkSideInputReader(sideInputs, getRuntimeContext());
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
index 8640801..9c7b636 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
@@ -17,14 +17,9 @@
*/
package org.apache.beam.runners.flink.translation.functions;
-import java.io.IOException;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
/**
* A {@link StepContext} for Flink Batch Runner execution.
@@ -32,35 +27,6 @@
public class FlinkNoOpStepContext implements StepContext {
@Override
- public String getStepName() {
- return null;
- }
-
- @Override
- public String getTransformName() {
- return null;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {
-
- }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window,
- Coder<W> windowCoder) throws IOException {
- }
-
- @Override
public StateInternals stateInternals() {
return null;
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
index db12a49..49e821c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.flink.translation.functions;
import java.util.Map;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.CombineFnBase;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -46,7 +46,7 @@
protected final WindowingStrategy<Object, W> windowingStrategy;
- protected final SerializedPipelineOptions serializedOptions;
+ protected final SerializablePipelineOptions serializedOptions;
protected final Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs;
@@ -59,7 +59,7 @@
this.combineFn = combineFn;
this.windowingStrategy = windowingStrategy;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
}
@@ -68,7 +68,7 @@
Iterable<WindowedValue<KV<K, InputT>>> elements,
Collector<WindowedValue<KV<K, AccumT>>> out) throws Exception {
- PipelineOptions options = serializedOptions.getPipelineOptions();
+ PipelineOptions options = serializedOptions.get();
FlinkSideInputReader sideInputReader =
new FlinkSideInputReader(sideInputs, getRuntimeContext());
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
index 53d71d8..6645b3a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.flink.translation.functions;
import java.util.Map;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.CombineFnBase;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -48,7 +48,7 @@
protected final Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs;
- protected final SerializedPipelineOptions serializedOptions;
+ protected final SerializablePipelineOptions serializedOptions;
public FlinkReduceFunction(
CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn,
@@ -61,7 +61,7 @@
this.windowingStrategy = windowingStrategy;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
}
@@ -70,7 +70,7 @@
Iterable<WindowedValue<KV<K, AccumT>>> elements,
Collector<WindowedValue<KV<K, OutputT>>> out) throws Exception {
- PipelineOptions options = serializedOptions.getPipelineOptions();
+ PipelineOptions options = serializedOptions.get();
FlinkSideInputReader sideInputReader =
new FlinkSideInputReader(sideInputs, getRuntimeContext());
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
index 6517bf2..412269c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
@@ -19,8 +19,9 @@
import static org.apache.flink.util.Preconditions.checkArgument;
-import java.util.Collections;
+import com.google.common.collect.Lists;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
@@ -30,9 +31,9 @@
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateNamespaces;
import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.FlinkPipelineOptions;
import org.apache.beam.runners.flink.metrics.DoFnRunnerWithMetricsUpdate;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
@@ -60,7 +61,7 @@
private String stepName;
private final WindowingStrategy<?, ?> windowingStrategy;
private final Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs;
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
private final Map<TupleTag<?>, Integer> outputMap;
private final TupleTag<OutputT> mainOutputTag;
private transient DoFnInvoker doFnInvoker;
@@ -78,7 +79,7 @@
this.stepName = stepName;
this.windowingStrategy = windowingStrategy;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
this.outputMap = outputMap;
this.mainOutputTag = mainOutputTag;
}
@@ -90,7 +91,7 @@
RuntimeContext runtimeContext = getRuntimeContext();
DoFnRunners.OutputManager outputManager;
- if (outputMap == null) {
+ if (outputMap.size() == 1) {
outputManager = new FlinkDoFnFunction.DoFnOutputManager(out);
} else {
// it has some additional Outputs
@@ -114,13 +115,14 @@
timerInternals.advanceProcessingTime(Instant.now());
timerInternals.advanceSynchronizedProcessingTime(Instant.now());
+ List<TupleTag<?>> additionalOutputTags = Lists.newArrayList(outputMap.keySet());
+
DoFnRunner<KV<K, V>, OutputT> doFnRunner = DoFnRunners.simpleRunner(
- serializedOptions.getPipelineOptions(), dofn,
+ serializedOptions.get(), dofn,
new FlinkSideInputReader(sideInputs, runtimeContext),
outputManager,
mainOutputTag,
- // see SimpleDoFnRunner, just use it to limit number of additional outputs
- Collections.<TupleTag<?>>emptyList(),
+ additionalOutputTags,
new FlinkNoOpStepContext() {
@Override
public StateInternals stateInternals() {
@@ -133,7 +135,7 @@
},
windowingStrategy);
- if ((serializedOptions.getPipelineOptions().as(FlinkPipelineOptions.class))
+ if ((serializedOptions.get().as(FlinkPipelineOptions.class))
.getEnableMetrics()) {
doFnRunner = new DoFnRunnerWithMetricsUpdate<>(stepName, doFnRunner, getRuntimeContext());
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
index e003119..c8dbac4 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
@@ -19,12 +19,15 @@
import java.io.EOFException;
import java.io.IOException;
+import java.util.Objects;
import org.apache.beam.runners.flink.translation.wrappers.DataInputViewWrapper;
import org.apache.beam.runners.flink.translation.wrappers.DataOutputViewWrapper;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -129,4 +132,79 @@
public int hashCode() {
return coder.hashCode();
}
+
+ @Override
+ public TypeSerializerConfigSnapshot snapshotConfiguration() {
+ return new CoderTypeSerializerConfigSnapshot<>(coder);
+ }
+
+ @Override
+ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+ if (snapshotConfiguration().equals(configSnapshot)) {
+ return CompatibilityResult.compatible();
+ }
+ return CompatibilityResult.requiresMigration();
+ }
+
+ /**
+ * TypeSerializerConfigSnapshot of CoderTypeSerializer. This uses the class name of the
+ * {@link Coder} to determine compatibility. This is a bit crude but better than using
+ * Java Serialization to (de)serialize the {@link Coder}.
+ */
+ public static class CoderTypeSerializerConfigSnapshot<T> extends TypeSerializerConfigSnapshot {
+
+ private static final int VERSION = 1;
+ private String coderName;
+
+ public CoderTypeSerializerConfigSnapshot() {
+ // empty constructor for satisfying IOReadableWritable which is used for deserialization
+ }
+
+ public CoderTypeSerializerConfigSnapshot(Coder<T> coder) {
+ this.coderName = coder.getClass().getName();
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ CoderTypeSerializerConfigSnapshot<?> that = (CoderTypeSerializerConfigSnapshot<?>) o;
+
+ return coderName != null ? coderName.equals(that.coderName) : that.coderName == null;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+ out.writeUTF(coderName);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ super.read(in);
+ this.coderName = in.readUTF();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(coderName);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "CoderTypeSerializer{"
+ + "coder=" + coder
+ + '}';
+ }
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
index c3b9794..c40eb46 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
@@ -20,13 +20,14 @@
import java.io.IOException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
/**
* {@link TypeSerializer} for values that were encoded using a {@link Coder}.
*/
-public final class EncodedValueSerializer extends TypeSerializer<byte[]> {
+public final class EncodedValueSerializer extends TypeSerializerSingleton<byte[]> {
private static final long serialVersionUID = 1L;
@@ -57,7 +58,6 @@
return -1;
}
-
@Override
public void serialize(byte[] record, DataOutputView target) throws IOException {
if (record == null) {
@@ -94,18 +94,4 @@
return obj instanceof EncodedValueSerializer;
}
- @Override
- public int hashCode() {
- return this.getClass().hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return obj instanceof EncodedValueSerializer;
- }
-
- @Override
- public TypeSerializer<byte[]> duplicate() {
- return this;
- }
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
deleted file mode 100644
index 40b6dd6..0000000
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.flink.translation.utils;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.common.ReflectHelpers;
-
-/**
- * Encapsulates the PipelineOptions in serialized form to ship them to the cluster.
- */
-public class SerializedPipelineOptions implements Serializable {
-
- private final byte[] serializedOptions;
-
- /** Lazily initialized copy of deserialized options. */
- private transient PipelineOptions pipelineOptions;
-
- public SerializedPipelineOptions(PipelineOptions options) {
- checkNotNull(options, "PipelineOptions must not be null.");
-
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- createMapper().writeValue(baos, options);
- this.serializedOptions = baos.toByteArray();
- } catch (Exception e) {
- throw new RuntimeException("Couldn't serialize PipelineOptions.", e);
- }
-
- }
-
- public PipelineOptions getPipelineOptions() {
- if (pipelineOptions == null) {
- try {
- pipelineOptions = createMapper().readValue(serializedOptions, PipelineOptions.class);
-
- FileSystems.setDefaultPipelineOptions(pipelineOptions);
- } catch (IOException e) {
- throw new RuntimeException("Couldn't deserialize the PipelineOptions.", e);
- }
- }
-
- return pipelineOptions;
- }
-
- /**
- * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
- * for user specified configuration injection into the ObjectMapper. This supports user custom
- * types on {@link PipelineOptions}.
- */
- private static ObjectMapper createMapper() {
- return new ObjectMapper().registerModules(
- ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
- }
-}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
index 27e6912..3f9d601 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
@@ -19,9 +19,9 @@
import java.io.IOException;
import java.util.List;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.metrics.FlinkMetricContainer;
import org.apache.beam.runners.flink.metrics.ReaderInvocationUtil;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -50,7 +50,7 @@
private final BoundedSource<T> initialSource;
private transient PipelineOptions options;
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
private transient BoundedSource.BoundedReader<T> reader;
private boolean inputAvailable = false;
@@ -61,12 +61,12 @@
String stepName, BoundedSource<T> initialSource, PipelineOptions options) {
this.stepName = stepName;
this.initialSource = initialSource;
- this.serializedOptions = new SerializedPipelineOptions(options);
+ this.serializedOptions = new SerializablePipelineOptions(options);
}
@Override
public void configure(Configuration configuration) {
- options = serializedOptions.getPipelineOptions();
+ options = serializedOptions.get();
}
@Override
@@ -76,7 +76,7 @@
readerInvoker =
new ReaderInvocationUtil<>(
stepName,
- serializedOptions.getPipelineOptions(),
+ serializedOptions.get(),
metricContainer);
reader = ((BoundedSource<T>) sourceInputSplit.getSource()).createReader(options);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index f35ba7a..d203ffb 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -19,20 +19,25 @@
import static org.apache.flink.util.Preconditions.checkArgument;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetNewDoFn;
import org.apache.beam.runners.core.NullSideInputReader;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
@@ -46,18 +51,22 @@
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTags;
import org.apache.beam.runners.core.StatefulDoFnRunner;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.FlinkPipelineOptions;
import org.apache.beam.runners.flink.metrics.DoFnRunnerWithMetricsUpdate;
import org.apache.beam.runners.flink.translation.types.CoderTypeSerializer;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkBroadcastStateInternals;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkKeyGroupStateInternals;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkSplitStateInternals;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkStateInternals;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.KeyGroupCheckpointedOperator;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.TimeDomain;
@@ -67,6 +76,7 @@
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -78,6 +88,7 @@
import org.apache.flink.runtime.state.KeyedStateCheckpointOutputStream;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.HeapInternalTimerService;
@@ -88,27 +99,28 @@
import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
+import org.apache.flink.util.OutputTag;
import org.joda.time.Instant;
/**
* Flink operator for executing {@link DoFn DoFns}.
*
* @param <InputT> the input type of the {@link DoFn}
- * @param <FnOutputT> the output type of the {@link DoFn}
- * @param <OutputT> the output type of the operator, this can be different from the fn output
- * type when we have additional tagged outputs
+ * @param <OutputT> the output type of the {@link DoFn}
*/
-public class DoFnOperator<InputT, FnOutputT, OutputT>
- extends AbstractStreamOperator<OutputT>
- implements OneInputStreamOperator<WindowedValue<InputT>, OutputT>,
- TwoInputStreamOperator<WindowedValue<InputT>, RawUnionValue, OutputT>,
+public class DoFnOperator<InputT, OutputT>
+ extends AbstractStreamOperator<WindowedValue<OutputT>>
+ implements OneInputStreamOperator<WindowedValue<InputT>, WindowedValue<OutputT>>,
+ TwoInputStreamOperator<WindowedValue<InputT>, RawUnionValue, WindowedValue<OutputT>>,
KeyGroupCheckpointedOperator, Triggerable<Object, TimerData> {
- protected DoFn<InputT, FnOutputT> doFn;
+ protected DoFn<InputT, OutputT> doFn;
- protected final SerializedPipelineOptions serializedOptions;
+ protected final SerializablePipelineOptions serializedOptions;
- protected final TupleTag<FnOutputT> mainOutputTag;
+ protected final TupleTag<OutputT> mainOutputTag;
protected final List<TupleTag<?>> additionalOutputTags;
protected final Collection<PCollectionView<?>> sideInputs;
@@ -118,24 +130,26 @@
protected final OutputManagerFactory<OutputT> outputManagerFactory;
- protected transient DoFnRunner<InputT, FnOutputT> doFnRunner;
- protected transient PushbackSideInputDoFnRunner<InputT, FnOutputT> pushbackDoFnRunner;
+ protected transient DoFnRunner<InputT, OutputT> doFnRunner;
+ protected transient PushbackSideInputDoFnRunner<InputT, OutputT> pushbackDoFnRunner;
protected transient SideInputHandler sideInputHandler;
protected transient SideInputReader sideInputReader;
- protected transient DoFnRunners.OutputManager outputManager;
+ protected transient BufferedOutputManager<OutputT> outputManager;
- private transient DoFnInvoker<InputT, FnOutputT> doFnInvoker;
+ private transient DoFnInvoker<InputT, OutputT> doFnInvoker;
protected transient long currentInputWatermark;
+ protected transient long currentSideInputWatermark;
+
protected transient long currentOutputWatermark;
private transient StateTag<BagState<WindowedValue<InputT>>> pushedBackTag;
- protected transient FlinkStateInternals<?> stateInternals;
+ protected transient FlinkStateInternals<?> keyedStateInternals;
private final String stepName;
@@ -145,19 +159,29 @@
private final TimerInternals.TimerDataCoder timerCoder;
+ private final long maxBundleSize;
+
+ private final long maxBundleTimeMills;
+
protected transient HeapInternalTimerService<?, TimerInternals.TimerData> timerService;
protected transient FlinkTimerInternals timerInternals;
- private transient StateInternals pushbackStateInternals;
+ private transient StateInternals nonKeyedStateInternals;
private transient Optional<Long> pushedBackWatermark;
+ // bundle control
+ private transient boolean bundleStarted = false;
+ private transient long elementCount;
+ private transient long lastFinishBundleTime;
+ private transient ScheduledFuture<?> checkFinishBundleTimer;
+
public DoFnOperator(
- DoFn<InputT, FnOutputT> doFn,
+ DoFn<InputT, OutputT> doFn,
String stepName,
Coder<WindowedValue<InputT>> inputCoder,
- TupleTag<FnOutputT> mainOutputTag,
+ TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> additionalOutputTags,
OutputManagerFactory<OutputT> outputManagerFactory,
WindowingStrategy<?, ?> windowingStrategy,
@@ -172,7 +196,7 @@
this.additionalOutputTags = additionalOutputTags;
this.sideInputTagMapping = sideInputTagMapping;
this.sideInputs = sideInputs;
- this.serializedOptions = new SerializedPipelineOptions(options);
+ this.serializedOptions = new SerializablePipelineOptions(options);
this.windowingStrategy = windowingStrategy;
this.outputManagerFactory = outputManagerFactory;
@@ -182,27 +206,56 @@
this.timerCoder =
TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
- }
- private ExecutionContext.StepContext createStepContext() {
- return new StepContext();
+ FlinkPipelineOptions flinkOptions = options.as(FlinkPipelineOptions.class);
+
+ this.maxBundleSize = flinkOptions.getMaxBundleSize();
+ this.maxBundleTimeMills = flinkOptions.getMaxBundleTimeMills();
}
// allow overriding this in WindowDoFnOperator because this one dynamically creates
// the DoFn
- protected DoFn<InputT, FnOutputT> getDoFn() {
+ protected DoFn<InputT, OutputT> getDoFn() {
return doFn;
}
@Override
+ public void setup(
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<StreamRecord<WindowedValue<OutputT>>> output) {
+
+ // make sure that FileSystems is initialized correctly
+ FlinkPipelineOptions options =
+ serializedOptions.get().as(FlinkPipelineOptions.class);
+ FileSystems.setDefaultPipelineOptions(options);
+
+ super.setup(containingTask, config, output);
+ }
+
+ @Override
public void open() throws Exception {
super.open();
setCurrentInputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis());
+ setCurrentSideInputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis());
setCurrentOutputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis());
+ FlinkPipelineOptions options =
+ serializedOptions.get().as(FlinkPipelineOptions.class);
sideInputReader = NullSideInputReader.of(sideInputs);
+ // maybe init by initializeState
+ if (nonKeyedStateInternals == null) {
+ if (keyCoder != null) {
+ nonKeyedStateInternals = new FlinkKeyGroupStateInternals<>(keyCoder,
+ getKeyedStateBackend());
+ } else {
+ nonKeyedStateInternals =
+ new FlinkSplitStateInternals<>(getOperatorStateBackend());
+ }
+ }
+
if (!sideInputs.isEmpty()) {
pushedBackTag = StateTags.bag("pushed-back-values", inputCoder);
@@ -214,26 +267,14 @@
sideInputHandler = new SideInputHandler(sideInputs, sideInputStateInternals);
sideInputReader = sideInputHandler;
- // maybe init by initializeState
- if (pushbackStateInternals == null) {
- if (keyCoder != null) {
- pushbackStateInternals = new FlinkKeyGroupStateInternals<>(keyCoder,
- getKeyedStateBackend());
- } else {
- pushbackStateInternals =
- new FlinkSplitStateInternals<Object>(getOperatorStateBackend());
- }
- }
-
pushedBackWatermark = Optional.absent();
-
}
- outputManager = outputManagerFactory.create(output);
+ outputManager = outputManagerFactory.create(output, nonKeyedStateInternals);
// StatefulPardo or WindowDoFn
if (keyCoder != null) {
- stateInternals = new FlinkStateInternals<>((KeyedStateBackend) getKeyedStateBackend(),
+ keyedStateInternals = new FlinkStateInternals<>((KeyedStateBackend) getKeyedStateBackend(),
keyCoder);
timerService = (HeapInternalTimerService<?, TimerInternals.TimerData>)
@@ -250,10 +291,10 @@
doFnInvoker.invokeSetup();
- ExecutionContext.StepContext stepContext = createStepContext();
+ StepContext stepContext = new FlinkStepContext();
doFnRunner = DoFnRunners.simpleRunner(
- serializedOptions.getPipelineOptions(),
+ options,
doFn,
sideInputReader,
outputManager,
@@ -298,11 +339,24 @@
stateCleaner);
}
- if ((serializedOptions.getPipelineOptions().as(FlinkPipelineOptions.class))
- .getEnableMetrics()) {
+ if (options.getEnableMetrics()) {
doFnRunner = new DoFnRunnerWithMetricsUpdate<>(stepName, doFnRunner, getRuntimeContext());
}
+ elementCount = 0L;
+ lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime();
+
+ // Schedule timer to check timeout of finish bundle.
+ long bundleCheckPeriod = (maxBundleTimeMills + 1) / 2;
+ checkFinishBundleTimer = getProcessingTimeService().scheduleAtFixedRate(
+ new ProcessingTimeCallback() {
+ @Override
+ public void onProcessingTime(long timestamp) throws Exception {
+ checkInvokeFinishBundleByTime();
+ }
+ },
+ bundleCheckPeriod, bundleCheckPeriod);
+
pushbackDoFnRunner =
SimplePushbackSideInputDoFnRunner.create(doFnRunner, sideInputs, sideInputHandler);
}
@@ -310,13 +364,29 @@
@Override
public void close() throws Exception {
super.close();
+
+ // sanity check: these should have been flushed out by +Inf watermarks
+ if (!sideInputs.isEmpty() && nonKeyedStateInternals != null) {
+ BagState<WindowedValue<InputT>> pushedBack =
+ nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag);
+
+ Iterable<WindowedValue<InputT>> pushedBackContents = pushedBack.read();
+ if (pushedBackContents != null) {
+ if (!Iterables.isEmpty(pushedBackContents)) {
+ String pushedBackString = Joiner.on(",").join(pushedBackContents);
+ throw new RuntimeException(
+ "Leftover pushed-back data: " + pushedBackString + ". This indicates a bug.");
+ }
+ }
+ }
+ checkFinishBundleTimer.cancel(true);
doFnInvoker.invokeTeardown();
}
- protected final long getPushbackWatermarkHold() {
+ private long getPushbackWatermarkHold() {
// if we don't have side inputs we never hold the watermark
if (sideInputs.isEmpty()) {
- return BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis();
+ return Long.MAX_VALUE;
}
try {
@@ -333,9 +403,9 @@
if (!pushedBackWatermark.isPresent()) {
BagState<WindowedValue<InputT>> pushedBack =
- pushbackStateInternals.state(StateNamespaces.global(), pushedBackTag);
+ nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag);
- long min = BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis();
+ long min = Long.MAX_VALUE;
for (WindowedValue<InputT> value : pushedBack.read()) {
min = Math.min(min, value.getTimestamp().getMillis());
}
@@ -346,9 +416,9 @@
@Override
public final void processElement(
StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception {
- doFnRunner.startBundle();
+ checkInvokeStartBundle();
doFnRunner.processElement(streamRecord.getValue());
- doFnRunner.finishBundle();
+ checkInvokeFinishBundleByCount();
}
private void setPushedBackWatermark(long watermark) {
@@ -358,12 +428,12 @@
@Override
public final void processElement1(
StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception {
- pushbackDoFnRunner.startBundle();
+ checkInvokeStartBundle();
Iterable<WindowedValue<InputT>> justPushedBack =
pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue());
BagState<WindowedValue<InputT>> pushedBack =
- pushbackStateInternals.state(StateNamespaces.global(), pushedBackTag);
+ nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag);
checkInitPushedBackWatermark();
@@ -373,13 +443,13 @@
pushedBack.add(pushedBackValue);
}
setPushedBackWatermark(min);
- pushbackDoFnRunner.finishBundle();
+ checkInvokeFinishBundleByCount();
}
@Override
public final void processElement2(
StreamRecord<RawUnionValue> streamRecord) throws Exception {
- pushbackDoFnRunner.startBundle();
+ checkInvokeStartBundle();
@SuppressWarnings("unchecked")
WindowedValue<Iterable<?>> value =
@@ -389,7 +459,7 @@
sideInputHandler.addSideInputValue(sideInput, value);
BagState<WindowedValue<InputT>> pushedBack =
- pushbackStateInternals.state(StateNamespaces.global(), pushedBackTag);
+ nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag);
List<WindowedValue<InputT>> newPushedBack = new ArrayList<>();
@@ -408,14 +478,14 @@
}
pushedBack.clear();
- long min = BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis();
+ long min = Long.MAX_VALUE;
for (WindowedValue<InputT> pushedBackValue : newPushedBack) {
min = Math.min(min, pushedBackValue.getTimestamp().getMillis());
pushedBack.add(pushedBackValue);
}
setPushedBackWatermark(min);
- pushbackDoFnRunner.finishBundle();
+ checkInvokeFinishBundleByCount();
// maybe output a new watermark
processWatermark1(new Watermark(currentInputWatermark));
@@ -428,71 +498,165 @@
@Override
public void processWatermark1(Watermark mark) throws Exception {
+
+ checkInvokeStartBundle();
+
+ // We do the check here because we are guaranteed to at least get the +Inf watermark on the
+ // main input when the job finishes.
+ if (currentSideInputWatermark >= BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ // this means we will never see any more side input
+ // we also do the check here because we might have received the side-input MAX watermark
+ // before receiving any main-input data
+ emitAllPushedBackData();
+ }
+
if (keyCoder == null) {
setCurrentInputWatermark(mark.getTimestamp());
long potentialOutputWatermark =
Math.min(getPushbackWatermarkHold(), currentInputWatermark);
if (potentialOutputWatermark > currentOutputWatermark) {
setCurrentOutputWatermark(potentialOutputWatermark);
- output.emitWatermark(new Watermark(currentOutputWatermark));
+ emitWatermark(currentOutputWatermark);
}
} else {
- // fireTimers, so we need startBundle.
- pushbackDoFnRunner.startBundle();
-
setCurrentInputWatermark(mark.getTimestamp());
// hold back by the pushed back values waiting for side inputs
- long actualInputWatermark = Math.min(getPushbackWatermarkHold(), mark.getTimestamp());
+ long pushedBackInputWatermark = Math.min(getPushbackWatermarkHold(), mark.getTimestamp());
- timerService.advanceWatermark(actualInputWatermark);
+ timerService.advanceWatermark(toFlinkRuntimeWatermark(pushedBackInputWatermark));
- Instant watermarkHold = stateInternals.watermarkHold();
+ Instant watermarkHold = keyedStateInternals.watermarkHold();
long combinedWatermarkHold = Math.min(watermarkHold.getMillis(), getPushbackWatermarkHold());
- long potentialOutputWatermark = Math.min(currentInputWatermark, combinedWatermarkHold);
+ long potentialOutputWatermark = Math.min(pushedBackInputWatermark, combinedWatermarkHold);
if (potentialOutputWatermark > currentOutputWatermark) {
setCurrentOutputWatermark(potentialOutputWatermark);
- output.emitWatermark(new Watermark(currentOutputWatermark));
+ emitWatermark(currentOutputWatermark);
}
- pushbackDoFnRunner.finishBundle();
}
}
+ private void emitWatermark(long watermark) {
+ // Must invoke finishBatch before emit the +Inf watermark otherwise there are some late events.
+ if (watermark >= BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ invokeFinishBundle();
+ }
+ output.emitWatermark(new Watermark(watermark));
+ }
+
@Override
public void processWatermark2(Watermark mark) throws Exception {
- if (mark.getTimestamp() == BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ checkInvokeStartBundle();
+
+ setCurrentSideInputWatermark(mark.getTimestamp());
+ if (mark.getTimestamp() >= BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
// this means we will never see any more side input
- pushbackDoFnRunner.startBundle();
-
- BagState<WindowedValue<InputT>> pushedBack =
- pushbackStateInternals.state(StateNamespaces.global(), pushedBackTag);
-
- Iterable<WindowedValue<InputT>> pushedBackContents = pushedBack.read();
- if (pushedBackContents != null) {
- for (WindowedValue<InputT> elem : pushedBackContents) {
-
- // we need to set the correct key in case the operator is
- // a (keyed) window operator
- setKeyContextElement1(new StreamRecord<>(elem));
-
- doFnRunner.processElement(elem);
- }
- }
-
- setPushedBackWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis());
-
- pushbackDoFnRunner.finishBundle();
+ emitAllPushedBackData();
// maybe output a new watermark
processWatermark1(new Watermark(currentInputWatermark));
}
+
+ }
+
+ /**
+ * Converts a Beam watermark to a Flink watermark. This is only relevant when considering what
+ * event-time timers to fire: in Beam, a watermark {@code T} says there will not be any elements
+ * with a timestamp {@code < T} in the future. A Flink watermark {@code T} says there will not be
+ * any elements with a timestamp {@code <= T} in the future. We correct this by subtracting
+ * {@code 1} from a Beam watermark before passing to any relevant Flink runtime components.
+ */
+ private static long toFlinkRuntimeWatermark(long beamWatermark) {
+ return beamWatermark - 1;
+ }
+
+ /**
+ * Emits all pushed-back data. This should be used once we know that there will not be
+ * any future side input, i.e. that there is no point in waiting.
+ */
+ private void emitAllPushedBackData() throws Exception {
+
+ BagState<WindowedValue<InputT>> pushedBack =
+ nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag);
+
+ Iterable<WindowedValue<InputT>> pushedBackContents = pushedBack.read();
+ if (pushedBackContents != null) {
+ for (WindowedValue<InputT> elem : pushedBackContents) {
+
+ // we need to set the correct key in case the operator is
+ // a (keyed) window operator
+ setKeyContextElement1(new StreamRecord<>(elem));
+
+ doFnRunner.processElement(elem);
+ }
+ }
+
+ pushedBack.clear();
+
+ setPushedBackWatermark(Long.MAX_VALUE);
+
+ }
+
+ /**
+ * Check whether invoke startBundle, if it is, need to output elements that were
+ * buffered as part of finishing a bundle in snapshot() first.
+ *
+ * <p>In order to avoid having {@link DoFnRunner#processElement(WindowedValue)} or
+ * {@link DoFnRunner#onTimer(String, BoundedWindow, Instant, TimeDomain)} not between
+ * StartBundle and FinishBundle, this method needs to be called in each processElement
+ * and each processWatermark and onProcessingTime. Do not need to call in onEventTime,
+ * because it has been guaranteed in the processWatermark.
+ */
+ private void checkInvokeStartBundle() {
+ if (!bundleStarted) {
+ outputManager.flushBuffer();
+ pushbackDoFnRunner.startBundle();
+ bundleStarted = true;
+ }
+ }
+
+ /**
+ * Check whether invoke finishBundle by elements count. Called in processElement.
+ */
+ private void checkInvokeFinishBundleByCount() {
+ elementCount++;
+ if (elementCount >= maxBundleSize) {
+ invokeFinishBundle();
+ }
+ }
+
+ /**
+ * Check whether invoke finishBundle by timeout.
+ */
+ private void checkInvokeFinishBundleByTime() {
+ long now = getProcessingTimeService().getCurrentProcessingTime();
+ if (now - lastFinishBundleTime >= maxBundleTimeMills) {
+ invokeFinishBundle();
+ }
+ }
+
+ private void invokeFinishBundle() {
+ if (bundleStarted) {
+ pushbackDoFnRunner.finishBundle();
+ bundleStarted = false;
+ elementCount = 0L;
+ lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime();
+ }
}
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
+
+ // Forced finish a bundle in checkpoint barrier otherwise may lose data.
+ // Careful, it use OperatorState or KeyGroupState to store outputs, So it
+ // must be called before their snapshot.
+ outputManager.openBuffer();
+ invokeFinishBundle();
+ outputManager.closeBuffer();
+
// copy from AbstractStreamOperator
if (getKeyedStateBackend() != null) {
KeyedStateCheckpointOutputStream out;
@@ -538,8 +702,8 @@
@Override
public void snapshotKeyGroupState(int keyGroupIndex, DataOutputStream out) throws Exception {
- if (!sideInputs.isEmpty() && keyCoder != null) {
- ((FlinkKeyGroupStateInternals) pushbackStateInternals).snapshotKeyGroupState(
+ if (keyCoder != null) {
+ ((FlinkKeyGroupStateInternals) nonKeyedStateInternals).snapshotKeyGroupState(
keyGroupIndex, out);
}
}
@@ -577,23 +741,26 @@
@Override
public void restoreKeyGroupState(int keyGroupIndex, DataInputStream in) throws Exception {
- if (!sideInputs.isEmpty() && keyCoder != null) {
- if (pushbackStateInternals == null) {
- pushbackStateInternals = new FlinkKeyGroupStateInternals<>(keyCoder,
+ if (keyCoder != null) {
+ if (nonKeyedStateInternals == null) {
+ nonKeyedStateInternals = new FlinkKeyGroupStateInternals<>(keyCoder,
getKeyedStateBackend());
}
- ((FlinkKeyGroupStateInternals) pushbackStateInternals)
+ ((FlinkKeyGroupStateInternals) nonKeyedStateInternals)
.restoreKeyGroupState(keyGroupIndex, in, getUserCodeClassloader());
}
}
@Override
public void onEventTime(InternalTimer<Object, TimerData> timer) throws Exception {
+ // We don't have to cal checkInvokeStartBundle() because it's already called in
+ // processWatermark*().
fireTimer(timer);
}
@Override
public void onProcessingTime(InternalTimer<Object, TimerData> timer) throws Exception {
+ checkInvokeStartBundle();
fireTimer(timer);
}
@@ -612,63 +779,195 @@
this.currentInputWatermark = currentInputWatermark;
}
+ private void setCurrentSideInputWatermark(long currentInputWatermark) {
+ this.currentSideInputWatermark = currentInputWatermark;
+ }
+
private void setCurrentOutputWatermark(long currentOutputWatermark) {
this.currentOutputWatermark = currentOutputWatermark;
}
/**
- * Factory for creating an {@link DoFnRunners.OutputManager} from
+ * Factory for creating an {@link BufferedOutputManager} from
* a Flink {@link Output}.
*/
interface OutputManagerFactory<OutputT> extends Serializable {
- DoFnRunners.OutputManager create(Output<StreamRecord<OutputT>> output);
+ BufferedOutputManager<OutputT> create(
+ Output<StreamRecord<WindowedValue<OutputT>>> output,
+ StateInternals stateInternals);
}
/**
- * Default implementation of {@link OutputManagerFactory} that creates an
- * {@link DoFnRunners.OutputManager} that only writes to
- * a single logical output.
+ * A {@link DoFnRunners.OutputManager} that can buffer its outputs.
+ * Use {@link FlinkSplitStateInternals} or {@link FlinkKeyGroupStateInternals}
+ * to keep buffer data.
*/
- public static class DefaultOutputManagerFactory<OutputT>
- implements OutputManagerFactory<OutputT> {
+ public static class BufferedOutputManager<OutputT> implements
+ DoFnRunners.OutputManager {
+
+ private TupleTag<OutputT> mainTag;
+ private Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags;
+ private Map<TupleTag<?>, Integer> tagsToIds;
+ private Map<Integer, TupleTag<?>> idsToTags;
+ protected Output<StreamRecord<WindowedValue<OutputT>>> output;
+
+ private boolean openBuffer = false;
+ private BagState<KV<Integer, WindowedValue<?>>> bufferState;
+
+ BufferedOutputManager(
+ Output<StreamRecord<WindowedValue<OutputT>>> output,
+ TupleTag<OutputT> mainTag,
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
+ final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders,
+ Map<TupleTag<?>, Integer> tagsToIds,
+ StateInternals stateInternals) {
+ this.output = output;
+ this.mainTag = mainTag;
+ this.tagsToOutputTags = tagsToOutputTags;
+ this.tagsToIds = tagsToIds;
+ this.idsToTags = new HashMap<>();
+ for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) {
+ idsToTags.put(entry.getValue(), entry.getKey());
+ }
+
+ ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder =
+ ImmutableMap.builder();
+ for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) {
+ idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey()));
+ }
+
+ StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag =
+ StateTags.bag("bundle-buffer-tag",
+ new TaggedKvCoder(idsToCodersBuilder.build()));
+ bufferState = stateInternals.state(StateNamespaces.global(), bufferTag);
+ }
+
+ void openBuffer() {
+ this.openBuffer = true;
+ }
+
+ void closeBuffer() {
+ this.openBuffer = false;
+ }
+
@Override
- public DoFnRunners.OutputManager create(final Output<StreamRecord<OutputT>> output) {
- return new DoFnRunners.OutputManager() {
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> value) {
- // with tagged outputs we can't get around this because we don't
- // know our own output type...
- @SuppressWarnings("unchecked")
- OutputT castValue = (OutputT) value;
- output.collect(new StreamRecord<>(castValue));
- }
- };
+ public <T> void output(TupleTag<T> tag, WindowedValue<T> value) {
+ if (!openBuffer) {
+ emit(tag, value);
+ } else {
+ bufferState.add(KV.<Integer, WindowedValue<?>>of(tagsToIds.get(tag), value));
+ }
+ }
+
+ /**
+ * Flush elements of bufferState to Flink Output. This method can't be invoke in
+ * {@link #snapshotState(StateSnapshotContext)}
+ */
+ void flushBuffer() {
+ for (KV<Integer, WindowedValue<?>> taggedElem : bufferState.read()) {
+ emit(idsToTags.get(taggedElem.getKey()), (WindowedValue) taggedElem.getValue());
+ }
+ bufferState.clear();
+ }
+
+ private <T> void emit(TupleTag<T> tag, WindowedValue<T> value) {
+ if (tag.equals(mainTag)) {
+ // with tagged outputs we can't get around this because we don't
+ // know our own output type...
+ @SuppressWarnings("unchecked")
+ WindowedValue<OutputT> castValue = (WindowedValue<OutputT>) value;
+ output.collect(new StreamRecord<>(castValue));
+ } else {
+ @SuppressWarnings("unchecked")
+ OutputTag<WindowedValue<T>> outputTag = (OutputTag) tagsToOutputTags.get(tag);
+ output.collect(outputTag, new StreamRecord<>(value));
+ }
+ }
+ }
+
+ /**
+ * Coder for KV of id and value. It will be serialized in Flink checkpoint.
+ */
+ private static class TaggedKvCoder extends StructuredCoder<KV<Integer, WindowedValue<?>>> {
+
+ private Map<Integer, Coder<WindowedValue<?>>> idsToCoders;
+
+ TaggedKvCoder(Map<Integer, Coder<WindowedValue<?>>> idsToCoders) {
+ this.idsToCoders = idsToCoders;
+ }
+
+ @Override
+ public void encode(KV<Integer, WindowedValue<?>> kv, OutputStream out)
+ throws IOException {
+ Coder<WindowedValue<?>> coder = idsToCoders.get(kv.getKey());
+ VarIntCoder.of().encode(kv.getKey(), out);
+ coder.encode(kv.getValue(), out);
+ }
+
+ @Override
+ public KV<Integer, WindowedValue<?>> decode(InputStream in)
+ throws IOException {
+ Integer id = VarIntCoder.of().decode(in);
+ Coder<WindowedValue<?>> coder = idsToCoders.get(id);
+ WindowedValue<?> value = coder.decode(in);
+ return KV.<Integer, WindowedValue<?>>of(id, value);
+ }
+
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return new ArrayList<>(idsToCoders.values());
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ for (Coder<?> coder : idsToCoders.values()) {
+ verifyDeterministic(this, "Coder must be deterministic", coder);
+ }
}
}
/**
* Implementation of {@link OutputManagerFactory} that creates an
- * {@link DoFnRunners.OutputManager} that can write to multiple logical
- * outputs by unioning them in a {@link RawUnionValue}.
+ * {@link BufferedOutputManager} that can write to multiple logical
+ * outputs by Flink side output.
*/
- public static class MultiOutputOutputManagerFactory
- implements OutputManagerFactory<RawUnionValue> {
+ public static class MultiOutputOutputManagerFactory<OutputT>
+ implements OutputManagerFactory<OutputT> {
- Map<TupleTag<?>, Integer> mapping;
+ private TupleTag<OutputT> mainTag;
+ private Map<TupleTag<?>, Integer> tagsToIds;
+ private Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags;
+ private Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders;
- public MultiOutputOutputManagerFactory(Map<TupleTag<?>, Integer> mapping) {
- this.mapping = mapping;
+ // There is no side output.
+ @SuppressWarnings("unchecked")
+ public MultiOutputOutputManagerFactory(
+ TupleTag<OutputT> mainTag, Coder<WindowedValue<OutputT>> mainCoder) {
+ this(mainTag,
+ new HashMap<TupleTag<?>, OutputTag<WindowedValue<?>>>(),
+ ImmutableMap.<TupleTag<?>, Coder<WindowedValue<?>>>builder()
+ .put(mainTag, (Coder) mainCoder).build(),
+ ImmutableMap.<TupleTag<?>, Integer>builder()
+ .put(mainTag, 0).build());
+ }
+
+ public MultiOutputOutputManagerFactory(
+ TupleTag<OutputT> mainTag,
+ Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders,
+ Map<TupleTag<?>, Integer> tagsToIds) {
+ this.mainTag = mainTag;
+ this.tagsToOutputTags = tagsToOutputTags;
+ this.tagsToCoders = tagsToCoders;
+ this.tagsToIds = tagsToIds;
}
@Override
- public DoFnRunners.OutputManager create(final Output<StreamRecord<RawUnionValue>> output) {
- return new DoFnRunners.OutputManager() {
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> value) {
- int intTag = mapping.get(tag);
- output.collect(new StreamRecord<>(new RawUnionValue(intTag, value)));
- }
- };
+ public BufferedOutputManager<OutputT> create(
+ Output<StreamRecord<WindowedValue<OutputT>>> output,
+ StateInternals stateInternals) {
+ return new BufferedOutputManager<>(
+ output, mainTag, tagsToOutputTags, tagsToCoders, tagsToIds, stateInternals);
}
}
@@ -676,37 +975,11 @@
* {@link StepContext} for running {@link DoFn DoFns} on Flink. This does not allow
* accessing state or timer internals.
*/
- protected class StepContext implements ExecutionContext.StepContext {
-
- @Override
- public String getStepName() {
- return null;
- }
-
- @Override
- public String getTransformName() {
- return null;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {}
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window,
- Coder<W> windowCoder) throws IOException {
- throw new UnsupportedOperationException("Writing side-input data is not supported.");
- }
+ protected class FlinkStepContext implements StepContext {
@Override
public StateInternals stateInternals() {
- return stateInternals;
+ return keyedStateInternals;
}
@Override
@@ -723,6 +996,9 @@
setTimer(TimerData.of(timerId, namespace, target, timeDomain));
}
+ /**
+ * @deprecated use {@link #setTimer(StateNamespace, String, Instant, TimeDomain)}.
+ */
@Deprecated
@Override
public void setTimer(TimerData timerKey) {
@@ -741,6 +1017,9 @@
}
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(StateNamespace namespace, String timerId) {
@@ -754,6 +1033,9 @@
"Canceling of a timer by ID is not yet supported.");
}
+ /**
+ * @deprecated use {@link #deleteTimer(StateNamespace, String, TimeDomain)}.
+ */
@Deprecated
@Override
public void deleteTimer(TimerData timerKey) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
index 968fc0a..b255bb4 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
@@ -26,12 +26,11 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import org.apache.beam.runners.core.ElementAndRestriction;
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.KeyedWorkItems;
import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
import org.apache.beam.runners.core.OutputWindowedValue;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateInternalsFactory;
import org.apache.beam.runners.core.TimerInternals;
@@ -43,6 +42,7 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -55,19 +55,16 @@
* the {@code @ProcessElement} method of a splittable {@link DoFn}.
*/
public class SplittableDoFnOperator<
- InputT, FnOutputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
- extends DoFnOperator<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, FnOutputT, OutputT> {
+ InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+ extends DoFnOperator<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> {
private transient ScheduledExecutorService executorService;
public SplittableDoFnOperator(
- DoFn<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, FnOutputT> doFn,
+ DoFn<KeyedWorkItem<String, KV<InputT, RestrictionT>>, OutputT> doFn,
String stepName,
- Coder<
- WindowedValue<
- KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>> inputCoder,
- TupleTag<FnOutputT> mainOutputTag,
+ Coder<WindowedValue<KeyedWorkItem<String, KV<InputT, RestrictionT>>>> inputCoder,
+ TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> additionalOutputTags,
OutputManagerFactory<OutputT> outputManagerFactory,
WindowingStrategy<?, ?> windowingStrategy,
@@ -87,21 +84,20 @@
sideInputs,
options,
keyCoder);
-
}
@Override
public void open() throws Exception {
super.open();
- checkState(doFn instanceof SplittableParDo.ProcessFn);
+ checkState(doFn instanceof ProcessFn);
StateInternalsFactory<String> stateInternalsFactory = new StateInternalsFactory<String>() {
@Override
public StateInternals stateInternalsForKey(String key) {
//this will implicitly be keyed by the key of the incoming
// element or by the key of a firing timer
- return (StateInternals) stateInternals;
+ return (StateInternals) keyedStateInternals;
}
};
TimerInternalsFactory<String> timerInternalsFactory = new TimerInternalsFactory<String>() {
@@ -114,16 +110,16 @@
executorService = Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory());
- ((SplittableParDo.ProcessFn) doFn).setStateInternalsFactory(stateInternalsFactory);
- ((SplittableParDo.ProcessFn) doFn).setTimerInternalsFactory(timerInternalsFactory);
- ((SplittableParDo.ProcessFn) doFn).setProcessElementInvoker(
+ ((ProcessFn) doFn).setStateInternalsFactory(stateInternalsFactory);
+ ((ProcessFn) doFn).setTimerInternalsFactory(timerInternalsFactory);
+ ((ProcessFn) doFn).setProcessElementInvoker(
new OutputAndTimeBoundedSplittableProcessElementInvoker<>(
doFn,
- serializedOptions.getPipelineOptions(),
- new OutputWindowedValue<FnOutputT>() {
+ serializedOptions.get(),
+ new OutputWindowedValue<OutputT>() {
@Override
public void outputWindowedValue(
- FnOutputT output,
+ OutputT output,
Instant timestamp,
Collection<? extends BoundedWindow> windows,
PaneInfo pane) {
@@ -151,8 +147,8 @@
@Override
public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) {
doFnRunner.processElement(WindowedValue.valueInGlobalWindow(
- KeyedWorkItems.<String, ElementAndRestriction<InputT, RestrictionT>>timersWorkItem(
- (String) stateInternals.getKey(),
+ KeyedWorkItems.<String, KV<InputT, RestrictionT>>timersWorkItem(
+ (String) keyedStateInternals.getKey(),
Collections.singletonList(timer.getNamespace()))));
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
index bf64ede..b1fb398 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
@@ -46,7 +46,7 @@
* Flink operator for executing window {@link DoFn DoFns}.
*/
public class WindowDoFnOperator<K, InputT, OutputT>
- extends DoFnOperator<KeyedWorkItem<K, InputT>, KV<K, OutputT>, WindowedValue<KV<K, OutputT>>> {
+ extends DoFnOperator<KeyedWorkItem<K, InputT>, KV<K, OutputT>> {
private final SystemReduceFn<K, InputT, ?, OutputT, BoundedWindow> systemReduceFn;
@@ -56,7 +56,7 @@
Coder<WindowedValue<KeyedWorkItem<K, InputT>>> inputCoder,
TupleTag<KV<K, OutputT>> mainOutputTag,
List<TupleTag<?>> additionalOutputTags,
- OutputManagerFactory<WindowedValue<KV<K, OutputT>>> outputManagerFactory,
+ OutputManagerFactory<KV<K, OutputT>> outputManagerFactory,
WindowingStrategy<?, ?> windowingStrategy,
Map<Integer, PCollectionView<?>> sideInputTagMapping,
Collection<PCollectionView<?>> sideInputs,
@@ -86,7 +86,7 @@
public StateInternals stateInternalsForKey(K key) {
//this will implicitly be keyed by the key of the incoming
// element or by the key of a firing timer
- return (StateInternals) stateInternals;
+ return (StateInternals) keyedStateInternals;
}
};
TimerInternalsFactory<K> timerInternalsFactory = new TimerInternalsFactory<K>() {
@@ -112,7 +112,7 @@
public void fireTimer(InternalTimer<?, TimerData> timer) {
doFnRunner.processElement(WindowedValue.valueInGlobalWindow(
KeyedWorkItems.<K, InputT>timersWorkItem(
- (K) stateInternals.getKey(),
+ (K) keyedStateInternals.getKey(),
Collections.singletonList(timer.getNamespace()))));
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
index 6d75688..5ddc46f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
@@ -20,9 +20,9 @@
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.List;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.metrics.FlinkMetricContainer;
import org.apache.beam.runners.flink.metrics.ReaderInvocationUtil;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -48,7 +48,7 @@
/**
* Keep the options so that we can initialize the readers.
*/
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
/**
* The split sources. We split them in the constructor to ensure that all parallel
@@ -74,7 +74,7 @@
BoundedSource<OutputT> source,
int parallelism) throws Exception {
this.stepName = stepName;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
long desiredBundleSize = source.getEstimatedSizeBytes(pipelineOptions) / parallelism;
@@ -109,13 +109,13 @@
ReaderInvocationUtil<OutputT, BoundedSource.BoundedReader<OutputT>> readerInvoker =
new ReaderInvocationUtil<>(
stepName,
- serializedOptions.getPipelineOptions(),
+ serializedOptions.get(),
metricContainer);
readers = new ArrayList<>();
// initialize readers from scratch
for (BoundedSource<OutputT> source : localSources) {
- readers.add(source.createReader(serializedOptions.getPipelineOptions()));
+ readers.add(source.createReader(serializedOptions.get()));
}
if (readers.size() == 1) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSocketSource.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSocketSource.java
index 910a33f..49e4ddc 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSocketSource.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSocketSource.java
@@ -123,7 +123,7 @@
}
@Override
- public Coder getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return DEFAULT_SOCKET_CODER;
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
index ec21699..817dd74 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
@@ -22,15 +22,16 @@
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.flink.metrics.FlinkMetricContainer;
import org.apache.beam.runners.flink.metrics.ReaderInvocationUtil;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -71,7 +72,7 @@
/**
* Keep the options so that we can initialize the localReaders.
*/
- private final SerializedPipelineOptions serializedOptions;
+ private final SerializablePipelineOptions serializedOptions;
/**
* For snapshot and restore.
@@ -140,7 +141,7 @@
UnboundedSource<OutputT, CheckpointMarkT> source,
int parallelism) throws Exception {
this.stepName = stepName;
- this.serializedOptions = new SerializedPipelineOptions(pipelineOptions);
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
if (source.requiresDeduping()) {
LOG.warn("Source {} requires deduping but Flink runner doesn't support this yet.", source);
@@ -188,7 +189,7 @@
stateForCheckpoint.get()) {
localSplitSources.add(restored.getKey());
localReaders.add(restored.getKey().createReader(
- serializedOptions.getPipelineOptions(), restored.getValue()));
+ serializedOptions.get(), restored.getValue()));
}
} else {
// initialize localReaders and localSources from scratch
@@ -197,7 +198,7 @@
UnboundedSource<OutputT, CheckpointMarkT> source =
splitSources.get(i);
UnboundedSource.UnboundedReader<OutputT> reader =
- source.createReader(serializedOptions.getPipelineOptions(), null);
+ source.createReader(serializedOptions.get(), null);
localSplitSources.add(source);
localReaders.add(reader);
}
@@ -220,7 +221,7 @@
ReaderInvocationUtil<OutputT, UnboundedSource.UnboundedReader<OutputT>> readerInvoker =
new ReaderInvocationUtil<>(
stepName,
- serializedOptions.getPipelineOptions(),
+ serializedOptions.get(),
metricContainer);
if (localReaders.size() == 0) {
@@ -283,6 +284,8 @@
}
}
+ setNextWatermarkTimer(this.runtimeContext);
+
// a flag telling us whether any of the localReaders had data
// if no reader had data, sleep for bit
boolean hadData = false;
@@ -434,6 +437,10 @@
}
}
context.emitWatermark(new Watermark(watermarkMillis));
+
+ if (watermarkMillis >= BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ this.isRunning = false;
+ }
}
setNextWatermarkTimer(this.runtimeContext);
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
index f44e668..6cc2429 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
@@ -49,11 +49,11 @@
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
+import org.apache.flink.api.common.state.OperatorStateStore;
import org.apache.flink.runtime.state.OperatorStateBackend;
/**
- * {@link StateInternals} that uses a Flink {@link DefaultOperatorStateBackend}
+ * {@link StateInternals} that uses a Flink {@link OperatorStateBackend}
* to manage the broadcast state.
* The state is the same on all parallel instances of the operator.
* So we just need store state of operator-0 in OperatorStateBackend.
@@ -64,13 +64,12 @@
public class FlinkBroadcastStateInternals<K> implements StateInternals {
private int indexInSubtaskGroup;
- private final DefaultOperatorStateBackend stateBackend;
+ private final OperatorStateBackend stateBackend;
// stateName -> <namespace, state>
private Map<String, Map<String, ?>> stateForNonZeroOperator;
public FlinkBroadcastStateInternals(int indexInSubtaskGroup, OperatorStateBackend stateBackend) {
- //TODO flink do not yet expose through public API
- this.stateBackend = (DefaultOperatorStateBackend) stateBackend;
+ this.stateBackend = stateBackend;
this.indexInSubtaskGroup = indexInSubtaskGroup;
if (indexInSubtaskGroup != 0) {
stateForNonZeroOperator = new HashMap<>();
@@ -178,10 +177,10 @@
private String name;
private final StateNamespace namespace;
private final ListStateDescriptor<Map<String, T>> flinkStateDescriptor;
- private final DefaultOperatorStateBackend flinkStateBackend;
+ private final OperatorStateStore flinkStateBackend;
AbstractBroadcastState(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
String name,
StateNamespace namespace,
Coder<T> coder) {
@@ -211,7 +210,7 @@
if (result != null) {
stateForNonZeroOperator.put(name, result);
// we don't need it anymore, must clear it.
- flinkStateBackend.getBroadcastOperatorState(
+ flinkStateBackend.getUnionListState(
flinkStateDescriptor).clear();
}
}
@@ -220,7 +219,7 @@
}
Map<String, T> getMapFromBroadcastState() throws Exception {
- ListState<Map<String, T>> state = flinkStateBackend.getBroadcastOperatorState(
+ ListState<Map<String, T>> state = flinkStateBackend.getUnionListState(
flinkStateDescriptor);
Iterable<Map<String, T>> iterable = state.get();
Map<String, T> ret = null;
@@ -239,7 +238,7 @@
*/
void updateMap(Map<String, T> map) throws Exception {
if (indexInSubtaskGroup == 0) {
- ListState<Map<String, T>> state = flinkStateBackend.getBroadcastOperatorState(
+ ListState<Map<String, T>> state = flinkStateBackend.getUnionListState(
flinkStateDescriptor);
state.clear();
if (map.size() > 0) {
@@ -304,7 +303,7 @@
private final StateTag<ValueState<T>> address;
FlinkBroadcastValueState(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
StateTag<ValueState<T>> address,
StateNamespace namespace,
Coder<T> coder) {
@@ -365,7 +364,7 @@
private final StateTag<BagState<T>> address;
FlinkBroadcastBagState(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
StateTag<BagState<T>> address,
StateNamespace namespace,
Coder<T> coder) {
@@ -454,7 +453,7 @@
private final Combine.CombineFn<InputT, AccumT, OutputT> combineFn;
FlinkCombiningState(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
StateTag<CombiningState<InputT, AccumT, OutputT>> address,
Combine.CombineFn<InputT, AccumT, OutputT> combineFn,
StateNamespace namespace,
@@ -572,7 +571,7 @@
private final FlinkBroadcastStateInternals<K> flinkStateInternals;
FlinkKeyedCombiningState(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
StateTag<CombiningState<InputT, AccumT, OutputT>> address,
Combine.CombineFn<InputT, AccumT, OutputT> combineFn,
StateNamespace namespace,
@@ -709,7 +708,7 @@
private final CombineWithContext.Context context;
FlinkCombiningStateWithContext(
- DefaultOperatorStateBackend flinkStateBackend,
+ OperatorStateBackend flinkStateBackend,
StateTag<CombiningState<InputT, AccumT, OutputT>> address,
CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn,
StateNamespace namespace,
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
index bb2a9ff..09e59fd 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
@@ -167,7 +167,7 @@
@Override
public void add(T input) {
try {
- flinkStateBackend.getOperatorState(descriptor).add(input);
+ flinkStateBackend.getListState(descriptor).add(input);
} catch (Exception e) {
throw new RuntimeException("Error updating state.", e);
}
@@ -181,7 +181,7 @@
@Override
public Iterable<T> read() {
try {
- Iterable<T> result = flinkStateBackend.getOperatorState(descriptor).get();
+ Iterable<T> result = flinkStateBackend.getListState(descriptor).get();
return result != null ? result : Collections.<T>emptyList();
} catch (Exception e) {
throw new RuntimeException("Error updating state.", e);
@@ -194,7 +194,7 @@
@Override
public Boolean read() {
try {
- Iterable<T> result = flinkStateBackend.getOperatorState(descriptor).get();
+ Iterable<T> result = flinkStateBackend.getListState(descriptor).get();
// PartitionableListState.get() return empty collection When there is no element,
// KeyedListState different. (return null)
return result == null || Iterators.size(result.iterator()) == 0;
@@ -214,7 +214,7 @@
@Override
public void clear() {
try {
- flinkStateBackend.getOperatorState(descriptor).clear();
+ flinkStateBackend.getListState(descriptor).clear();
} catch (Exception e) {
throw new RuntimeException("Error reading state.", e);
}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
index 9cb742ee..bbe79db 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.runners.flink.translation.wrappers.streaming.state;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -25,7 +27,7 @@
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
+import org.apache.beam.runners.flink.translation.types.CoderTypeSerializer;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.InstantCoder;
@@ -33,6 +35,7 @@
import org.apache.beam.sdk.state.CombiningState;
import org.apache.beam.sdk.state.MapState;
import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ReadableStates;
import org.apache.beam.sdk.state.SetState;
import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.StateContext;
@@ -46,7 +49,9 @@
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.CombineContextFactory;
import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeutils.base.BooleanSerializer;
import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.runtime.state.KeyedStateBackend;
import org.joda.time.Instant;
@@ -126,17 +131,17 @@
@Override
public <T> SetState<T> bindSet(
StateTag<SetState<T>> address, Coder<T> elemCoder) {
- throw new UnsupportedOperationException(
- String.format("%s is not supported", SetState.class.getSimpleName()));
+ return new FlinkSetState<>(
+ flinkStateBackend, address, namespace, elemCoder);
}
@Override
public <KeyT, ValueT> MapState<KeyT, ValueT> bindMap(
- StateTag<MapState<KeyT, ValueT>> spec,
+ StateTag<MapState<KeyT, ValueT>> address,
Coder<KeyT> mapKeyCoder,
Coder<ValueT> mapValueCoder) {
- throw new UnsupportedOperationException(
- String.format("%s is not supported", MapState.class.getSimpleName()));
+ return new FlinkMapState<>(
+ flinkStateBackend, address, namespace, mapKeyCoder, mapValueCoder);
}
@Override
@@ -194,9 +199,8 @@
this.address = address;
this.flinkStateBackend = flinkStateBackend;
- CoderTypeInformation<T> typeInfo = new CoderTypeInformation<>(coder);
-
- flinkStateDescriptor = new ValueStateDescriptor<>(address.getId(), typeInfo, null);
+ flinkStateDescriptor = new ValueStateDescriptor<>(
+ address.getId(), new CoderTypeSerializer<>(coder));
}
@Override
@@ -280,9 +284,8 @@
this.address = address;
this.flinkStateBackend = flinkStateBackend;
- CoderTypeInformation<T> typeInfo = new CoderTypeInformation<>(coder);
-
- flinkStateDescriptor = new ListStateDescriptor<>(address.getId(), typeInfo);
+ flinkStateDescriptor = new ListStateDescriptor<>(
+ address.getId(), new CoderTypeSerializer<>(coder));
}
@Override
@@ -310,7 +313,7 @@
StringSerializer.INSTANCE,
flinkStateDescriptor).get();
- return result != null ? result : Collections.<T>emptyList();
+ return result != null ? ImmutableList.copyOf(result) : Collections.<T>emptyList();
} catch (Exception e) {
throw new RuntimeException("Error reading state.", e);
}
@@ -396,9 +399,8 @@
this.combineFn = combineFn;
this.flinkStateBackend = flinkStateBackend;
- CoderTypeInformation<AccumT> typeInfo = new CoderTypeInformation<>(accumCoder);
-
- flinkStateDescriptor = new ValueStateDescriptor<>(address.getId(), typeInfo, null);
+ flinkStateDescriptor = new ValueStateDescriptor<>(
+ address.getId(), new CoderTypeSerializer<>(accumCoder));
}
@Override
@@ -543,179 +545,6 @@
}
}
- private static class FlinkKeyedCombiningState<K, InputT, AccumT, OutputT>
- implements CombiningState<InputT, AccumT, OutputT> {
-
- private final StateNamespace namespace;
- private final StateTag<CombiningState<InputT, AccumT, OutputT>> address;
- private final Combine.CombineFn<InputT, AccumT, OutputT> combineFn;
- private final ValueStateDescriptor<AccumT> flinkStateDescriptor;
- private final KeyedStateBackend<ByteBuffer> flinkStateBackend;
- private final FlinkStateInternals<K> flinkStateInternals;
-
- FlinkKeyedCombiningState(
- KeyedStateBackend<ByteBuffer> flinkStateBackend,
- StateTag<CombiningState<InputT, AccumT, OutputT>> address,
- Combine.CombineFn<InputT, AccumT, OutputT> combineFn,
- StateNamespace namespace,
- Coder<AccumT> accumCoder,
- FlinkStateInternals<K> flinkStateInternals) {
-
- this.namespace = namespace;
- this.address = address;
- this.combineFn = combineFn;
- this.flinkStateBackend = flinkStateBackend;
- this.flinkStateInternals = flinkStateInternals;
-
- CoderTypeInformation<AccumT> typeInfo = new CoderTypeInformation<>(accumCoder);
-
- flinkStateDescriptor = new ValueStateDescriptor<>(address.getId(), typeInfo, null);
- }
-
- @Override
- public CombiningState<InputT, AccumT, OutputT> readLater() {
- return this;
- }
-
- @Override
- public void add(InputT value) {
- try {
- org.apache.flink.api.common.state.ValueState<AccumT> state =
- flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor);
-
- AccumT current = state.value();
- if (current == null) {
- current = combineFn.createAccumulator();
- }
- current = combineFn.addInput(current, value);
- state.update(current);
- } catch (RuntimeException re) {
- throw re;
- } catch (Exception e) {
- throw new RuntimeException("Error adding to state." , e);
- }
- }
-
- @Override
- public void addAccum(AccumT accum) {
- try {
- org.apache.flink.api.common.state.ValueState<AccumT> state =
- flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor);
-
- AccumT current = state.value();
- if (current == null) {
- state.update(accum);
- } else {
- current = combineFn.mergeAccumulators(Lists.newArrayList(current, accum));
- state.update(current);
- }
- } catch (Exception e) {
- throw new RuntimeException("Error adding to state.", e);
- }
- }
-
- @Override
- public AccumT getAccum() {
- try {
- return flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor).value();
- } catch (Exception e) {
- throw new RuntimeException("Error reading state.", e);
- }
- }
-
- @Override
- public AccumT mergeAccumulators(Iterable<AccumT> accumulators) {
- return combineFn.mergeAccumulators(accumulators);
- }
-
- @Override
- public OutputT read() {
- try {
- org.apache.flink.api.common.state.ValueState<AccumT> state =
- flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor);
-
- AccumT accum = state.value();
- if (accum != null) {
- return combineFn.extractOutput(accum);
- } else {
- return combineFn.extractOutput(combineFn.createAccumulator());
- }
- } catch (Exception e) {
- throw new RuntimeException("Error reading state.", e);
- }
- }
-
- @Override
- public ReadableState<Boolean> isEmpty() {
- return new ReadableState<Boolean>() {
- @Override
- public Boolean read() {
- try {
- return flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor).value() == null;
- } catch (Exception e) {
- throw new RuntimeException("Error reading state.", e);
- }
-
- }
-
- @Override
- public ReadableState<Boolean> readLater() {
- return this;
- }
- };
- }
-
- @Override
- public void clear() {
- try {
- flinkStateBackend.getPartitionedState(
- namespace.stringKey(),
- StringSerializer.INSTANCE,
- flinkStateDescriptor).clear();
- } catch (Exception e) {
- throw new RuntimeException("Error clearing state.", e);
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- FlinkKeyedCombiningState<?, ?, ?, ?> that =
- (FlinkKeyedCombiningState<?, ?, ?, ?>) o;
-
- return namespace.equals(that.namespace) && address.equals(that.address);
-
- }
-
- @Override
- public int hashCode() {
- int result = namespace.hashCode();
- result = 31 * result + address.hashCode();
- return result;
- }
- }
-
private static class FlinkCombiningStateWithContext<K, InputT, AccumT, OutputT>
implements CombiningState<InputT, AccumT, OutputT> {
@@ -743,9 +572,8 @@
this.flinkStateInternals = flinkStateInternals;
this.context = context;
- CoderTypeInformation<AccumT> typeInfo = new CoderTypeInformation<>(accumCoder);
-
- flinkStateDescriptor = new ValueStateDescriptor<>(address.getId(), typeInfo, null);
+ flinkStateDescriptor = new ValueStateDescriptor<>(
+ address.getId(), new CoderTypeSerializer<>(accumCoder));
}
@Override
@@ -821,7 +649,11 @@
flinkStateDescriptor);
AccumT accum = state.value();
- return combineFn.extractOutput(accum, context);
+ if (accum != null) {
+ return combineFn.extractOutput(accum, context);
+ } else {
+ return combineFn.extractOutput(combineFn.createAccumulator(context), context);
+ }
} catch (Exception e) {
throw new RuntimeException("Error reading state.", e);
}
@@ -907,8 +739,8 @@
this.flinkStateBackend = flinkStateBackend;
this.flinkStateInternals = flinkStateInternals;
- CoderTypeInformation<Instant> typeInfo = new CoderTypeInformation<>(InstantCoder.of());
- flinkStateDescriptor = new ValueStateDescriptor<>(address.getId(), typeInfo, null);
+ flinkStateDescriptor = new ValueStateDescriptor<>(
+ address.getId(), new CoderTypeSerializer<>(InstantCoder.of()));
}
@Override
@@ -1025,4 +857,337 @@
return result;
}
}
+
+ private static class FlinkMapState<KeyT, ValueT> implements MapState<KeyT, ValueT> {
+
+ private final StateNamespace namespace;
+ private final StateTag<MapState<KeyT, ValueT>> address;
+ private final MapStateDescriptor<KeyT, ValueT> flinkStateDescriptor;
+ private final KeyedStateBackend<ByteBuffer> flinkStateBackend;
+
+ FlinkMapState(
+ KeyedStateBackend<ByteBuffer> flinkStateBackend,
+ StateTag<MapState<KeyT, ValueT>> address,
+ StateNamespace namespace,
+ Coder<KeyT> mapKeyCoder, Coder<ValueT> mapValueCoder) {
+ this.namespace = namespace;
+ this.address = address;
+ this.flinkStateBackend = flinkStateBackend;
+ this.flinkStateDescriptor = new MapStateDescriptor<>(address.getId(),
+ new CoderTypeSerializer<>(mapKeyCoder), new CoderTypeSerializer<>(mapValueCoder));
+ }
+
+ @Override
+ public ReadableState<ValueT> get(final KeyT input) {
+ try {
+ return ReadableStates.immediate(
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).get(input));
+ } catch (Exception e) {
+ throw new RuntimeException("Error get from state.", e);
+ }
+ }
+
+ @Override
+ public void put(KeyT key, ValueT value) {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).put(key, value);
+ } catch (Exception e) {
+ throw new RuntimeException("Error put kv to state.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<ValueT> putIfAbsent(final KeyT key, final ValueT value) {
+ try {
+ ValueT current = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).get(key);
+
+ if (current == null) {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).put(key, value);
+ }
+ return ReadableStates.immediate(current);
+ } catch (Exception e) {
+ throw new RuntimeException("Error put kv to state.", e);
+ }
+ }
+
+ @Override
+ public void remove(KeyT key) {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).remove(key);
+ } catch (Exception e) {
+ throw new RuntimeException("Error remove map state key.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Iterable<KeyT>> keys() {
+ return new ReadableState<Iterable<KeyT>>() {
+ @Override
+ public Iterable<KeyT> read() {
+ try {
+ Iterable<KeyT> result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).keys();
+ return result != null ? ImmutableList.copyOf(result) : Collections.<KeyT>emptyList();
+ } catch (Exception e) {
+ throw new RuntimeException("Error get map state keys.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Iterable<KeyT>> readLater() {
+ return this;
+ }
+ };
+ }
+
+ @Override
+ public ReadableState<Iterable<ValueT>> values() {
+ return new ReadableState<Iterable<ValueT>>() {
+ @Override
+ public Iterable<ValueT> read() {
+ try {
+ Iterable<ValueT> result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).values();
+ return result != null ? ImmutableList.copyOf(result) : Collections.<ValueT>emptyList();
+ } catch (Exception e) {
+ throw new RuntimeException("Error get map state values.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Iterable<ValueT>> readLater() {
+ return this;
+ }
+ };
+ }
+
+ @Override
+ public ReadableState<Iterable<Map.Entry<KeyT, ValueT>>> entries() {
+ return new ReadableState<Iterable<Map.Entry<KeyT, ValueT>>>() {
+ @Override
+ public Iterable<Map.Entry<KeyT, ValueT>> read() {
+ try {
+ Iterable<Map.Entry<KeyT, ValueT>> result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).entries();
+ return result != null
+ ? ImmutableList.copyOf(result)
+ : Collections.<Map.Entry<KeyT, ValueT>>emptyList();
+ } catch (Exception e) {
+ throw new RuntimeException("Error get map state entries.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Iterable<Map.Entry<KeyT, ValueT>>> readLater() {
+ return this;
+ }
+ };
+ }
+
+ @Override
+ public void clear() {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).clear();
+ } catch (Exception e) {
+ throw new RuntimeException("Error clearing state.", e);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ FlinkMapState<?, ?> that = (FlinkMapState<?, ?>) o;
+
+ return namespace.equals(that.namespace) && address.equals(that.address);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = namespace.hashCode();
+ result = 31 * result + address.hashCode();
+ return result;
+ }
+ }
+
+ private static class FlinkSetState<T> implements SetState<T> {
+
+ private final StateNamespace namespace;
+ private final StateTag<SetState<T>> address;
+ private final MapStateDescriptor<T, Boolean> flinkStateDescriptor;
+ private final KeyedStateBackend<ByteBuffer> flinkStateBackend;
+
+ FlinkSetState(
+ KeyedStateBackend<ByteBuffer> flinkStateBackend,
+ StateTag<SetState<T>> address,
+ StateNamespace namespace,
+ Coder<T> coder) {
+ this.namespace = namespace;
+ this.address = address;
+ this.flinkStateBackend = flinkStateBackend;
+ this.flinkStateDescriptor = new MapStateDescriptor<>(address.getId(),
+ new CoderTypeSerializer<>(coder), new BooleanSerializer());
+ }
+
+ @Override
+ public ReadableState<Boolean> contains(final T t) {
+ try {
+ Boolean result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).get(t);
+ return ReadableStates.immediate(result != null ? result : false);
+ } catch (Exception e) {
+ throw new RuntimeException("Error contains value from state.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Boolean> addIfAbsent(final T t) {
+ try {
+ org.apache.flink.api.common.state.MapState<T, Boolean> state =
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor);
+ boolean alreadyContained = state.contains(t);
+ if (!alreadyContained) {
+ state.put(t, true);
+ }
+ return ReadableStates.immediate(!alreadyContained);
+ } catch (Exception e) {
+ throw new RuntimeException("Error addIfAbsent value to state.", e);
+ }
+ }
+
+ @Override
+ public void remove(T t) {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).remove(t);
+ } catch (Exception e) {
+ throw new RuntimeException("Error remove value to state.", e);
+ }
+ }
+
+ @Override
+ public SetState<T> readLater() {
+ return this;
+ }
+
+ @Override
+ public void add(T value) {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).put(value, true);
+ } catch (Exception e) {
+ throw new RuntimeException("Error add value to state.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return new ReadableState<Boolean>() {
+ @Override
+ public Boolean read() {
+ try {
+ Iterable<T> result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).keys();
+ return result == null || Iterables.isEmpty(result);
+ } catch (Exception e) {
+ throw new RuntimeException("Error isEmpty from state.", e);
+ }
+ }
+
+ @Override
+ public ReadableState<Boolean> readLater() {
+ return this;
+ }
+ };
+ }
+
+ @Override
+ public Iterable<T> read() {
+ try {
+ Iterable<T> result = flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).keys();
+ return result != null ? ImmutableList.copyOf(result) : Collections.<T>emptyList();
+ } catch (Exception e) {
+ throw new RuntimeException("Error read from state.", e);
+ }
+ }
+
+ @Override
+ public void clear() {
+ try {
+ flinkStateBackend.getPartitionedState(
+ namespace.stringKey(),
+ StringSerializer.INSTANCE,
+ flinkStateDescriptor).clear();
+ } catch (Exception e) {
+ throw new RuntimeException("Error clearing state.", e);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ FlinkSetState<?> that = (FlinkSetState<?>) o;
+
+ return namespace.equals(that.namespace) && address.equals(that.address);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = namespace.hashCode();
+ result = 31 * result + address.hashCode();
+ return result;
+ }
+ }
+
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
index 8382a2d..57086df 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
@@ -17,33 +17,10 @@
*/
package org.apache.beam.runners.flink;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.auto.service.AutoService;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.util.Collections;
import java.util.HashMap;
-import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.runners.flink.translation.wrappers.streaming.DoFnOperator;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
@@ -60,12 +37,10 @@
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
import org.joda.time.Instant;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
/**
@@ -73,9 +48,7 @@
*/
public class PipelineOptionsTest {
- /**
- * Pipeline options.
- */
+ /** Pipeline options. */
public interface MyOptions extends FlinkPipelineOptions {
@Description("Bla bla bla")
@Default.String("Hello")
@@ -83,66 +56,20 @@
void setTestOption(String value);
}
- private static MyOptions options;
- private static SerializedPipelineOptions serializedOptions;
-
- private static final String[] args = new String[]{"--testOption=nothing"};
-
- @BeforeClass
- public static void beforeTest() {
- options = PipelineOptionsFactory.fromArgs(args).as(MyOptions.class);
- serializedOptions = new SerializedPipelineOptions(options);
- }
-
- @Test
- public void testDeserialization() {
- MyOptions deserializedOptions = serializedOptions.getPipelineOptions().as(MyOptions.class);
- assertEquals("nothing", deserializedOptions.getTestOption());
- }
-
- @Test
- public void testIgnoredFieldSerialization() {
- FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class);
- options.setStateBackend(new MemoryStateBackend());
-
- FlinkPipelineOptions deserialized =
- new SerializedPipelineOptions(options).getPipelineOptions().as(FlinkPipelineOptions.class);
-
- assertNull(deserialized.getStateBackend());
- }
-
- @Test
- public void testEnableMetrics() {
- FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class);
- options.setEnableMetrics(false);
- assertFalse(options.getEnableMetrics());
- }
-
- @Test
- public void testCaching() {
- PipelineOptions deserializedOptions =
- serializedOptions.getPipelineOptions().as(PipelineOptions.class);
-
- assertNotNull(deserializedOptions);
- assertTrue(deserializedOptions == serializedOptions.getPipelineOptions());
- assertTrue(deserializedOptions == serializedOptions.getPipelineOptions());
- assertTrue(deserializedOptions == serializedOptions.getPipelineOptions());
- }
-
- @Test(expected = Exception.class)
- public void testNonNull() {
- new SerializedPipelineOptions(null);
- }
+ private static MyOptions options =
+ PipelineOptionsFactory.fromArgs("--testOption=nothing").as(MyOptions.class);
@Test(expected = Exception.class)
public void parDoBaseClassPipelineOptionsNullTest() {
- DoFnOperator<String, String, String> doFnOperator = new DoFnOperator<>(
+ TupleTag<String> mainTag = new TupleTag<>("main-output");
+ Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
new TestDoFn(),
"stepName",
- WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()),
- new TupleTag<String>("main-output"),
+ coder,
+ mainTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<String>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, coder),
WindowingStrategy.globalDefault(),
new HashMap<Integer, PCollectionView<?>>(),
Collections.<PCollectionView<?>>emptyList(),
@@ -157,13 +84,16 @@
@Test
public void parDoBaseClassPipelineOptionsSerializationTest() throws Exception {
- DoFnOperator<String, String, String> doFnOperator = new DoFnOperator<>(
+ TupleTag<String> mainTag = new TupleTag<>("main-output");
+
+ Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
new TestDoFn(),
"stepName",
- WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()),
- new TupleTag<String>("main-output"),
+ coder,
+ mainTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<String>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, coder),
WindowingStrategy.globalDefault(),
new HashMap<Integer, PCollectionView<?>>(),
Collections.<PCollectionView<?>>emptyList(),
@@ -173,16 +103,14 @@
final byte[] serialized = SerializationUtils.serialize(doFnOperator);
@SuppressWarnings("unchecked")
- DoFnOperator<Object, Object, Object> deserialized =
- (DoFnOperator<Object, Object, Object>) SerializationUtils.deserialize(serialized);
+ DoFnOperator<Object, Object> deserialized = SerializationUtils.deserialize(serialized);
TypeInformation<WindowedValue<Object>> typeInformation = TypeInformation.of(
new TypeHint<WindowedValue<Object>>() {});
- OneInputStreamOperatorTestHarness<WindowedValue<Object>, Object> testHarness =
+ OneInputStreamOperatorTestHarness<WindowedValue<Object>, WindowedValue<Object>> testHarness =
new OneInputStreamOperatorTestHarness<>(deserialized,
typeInformation.createSerializer(new ExecutionConfig()));
-
testHarness.open();
// execute once to access options
@@ -197,18 +125,7 @@
}
- @Test
- public void testExternalizedCheckpointsConfigs() {
- String[] args = new String[] { "--externalizedCheckpointsEnabled=true",
- "--retainExternalizedCheckpointsOnCancellation=false" };
- final FlinkPipelineOptions options = PipelineOptionsFactory.fromArgs(args)
- .as(FlinkPipelineOptions.class);
- assertEquals(options.isExternalizedCheckpointsEnabled(), true);
- assertEquals(options.getRetainExternalizedCheckpointsOnCancellation(), false);
- }
-
private static class TestDoFn extends DoFn<String, String> {
-
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
Assert.assertNotNull(c.getPipelineOptions());
@@ -217,74 +134,4 @@
c.getPipelineOptions().as(MyOptions.class).getTestOption());
}
}
-
- /** PipelineOptions used to test auto registration of Jackson modules. */
- public interface JacksonIncompatibleOptions extends PipelineOptions {
- JacksonIncompatible getJacksonIncompatible();
- void setJacksonIncompatible(JacksonIncompatible value);
- }
-
- /** A Jackson {@link Module} to test auto-registration of modules. */
- @AutoService(Module.class)
- public static class RegisteredTestModule extends SimpleModule {
- public RegisteredTestModule() {
- super("RegisteredTestModule");
- setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
- }
- }
-
- /** A class which Jackson does not know how to serialize/deserialize. */
- public static class JacksonIncompatible {
- private final String value;
- public JacksonIncompatible(String value) {
- this.value = value;
- }
- }
-
- /** A Jackson mixin used to add annotations to other classes. */
- @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
- @JsonSerialize(using = JacksonIncompatibleSerializer.class)
- public static final class JacksonIncompatibleMixin {}
-
- /** A Jackson deserializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleDeserializer extends
- JsonDeserializer<JacksonIncompatible> {
-
- @Override
- public JacksonIncompatible deserialize(JsonParser jsonParser,
- DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- return new JacksonIncompatible(jsonParser.readValueAs(String.class));
- }
- }
-
- /** A Jackson serializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
-
- @Override
- public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
- jsonGenerator.writeString(jacksonIncompatible.value);
- }
- }
-
- @Test
- public void testSerializingPipelineOptionsWithCustomUserType() throws Exception {
- String expectedValue = "testValue";
- PipelineOptions options = PipelineOptionsFactory
- .fromArgs("--jacksonIncompatible=\"" + expectedValue + "\"")
- .as(JacksonIncompatibleOptions.class);
- SerializedPipelineOptions context = new SerializedPipelineOptions(options);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {
- outputStream.writeObject(context);
- }
- try (ObjectInputStream inputStream =
- new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
- SerializedPipelineOptions copy = (SerializedPipelineOptions) inputStream.readObject();
- assertEquals(expectedValue,
- copy.getPipelineOptions().as(JacksonIncompatibleOptions.class)
- .getJacksonIncompatible().value);
- }
- }
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
index 79bc0e0..ad17de8 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
@@ -33,6 +33,7 @@
import org.apache.beam.runners.core.StatefulDoFnRunner;
import org.apache.beam.runners.flink.FlinkPipelineOptions;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
+import org.apache.beam.runners.flink.translation.types.CoderTypeSerializer;
import org.apache.beam.runners.flink.translation.wrappers.streaming.DoFnOperator;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -51,6 +52,7 @@
import org.apache.beam.sdk.transforms.join.RawUnionValue;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -61,10 +63,12 @@
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness;
import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness;
+import org.apache.flink.util.OutputTag;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Test;
@@ -105,25 +109,24 @@
@SuppressWarnings("unchecked")
public void testSingleOutput() throws Exception {
- WindowedValue.ValueOnlyWindowedValueCoder<String> windowedValueCoder =
- WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
+ Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
TupleTag<String> outputTag = new TupleTag<>("main-output");
- DoFnOperator<String, String, String> doFnOperator = new DoFnOperator<>(
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
new IdentityDoFn<String>(),
"stepName",
- windowedValueCoder,
+ coder,
outputTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder),
WindowingStrategy.globalDefault(),
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
PipelineOptionsFactory.as(FlinkPipelineOptions.class),
null);
- OneInputStreamOperatorTestHarness<WindowedValue<String>, String> testHarness =
+ OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness =
new OneInputStreamOperatorTestHarness<>(doFnOperator);
testHarness.open();
@@ -141,32 +144,45 @@
@SuppressWarnings("unchecked")
public void testMultiOutputOutput() throws Exception {
- WindowedValue.ValueOnlyWindowedValueCoder<String> windowedValueCoder =
+ WindowedValue.ValueOnlyWindowedValueCoder<String> coder =
WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
TupleTag<String> mainOutput = new TupleTag<>("main-output");
TupleTag<String> additionalOutput1 = new TupleTag<>("output-1");
TupleTag<String> additionalOutput2 = new TupleTag<>("output-2");
- ImmutableMap<TupleTag<?>, Integer> outputMapping = ImmutableMap.<TupleTag<?>, Integer>builder()
- .put(mainOutput, 1)
- .put(additionalOutput1, 2)
- .put(additionalOutput2, 3)
- .build();
+ ImmutableMap<TupleTag<?>, OutputTag<?>> tagsToOutputTags =
+ ImmutableMap.<TupleTag<?>, OutputTag<?>>builder()
+ .put(additionalOutput1, new OutputTag<String>(additionalOutput1.getId()){})
+ .put(additionalOutput2, new OutputTag<String>(additionalOutput2.getId()){})
+ .build();
+ ImmutableMap<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders =
+ ImmutableMap.<TupleTag<?>, Coder<WindowedValue<?>>>builder()
+ .put(mainOutput, (Coder) coder)
+ .put(additionalOutput1, coder)
+ .put(additionalOutput2, coder)
+ .build();
+ ImmutableMap<TupleTag<?>, Integer> tagsToIds =
+ ImmutableMap.<TupleTag<?>, Integer>builder()
+ .put(mainOutput, 0)
+ .put(additionalOutput1, 1)
+ .put(additionalOutput2, 2)
+ .build();
- DoFnOperator<String, String, RawUnionValue> doFnOperator = new DoFnOperator<>(
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
new MultiOutputDoFn(additionalOutput1, additionalOutput2),
"stepName",
- windowedValueCoder,
+ coder,
mainOutput,
ImmutableList.<TupleTag<?>>of(additionalOutput1, additionalOutput2),
- new DoFnOperator.MultiOutputOutputManagerFactory(outputMapping),
+ new DoFnOperator.MultiOutputOutputManagerFactory(
+ mainOutput, tagsToOutputTags, tagsToCoders, tagsToIds),
WindowingStrategy.globalDefault(),
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
PipelineOptionsFactory.as(FlinkPipelineOptions.class),
null);
- OneInputStreamOperatorTestHarness<WindowedValue<String>, RawUnionValue> testHarness =
+ OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness =
new OneInputStreamOperatorTestHarness<>(doFnOperator);
testHarness.open();
@@ -176,17 +192,138 @@
testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("hello")));
assertThat(
- this.stripStreamRecordFromRawUnion(testHarness.getOutput()),
+ this.stripStreamRecord(testHarness.getOutput()),
contains(
- new RawUnionValue(2, WindowedValue.valueInGlobalWindow("extra: one")),
- new RawUnionValue(3, WindowedValue.valueInGlobalWindow("extra: two")),
- new RawUnionValue(1, WindowedValue.valueInGlobalWindow("got: hello")),
- new RawUnionValue(2, WindowedValue.valueInGlobalWindow("got: hello")),
- new RawUnionValue(3, WindowedValue.valueInGlobalWindow("got: hello"))));
+ WindowedValue.valueInGlobalWindow("got: hello")));
+
+ assertThat(
+ this.stripStreamRecord(testHarness.getSideOutput(tagsToOutputTags.get(additionalOutput1))),
+ contains(
+ WindowedValue.valueInGlobalWindow("extra: one"),
+ WindowedValue.valueInGlobalWindow("got: hello")));
+
+ assertThat(
+ this.stripStreamRecord(testHarness.getSideOutput(tagsToOutputTags.get(additionalOutput2))),
+ contains(
+ WindowedValue.valueInGlobalWindow("extra: two"),
+ WindowedValue.valueInGlobalWindow("got: hello")));
testHarness.close();
}
+ /**
+ * This test specifically verifies that we correctly map Flink watermarks to Beam watermarks. In
+ * Beam, a watermark {@code T} guarantees there will not be elements with a timestamp
+ * {@code < T} in the future. In Flink, a watermark {@code T} guarantees there will not be
+ * elements with a timestamp {@code <= T} in the future. We have to make sure to take this into
+ * account when firing timers.
+ *
+ * <p>This not test the timer API in general or processing-time timers because there are generic
+ * tests for this in {@code ParDoTest}.
+ */
+ @Test
+ public void testWatermarkContract() throws Exception {
+
+ final Instant timerTimestamp = new Instant(1000);
+ final String outputMessage = "Timer fired";
+
+ WindowingStrategy<Object, IntervalWindow> windowingStrategy =
+ WindowingStrategy.of(FixedWindows.of(new Duration(10_000)));
+
+ DoFn<Integer, String> fn = new DoFn<Integer, String>() {
+ private static final String EVENT_TIMER_ID = "eventTimer";
+
+ @TimerId(EVENT_TIMER_ID)
+ private final TimerSpec eventTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void processElement(ProcessContext context, @TimerId(EVENT_TIMER_ID) Timer timer) {
+ timer.set(timerTimestamp);
+ }
+
+ @OnTimer(EVENT_TIMER_ID)
+ public void onEventTime(OnTimerContext context) {
+ assertEquals(
+ "Timer timestamp must match set timestamp.", timerTimestamp, context.timestamp());
+ context.outputWithTimestamp(outputMessage, context.timestamp());
+ }
+ };
+
+ WindowedValue.FullWindowedValueCoder<Integer> inputCoder =
+ WindowedValue.getFullCoder(
+ VarIntCoder.of(),
+ windowingStrategy.getWindowFn().windowCoder());
+
+ WindowedValue.FullWindowedValueCoder<String> outputCoder =
+ WindowedValue.getFullCoder(
+ StringUtf8Coder.of(),
+ windowingStrategy.getWindowFn().windowCoder());
+
+
+ TupleTag<String> outputTag = new TupleTag<>("main-output");
+
+ DoFnOperator<Integer, String> doFnOperator = new DoFnOperator<>(
+ fn,
+ "stepName",
+ inputCoder,
+ outputTag,
+ Collections.<TupleTag<?>>emptyList(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, outputCoder),
+ windowingStrategy,
+ new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
+ Collections.<PCollectionView<?>>emptyList(), /* side inputs */
+ PipelineOptionsFactory.as(FlinkPipelineOptions.class),
+ VarIntCoder.of() /* key coder */);
+
+ OneInputStreamOperatorTestHarness<WindowedValue<Integer>, WindowedValue<String>> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(
+ doFnOperator,
+ new KeySelector<WindowedValue<Integer>, Integer>() {
+ @Override
+ public Integer getKey(WindowedValue<Integer> integerWindowedValue) throws Exception {
+ return integerWindowedValue.getValue();
+ }
+ },
+ new CoderTypeInformation<>(VarIntCoder.of()));
+
+ testHarness.setup(new CoderTypeSerializer<>(outputCoder));
+
+ testHarness.open();
+
+ testHarness.processWatermark(0);
+
+ IntervalWindow window1 = new IntervalWindow(new Instant(0), Duration.millis(10_000));
+
+ // this should register a timer
+ testHarness.processElement(
+ new StreamRecord<>(WindowedValue.of(13, new Instant(0), window1, PaneInfo.NO_FIRING)));
+
+ assertThat(
+ this.<String>stripStreamRecordFromWindowedValue(testHarness.getOutput()),
+ emptyIterable());
+
+ // this does not yet fire the timer (in vanilla Flink it would)
+ testHarness.processWatermark(timerTimestamp.getMillis());
+
+ assertThat(
+ this.<String>stripStreamRecordFromWindowedValue(testHarness.getOutput()),
+ emptyIterable());
+
+ testHarness.getOutput().clear();
+
+ // this must fire the timer
+ testHarness.processWatermark(timerTimestamp.getMillis() + 1);
+
+ assertThat(
+ this.<String>stripStreamRecordFromWindowedValue(testHarness.getOutput()),
+ contains(
+ WindowedValue.of(
+ outputMessage, new Instant(timerTimestamp), window1, PaneInfo.NO_FIRING)));
+
+ testHarness.close();
+ }
+
+
@Test
public void testLateDroppingForStatefulFn() throws Exception {
@@ -205,20 +342,20 @@
}
};
- WindowedValue.FullWindowedValueCoder<Integer> windowedValueCoder =
- WindowedValue.getFullCoder(
- VarIntCoder.of(),
- windowingStrategy.getWindowFn().windowCoder());
+ Coder<WindowedValue<Integer>> inputCoder = WindowedValue.getFullCoder(
+ VarIntCoder.of(), windowingStrategy.getWindowFn().windowCoder());
+ Coder<WindowedValue<String>> outputCoder = WindowedValue.getFullCoder(
+ StringUtf8Coder.of(), windowingStrategy.getWindowFn().windowCoder());
TupleTag<String> outputTag = new TupleTag<>("main-output");
- DoFnOperator<Integer, String, WindowedValue<String>> doFnOperator = new DoFnOperator<>(
+ DoFnOperator<Integer, String> doFnOperator = new DoFnOperator<>(
fn,
"stepName",
- windowedValueCoder,
+ inputCoder,
outputTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<WindowedValue<String>>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, outputCoder),
windowingStrategy,
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
@@ -317,22 +454,21 @@
}
};
- WindowedValue.FullWindowedValueCoder<KV<String, Integer>> windowedValueCoder =
+ WindowedValue.FullWindowedValueCoder<KV<String, Integer>> coder =
WindowedValue.getFullCoder(
KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()),
windowingStrategy.getWindowFn().windowCoder());
TupleTag<KV<String, Integer>> outputTag = new TupleTag<>("main-output");
- DoFnOperator<
- KV<String, Integer>, KV<String, Integer>, WindowedValue<KV<String, Integer>>> doFnOperator =
+ DoFnOperator<KV<String, Integer>, KV<String, Integer>> doFnOperator =
new DoFnOperator<>(
fn,
"stepName",
- windowedValueCoder,
+ coder,
outputTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<WindowedValue<KV<String, Integer>>>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder),
windowingStrategy,
new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
Collections.<PCollectionView<?>>emptyList(), /* side inputs */
@@ -384,11 +520,13 @@
// this should trigger both the window.maxTimestamp() timer and the GC timer
// this tests that the GC timer fires after the user timer
+ // we have to add 1 here because Flink timers fire when watermark >= timestamp while Beam
+ // timers fire when watermark > timestamp
testHarness.processWatermark(
window1.maxTimestamp()
.plus(windowingStrategy.getAllowedLateness())
.plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS)
- .getMillis());
+ .getMillis() + 1);
assertThat(
this.<KV<String, Integer>>stripStreamRecordFromWindowedValue(testHarness.getOutput()),
@@ -406,8 +544,7 @@
public void testSideInputs(boolean keyed) throws Exception {
- WindowedValue.ValueOnlyWindowedValueCoder<String> windowedValueCoder =
- WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
+ Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
TupleTag<String> outputTag = new TupleTag<>("main-output");
@@ -422,21 +559,21 @@
keyCoder = StringUtf8Coder.of();
}
- DoFnOperator<String, String, String> doFnOperator = new DoFnOperator<>(
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
new IdentityDoFn<String>(),
"stepName",
- windowedValueCoder,
+ coder,
outputTag,
Collections.<TupleTag<?>>emptyList(),
- new DoFnOperator.DefaultOutputManagerFactory<String>(),
+ new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder),
WindowingStrategy.globalDefault(),
sideInputMapping, /* side-input mapping */
ImmutableList.<PCollectionView<?>>of(view1, view2), /* side inputs */
PipelineOptionsFactory.as(FlinkPipelineOptions.class),
keyCoder);
- TwoInputStreamOperatorTestHarness<WindowedValue<String>, RawUnionValue, String> testHarness =
- new TwoInputStreamOperatorTestHarness<>(doFnOperator);
+ TwoInputStreamOperatorTestHarness<WindowedValue<String>, RawUnionValue, WindowedValue<String>>
+ testHarness = new TwoInputStreamOperatorTestHarness<>(doFnOperator);
if (keyed) {
// we use a dummy key for the second input since it is considered to be broadcast
@@ -506,6 +643,105 @@
testSideInputs(true);
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testBundle() throws Exception {
+
+ WindowedValue.ValueOnlyWindowedValueCoder<String> windowedValueCoder =
+ WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
+
+ TupleTag<String> outputTag = new TupleTag<>("main-output");
+ FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class);
+ options.setMaxBundleSize(2L);
+ options.setMaxBundleTimeMills(10L);
+
+ IdentityDoFn<String> doFn = new IdentityDoFn<String>() {
+ @FinishBundle
+ public void finishBundle(FinishBundleContext context) {
+ context.output(
+ "finishBundle", BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE);
+ }
+ };
+
+ DoFnOperator.MultiOutputOutputManagerFactory<String> outputManagerFactory =
+ new DoFnOperator.MultiOutputOutputManagerFactory(
+ outputTag,
+ WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE));
+
+ DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
+ doFn,
+ "stepName",
+ windowedValueCoder,
+ outputTag,
+ Collections.<TupleTag<?>>emptyList(),
+ outputManagerFactory,
+ WindowingStrategy.globalDefault(),
+ new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
+ Collections.<PCollectionView<?>>emptyList(), /* side inputs */
+ options,
+ null);
+
+ OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(doFnOperator);
+
+ testHarness.open();
+
+ testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("a")));
+ testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("b")));
+ testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("c")));
+
+ // draw a snapshot
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+
+ // There is a finishBundle in snapshot()
+ // Elements will be buffered as part of finishing a bundle in snapshot()
+ assertThat(
+ this.<String>stripStreamRecordFromWindowedValue(testHarness.getOutput()),
+ contains(
+ WindowedValue.valueInGlobalWindow("a"),
+ WindowedValue.valueInGlobalWindow("b"),
+ WindowedValue.valueInGlobalWindow("finishBundle"),
+ WindowedValue.valueInGlobalWindow("c")));
+
+ testHarness.close();
+
+ DoFnOperator<String, String> newDoFnOperator = new DoFnOperator<>(
+ doFn,
+ "stepName",
+ windowedValueCoder,
+ outputTag,
+ Collections.<TupleTag<?>>emptyList(),
+ outputManagerFactory,
+ WindowingStrategy.globalDefault(),
+ new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */
+ Collections.<PCollectionView<?>>emptyList(), /* side inputs */
+ options,
+ null);
+
+ OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> newHarness =
+ new OneInputStreamOperatorTestHarness<>(newDoFnOperator);
+
+ // restore snapshot
+ newHarness.initializeState(snapshot);
+
+ newHarness.open();
+
+ // startBundle will output the buffered elements.
+ newHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("d")));
+
+ // check finishBundle by timeout
+ newHarness.setProcessingTime(10);
+
+ assertThat(
+ this.<String>stripStreamRecordFromWindowedValue(newHarness.getOutput()),
+ contains(
+ WindowedValue.valueInGlobalWindow("finishBundle"),
+ WindowedValue.valueInGlobalWindow("d"),
+ WindowedValue.valueInGlobalWindow("finishBundle")));
+
+ newHarness.close();
+ }
+
private <T> Iterable<WindowedValue<T>> stripStreamRecordFromWindowedValue(
Iterable<Object> input) {
@@ -527,19 +763,19 @@
});
}
- private Iterable<RawUnionValue> stripStreamRecordFromRawUnion(Iterable<Object> input) {
+ private Iterable<WindowedValue<String>> stripStreamRecord(Iterable<?> input) {
return FluentIterable.from(input).filter(new Predicate<Object>() {
@Override
public boolean apply(@Nullable Object o) {
- return o instanceof StreamRecord && ((StreamRecord) o).getValue() instanceof RawUnionValue;
+ return o instanceof StreamRecord;
}
- }).transform(new Function<Object, RawUnionValue>() {
+ }).transform(new Function<Object, WindowedValue<String>>() {
@Nullable
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
- public RawUnionValue apply(@Nullable Object o) {
- if (o instanceof StreamRecord && ((StreamRecord) o).getValue() instanceof RawUnionValue) {
- return (RawUnionValue) ((StreamRecord) o).getValue();
+ public WindowedValue<String> apply(@Nullable Object o) {
+ if (o instanceof StreamRecord) {
+ return (WindowedValue<String>) ((StreamRecord) o).getValue();
}
throw new RuntimeException("unreachable");
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
index 2b96d91..3409d27 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
@@ -17,229 +17,87 @@
*/
package org.apache.beam.runners.flink.streaming;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertThat;
-
-import java.util.Arrays;
-import org.apache.beam.runners.core.StateMerging;
-import org.apache.beam.runners.core.StateNamespace;
-import org.apache.beam.runners.core.StateNamespaceForTest;
-import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.runners.core.StateTags;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkBroadcastStateInternals;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.state.BagState;
-import org.apache.beam.sdk.state.CombiningState;
-import org.apache.beam.sdk.state.GroupingState;
-import org.apache.beam.sdk.state.ReadableState;
-import org.apache.beam.sdk.state.ValueState;
-import org.apache.beam.sdk.transforms.Sum;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.hamcrest.Matchers;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for {@link FlinkBroadcastStateInternals}. This is based on the tests for
- * {@code InMemoryStateInternals}.
+ * {@code StateInternalsTest}.
+ *
+ * <p>Just test value, bag and combining.
*/
@RunWith(JUnit4.class)
-public class FlinkBroadcastStateInternalsTest {
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
- private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
+public class FlinkBroadcastStateInternalsTest extends StateInternalsTest {
- private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
- StateTags.value("stringValue", StringUtf8Coder.of());
- private static final StateTag<CombiningState<Integer, int[], Integer>>
- SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
- "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
-
- FlinkBroadcastStateInternals<String> underTest;
-
- @Before
- public void initStateInternals() {
+ @Override
+ protected StateInternals createStateInternals() {
MemoryStateBackend backend = new MemoryStateBackend();
try {
OperatorStateBackend operatorStateBackend =
backend.createOperatorStateBackend(new DummyEnvironment("test", 1, 0), "");
- underTest = new FlinkBroadcastStateInternals<>(1, operatorStateBackend);
-
+ return new FlinkBroadcastStateInternals<>(1, operatorStateBackend);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
- @Test
- public void testValue() throws Exception {
- ValueState<String> value = underTest.state(NAMESPACE_1, STRING_VALUE_ADDR);
+ @Override
+ @Ignore
+ public void testSet() {}
- assertEquals(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
- assertNotEquals(
- underTest.state(NAMESPACE_2, STRING_VALUE_ADDR),
- value);
+ @Override
+ @Ignore
+ public void testSetIsEmpty() {}
- assertThat(value.read(), Matchers.nullValue());
- value.write("hello");
- assertThat(value.read(), Matchers.equalTo("hello"));
- value.write("world");
- assertThat(value.read(), Matchers.equalTo("world"));
+ @Override
+ @Ignore
+ public void testMergeSetIntoSource() {}
- value.clear();
- assertThat(value.read(), Matchers.nullValue());
- assertEquals(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
+ @Override
+ @Ignore
+ public void testMergeSetIntoNewNamespace() {}
- }
+ @Override
+ @Ignore
+ public void testMap() {}
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ @Override
+ @Ignore
+ public void testWatermarkEarliestState() {}
- assertEquals(value, underTest.state(NAMESPACE_1, STRING_BAG_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, STRING_BAG_ADDR)));
+ @Override
+ @Ignore
+ public void testWatermarkLatestState() {}
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello"));
+ @Override
+ @Ignore
+ public void testWatermarkEndOfWindowState() {}
- value.add("world");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello", "world"));
+ @Override
+ @Ignore
+ public void testWatermarkStateIsEmpty() {}
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertEquals(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), value);
+ @Override
+ @Ignore
+ public void testMergeEarliestWatermarkIntoSource() {}
- }
+ @Override
+ @Ignore
+ public void testMergeLatestWatermarkIntoSource() {}
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ @Override
+ @Ignore
+ public void testSetReadable() {}
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeBagIntoSource() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
-
- // Reading the merged bag gets both the contents
- assertThat(bag1.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeBagIntoNewNamespace() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
- BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
-
- // Reading the merged bag gets both the contents
- assertThat(bag3.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag1.read(), Matchers.emptyIterable());
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testCombiningValue() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR)));
-
- assertThat(value.read(), Matchers.equalTo(0));
- value.add(2);
- assertThat(value.read(), Matchers.equalTo(2));
-
- value.add(3);
- assertThat(value.read(), Matchers.equalTo(5));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(0));
- assertEquals(underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR), value);
- }
-
- @Test
- public void testCombiningIsEmpty() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(5);
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeCombiningValueIntoSource() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- assertThat(value1.read(), Matchers.equalTo(11));
- assertThat(value2.read(), Matchers.equalTo(10));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1);
-
- assertThat(value1.read(), Matchers.equalTo(21));
- assertThat(value2.read(), Matchers.equalTo(0));
- }
-
- @Test
- public void testMergeCombiningValueIntoNewNamespace() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value3 =
- underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3);
-
- // Merging clears the old values and updates the result value.
- assertThat(value1.read(), Matchers.equalTo(0));
- assertThat(value2.read(), Matchers.equalTo(0));
- assertThat(value3.read(), Matchers.equalTo(21));
- }
+ @Override
+ @Ignore
+ public void testMapReadable() {}
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
index 4012373..aed14f3 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.runners.flink.streaming;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import java.io.ByteArrayInputStream;
@@ -26,8 +24,8 @@
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.apache.beam.runners.core.StateMerging;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateNamespaceForTest;
import org.apache.beam.runners.core.StateTag;
@@ -35,7 +33,6 @@
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkKeyGroupStateInternals;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.state.BagState;
-import org.apache.beam.sdk.state.ReadableState;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
@@ -47,40 +44,203 @@
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KeyedStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.apache.flink.streaming.api.operators.KeyContext;
import org.hamcrest.Matchers;
-import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.junit.runners.Suite;
/**
* Tests for {@link FlinkKeyGroupStateInternals}. This is based on the tests for
- * {@code InMemoryStateInternals}.
+ * {@code StateInternalsTest}.
*/
-@RunWith(JUnit4.class)
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ FlinkKeyGroupStateInternalsTest.StandardStateInternalsTests.class,
+ FlinkKeyGroupStateInternalsTest.OtherTests.class
+})
public class FlinkKeyGroupStateInternalsTest {
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
- private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
-
- FlinkKeyGroupStateInternals<String> underTest;
- private KeyedStateBackend keyedStateBackend;
-
- @Before
- public void initStateInternals() {
- try {
- keyedStateBackend = getKeyedStateBackend(2, new KeyGroupRange(0, 1));
- underTest = new FlinkKeyGroupStateInternals<>(StringUtf8Coder.of(), keyedStateBackend);
- } catch (Exception e) {
- throw new RuntimeException(e);
+ /**
+ * A standard StateInternals test. Just test BagState.
+ */
+ @RunWith(JUnit4.class)
+ public static class StandardStateInternalsTests extends StateInternalsTest {
+ @Override
+ protected StateInternals createStateInternals() {
+ KeyedStateBackend keyedStateBackend =
+ getKeyedStateBackend(2, new KeyGroupRange(0, 1));
+ return new FlinkKeyGroupStateInternals<>(StringUtf8Coder.of(), keyedStateBackend);
}
+
+ @Override
+ @Ignore
+ public void testValue() {}
+
+ @Override
+ @Ignore
+ public void testSet() {}
+
+ @Override
+ @Ignore
+ public void testSetIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeSetIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeSetIntoNewNamespace() {}
+
+ @Override
+ @Ignore
+ public void testMap() {}
+
+ @Override
+ @Ignore
+ public void testCombiningValue() {}
+
+ @Override
+ @Ignore
+ public void testCombiningIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeCombiningValueIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeCombiningValueIntoNewNamespace() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkEarliestState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkLatestState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkEndOfWindowState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkStateIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeEarliestWatermarkIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeLatestWatermarkIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testSetReadable() {}
+
+ @Override
+ @Ignore
+ public void testMapReadable() {}
}
- private KeyedStateBackend getKeyedStateBackend(int numberOfKeyGroups,
+ /**
+ * A specific test of FlinkKeyGroupStateInternalsTest.
+ */
+ @RunWith(JUnit4.class)
+ public static class OtherTests {
+
+ private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
+ private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
+ private static final StateTag<BagState<String>> STRING_BAG_ADDR =
+ StateTags.bag("stringBag", StringUtf8Coder.of());
+
+ @Test
+ public void testKeyGroupAndCheckpoint() throws Exception {
+ // assign to keyGroup 0
+ ByteBuffer key0 = ByteBuffer.wrap(
+ CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "11111111"));
+ // assign to keyGroup 1
+ ByteBuffer key1 = ByteBuffer.wrap(
+ CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "22222222"));
+ FlinkKeyGroupStateInternals<String> allState;
+ {
+ KeyedStateBackend<ByteBuffer> keyedStateBackend =
+ getKeyedStateBackend(2, new KeyGroupRange(0, 1));
+ allState = new FlinkKeyGroupStateInternals<>(
+ StringUtf8Coder.of(), keyedStateBackend);
+ BagState<String> valueForNamespace0 = allState.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> valueForNamespace1 = allState.state(NAMESPACE_2, STRING_BAG_ADDR);
+ keyedStateBackend.setCurrentKey(key0);
+ valueForNamespace0.add("0");
+ valueForNamespace1.add("2");
+ keyedStateBackend.setCurrentKey(key1);
+ valueForNamespace0.add("1");
+ valueForNamespace1.add("3");
+ assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0", "1"));
+ assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2", "3"));
+ }
+
+ ClassLoader classLoader = FlinkKeyGroupStateInternalsTest.class.getClassLoader();
+
+ // 1. scale up
+ ByteArrayOutputStream out0 = new ByteArrayOutputStream();
+ allState.snapshotKeyGroupState(0, new DataOutputStream(out0));
+ DataInputStream in0 = new DataInputStream(
+ new ByteArrayInputStream(out0.toByteArray()));
+ {
+ KeyedStateBackend<ByteBuffer> keyedStateBackend =
+ getKeyedStateBackend(2, new KeyGroupRange(0, 0));
+ FlinkKeyGroupStateInternals<String> state0 =
+ new FlinkKeyGroupStateInternals<>(
+ StringUtf8Coder.of(), keyedStateBackend);
+ state0.restoreKeyGroupState(0, in0, classLoader);
+ BagState<String> valueForNamespace0 = state0.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> valueForNamespace1 = state0.state(NAMESPACE_2, STRING_BAG_ADDR);
+ assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0"));
+ assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2"));
+ }
+
+ ByteArrayOutputStream out1 = new ByteArrayOutputStream();
+ allState.snapshotKeyGroupState(1, new DataOutputStream(out1));
+ DataInputStream in1 = new DataInputStream(
+ new ByteArrayInputStream(out1.toByteArray()));
+ {
+ KeyedStateBackend<ByteBuffer> keyedStateBackend =
+ getKeyedStateBackend(2, new KeyGroupRange(1, 1));
+ FlinkKeyGroupStateInternals<String> state1 =
+ new FlinkKeyGroupStateInternals<>(
+ StringUtf8Coder.of(), keyedStateBackend);
+ state1.restoreKeyGroupState(1, in1, classLoader);
+ BagState<String> valueForNamespace0 = state1.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> valueForNamespace1 = state1.state(NAMESPACE_2, STRING_BAG_ADDR);
+ assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("1"));
+ assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("3"));
+ }
+
+ // 2. scale down
+ {
+ KeyedStateBackend<ByteBuffer> keyedStateBackend =
+ getKeyedStateBackend(2, new KeyGroupRange(0, 1));
+ FlinkKeyGroupStateInternals<String> newAllState = new FlinkKeyGroupStateInternals<>(
+ StringUtf8Coder.of(), keyedStateBackend);
+ in0.reset();
+ in1.reset();
+ newAllState.restoreKeyGroupState(0, in0, classLoader);
+ newAllState.restoreKeyGroupState(1, in1, classLoader);
+ BagState<String> valueForNamespace0 = newAllState.state(NAMESPACE_1, STRING_BAG_ADDR);
+ BagState<String> valueForNamespace1 = newAllState.state(NAMESPACE_2, STRING_BAG_ADDR);
+ assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0", "1"));
+ assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2", "3"));
+ }
+ }
+
+ }
+
+ private static KeyedStateBackend<ByteBuffer> getKeyedStateBackend(int numberOfKeyGroups,
KeyGroupRange keyGroupRange) {
MemoryStateBackend backend = new MemoryStateBackend();
try {
@@ -100,163 +260,4 @@
}
}
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- assertEquals(value, underTest.state(NAMESPACE_1, STRING_BAG_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, STRING_BAG_ADDR)));
-
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello"));
-
- value.add("world");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello", "world"));
-
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertEquals(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), value);
-
- }
-
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeBagIntoSource() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
-
- // Reading the merged bag gets both the contents
- assertThat(bag1.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeBagIntoNewNamespace() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
- BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
-
- // Reading the merged bag gets both the contents
- assertThat(bag3.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag1.read(), Matchers.emptyIterable());
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testKeyGroupAndCheckpoint() throws Exception {
- // assign to keyGroup 0
- ByteBuffer key0 = ByteBuffer.wrap(
- CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "11111111"));
- // assign to keyGroup 1
- ByteBuffer key1 = ByteBuffer.wrap(
- CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "22222222"));
- FlinkKeyGroupStateInternals<String> allState;
- {
- KeyedStateBackend keyedStateBackend = getKeyedStateBackend(2, new KeyGroupRange(0, 1));
- allState = new FlinkKeyGroupStateInternals<>(
- StringUtf8Coder.of(), keyedStateBackend);
- BagState<String> valueForNamespace0 = allState.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> valueForNamespace1 = allState.state(NAMESPACE_2, STRING_BAG_ADDR);
- keyedStateBackend.setCurrentKey(key0);
- valueForNamespace0.add("0");
- valueForNamespace1.add("2");
- keyedStateBackend.setCurrentKey(key1);
- valueForNamespace0.add("1");
- valueForNamespace1.add("3");
- assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0", "1"));
- assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2", "3"));
- }
-
- ClassLoader classLoader = FlinkKeyGroupStateInternalsTest.class.getClassLoader();
-
- // 1. scale up
- ByteArrayOutputStream out0 = new ByteArrayOutputStream();
- allState.snapshotKeyGroupState(0, new DataOutputStream(out0));
- DataInputStream in0 = new DataInputStream(
- new ByteArrayInputStream(out0.toByteArray()));
- {
- KeyedStateBackend keyedStateBackend = getKeyedStateBackend(2, new KeyGroupRange(0, 0));
- FlinkKeyGroupStateInternals<String> state0 =
- new FlinkKeyGroupStateInternals<>(
- StringUtf8Coder.of(), keyedStateBackend);
- state0.restoreKeyGroupState(0, in0, classLoader);
- BagState<String> valueForNamespace0 = state0.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> valueForNamespace1 = state0.state(NAMESPACE_2, STRING_BAG_ADDR);
- assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0"));
- assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2"));
- }
-
- ByteArrayOutputStream out1 = new ByteArrayOutputStream();
- allState.snapshotKeyGroupState(1, new DataOutputStream(out1));
- DataInputStream in1 = new DataInputStream(
- new ByteArrayInputStream(out1.toByteArray()));
- {
- KeyedStateBackend keyedStateBackend = getKeyedStateBackend(2, new KeyGroupRange(1, 1));
- FlinkKeyGroupStateInternals<String> state1 =
- new FlinkKeyGroupStateInternals<>(
- StringUtf8Coder.of(), keyedStateBackend);
- state1.restoreKeyGroupState(1, in1, classLoader);
- BagState<String> valueForNamespace0 = state1.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> valueForNamespace1 = state1.state(NAMESPACE_2, STRING_BAG_ADDR);
- assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("1"));
- assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("3"));
- }
-
- // 2. scale down
- {
- KeyedStateBackend keyedStateBackend = getKeyedStateBackend(2, new KeyGroupRange(0, 1));
- FlinkKeyGroupStateInternals<String> newAllState = new FlinkKeyGroupStateInternals<>(
- StringUtf8Coder.of(), keyedStateBackend);
- in0.reset();
- in1.reset();
- newAllState.restoreKeyGroupState(0, in0, classLoader);
- newAllState.restoreKeyGroupState(1, in1, classLoader);
- BagState<String> valueForNamespace0 = newAllState.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> valueForNamespace1 = newAllState.state(NAMESPACE_2, STRING_BAG_ADDR);
- assertThat(valueForNamespace0.read(), Matchers.containsInAnyOrder("0", "1"));
- assertThat(valueForNamespace1.read(), Matchers.containsInAnyOrder("2", "3"));
- }
-
- }
-
- private static class TestKeyContext implements KeyContext {
-
- private Object key;
-
- @Override
- public void setCurrentKey(Object key) {
- this.key = key;
- }
-
- @Override
- public Object getCurrentKey() {
- return key;
- }
- }
-
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
index 17cd3f5..667b5ba 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
@@ -17,85 +17,115 @@
*/
package org.apache.beam.runners.flink.streaming;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-
-import org.apache.beam.runners.core.StateNamespace;
-import org.apache.beam.runners.core.StateNamespaceForTest;
-import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.runners.core.StateTags;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkSplitStateInternals;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.state.BagState;
-import org.apache.beam.sdk.state.ReadableState;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.hamcrest.Matchers;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for {@link FlinkSplitStateInternals}. This is based on the tests for
- * {@code InMemoryStateInternals}.
+ * {@code StateInternalsTest}.
+ *
+ * <p>Just test testBag and testBagIsEmpty.
*/
@RunWith(JUnit4.class)
-public class FlinkSplitStateInternalsTest {
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
+public class FlinkSplitStateInternalsTest extends StateInternalsTest {
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
-
- FlinkSplitStateInternals<String> underTest;
-
- @Before
- public void initStateInternals() {
+ @Override
+ protected StateInternals createStateInternals() {
MemoryStateBackend backend = new MemoryStateBackend();
try {
OperatorStateBackend operatorStateBackend =
backend.createOperatorStateBackend(new DummyEnvironment("test", 1, 0), "");
- underTest = new FlinkSplitStateInternals<>(operatorStateBackend);
-
+ return new FlinkSplitStateInternals<>(operatorStateBackend);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ @Override
+ @Ignore
+ public void testMergeBagIntoSource() {}
- assertEquals(value, underTest.state(NAMESPACE_1, STRING_BAG_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, STRING_BAG_ADDR)));
+ @Override
+ @Ignore
+ public void testMergeBagIntoNewNamespace() {}
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello"));
+ @Override
+ @Ignore
+ public void testValue() {}
- value.add("world");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello", "world"));
+ @Override
+ @Ignore
+ public void testSet() {}
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertEquals(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), value);
+ @Override
+ @Ignore
+ public void testSetIsEmpty() {}
- }
+ @Override
+ @Ignore
+ public void testMergeSetIntoSource() {}
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
+ @Override
+ @Ignore
+ public void testMergeSetIntoNewNamespace() {}
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
+ @Override
+ @Ignore
+ public void testMap() {}
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
+ @Override
+ @Ignore
+ public void testCombiningValue() {}
+
+ @Override
+ @Ignore
+ public void testCombiningIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeCombiningValueIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeCombiningValueIntoNewNamespace() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkEarliestState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkLatestState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkEndOfWindowState() {}
+
+ @Override
+ @Ignore
+ public void testWatermarkStateIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeEarliestWatermarkIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeLatestWatermarkIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testSetReadable() {}
+
+ @Override
+ @Ignore
+ public void testMapReadable() {}
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
index 35d2b78..b8d41de 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
@@ -17,31 +17,11 @@
*/
package org.apache.beam.runners.flink.streaming;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertThat;
-
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.apache.beam.runners.core.StateMerging;
-import org.apache.beam.runners.core.StateNamespace;
-import org.apache.beam.runners.core.StateNamespaceForTest;
-import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.runners.core.StateTags;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkStateInternals;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.state.BagState;
-import org.apache.beam.sdk.state.CombiningState;
-import org.apache.beam.sdk.state.GroupingState;
-import org.apache.beam.sdk.state.ReadableState;
-import org.apache.beam.sdk.state.ValueState;
-import org.apache.beam.sdk.state.WatermarkHoldState;
-import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
@@ -52,42 +32,17 @@
import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.hamcrest.Matchers;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
- * Tests for {@link FlinkStateInternals}. This is based on the tests for
- * {@code InMemoryStateInternals}.
+ * Tests for {@link FlinkStateInternals}. This is based on {@link StateInternalsTest}.
*/
@RunWith(JUnit4.class)
-public class FlinkStateInternalsTest {
- private static final BoundedWindow WINDOW_1 = new IntervalWindow(new Instant(0), new Instant(10));
- private static final StateNamespace NAMESPACE_1 = new StateNamespaceForTest("ns1");
- private static final StateNamespace NAMESPACE_2 = new StateNamespaceForTest("ns2");
- private static final StateNamespace NAMESPACE_3 = new StateNamespaceForTest("ns3");
+public class FlinkStateInternalsTest extends StateInternalsTest {
- private static final StateTag<ValueState<String>> STRING_VALUE_ADDR =
- StateTags.value("stringValue", StringUtf8Coder.of());
- private static final StateTag<CombiningState<Integer, int[], Integer>>
- SUM_INTEGER_ADDR = StateTags.combiningValueFromInputInternal(
- "sumInteger", VarIntCoder.of(), Sum.ofIntegers());
- private static final StateTag<BagState<String>> STRING_BAG_ADDR =
- StateTags.bag("stringBag", StringUtf8Coder.of());
- private static final StateTag<WatermarkHoldState> WATERMARK_EARLIEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_LATEST_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST);
- private static final StateTag<WatermarkHoldState> WATERMARK_EOW_ADDR =
- StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW);
-
- FlinkStateInternals<String> underTest;
-
- @Before
- public void initStateInternals() {
+ @Override
+ protected StateInternals createStateInternals() {
MemoryStateBackend backend = new MemoryStateBackend();
try {
AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend(
@@ -98,296 +53,14 @@
1,
new KeyGroupRange(0, 0),
new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
- underTest = new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of());
keyedStateBackend.setCurrentKey(
ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "Hello")));
+
+ return new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
- @Test
- public void testValue() throws Exception {
- ValueState<String> value = underTest.state(NAMESPACE_1, STRING_VALUE_ADDR);
-
- assertEquals(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
- assertNotEquals(
- underTest.state(NAMESPACE_2, STRING_VALUE_ADDR),
- value);
-
- assertThat(value.read(), Matchers.nullValue());
- value.write("hello");
- assertThat(value.read(), Matchers.equalTo("hello"));
- value.write("world");
- assertThat(value.read(), Matchers.equalTo("world"));
-
- value.clear();
- assertThat(value.read(), Matchers.nullValue());
- assertEquals(underTest.state(NAMESPACE_1, STRING_VALUE_ADDR), value);
-
- }
-
- @Test
- public void testBag() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- assertEquals(value, underTest.state(NAMESPACE_1, STRING_BAG_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, STRING_BAG_ADDR)));
-
- assertThat(value.read(), Matchers.emptyIterable());
- value.add("hello");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello"));
-
- value.add("world");
- assertThat(value.read(), Matchers.containsInAnyOrder("hello", "world"));
-
- value.clear();
- assertThat(value.read(), Matchers.emptyIterable());
- assertEquals(underTest.state(NAMESPACE_1, STRING_BAG_ADDR), value);
-
- }
-
- @Test
- public void testBagIsEmpty() throws Exception {
- BagState<String> value = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add("hello");
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeBagIntoSource() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2), bag1);
-
- // Reading the merged bag gets both the contents
- assertThat(bag1.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testMergeBagIntoNewNamespace() throws Exception {
- BagState<String> bag1 = underTest.state(NAMESPACE_1, STRING_BAG_ADDR);
- BagState<String> bag2 = underTest.state(NAMESPACE_2, STRING_BAG_ADDR);
- BagState<String> bag3 = underTest.state(NAMESPACE_3, STRING_BAG_ADDR);
-
- bag1.add("Hello");
- bag2.add("World");
- bag1.add("!");
-
- StateMerging.mergeBags(Arrays.asList(bag1, bag2, bag3), bag3);
-
- // Reading the merged bag gets both the contents
- assertThat(bag3.read(), Matchers.containsInAnyOrder("Hello", "World", "!"));
- assertThat(bag1.read(), Matchers.emptyIterable());
- assertThat(bag2.read(), Matchers.emptyIterable());
- }
-
- @Test
- public void testCombiningValue() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR)));
-
- assertThat(value.read(), Matchers.equalTo(0));
- value.add(2);
- assertThat(value.read(), Matchers.equalTo(2));
-
- value.add(3);
- assertThat(value.read(), Matchers.equalTo(5));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(0));
- assertEquals(underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR), value);
- }
-
- @Test
- public void testCombiningIsEmpty() throws Exception {
- GroupingState<Integer, Integer> value = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(5);
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeCombiningValueIntoSource() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- assertThat(value1.read(), Matchers.equalTo(11));
- assertThat(value2.read(), Matchers.equalTo(10));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1);
-
- assertThat(value1.read(), Matchers.equalTo(21));
- assertThat(value2.read(), Matchers.equalTo(0));
- }
-
- @Test
- public void testMergeCombiningValueIntoNewNamespace() throws Exception {
- CombiningState<Integer, int[], Integer> value1 =
- underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value2 =
- underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR);
- CombiningState<Integer, int[], Integer> value3 =
- underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR);
-
- value1.add(5);
- value2.add(10);
- value1.add(6);
-
- StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3);
-
- // Merging clears the old values and updates the result value.
- assertThat(value1.read(), Matchers.equalTo(0));
- assertThat(value2.read(), Matchers.equalTo(0));
- assertThat(value3.read(), Matchers.equalTo(21));
- }
-
- @Test
- public void testWatermarkEarliestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), Matchers.equalTo(new Instant(1000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), value);
- }
-
- @Test
- public void testWatermarkLatestState() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.add(new Instant(3000));
- assertThat(value.read(), Matchers.equalTo(new Instant(3000)));
-
- value.add(new Instant(1000));
- assertThat(value.read(), Matchers.equalTo(new Instant(3000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), value);
- }
-
- @Test
- public void testWatermarkEndOfWindowState() throws Exception {
- WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR);
-
- // State instances are cached, but depend on the namespace.
- assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR));
- assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR)));
-
- assertThat(value.read(), Matchers.nullValue());
- value.add(new Instant(2000));
- assertThat(value.read(), Matchers.equalTo(new Instant(2000)));
-
- value.clear();
- assertThat(value.read(), Matchers.equalTo(null));
- assertEquals(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), value);
- }
-
- @Test
- public void testWatermarkStateIsEmpty() throws Exception {
- WatermarkHoldState value =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
-
- assertThat(value.isEmpty().read(), Matchers.is(true));
- ReadableState<Boolean> readFuture = value.isEmpty();
- value.add(new Instant(1000));
- assertThat(readFuture.read(), Matchers.is(false));
-
- value.clear();
- assertThat(readFuture.read(), Matchers.is(true));
- }
-
- @Test
- public void testMergeEarliestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the merged value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value1, WINDOW_1);
-
- assertThat(value1.read(), Matchers.equalTo(new Instant(2000)));
- assertThat(value2.read(), Matchers.equalTo(null));
- }
-
- @Test
- public void testMergeLatestWatermarkIntoSource() throws Exception {
- WatermarkHoldState value1 =
- underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value2 =
- underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR);
- WatermarkHoldState value3 =
- underTest.state(NAMESPACE_3, WATERMARK_LATEST_ADDR);
-
- value1.add(new Instant(3000));
- value2.add(new Instant(5000));
- value1.add(new Instant(4000));
- value2.add(new Instant(2000));
-
- // Merging clears the old values and updates the result value.
- StateMerging.mergeWatermarks(Arrays.asList(value1, value2), value3, WINDOW_1);
-
- // Merging clears the old values and updates the result value.
- assertThat(value3.read(), Matchers.equalTo(new Instant(5000)));
- assertThat(value1.read(), Matchers.equalTo(null));
- assertThat(value2.read(), Matchers.equalTo(null));
- }
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
index 3a08088..df6c4d1 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
@@ -133,18 +133,8 @@
public Coder<CounterMark> getCheckpointMarkCoder() {
return DelegateCoder.of(
VarIntCoder.of(),
- new DelegateCoder.CodingFunction<CounterMark, Integer>() {
- @Override
- public Integer apply(CounterMark input) {
- return input.current;
- }
- },
- new DelegateCoder.CodingFunction<Integer, CounterMark>() {
- @Override
- public CounterMark apply(Integer input) {
- return new CounterMark(input);
- }
- });
+ new FromCounterMark(),
+ new ToCounterMark());
}
@Override
@@ -245,10 +235,41 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<KV<Integer, Integer>> getDefaultOutputCoder() {
+ public Coder<KV<Integer, Integer>> getOutputCoder() {
return KvCoder.of(VarIntCoder.of(), VarIntCoder.of());
}
+
+ private class FromCounterMark implements DelegateCoder.CodingFunction<CounterMark, Integer> {
+ @Override
+ public Integer apply(CounterMark input) {
+ return input.current;
+ }
+
+ @Override
+ public int hashCode() {
+ return FromCounterMark.class.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof FromCounterMark;
+ }
+ }
+
+ private class ToCounterMark implements DelegateCoder.CodingFunction<Integer, CounterMark> {
+ @Override
+ public CounterMark apply(Integer input) {
+ return new CounterMark(input);
+ }
+
+ @Override
+ public int hashCode() {
+ return ToCounterMark.class.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof ToCounterMark;
+ }
+ }
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
index 500fa66..bb2be60 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
@@ -20,49 +20,39 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import com.google.common.base.Joiner;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.beam.runners.flink.translation.wrappers.streaming.io.UnboundedSourceWrapper;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.ValueWithRecordId;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.state.ListState;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.OperatorStateStore;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
-import org.apache.flink.runtime.state.StateInitializationContext;
-import org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl;
import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.streaming.runtime.tasks.StreamTask;
-import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
+import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.OutputTag;
+import org.joda.time.Instant;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.Matchers;
/**
* Tests for {@link UnboundedSourceWrapper}.
@@ -102,7 +92,7 @@
* If numSplits > numTasks the source has one source will manage multiple readers.
*/
@Test
- public void testReaders() throws Exception {
+ public void testValueEmission() throws Exception {
final int numElements = 20;
final Object checkpointLock = new Object();
PipelineOptions options = PipelineOptionsFactory.create();
@@ -122,11 +112,20 @@
KV<Integer, Integer>,
TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(sourceOperator, numTasks);
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
+
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
try {
- sourceOperator.open();
+ testHarness.open();
sourceOperator.run(checkpointLock,
+ new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
private int count = 0;
@@ -135,6 +134,11 @@
}
@Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> streamRecord) {
+ collect((StreamRecord) streamRecord);
+ }
+
+ @Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
}
@@ -164,6 +168,111 @@
}
/**
+ * Creates a {@link UnboundedSourceWrapper} that has one or multiple readers per source.
+ * If numSplits > numTasks the source has one source will manage multiple readers.
+ *
+ * <p>This test verifies that watermark are correctly forwarded.
+ */
+ @Test(timeout = 30_000)
+ public void testWatermarkEmission() throws Exception {
+ final int numElements = 500;
+ final Object checkpointLock = new Object();
+ PipelineOptions options = PipelineOptionsFactory.create();
+
+ // this source will emit exactly NUM_ELEMENTS across all parallel readers,
+ // afterwards it will stall. We check whether we also receive NUM_ELEMENTS
+ // elements later.
+ TestCountingSource source = new TestCountingSource(numElements);
+ UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark> flinkWrapper =
+ new UnboundedSourceWrapper<>("stepName", options, source, numSplits);
+
+ assertEquals(numSplits, flinkWrapper.getSplitSources().size());
+
+ final StreamSource<WindowedValue<
+ ValueWithRecordId<KV<Integer, Integer>>>,
+ UnboundedSourceWrapper<
+ KV<Integer, Integer>,
+ TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
+
+ final AbstractStreamOperatorTestHarness<
+ WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>> testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
+
+ testHarness.setProcessingTime(Instant.now().getMillis());
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ final ConcurrentLinkedQueue<Object> caughtExceptions = new ConcurrentLinkedQueue<>();
+
+ // use the AtomicBoolean just for the set()/get() functionality for communicating
+ // with the outer Thread
+ final AtomicBoolean seenWatermark = new AtomicBoolean(false);
+
+ Thread sourceThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ testHarness.open();
+ sourceOperator.run(checkpointLock,
+ new TestStreamStatusMaintainer(),
+ new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
+
+ @Override
+ public void emitWatermark(Watermark watermark) {
+ if (watermark.getTimestamp() >= numElements / 2) {
+ seenWatermark.set(true);
+ }
+ }
+
+ @Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> streamRecord) {
+ }
+
+ @Override
+ public void emitLatencyMarker(LatencyMarker latencyMarker) {
+ }
+
+ @Override
+ public void collect(StreamRecord<WindowedValue<
+ ValueWithRecordId<KV<Integer, Integer>>>> windowedValueStreamRecord) {
+ }
+
+ @Override
+ public void close() {
+
+ }
+ });
+ } catch (Exception e) {
+ System.out.println("Caught exception: " + e);
+ caughtExceptions.add(e);
+ }
+ }
+ };
+
+ sourceThread.start();
+
+ while (true) {
+ if (!caughtExceptions.isEmpty()) {
+ fail("Caught exception(s): " + Joiner.on(",").join(caughtExceptions));
+ }
+ if (seenWatermark.get()) {
+ break;
+ }
+ Thread.sleep(10);
+
+ // need to advance this so that the watermark timers in the source wrapper fire
+ testHarness.setProcessingTime(Instant.now().getMillis());
+ }
+
+ sourceOperator.cancel();
+ sourceThread.join();
+ }
+
+
+ /**
* Verify that snapshot/restore work as expected. We bring up a source and cancel
* after seeing a certain number of elements. Then we snapshot that source,
* bring up a completely new source that we restore from the snapshot and verify
@@ -191,30 +300,24 @@
TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
- OperatorStateStore backend = mock(OperatorStateStore.class);
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
- TestingListState<KV<UnboundedSource, TestCountingSource.CounterMark>>
- listState = new TestingListState<>();
-
- when(backend.getOperatorState(Matchers.any(ListStateDescriptor.class)))
- .thenReturn(listState);
-
- StateInitializationContext initializationContext = mock(StateInitializationContext.class);
-
- when(initializationContext.getOperatorStateStore()).thenReturn(backend);
- when(initializationContext.isRestored()).thenReturn(false, true);
-
- flinkWrapper.initializeState(initializationContext);
-
- setupSourceOperator(sourceOperator, numTasks);
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
final Set<KV<Integer, Integer>> emittedElements = new HashSet<>();
boolean readFirstBatchOfElements = false;
try {
- sourceOperator.open();
+ testHarness.open();
sourceOperator.run(checkpointLock,
+ new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
private int count = 0;
@@ -223,6 +326,11 @@
}
@Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> streamRecord) {
+ collect((StreamRecord) streamRecord);
+ }
+
+ @Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
}
@@ -250,21 +358,12 @@
assertTrue("Did not successfully read first batch of elements.", readFirstBatchOfElements);
// draw a snapshot
- flinkWrapper.snapshotState(new StateSnapshotContextSynchronousImpl(0, 0));
-
- // test snapshot offsets
- assertEquals(flinkWrapper.getLocalSplitSources().size(),
- listState.getList().size());
- int totalEmit = 0;
- for (KV<UnboundedSource, TestCountingSource.CounterMark> kv : listState.get()) {
- totalEmit += kv.getValue().current + 1;
- }
- assertEquals(numElements / 2, totalEmit);
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
// test that finalizeCheckpoint on CheckpointMark is called
final ArrayList<Integer> finalizeList = new ArrayList<>();
TestCountingSource.setFinalizeTracker(finalizeList);
- flinkWrapper.notifyCheckpointComplete(0);
+ testHarness.notifyOfCompletedCheckpoint(0);
assertEquals(flinkWrapper.getLocalSplitSources().size(), finalizeList.size());
// create a completely new source but restore from the snapshot
@@ -282,17 +381,27 @@
TestCountingSource.CounterMark>> restoredSourceOperator =
new StreamSource<>(restoredFlinkWrapper);
- setupSourceOperator(restoredSourceOperator, numTasks);
+ // set parallelism to 1 to ensure that our testing operator gets all checkpointed state
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ restoredTestHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ restoredSourceOperator,
+ numTasks /* max parallelism */,
+ 1 /* parallelism */,
+ 0 /* subtask index */);
+
+ restoredTestHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
// restore snapshot
- restoredFlinkWrapper.initializeState(initializationContext);
+ restoredTestHarness.initializeState(snapshot);
boolean readSecondBatchOfElements = false;
// run again and verify that we see the other elements
try {
- restoredSourceOperator.open();
+ restoredTestHarness.open();
restoredSourceOperator.run(checkpointLock,
+ new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
private int count = 0;
@@ -301,6 +410,11 @@
}
@Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> streamRecord) {
+ collect((StreamRecord) streamRecord);
+ }
+
+ @Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
}
@@ -324,7 +438,9 @@
readSecondBatchOfElements = true;
}
- assertEquals(Math.max(1, numSplits / numTasks), flinkWrapper.getLocalSplitSources().size());
+ assertEquals(
+ Math.max(1, numSplits / numTasks),
+ restoredFlinkWrapper.getLocalSplitSources().size());
assertTrue("Did not successfully read second batch of elements.", readSecondBatchOfElements);
@@ -343,68 +459,57 @@
return null;
}
};
+
UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark> flinkWrapper =
new UnboundedSourceWrapper<>("stepName", options, source, numSplits);
- OperatorStateStore backend = mock(OperatorStateStore.class);
+ StreamSource<
+ WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
+ UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
+ sourceOperator = new StreamSource<>(flinkWrapper);
- TestingListState<KV<UnboundedSource, TestCountingSource.CounterMark>>
- listState = new TestingListState<>();
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
- when(backend.getOperatorState(Matchers.any(ListStateDescriptor.class)))
- .thenReturn(listState);
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
- StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+ testHarness.open();
- when(initializationContext.getOperatorStateStore()).thenReturn(backend);
- when(initializationContext.isRestored()).thenReturn(false, true);
-
- flinkWrapper.initializeState(initializationContext);
-
- StreamSource sourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(sourceOperator, numTasks);
- sourceOperator.open();
-
- flinkWrapper.snapshotState(new StateSnapshotContextSynchronousImpl(0, 0));
-
- assertEquals(0, listState.getList().size());
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
UnboundedSourceWrapper<
KV<Integer, Integer>, TestCountingSource.CounterMark> restoredFlinkWrapper =
- new UnboundedSourceWrapper<>("stepName", options, new TestCountingSource(numElements),
- numSplits);
+ new UnboundedSourceWrapper<>(
+ "stepName", options, new TestCountingSource(numElements), numSplits);
- StreamSource restoredSourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(restoredSourceOperator, numTasks);
- sourceOperator.open();
+ StreamSource<
+ WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
+ UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
+ restoredSourceOperator =
+ new StreamSource<>(restoredFlinkWrapper);
- restoredFlinkWrapper.initializeState(initializationContext);
+ // set parallelism to 1 to ensure that our testing operator gets all checkpointed state
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ restoredTestHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ restoredSourceOperator,
+ numTasks /* max parallelism */,
+ 1 /* parallelism */,
+ 0 /* subtask index */);
- assertEquals(Math.max(1, numSplits / numTasks), flinkWrapper.getLocalSplitSources().size());
+ restoredTestHarness.setup();
+ restoredTestHarness.initializeState(snapshot);
+ restoredTestHarness.open();
- }
+ // when the source checkpointed a null we don't re-initialize the splits, that is we
+ // will have no splits.
+ assertEquals(0, restoredFlinkWrapper.getLocalSplitSources().size());
- @SuppressWarnings("unchecked")
- private static <T> void setupSourceOperator(StreamSource<T, ?> operator, int numSubTasks) {
- ExecutionConfig executionConfig = new ExecutionConfig();
- StreamConfig cfg = new StreamConfig(new Configuration());
-
- cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
-
- Environment env = new DummyEnvironment("MockTwoInputTask", numSubTasks, 0);
-
- StreamTask<?, ?> mockTask = mock(StreamTask.class);
- when(mockTask.getName()).thenReturn("Mock Task");
- when(mockTask.getCheckpointLock()).thenReturn(new Object());
- when(mockTask.getConfiguration()).thenReturn(cfg);
- when(mockTask.getEnvironment()).thenReturn(env);
- when(mockTask.getExecutionConfig()).thenReturn(executionConfig);
- when(mockTask.getAccumulatorMap())
- .thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap());
- TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
- when(mockTask.getProcessingTimeService()).thenReturn(testProcessingTimeService);
-
- operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
/**
@@ -437,29 +542,20 @@
}
- private static final class TestingListState<T> implements ListState<T> {
-
- private final List<T> list = new ArrayList<>();
+ private static final class TestStreamStatusMaintainer implements StreamStatusMaintainer {
+ StreamStatus currentStreamStatus = StreamStatus.ACTIVE;
@Override
- public void clear() {
- list.clear();
+ public void toggleStreamStatus(StreamStatus streamStatus) {
+ if (!currentStreamStatus.equals(streamStatus)) {
+ currentStreamStatus = streamStatus;
+ }
}
@Override
- public Iterable<T> get() throws Exception {
- return list;
+ public StreamStatus getStreamStatus() {
+ return currentStreamStatus;
}
-
- @Override
- public void add(T value) throws Exception {
- list.add(value);
- }
-
- public List<T> getList() {
- return list;
- }
-
}
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializerTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializerTest.java
new file mode 100644
index 0000000..b0c40de
--- /dev/null
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializerTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.translation.types;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.beam.runners.flink.translation.types.CoderTypeSerializer.CoderTypeSerializerConfigSnapshot;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.flink.api.common.typeutils.ComparatorTestBase;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.junit.Test;
+
+/**
+ * Tests {@link CoderTypeSerializer}.
+ */
+public class CoderTypeSerializerTest {
+
+ @Test
+ public void shouldWriteAndReadSnapshotForAnonymousClassCoder() throws Exception {
+ AtomicCoder<String> anonymousClassCoder = new AtomicCoder<String>() {
+
+ @Override
+ public void encode(String value, OutputStream outStream)
+ throws CoderException, IOException {
+
+ }
+
+ @Override
+ public String decode(InputStream inStream) throws CoderException, IOException {
+ return "";
+ }
+ };
+
+ testWriteAndReadConfigSnapshot(anonymousClassCoder);
+ }
+
+ @Test
+ public void shouldWriteAndReadSnapshotForConcreteClassCoder() throws Exception {
+ Coder<String> concreteClassCoder = StringUtf8Coder.of();
+ testWriteAndReadConfigSnapshot(concreteClassCoder);
+ }
+
+ private void testWriteAndReadConfigSnapshot(Coder<String> coder) throws IOException {
+ CoderTypeSerializer<String> serializer = new CoderTypeSerializer<>(coder);
+
+ TypeSerializerConfigSnapshot writtenSnapshot = serializer.snapshotConfiguration();
+ ComparatorTestBase.TestOutputView outView = new ComparatorTestBase.TestOutputView();
+ writtenSnapshot.write(outView);
+
+ TypeSerializerConfigSnapshot readSnapshot = new CoderTypeSerializerConfigSnapshot<>();
+ readSnapshot.read(outView.getInputView());
+
+ assertThat(readSnapshot, is(writtenSnapshot));
+ }
+}
+
diff --git a/runners/gcp/gcemd/Dockerfile b/runners/gcp/gcemd/Dockerfile
new file mode 100644
index 0000000..b8fa8aa
--- /dev/null
+++ b/runners/gcp/gcemd/Dockerfile
@@ -0,0 +1,30 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM debian:stretch
+MAINTAINER "Apache Beam <dev@beam.apache.org>"
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ ca-certificates \
+ && \
+ rm -rf /var/lib/apt/lists/*
+
+ADD target/linux_amd64/gcemd /opt/apache/beam/
+
+ENTRYPOINT ["/opt/apache/beam/gcemd"]
diff --git a/runners/gcp/gcemd/main.go b/runners/gcp/gcemd/main.go
new file mode 100644
index 0000000..6c12907
--- /dev/null
+++ b/runners/gcp/gcemd/main.go
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// gcemd is a metadata-configured provisioning server for GCE.
+package main
+
+import (
+ "flag"
+ "log"
+ "net"
+
+ "cloud.google.com/go/compute/metadata"
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/provision"
+ "golang.org/x/net/context"
+ "google.golang.org/grpc"
+)
+
+var (
+ endpoint = flag.String("endpoint", "", "Server endpoint to expose.")
+)
+
+func main() {
+ flag.Parse()
+ if *endpoint == "" {
+ log.Fatal("No endpoint provided. Use --endpoint=localhost:12345")
+ }
+ if !metadata.OnGCE() {
+ log.Fatal("Not running on GCE")
+ }
+
+ log.Printf("Starting provisioning server on %v", *endpoint)
+
+ jobID, err := metadata.InstanceAttributeValue("job_id")
+ if err != nil {
+ log.Fatalf("Failed to find job ID: %v", err)
+ }
+ jobName, err := metadata.InstanceAttributeValue("job_name")
+ if err != nil {
+ log.Fatalf("Failed to find job name: %v", err)
+ }
+ opt, err := metadata.InstanceAttributeValue("sdk_pipeline_options")
+ if err != nil {
+ log.Fatalf("Failed to find SDK pipeline options: %v", err)
+ }
+ options, err := provision.JSONToProto(opt)
+ if err != nil {
+ log.Fatalf("Failed to parse SDK pipeline options: %v", err)
+ }
+
+ info := &pb.ProvisionInfo{
+ JobId: jobID,
+ JobName: jobName,
+ PipelineOptions: options,
+ }
+
+ gs := grpc.NewServer()
+ pb.RegisterProvisionServiceServer(gs, &server{info: info})
+
+ listener, err := net.Listen("tcp", *endpoint)
+ if err != nil {
+ log.Fatalf("Failed to listen to %v: %v", *endpoint, err)
+ }
+ log.Fatalf("Server failed: %v", gs.Serve(listener))
+}
+
+type server struct {
+ info *pb.ProvisionInfo
+}
+
+func (s *server) GetProvisionInfo(ctx context.Context, req *pb.GetProvisionInfoRequest) (*pb.GetProvisionInfoResponse, error) {
+ return &pb.GetProvisionInfoResponse{Info: s.info}, nil
+}
diff --git a/runners/gcp/gcemd/pom.xml b/runners/gcp/gcemd/pom.xml
new file mode 100644
index 0000000..2ade872
--- /dev/null
+++ b/runners/gcp/gcemd/pom.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gcp-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-gcp-gcemd</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: Runners :: Google Cloud Platform :: GCE metadata provisioning</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+ <go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.base}/github.com/apache/beam/cmd/gcemd</outputDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>*.go</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- CAVEAT: for latest shared files, run mvn install in sdks/go -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ <classifier>pkg-sources</classifier>
+ <overWrite>true</overWrite>
+ <outputDirectory>${go.source.dir}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>cloud.google.com/go/compute/metadata</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/gcemd</package>
+ </packages>
+ <resultName>gcemd</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/gcemd</package>
+ </packages>
+ <resultName>linux_amd64/gcemd</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <configuration>
+ <repository>${docker-repository-root}/gcemd</repository>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/runners/gcp/gcsproxy/Dockerfile b/runners/gcp/gcsproxy/Dockerfile
new file mode 100644
index 0000000..5ff9141
--- /dev/null
+++ b/runners/gcp/gcsproxy/Dockerfile
@@ -0,0 +1,30 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM debian:stretch
+MAINTAINER "Apache Beam <dev@beam.apache.org>"
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ ca-certificates \
+ && \
+ rm -rf /var/lib/apt/lists/*
+
+ADD target/linux_amd64/gcsproxy /opt/apache/beam/
+
+ENTRYPOINT ["/opt/apache/beam/gcsproxy"]
diff --git a/runners/gcp/gcsproxy/main.go b/runners/gcp/gcsproxy/main.go
new file mode 100644
index 0000000..4123b6b
--- /dev/null
+++ b/runners/gcp/gcsproxy/main.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// gcsproxy is an artifact server backed by GCS and can run in either retrieval
+// (read) or staging (write) mode.
+package main
+
+import (
+ "context"
+ "flag"
+ "log"
+ "net"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/artifact/gcsproxy"
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "google.golang.org/grpc"
+)
+
+const (
+ retrieve = "retrieve"
+ stage = "stage"
+)
+
+var (
+ mode = flag.String("mode", retrieve, "Proxy mode: retrieve or stage.")
+ endpoint = flag.String("endpoint", "", "Server endpoint to expose.")
+ manifest = flag.String("manifest", "", "Location of proxy manifest.")
+)
+
+func main() {
+ flag.Parse()
+ if *manifest == "" {
+ log.Fatal("No proxy manifest location provided. Use --manifest=gs://foo/bar")
+ }
+ if *endpoint == "" {
+ log.Fatal("No endpoint provided. Use --endpoint=localhost:12345")
+ }
+
+ gs := grpc.NewServer()
+
+ switch *mode {
+ case retrieve:
+ // Retrieval mode. We download the manifest -- but not the
+ // artifacts -- eagerly.
+
+ log.Printf("Starting retrieval proxy from %v on %v", *manifest, *endpoint)
+
+ md, err := gcsproxy.ReadProxyManifest(context.Background(), *manifest)
+ if err != nil {
+ log.Fatalf("Failed to obtain proxy manifest %v: %v", *manifest, err)
+ }
+ proxy, err := gcsproxy.NewRetrievalServer(md)
+ if err != nil {
+ log.Fatalf("Failed to create artifact server: %v", err)
+ }
+ pb.RegisterArtifactRetrievalServiceServer(gs, proxy)
+
+ case stage:
+ // Staging proxy. We update the blobs next to the manifest
+ // in a blobs "directory".
+
+ log.Printf("Starting staging proxy to %v on %v", *manifest, *endpoint)
+
+ proxy, err := gcsproxy.NewStagingServer(*manifest)
+ if err != nil {
+ log.Fatalf("Failed to create artifact server: %v", err)
+ }
+ pb.RegisterArtifactStagingServiceServer(gs, proxy)
+
+ default:
+ log.Fatalf("Invalid mode: '%v', want '%v' or '%v'", *mode, retrieve, stage)
+ }
+
+ listener, err := net.Listen("tcp", *endpoint)
+ if err != nil {
+ log.Fatalf("Failed to listen to %v: %v", *endpoint, err)
+ }
+ log.Fatalf("Server failed: %v", gs.Serve(listener))
+}
diff --git a/runners/gcp/gcsproxy/pom.xml b/runners/gcp/gcsproxy/pom.xml
new file mode 100644
index 0000000..f2c562d
--- /dev/null
+++ b/runners/gcp/gcsproxy/pom.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gcp-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-gcp-gcsproxy</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: Runners :: Google Cloud Platform :: GCS artifact proxy</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+ <go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.base}/github.com/apache/beam/cmd/gcsproxy</outputDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>*.go</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- CAVEAT: for latest shared files, run mvn install in sdks/go -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ <classifier>pkg-sources</classifier>
+ <overWrite>true</overWrite>
+ <outputDirectory>${go.source.dir}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>google.golang.org/api/storage/v1</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/gcsproxy</package>
+ </packages>
+ <resultName>gcsproxy</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/gcsproxy</package>
+ </packages>
+ <resultName>linux_amd64/gcsproxy</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <configuration>
+ <repository>${docker-repository-root}/gcsproxy</repository>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/runners/gcp/pom.xml b/runners/gcp/pom.xml
new file mode 100644
index 0000000..eda19d8
--- /dev/null
+++ b/runners/gcp/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-gcp-parent</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: Runners :: Google Cloud Platform</name>
+
+ <modules>
+ <module>gcemd</module>
+ <module>gcsproxy</module>
+ </modules>
+</project>
diff --git a/runners/gearpump/README.md b/runners/gearpump/README.md
new file mode 100644
index 0000000..e8ce794
--- /dev/null
+++ b/runners/gearpump/README.md
@@ -0,0 +1,61 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+## Gearpump Beam Runner
+
+The Gearpump Beam runner allows users to execute pipelines written using the Apache Beam programming API with Apache Gearpump (incubating) as an execution engine.
+
+##Getting Started
+
+The following shows how to run the WordCount example that is provided with the source code on Beam.
+
+###Installing Beam
+
+To get the latest version of Beam with Gearpump-Runner, first clone the Beam repository:
+
+```
+git clone https://github.com/apache/beam
+git checkout gearpump-runner
+```
+
+Then switch to the newly created directory and run Maven to build the Apache Beam:
+
+```
+cd beam
+mvn clean install -DskipTests
+```
+
+Now Apache Beam and the Gearpump Runner are installed in your local Maven repository.
+
+###Running Wordcount Example
+
+Download something to count:
+
+```
+curl http://www.gutenberg.org/cache/epub/1128/pg1128.txt > /tmp/kinglear.txt
+```
+
+Run the pipeline, using the Gearpump runner:
+
+```
+cd examples/java
+mvn exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount -Dexec.args="--inputFile=/tmp/kinglear.txt --output=/tmp/wordcounts.txt --runner=TestGearpumpRunner" -Pgearpump-runner
+```
+
+Once completed, check the output file /tmp/wordcounts.txt-00000-of-00001
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
new file mode 100644
index 0000000..bb51745
--- /dev/null
+++ b/runners/gearpump/pom.xml
@@ -0,0 +1,277 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-gearpump</artifactId>
+
+ <name>Apache Beam :: Runners :: Gearpump</name>
+ <packaging>jar</packaging>
+
+ <repositories>
+ <repository>
+ <id>apache-repo</id>
+ <name>apache maven repo</name>
+ <url>https://repository.apache.org/content/repositories/releases</url>
+ </repository>
+ </repositories>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <gearpump.version>0.8.4</gearpump.version>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>local-validates-runner-tests</id>
+ <activation><activeByDefault>false</activeByDefault></activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>validates-runner-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
+ <excludedGroups>
+ org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
+ org.apache.beam.sdk.testing.UsesStatefulParDo,
+ org.apache.beam.sdk.testing.UsesTimersInParDo,
+ org.apache.beam.sdk.testing.UsesSplittableParDo,
+ org.apache.beam.sdk.testing.UsesAttemptedMetrics,
+ org.apache.beam.sdk.testing.UsesCommittedMetrics,
+ org.apache.beam.sdk.testing.UsesTestStream,
+ org.apache.beam.sdk.testing.UsesCustomWindowMerging
+ </excludedGroups>
+ <parallel>none</parallel>
+ <failIfNoTests>true</failIfNoTests>
+ <dependenciesToScan>
+ <dependency>org.apache.beam:beam-sdks-java-core</dependency>
+ </dependenciesToScan>
+ <systemPropertyVariables>
+ <beamTestPipelineOptions>
+ [
+ "--runner=TestGearpumpRunner",
+ "--streaming=true"
+ ]
+ </beamTestPipelineOptions>
+ </systemPropertyVariables>
+ <threadCount>4</threadCount>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gearpump</groupId>
+ <artifactId>gearpump-streaming_2.11</artifactId>
+ <version>${gearpump.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gearpump</groupId>
+ <artifactId>gearpump-core_2.11</artifactId>
+ <version>${gearpump.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ <version>1.3.0</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.11.8</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.collections</groupId>
+ <artifactId>google-collections</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-construction-java</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- JAR Packaging -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+
+ <!-- Java compiler -->
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ <testSource>1.8</testSource>
+ <testTarget>1.8</testTarget>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <enforceBytecodeVersion>
+ <maxJdkVersion>1.8</maxJdkVersion>
+ </enforceBytecodeVersion>
+ <requireJavaVersion>
+ <version>[1.8,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- uber jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineOptions.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineOptions.java
new file mode 100644
index 0000000..e02cbbc
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineOptions.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import java.util.Map;
+
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+import org.apache.gearpump.cluster.client.ClientContext;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+
+/**
+ * Options that configure the Gearpump pipeline.
+ */
+public interface GearpumpPipelineOptions extends PipelineOptions {
+
+ @Description("set unique application name for Gearpump runner")
+ void setApplicationName(String name);
+
+ String getApplicationName();
+
+ @Description("set parallelism for Gearpump processor")
+ void setParallelism(int parallelism);
+
+ @Default.Integer(1)
+ int getParallelism();
+
+ @Description("register Kryo serializers")
+ void setSerializers(Map<String, String> serializers);
+
+ @JsonIgnore
+ Map<String, String> getSerializers();
+
+ @Description("set EmbeddedCluster for tests")
+ void setEmbeddedCluster(EmbeddedCluster cluster);
+
+ @JsonIgnore
+ EmbeddedCluster getEmbeddedCluster();
+
+ void setClientContext(ClientContext clientContext);
+
+ @JsonIgnore
+ @Description("get client context to query application status")
+ ClientContext getClientContext();
+
+}
+
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
new file mode 100644
index 0000000..4a7e589
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.metrics.MetricResults;
+
+import org.apache.gearpump.cluster.ApplicationStatus;
+import org.apache.gearpump.cluster.MasterToAppMaster.AppMasterData;
+import org.apache.gearpump.cluster.client.ClientContext;
+import org.apache.gearpump.cluster.client.RunningApplication;
+import org.joda.time.Duration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+/**
+ * Result of executing a {@link Pipeline} with Gearpump.
+ */
+public class GearpumpPipelineResult implements PipelineResult {
+
+ private final ClientContext client;
+ private final RunningApplication app;
+ private boolean finished = false;
+
+ public GearpumpPipelineResult(ClientContext client, RunningApplication app) {
+ this.client = client;
+ this.app = app;
+ }
+
+ @Override
+ public State getState() {
+ if (!finished) {
+ return getGearpumpState();
+ } else {
+ return State.DONE;
+ }
+ }
+
+ @Override
+ public State cancel() throws IOException {
+ if (!finished) {
+ app.shutDown();
+ finished = true;
+ return State.CANCELLED;
+ } else {
+ return State.DONE;
+ }
+ }
+
+ @Override
+ public State waitUntilFinish(Duration duration) {
+ return waitUntilFinish();
+ }
+
+ @Override
+ public State waitUntilFinish() {
+ if (!finished) {
+ app.waitUntilFinish();
+ finished = true;
+ }
+ return State.DONE;
+ }
+
+ @Override
+ public MetricResults metrics() {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support querying metrics", getClass().getSimpleName()));
+ }
+
+ private State getGearpumpState() {
+ ApplicationStatus status = null;
+ List<AppMasterData> apps =
+ JavaConverters.<AppMasterData>seqAsJavaListConverter(
+ (Seq<AppMasterData>) client.listApps().appMasters()).asJava();
+ for (AppMasterData appData: apps) {
+ if (appData.appId() == app.appId()) {
+ status = appData.status();
+ }
+ }
+ if (null == status || status instanceof ApplicationStatus.NONEXIST$) {
+ return State.UNKNOWN;
+ } else if (status instanceof ApplicationStatus.ACTIVE$) {
+ return State.RUNNING;
+ } else if (status instanceof ApplicationStatus.SUCCEEDED$) {
+ return State.DONE;
+ } else {
+ return State.FAILED;
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
new file mode 100644
index 0000000..5febf3c
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValueFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.beam.runners.gearpump.translators.GearpumpPipelineTranslator;
+import org.apache.beam.runners.gearpump.translators.TranslationContext;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsValidator;
+
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.UserConfig;
+import org.apache.gearpump.cluster.client.ClientContext;
+import org.apache.gearpump.cluster.client.RunningApplication;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStreamApp;
+
+/**
+ * A {@link PipelineRunner} that executes the operations in the
+ * pipeline by first translating them to Gearpump Stream DSL
+ * and then executing them on a Gearpump cluster.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
+
+ private final GearpumpPipelineOptions options;
+
+ private static final String GEARPUMP_SERIALIZERS = "gearpump.serializers";
+ private static final String DEFAULT_APPNAME = "beam_gearpump_app";
+
+ public GearpumpRunner(GearpumpPipelineOptions options) {
+ this.options = options;
+ }
+
+ public static GearpumpRunner fromOptions(PipelineOptions options) {
+ GearpumpPipelineOptions pipelineOptions =
+ PipelineOptionsValidator.validate(GearpumpPipelineOptions.class, options);
+ return new GearpumpRunner(pipelineOptions);
+ }
+
+ @Override
+ public GearpumpPipelineResult run(Pipeline pipeline) {
+ String appName = options.getApplicationName();
+ if (null == appName) {
+ appName = DEFAULT_APPNAME;
+ }
+ Config config = registerSerializers(ClusterConfig.defaultConfig(),
+ options.getSerializers());
+ ClientContext clientContext = getClientContext(options, config);
+ options.setClientContext(clientContext);
+ UserConfig userConfig = UserConfig.empty();
+ JavaStreamApp streamApp = new JavaStreamApp(
+ appName, clientContext, userConfig);
+ TranslationContext translationContext = new TranslationContext(streamApp, options);
+ GearpumpPipelineTranslator translator = new GearpumpPipelineTranslator(translationContext);
+ translator.translate(pipeline);
+ RunningApplication app = streamApp.submit();
+
+ return new GearpumpPipelineResult(clientContext, app);
+ }
+
+ private ClientContext getClientContext(GearpumpPipelineOptions options, Config config) {
+ EmbeddedCluster cluster = options.getEmbeddedCluster();
+ if (cluster != null) {
+ return cluster.newClientContext();
+ } else {
+ return ClientContext.apply(config);
+ }
+ }
+
+ /**
+ * register class with default kryo serializers.
+ */
+ private Config registerSerializers(Config config, Map<String, String> userSerializers) {
+ Map<String, String> serializers = new HashMap<>();
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$ValueInGlobalWindow", "");
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInSingleWindow", "");
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInGlobalWindow", "");
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInMultipleWindows", "");
+ serializers.put("org.apache.beam.sdk.transforms.windowing.PaneInfo", "");
+ serializers.put("org.apache.beam.sdk.transforms.windowing.PaneInfo$Timing", "");
+ serializers.put("org.joda.time.Instant", "");
+ serializers.put("org.apache.beam.sdk.values.KV", "");
+ serializers.put("org.apache.beam.sdk.transforms.windowing.IntervalWindow", "");
+ serializers.put("org.apache.beam.sdk.values.TimestampedValue", "");
+ serializers.put(
+ "org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils$RawUnionValue", "");
+
+ if (userSerializers != null && !userSerializers.isEmpty()) {
+ serializers.putAll(userSerializers);
+ }
+
+ return config.withValue(GEARPUMP_SERIALIZERS, ConfigValueFactory.fromMap(serializers));
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
new file mode 100644
index 0000000..5152105
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
+import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
+
+/**
+ * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the
+ * {@link GearpumpRunner}.
+ *
+ * {@link AutoService} will register Gearpump's implementations of the {@link PipelineRunner}
+ * and {@link PipelineOptions} as available pipeline runner services.
+ */
+public class GearpumpRunnerRegistrar {
+ private GearpumpRunnerRegistrar() { }
+
+ /**
+ * Registers the {@link GearpumpRunner}.
+ */
+ @AutoService(PipelineRunnerRegistrar.class)
+ public static class Runner implements PipelineRunnerRegistrar {
+
+ @Override
+ public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
+ return ImmutableList.<Class<? extends PipelineRunner<?>>>of(
+ GearpumpRunner.class,
+ TestGearpumpRunner.class);
+ }
+ }
+
+ /**
+ * Registers the {@link GearpumpPipelineOptions}.
+ */
+ @AutoService(PipelineOptionsRegistrar.class)
+ public static class Options implements PipelineOptionsRegistrar {
+
+ @Override
+ public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
+ return ImmutableList.<Class<? extends PipelineOptions>>of(GearpumpPipelineOptions.class);
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
new file mode 100644
index 0000000..0a88849
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValueFactory;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsValidator;
+
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.apache.gearpump.util.Constants;
+
+/**
+ * Gearpump {@link PipelineRunner} for tests, which uses {@link EmbeddedCluster}.
+ */
+public class TestGearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
+
+ private final GearpumpRunner delegate;
+ private final EmbeddedCluster cluster;
+
+ private TestGearpumpRunner(GearpumpPipelineOptions options) {
+ Config config = ClusterConfig.master(null);
+ config = config.withValue(Constants.APPLICATION_TOTAL_RETRIES(),
+ ConfigValueFactory.fromAnyRef(0));
+ cluster = new EmbeddedCluster(config);
+ cluster.start();
+ options.setEmbeddedCluster(cluster);
+ delegate = GearpumpRunner.fromOptions(options);
+ }
+
+ public static TestGearpumpRunner fromOptions(PipelineOptions options) {
+ GearpumpPipelineOptions pipelineOptions =
+ PipelineOptionsValidator.validate(GearpumpPipelineOptions.class, options);
+ return new TestGearpumpRunner(pipelineOptions);
+ }
+
+ @Override
+ public GearpumpPipelineResult run(Pipeline pipeline) {
+ GearpumpPipelineResult result = delegate.run(pipeline);
+ result.waitUntilFinish();
+ cluster.stop();
+ return result;
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
new file mode 100644
index 0000000..5013616
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal implementation of the Beam runner for Apache Gearpump.
+ */
+package org.apache.beam.runners.gearpump;
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
new file mode 100644
index 0000000..559cb28
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import java.util.List;
+
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+/**
+ * CreateGearpumpPCollectionView bridges input stream to down stream
+ * transforms.
+ */
+public class CreateGearpumpPCollectionViewTranslator<ElemT, ViewT> implements
+ TransformTranslator<CreateStreamingGearpumpView.CreateGearpumpPCollectionView<ElemT, ViewT>> {
+
+ private static final long serialVersionUID = -3955521308055056034L;
+
+ @Override
+ public void translate(
+ CreateStreamingGearpumpView.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
+ TranslationContext context) {
+ JavaStream<WindowedValue<List<ElemT>>> inputStream =
+ context.getInputStream(context.getInput());
+ PCollectionView<ViewT> view = transform.getView();
+ context.setOutputStream(view, inputStream);
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
new file mode 100644
index 0000000..3ebe5c8
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Gearpump streaming overrides for various view (side input) transforms. */
+class CreateStreamingGearpumpView<ElemT, ViewT>
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
+
+ public CreateStreamingGearpumpView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ @Override
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ input
+ .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<ElemT, ViewT>of(view));
+ return input;
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link CreateStreamingGearpumpView}. This combiner requires that
+ * the input {@link PCollection} fits in memory. For a large {@link PCollection} this is
+ * expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<T>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
+
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateGearpumpPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollection<List<ElemT>>> {
+ private PCollectionView<ViewT> view;
+
+ private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateGearpumpPCollectionView<>(view);
+ }
+
+ @Override
+ public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) {
+ return PCollection.<List<ElemT>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+ }
+
+ public static class Factory<ElemT, ViewT>
+ implements PTransformOverrideFactory<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
+ public Factory() {}
+
+ @Override
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
+ AppliedPTransform<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
+ transform) {
+ return PTransformReplacement.of(
+ (PCollection<ElemT>) Iterables.getOnlyElement(transform.getInputs().values()),
+ new CreateStreamingGearpumpView<ElemT, ViewT>(transform.getTransform().getView()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
new file mode 100644
index 0000000..8cc0058
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Lists;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.runners.gearpump.translators.io.ValuesSource;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.values.PCollection;
+
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+/**
+ * Flatten.FlattenPCollectionList is translated to Gearpump merge function.
+ */
+public class FlattenPCollectionsTranslator<T> implements
+ TransformTranslator<Flatten.PCollections<T>> {
+
+ private static final long serialVersionUID = -5552148802472944759L;
+
+ @Override
+ public void translate(Flatten.PCollections<T> transform, TranslationContext context) {
+ JavaStream<T> merged = null;
+ Set<PCollection<T>> unique = new HashSet<>();
+ for (PValue input: context.getInputs().values()) {
+ PCollection<T> collection = (PCollection<T>) input;
+ JavaStream<T> inputStream = context.getInputStream(collection);
+ if (null == merged) {
+ merged = inputStream;
+ } else {
+ // duplicate edges are not allowed in Gearpump graph
+ // so we route through a dummy node
+ if (unique.contains(collection)) {
+ inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
+ }
+
+ merged = merged.merge(inputStream, 1, transform.getName());
+ }
+ unique.add(collection);
+ }
+
+ if (null == merged) {
+ UnboundedSourceWrapper<String, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
+ new ValuesSource<>(Lists.newArrayList("dummy"),
+ StringUtf8Coder.of()), context.getPipelineOptions());
+ merged = context.getSourceStream(unboundedSourceWrapper);
+ }
+ context.setOutputStream(context.getOutput(), merged);
+ }
+
+ private static class DummyFunction<T> extends MapFunction<T, T> {
+
+ private static final long serialVersionUID = 5454396869997290471L;
+
+ @Override
+ public T map(T t) {
+ return t;
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
new file mode 100644
index 0000000..ca98aac
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.PTransformOverride;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.PValue;
+
+import org.apache.gearpump.util.Graph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link GearpumpPipelineTranslator} knows how to translate {@link Pipeline} objects
+ * into Gearpump {@link Graph}.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ GearpumpPipelineTranslator.class);
+
+ /**
+ * A map from {@link PTransform} subclass to the corresponding
+ * {@link TransformTranslator} to use to translate that transform.
+ */
+ private static final Map<Class<? extends PTransform>, TransformTranslator>
+ transformTranslators = new HashMap<>();
+
+ private final TranslationContext translationContext;
+
+ static {
+ // register TransformTranslators
+ registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
+ registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
+ registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
+ registerTransformTranslator(Flatten.PCollections.class,
+ new FlattenPCollectionsTranslator());
+ registerTransformTranslator(ParDo.MultiOutput.class, new ParDoMultiOutputTranslator());
+ registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
+ registerTransformTranslator(CreateStreamingGearpumpView.CreateGearpumpPCollectionView.class,
+ new CreateGearpumpPCollectionViewTranslator());
+ }
+
+ public GearpumpPipelineTranslator(TranslationContext translationContext) {
+ this.translationContext = translationContext;
+ }
+
+ public void translate(Pipeline pipeline) {
+ List<PTransformOverride> overrides =
+ ImmutableList.<PTransformOverride>builder()
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
+ new CreateStreamingGearpumpView.Factory()))
+ .build();
+
+ pipeline.replaceAll(overrides);
+ pipeline.traverseTopologically(this);
+ }
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ LOG.debug("entering composite transform {}", node.getTransform());
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void leaveCompositeTransform(TransformHierarchy.Node node) {
+ LOG.debug("leaving composite transform {}", node.getTransform());
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ LOG.debug("visiting transform {}", node.getTransform());
+ PTransform transform = node.getTransform();
+ TransformTranslator translator = getTransformTranslator(transform.getClass());
+ if (null == translator) {
+ throw new IllegalStateException(
+ "no translator registered for " + transform);
+ }
+ translationContext.setCurrentTransform(node, getPipeline());
+ translator.translate(transform, translationContext);
+ }
+
+ @Override
+ public void visitValue(PValue value, TransformHierarchy.Node producer) {
+ LOG.debug("visiting value {}", value);
+ }
+
+ /**
+ * Records that instances of the specified PTransform class
+ * should be translated by default by the corresponding
+ * {@link TransformTranslator}.
+ */
+ private static <TransformT extends PTransform> void registerTransformTranslator(
+ Class<TransformT> transformClass,
+ TransformTranslator<? extends TransformT> transformTranslator) {
+ if (transformTranslators.put(transformClass, transformTranslator) != null) {
+ throw new IllegalArgumentException(
+ "defining multiple translators for " + transformClass);
+ }
+ }
+
+ /**
+ * Returns the {@link TransformTranslator} to use for instances of the
+ * specified PTransform class, or null if none registered.
+ */
+ private <TransformT extends PTransform>
+ TransformTranslator<TransformT> getTransformTranslator(Class<TransformT> transformClass) {
+ return transformTranslators.get(transformClass);
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
new file mode 100644
index 0000000..bea5a74
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.gearpump.streaming.dsl.api.functions.FoldFunction;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.javaapi.functions.GroupByFunction;
+import org.apache.gearpump.streaming.dsl.window.api.Discarding$;
+import org.apache.gearpump.streaming.dsl.window.api.EventTimeTrigger$;
+import org.apache.gearpump.streaming.dsl.window.api.WindowFunction;
+import org.apache.gearpump.streaming.dsl.window.api.Windows;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.joda.time.Instant;
+
+/**
+ * {@link GroupByKey} is translated to Gearpump groupBy function.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKey<K, V>> {
+
+ private static final long serialVersionUID = -8742202583992787659L;
+
+ @Override
+ public void translate(GroupByKey<K, V> transform, TranslationContext context) {
+ PCollection<KV<K, V>> input = (PCollection<KV<K, V>>) context.getInput();
+ Coder<K> inputKeyCoder = ((KvCoder<K, V>) input.getCoder()).getKeyCoder();
+ JavaStream<WindowedValue<KV<K, V>>> inputStream =
+ context.getInputStream(input);
+ int parallelism = context.getPipelineOptions().getParallelism();
+ TimestampCombiner timestampCombiner = input.getWindowingStrategy().getTimestampCombiner();
+ WindowFn<KV<K, V>, BoundedWindow> windowFn = (WindowFn<KV<K, V>, BoundedWindow>)
+ input.getWindowingStrategy().getWindowFn();
+ JavaStream<WindowedValue<KV<K, List<V>>>> outputStream = inputStream
+ .window(Windows.apply(
+ new GearpumpWindowFn(windowFn.isNonMerging()),
+ EventTimeTrigger$.MODULE$, Discarding$.MODULE$, windowFn.toString()))
+ .groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
+ .map(new KeyedByTimestamp<K, V>(windowFn, timestampCombiner), "keyed_by_timestamp")
+ .fold(new Merge<>(windowFn, timestampCombiner), "merge")
+ .map(new Values<K, V>(), "values");
+
+ context.setOutputStream(context.getOutput(), outputStream);
+ }
+
+ /**
+ * A transform used internally to translate Beam's Window to Gearpump's Window.
+ */
+ protected static class GearpumpWindowFn<T, W extends BoundedWindow>
+ implements WindowFunction, Serializable {
+
+ private final boolean isNonMerging;
+
+ public GearpumpWindowFn(boolean isNonMerging) {
+ this.isNonMerging = isNonMerging;
+ }
+
+ @Override
+ public <T> Window[] apply(Context<T> context) {
+ try {
+ Object element = context.element();
+ if (element instanceof TranslatorUtils.RawUnionValue) {
+ element = ((TranslatorUtils.RawUnionValue) element).getValue();
+ }
+ return toGearpumpWindows(((WindowedValue<T>) element).getWindows()
+ .toArray(new BoundedWindow[0]));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isNonMerging() {
+ return isNonMerging;
+ }
+
+ private Window[] toGearpumpWindows(BoundedWindow[] windows) {
+ Window[] gwins = new Window[windows.length];
+ for (int i = 0; i < windows.length; i++) {
+ gwins[i] = TranslatorUtils.boundedWindowToGearpumpWindow(windows[i]);
+ }
+ return gwins;
+ }
+ }
+
+ /**
+ * A transform used internally to group KV message by its key.
+ */
+ protected static class GroupByFn<K, V> extends
+ GroupByFunction<WindowedValue<KV<K, V>>, ByteBuffer> {
+
+ private static final long serialVersionUID = -807905402490735530L;
+ private final Coder<K> keyCoder;
+
+ GroupByFn(Coder<K> keyCoder) {
+ this.keyCoder = keyCoder;
+ }
+
+ @Override
+ public ByteBuffer groupBy(WindowedValue<KV<K, V>> wv) {
+ try {
+ return ByteBuffer.wrap(CoderUtils.encodeToByteArray(keyCoder, wv.getValue().getKey()));
+ } catch (CoderException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * A transform used internally to transform WindowedValue to KV.
+ */
+ protected static class KeyedByTimestamp<K, V>
+ extends MapFunction<WindowedValue<KV<K, V>>,
+ KV<Instant, WindowedValue<KV<K, V>>>> {
+
+ private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
+ private final TimestampCombiner timestampCombiner;
+
+ public KeyedByTimestamp(WindowFn<KV<K, V>, BoundedWindow> windowFn,
+ TimestampCombiner timestampCombiner) {
+ this.windowFn = windowFn;
+ this.timestampCombiner = timestampCombiner;
+ }
+
+ @Override
+ public KV<org.joda.time.Instant, WindowedValue<KV<K, V>>> map(
+ WindowedValue<KV<K, V>> wv) {
+ BoundedWindow window = Iterables.getOnlyElement(wv.getWindows());
+ Instant timestamp = timestampCombiner.assign(window
+ , windowFn.getOutputTime(wv.getTimestamp(), window));
+ return KV.of(timestamp, wv);
+ }
+ }
+
+ /**
+ * A transform used internally by Gearpump which encapsulates the merge logic.
+ */
+ protected static class Merge<K, V> extends
+ FoldFunction<KV<Instant, WindowedValue<KV<K, V>>>,
+ KV<Instant, WindowedValue<KV<K, List<V>>>>> {
+
+ private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
+ private final TimestampCombiner timestampCombiner;
+
+ Merge(WindowFn<KV<K, V>, BoundedWindow> windowFn,
+ TimestampCombiner timestampCombiner) {
+ this.windowFn = windowFn;
+ this.timestampCombiner = timestampCombiner;
+ }
+
+ @Override
+ public KV<Instant, WindowedValue<KV<K, List<V>>>> init() {
+ return KV.of(null, null);
+ }
+
+ @Override
+ public KV<Instant, WindowedValue<KV<K, List<V>>>> fold(
+ KV<Instant, WindowedValue<KV<K, List<V>>>> accum,
+ KV<Instant, WindowedValue<KV<K, V>>> iter) {
+ if (accum.getKey() == null) {
+ WindowedValue<KV<K, V>> wv = iter.getValue();
+ KV<K, V> kv = wv.getValue();
+ V v = kv.getValue();
+ List<V> nv = Lists.newArrayList(v);
+ return KV.of(iter.getKey(), wv.withValue(KV.of(kv.getKey(), nv)));
+ }
+
+ Instant t1 = accum.getKey();
+ Instant t2 = iter.getKey();
+
+ final WindowedValue<KV<K, List<V>>> wv1 = accum.getValue();
+ final WindowedValue<KV<K, V>> wv2 = iter.getValue();
+ wv1.getValue().getValue().add(wv2.getValue().getValue());
+
+ final List<BoundedWindow> mergedWindows = new ArrayList<>();
+ if (!windowFn.isNonMerging()) {
+ try {
+ windowFn.mergeWindows(windowFn.new MergeContext() {
+
+ @Override
+ public Collection<BoundedWindow> windows() {
+ ArrayList<BoundedWindow> windows = new ArrayList<>();
+ windows.addAll(wv1.getWindows());
+ windows.addAll(wv2.getWindows());
+ return windows;
+ }
+
+ @Override
+ public void merge(Collection<BoundedWindow> toBeMerged,
+ BoundedWindow mergeResult) throws Exception {
+ mergedWindows.add(mergeResult);
+ }
+ });
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ mergedWindows.addAll(wv1.getWindows());
+ }
+
+ Instant timestamp = timestampCombiner.combine(t1, t2);
+ return KV.of(timestamp,
+ WindowedValue.of(wv1.getValue(), timestamp,
+ mergedWindows, wv1.getPane()));
+ }
+ }
+
+ private static class Values<K, V> extends
+ MapFunction<KV<Instant, WindowedValue<KV<K, List<V>>>>,
+ WindowedValue<KV<K, List<V>>>> {
+
+ @Override
+ public WindowedValue<KV<K, List<V>>> map(KV<org.joda.time.Instant,
+ WindowedValue<KV<K, List<V>>>> kv) {
+ Instant timestamp = kv.getKey();
+ WindowedValue<KV<K, List<V>>> wv = kv.getValue();
+ return WindowedValue.of(wv.getValue(), timestamp, wv.getWindows(), wv.getPane());
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
new file mode 100644
index 0000000..d92979b
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+/**
+ * {@link ParDo.MultiOutput} is translated to Gearpump flatMap function
+ * with {@link DoFn} wrapped in {@link DoFnFunction}. The outputs are
+ * further filtered with Gearpump filter function by output tag
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class ParDoMultiOutputTranslator<InputT, OutputT> implements
+ TransformTranslator<ParDo.MultiOutput<InputT, OutputT>> {
+
+ private static final long serialVersionUID = -6023461558200028849L;
+
+ @Override
+ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationContext context) {
+ PCollection<InputT> inputT = (PCollection<InputT>) context.getInput();
+ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
+ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
+ Map<String, PCollectionView<?>> tagsToSideInputs =
+ TranslatorUtils.getTagsToSideInputs(sideInputs);
+
+ Map<TupleTag<?>, PValue> outputs = context.getOutputs();
+ final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
+ List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
+ for (TupleTag<?> tag: outputs.keySet()) {
+ if (tag != null && !tag.getId().equals(mainOutput.getId())) {
+ sideOutputs.add(tag);
+ }
+ }
+
+ JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
+ context, inputStream, tagsToSideInputs);
+
+ JavaStream<TranslatorUtils.RawUnionValue> outputStream =
+ TranslatorUtils.toList(unionStream).flatMap(
+ new DoFnFunction<>(
+ context.getPipelineOptions(),
+ transform.getFn(),
+ inputT.getWindowingStrategy(),
+ sideInputs,
+ tagsToSideInputs,
+ mainOutput,
+ sideOutputs), transform.getName());
+ for (Map.Entry<TupleTag<?>, PValue> output: outputs.entrySet()) {
+ JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
+ .filter(new FilterByOutputTag(output.getKey().getId()),
+ "filter_by_output_tag")
+ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
+ context.setOutputStream(output.getValue(), taggedStream);
+ }
+ }
+
+ private static class FilterByOutputTag extends FilterFunction<TranslatorUtils.RawUnionValue> {
+
+ private static final long serialVersionUID = 7276155265895637526L;
+ private final String tag;
+
+ FilterByOutputTag(String tag) {
+ this.tag = tag;
+ }
+
+ @Override
+ public boolean filter(TranslatorUtils.RawUnionValue value) {
+ return value.getUnionTag().equals(tag);
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
new file mode 100644
index 0000000..8f71a8e
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import org.apache.beam.runners.gearpump.translators.io.BoundedSourceWrapper;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+
+/**
+ * {@link Read.Bounded} is translated to Gearpump source function
+ * and {@link BoundedSource} is wrapped into Gearpump {@link DataSource}.
+ */
+public class ReadBoundedTranslator <T> implements TransformTranslator<Read.Bounded<T>> {
+
+ private static final long serialVersionUID = -3899020490896998330L;
+
+ @Override
+ public void translate(Read.Bounded<T> transform, TranslationContext context) {
+ BoundedSource<T> boundedSource = transform.getSource();
+ BoundedSourceWrapper<T> sourceWrapper = new BoundedSourceWrapper<>(boundedSource,
+ context.getPipelineOptions());
+ JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(sourceWrapper);
+
+ context.setOutputStream(context.getOutput(), sourceStream);
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
new file mode 100644
index 0000000..0462c57
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.util.WindowedValue;
+
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+
+/**
+ * {@link Read.Unbounded} is translated to Gearpump source function
+ * and {@link UnboundedSource} is wrapped into Gearpump {@link DataSource}.
+ */
+
+public class ReadUnboundedTranslator<T> implements TransformTranslator<Read.Unbounded<T>> {
+
+ private static final long serialVersionUID = 3529494817859948619L;
+
+ @Override
+ public void translate(Read.Unbounded<T> transform, TranslationContext context) {
+ UnboundedSource<T, ?> unboundedSource = transform.getSource();
+ UnboundedSourceWrapper<T, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
+ unboundedSource, context.getPipelineOptions());
+ JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper);
+
+ context.setOutputStream(context.getOutput(), sourceStream);
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
new file mode 100644
index 0000000..c7becad
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import java.io.Serializable;
+
+import org.apache.beam.sdk.transforms.PTransform;
+
+/**
+ * Translates {@link PTransform} to Gearpump functions.
+ */
+public interface TransformTranslator<T extends PTransform> extends Serializable {
+ void translate(T transform, TranslationContext context);
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
new file mode 100644
index 0000000..42b7a53
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Iterables;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.beam.runners.core.construction.TransformInputs;
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.values.PValue;
+
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.gearpump.cluster.UserConfig;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStreamApp;
+import org.apache.gearpump.streaming.source.DataSource;
+
+/**
+ * Maintains context data for {@link TransformTranslator}s.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class TranslationContext {
+
+ private final JavaStreamApp streamApp;
+ private final GearpumpPipelineOptions pipelineOptions;
+ private AppliedPTransform<?, ?, ?> currentTransform;
+ private final Map<PValue, JavaStream<?>> streams = new HashMap<>();
+
+ public TranslationContext(JavaStreamApp streamApp, GearpumpPipelineOptions pipelineOptions) {
+ this.streamApp = streamApp;
+ this.pipelineOptions = pipelineOptions;
+ }
+
+ public void setCurrentTransform(TransformHierarchy.Node treeNode, Pipeline pipeline) {
+ this.currentTransform = treeNode.toAppliedPTransform(pipeline);
+ }
+
+ public GearpumpPipelineOptions getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ public <InputT> JavaStream<InputT> getInputStream(PValue input) {
+ return (JavaStream<InputT>) streams.get(input);
+ }
+
+ public <OutputT> void setOutputStream(PValue output, JavaStream<OutputT> outputStream) {
+ if (!streams.containsKey(output)) {
+ streams.put(output, outputStream);
+ } else {
+ throw new RuntimeException("set stream for duplicated output " + output);
+ }
+ }
+
+ public Map<TupleTag<?>, PValue> getInputs() {
+ return getCurrentTransform().getInputs();
+ }
+
+ public PValue getInput() {
+ return Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+ }
+
+ public Map<TupleTag<?>, PValue> getOutputs() {
+ return getCurrentTransform().getOutputs();
+ }
+
+ public PValue getOutput() {
+ return Iterables.getOnlyElement(getOutputs().values());
+ }
+
+ private AppliedPTransform<?, ?, ?> getCurrentTransform() {
+ checkArgument(
+ currentTransform != null,
+ "current transform not set");
+ return currentTransform;
+ }
+
+ public <T> JavaStream<T> getSourceStream(DataSource dataSource) {
+ return streamApp.source(dataSource, pipelineOptions.getParallelism(),
+ UserConfig.empty(), "source");
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
new file mode 100644
index 0000000..d144b95
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
+import org.joda.time.Instant;
+
+/**
+ * {@link Window.Assign} is translated to Gearpump flatMap function.
+ */
+@SuppressWarnings("unchecked")
+public class WindowAssignTranslator<T> implements TransformTranslator<Window.Assign<T>> {
+
+ private static final long serialVersionUID = -964887482120489061L;
+
+ @Override
+ public void translate(Window.Assign<T> transform, TranslationContext context) {
+ PCollection<T> input = (PCollection<T>) context.getInput();
+ PCollection<T> output = (PCollection<T>) context.getOutput();
+ JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input);
+ WindowingStrategy<?, ?> outputStrategy = output.getWindowingStrategy();
+ WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
+ JavaStream<WindowedValue<T>> outputStream =
+ inputStream
+ .flatMap(new AssignWindows(windowFn), "assign_windows");
+
+ context.setOutputStream(output, outputStream);
+ }
+
+ /**
+ * A Function used internally by Gearpump to wrap the actual Beam's WindowFn.
+ */
+ protected static class AssignWindows<T> extends
+ FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
+
+ private static final long serialVersionUID = 7284565861938681360L;
+ private final WindowFn<T, BoundedWindow> windowFn;
+
+ AssignWindows(WindowFn<T, BoundedWindow> windowFn) {
+ this.windowFn = windowFn;
+ }
+
+ @Override
+ public Iterator<WindowedValue<T>> flatMap(final WindowedValue<T> value) {
+ try {
+ Collection<BoundedWindow> windows = windowFn.assignWindows(windowFn.new AssignContext() {
+ @Override
+ public T element() {
+ return value.getValue();
+ }
+
+ @Override
+ public Instant timestamp() {
+ return value.getTimestamp();
+ }
+
+ @Override
+ public BoundedWindow window() {
+ return Iterables.getOnlyElement(value.getWindows());
+ }
+ });
+ List<WindowedValue<T>> values = new ArrayList<>(windows.size());
+ for (BoundedWindow win: windows) {
+ values.add(
+ WindowedValue.of(value.getValue(), value.getTimestamp(), win, value.getPane()));
+ }
+ return values.iterator();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
new file mode 100644
index 0000000..fde265a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.functions;
+
+import com.google.common.collect.Iterables;
+
+import com.google.common.collect.Lists;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.InMemoryStateInternals;
+import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.SideInputHandler;
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.utils.DoFnRunnerFactory;
+import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils.RawUnionValue;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
+
+/**
+ * Gearpump {@link FlatMapFunction} wrapper over Beam {@link DoFn}.
+ */
+@SuppressWarnings("unchecked")
+public class DoFnFunction<InputT, OutputT> extends
+ FlatMapFunction<List<RawUnionValue>, RawUnionValue> {
+
+ private static final long serialVersionUID = -5701440128544343353L;
+ private final DoFnRunnerFactory<InputT, OutputT> doFnRunnerFactory;
+ private final DoFn<InputT, OutputT> doFn;
+ private transient DoFnInvoker<InputT, OutputT> doFnInvoker;
+ private transient PushbackSideInputDoFnRunner<InputT, OutputT> doFnRunner;
+ private transient SideInputHandler sideInputReader;
+ private transient List<WindowedValue<InputT>> pushedBackValues;
+ private final Collection<PCollectionView<?>> sideInputs;
+ private final Map<String, PCollectionView<?>> tagsToSideInputs;
+ private final TupleTag<OutputT> mainOutput;
+ private final List<TupleTag<?>> sideOutputs;
+ private final DoFnOutputManager outputManager;
+
+ public DoFnFunction(
+ GearpumpPipelineOptions pipelineOptions,
+ DoFn<InputT, OutputT> doFn,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Collection<PCollectionView<?>> sideInputs,
+ Map<String, PCollectionView<?>> sideInputTagMapping,
+ TupleTag<OutputT> mainOutput,
+ List<TupleTag<?>> sideOutputs) {
+ this.doFn = doFn;
+ this.outputManager = new DoFnOutputManager();
+ this.doFnRunnerFactory = new DoFnRunnerFactory<>(
+ pipelineOptions,
+ doFn,
+ sideInputs,
+ outputManager,
+ mainOutput,
+ sideOutputs,
+ new NoOpStepContext(),
+ windowingStrategy
+ );
+ this.sideInputs = sideInputs;
+ this.tagsToSideInputs = sideInputTagMapping;
+ this.mainOutput = mainOutput;
+ this.sideOutputs = sideOutputs;
+ }
+
+ @Override
+ public void setup() {
+ sideInputReader = new SideInputHandler(sideInputs,
+ InMemoryStateInternals.<Void>forKey(null));
+ doFnInvoker = DoFnInvokers.invokerFor(doFn);
+ doFnInvoker.invokeSetup();
+
+ doFnRunner = doFnRunnerFactory.createRunner(sideInputReader);
+
+ pushedBackValues = new LinkedList<>();
+ outputManager.setup(mainOutput, sideOutputs);
+ }
+
+ @Override
+ public void teardown() {
+ doFnInvoker.invokeTeardown();
+ }
+
+ @Override
+ public Iterator<TranslatorUtils.RawUnionValue> flatMap(List<RawUnionValue> inputs) {
+ outputManager.clear();
+
+ doFnRunner.startBundle();
+
+ for (RawUnionValue unionValue: inputs) {
+ final String tag = unionValue.getUnionTag();
+ if (tag.equals("0")) {
+ // main input
+ pushedBackValues.add((WindowedValue<InputT>) unionValue.getValue());
+ } else {
+ // side input
+ PCollectionView<?> sideInput = tagsToSideInputs.get(unionValue.getUnionTag());
+ WindowedValue<Iterable<?>> sideInputValue =
+ (WindowedValue<Iterable<?>>) unionValue.getValue();
+ sideInputReader.addSideInputValue(sideInput, sideInputValue);
+ }
+ }
+
+ for (PCollectionView<?> sideInput: sideInputs) {
+ for (WindowedValue<InputT> value : pushedBackValues) {
+ for (BoundedWindow win: value.getWindows()) {
+ BoundedWindow sideInputWindow =
+ sideInput.getWindowMappingFn().getSideInputWindow(win);
+ if (!sideInputReader.isReady(sideInput, sideInputWindow)) {
+ Object emptyValue = WindowedValue.of(
+ Lists.newArrayList(), value.getTimestamp(), sideInputWindow, value.getPane());
+ sideInputReader.addSideInputValue(sideInput, (WindowedValue<Iterable<?>>) emptyValue);
+ }
+ }
+ }
+ }
+
+ List<WindowedValue<InputT>> nextPushedBackValues = new LinkedList<>();
+ for (WindowedValue<InputT> value : pushedBackValues) {
+ Iterable<WindowedValue<InputT>> values = doFnRunner.processElementInReadyWindows(value);
+ Iterables.addAll(nextPushedBackValues, values);
+ }
+ pushedBackValues.clear();
+ Iterables.addAll(pushedBackValues, nextPushedBackValues);
+
+ doFnRunner.finishBundle();
+
+ return outputManager.getOutputs();
+ }
+
+ private static class DoFnOutputManager implements DoFnRunners.OutputManager, Serializable {
+
+ private static final long serialVersionUID = 4967375172737408160L;
+ private transient List<RawUnionValue> outputs;
+ private transient Set<TupleTag<?>> outputTags;
+
+ @Override
+ public <T> void output(TupleTag<T> outputTag, WindowedValue<T> output) {
+ if (outputTags.contains(outputTag)) {
+ outputs.add(new RawUnionValue(outputTag.getId(), output));
+ }
+ }
+
+ void setup(TupleTag<?> mainOutput, List<TupleTag<?>> sideOutputs) {
+ outputs = new LinkedList<>();
+ outputTags = new HashSet<>();
+ outputTags.add(mainOutput);
+ outputTags.addAll(sideOutputs);
+ }
+
+ void clear() {
+ outputs.clear();
+ }
+
+ Iterator<RawUnionValue> getOutputs() {
+ return outputs.iterator();
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
new file mode 100644
index 0000000..cba2363
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific wrappers for Beam DoFn.
+ */
+package org.apache.beam.runners.gearpump.translators.functions;
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
new file mode 100644
index 0000000..2c18735
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import java.io.IOException;
+
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+/**
+ * wrapper over BoundedSource for Gearpump DataSource API.
+ */
+public class BoundedSourceWrapper<T> extends GearpumpSource<T> {
+
+ private static final long serialVersionUID = 8199570485738786123L;
+ private final BoundedSource<T> source;
+
+ public BoundedSourceWrapper(BoundedSource<T> source, PipelineOptions options) {
+ super(options);
+ this.source = source;
+ }
+
+
+ @Override
+ protected Source.Reader<T> createReader(PipelineOptions options) throws IOException {
+ return source.createReader(options);
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
new file mode 100644
index 0000000..3766195
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import java.io.IOException;
+import java.time.Instant;
+
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+
+import org.apache.gearpump.DefaultMessage;
+import org.apache.gearpump.Message;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.apache.gearpump.streaming.source.Watermark;
+import org.apache.gearpump.streaming.task.TaskContext;
+
+/**
+ * common methods for {@link BoundedSourceWrapper} and {@link UnboundedSourceWrapper}.
+ */
+public abstract class GearpumpSource<T> implements DataSource {
+
+ private final SerializablePipelineOptions serializedOptions;
+
+ private Source.Reader<T> reader;
+ private boolean available = false;
+
+ GearpumpSource(PipelineOptions options) {
+ this.serializedOptions = new SerializablePipelineOptions(options);
+ }
+
+ protected abstract Source.Reader<T> createReader(PipelineOptions options) throws IOException;
+
+ @Override
+ public void open(TaskContext context, Instant startTime) {
+ try {
+ PipelineOptions options = serializedOptions.get();
+ this.reader = createReader(options);
+ this.available = reader.start();
+ } catch (Exception e) {
+ close();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Message read() {
+ Message message = null;
+ try {
+ if (available) {
+ T data = reader.getCurrent();
+ org.joda.time.Instant timestamp = reader.getCurrentTimestamp();
+ message = new DefaultMessage(
+ WindowedValue.timestampedValueInGlobalWindow(data, timestamp),
+ timestamp.getMillis());
+ }
+ available = reader.advance();
+ } catch (Exception e) {
+ close();
+ throw new RuntimeException(e);
+ }
+ return message;
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (reader != null) {
+ reader.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Instant getWatermark() {
+ if (reader instanceof UnboundedSource.UnboundedReader) {
+ org.joda.time.Instant watermark =
+ ((UnboundedSource.UnboundedReader) reader).getWatermark();
+ if (watermark == BoundedWindow.TIMESTAMP_MAX_VALUE) {
+ return Watermark.MAX();
+ } else {
+ return TranslatorUtils.jodaTimeToJava8Time(watermark);
+ }
+ } else {
+ if (available) {
+ return Watermark.MIN();
+ } else {
+ return Watermark.MAX();
+ }
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
new file mode 100644
index 0000000..cb912c1
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import java.io.IOException;
+
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+/**
+ * wrapper over UnboundedSource for Gearpump DataSource API.
+ */
+public class UnboundedSourceWrapper<OutputT, CheckpointMarkT extends UnboundedSource.CheckpointMark>
+ extends GearpumpSource<OutputT> {
+
+ private static final long serialVersionUID = -2453956849834747150L;
+ private final UnboundedSource<OutputT, CheckpointMarkT> source;
+
+ public UnboundedSourceWrapper(UnboundedSource<OutputT, CheckpointMarkT> source,
+ PipelineOptions options) {
+ super(options);
+ this.source = source;
+ }
+
+ @Override
+ protected Source.Reader<OutputT> createReader(PipelineOptions options) throws IOException {
+ return source.createReader(options, null);
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
new file mode 100644
index 0000000..c018037
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+
+/**
+ * unbounded source that reads from a Java {@link Iterable}.
+ */
+public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.CheckpointMark> {
+
+ private static final long serialVersionUID = 9113026175795235710L;
+ private final byte[] values;
+ private final IterableCoder<T> iterableCoder;
+
+ public ValuesSource(Iterable<T> values, Coder<T> coder) {
+ this.iterableCoder = IterableCoder.of(coder);
+ this.values = encode(values, iterableCoder);
+ }
+
+ private byte[] encode(Iterable<T> values, IterableCoder<T> coder) {
+ try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
+ coder.encode(values, stream, Coder.Context.OUTER);
+ return stream.toByteArray();
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private Iterable<T> decode(byte[] bytes) throws IOException{
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes)) {
+ return iterableCoder.decode(inputStream, Coder.Context.OUTER);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public java.util.List<? extends UnboundedSource<T, CheckpointMark>> split(
+ int desiredNumSplits, PipelineOptions options) throws Exception {
+ return Collections.singletonList(this);
+ }
+
+ @Override
+ public UnboundedReader<T> createReader(PipelineOptions options,
+ @Nullable CheckpointMark checkpointMark) {
+ try {
+ return new ValuesReader<>(decode(values), this);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Nullable
+ @Override
+ public Coder<CheckpointMark> getCheckpointMarkCoder() {
+ return null;
+ }
+
+ @Override
+ public Coder<T> getDefaultOutputCoder() {
+ return iterableCoder.getElemCoder();
+ }
+
+ private static class ValuesReader<T> extends UnboundedReader<T> {
+ private final UnboundedSource<T, CheckpointMark> source;
+ private final Iterable<T> values;
+ private transient Iterator<T> iterator;
+ private T current;
+
+ ValuesReader(Iterable<T> values,
+ UnboundedSource<T, CheckpointMark> source) {
+ this.values = values;
+ this.source = source;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ if (null == iterator) {
+ iterator = values.iterator();
+ }
+ return advance();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ if (iterator.hasNext()) {
+ current = iterator.next();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public T getCurrent() throws NoSuchElementException {
+ return current;
+ }
+
+ @Override
+ public Instant getCurrentTimestamp() throws NoSuchElementException {
+ return getTimestamp(current);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public Instant getWatermark() {
+ if (iterator.hasNext()) {
+ return getTimestamp(current);
+ } else {
+ return BoundedWindow.TIMESTAMP_MAX_VALUE;
+ }
+ }
+
+ @Override
+ public CheckpointMark getCheckpointMark() {
+ return null;
+ }
+
+ @Override
+ public UnboundedSource<T, ?> getCurrentSource() {
+ return source;
+ }
+
+ private Instant getTimestamp(Object value) {
+ if (value instanceof TimestampedValue) {
+ return ((TimestampedValue) value).getTimestamp();
+ } else {
+ return Instant.now();
+ }
+ }
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
new file mode 100644
index 0000000..dfdf51a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific wrappers for Beam I/O.
+ */
+package org.apache.beam.runners.gearpump.translators.io;
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
new file mode 100644
index 0000000..612096a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific translators.
+ */
+package org.apache.beam.runners.gearpump.translators;
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
new file mode 100644
index 0000000..6557c8b
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.utils;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.beam.runners.core.DoFnRunner;
+import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
+import org.apache.beam.runners.core.SimpleDoFnRunner;
+import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.StepContext;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * a serializable {@link SimpleDoFnRunner}.
+ */
+public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
+
+ private static final long serialVersionUID = -4109539010014189725L;
+ private final DoFn<InputT, OutputT> fn;
+ private final SerializablePipelineOptions serializedOptions;
+ private final Collection<PCollectionView<?>> sideInputs;
+ private final DoFnRunners.OutputManager outputManager;
+ private final TupleTag<OutputT> mainOutputTag;
+ private final List<TupleTag<?>> sideOutputTags;
+ private final StepContext stepContext;
+ private final WindowingStrategy<?, ?> windowingStrategy;
+
+ public DoFnRunnerFactory(
+ GearpumpPipelineOptions pipelineOptions,
+ DoFn<InputT, OutputT> doFn,
+ Collection<PCollectionView<?>> sideInputs,
+ DoFnRunners.OutputManager outputManager,
+ TupleTag<OutputT> mainOutputTag,
+ List<TupleTag<?>> sideOutputTags,
+ StepContext stepContext,
+ WindowingStrategy<?, ?> windowingStrategy) {
+ this.fn = doFn;
+ this.serializedOptions = new SerializablePipelineOptions(pipelineOptions);
+ this.sideInputs = sideInputs;
+ this.outputManager = outputManager;
+ this.mainOutputTag = mainOutputTag;
+ this.sideOutputTags = sideOutputTags;
+ this.stepContext = stepContext;
+ this.windowingStrategy = windowingStrategy;
+ }
+
+ public PushbackSideInputDoFnRunner<InputT, OutputT> createRunner(
+ ReadyCheckingSideInputReader sideInputReader) {
+ PipelineOptions options = serializedOptions.get();
+ DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner(
+ options, fn, sideInputReader, outputManager, mainOutputTag,
+ sideOutputTags, stepContext, windowingStrategy);
+ return SimplePushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
new file mode 100644
index 0000000..b795ed9
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.utils;
+
+import java.io.Serializable;
+
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
+import org.apache.beam.runners.core.TimerInternals;
+
+/**
+ * serializable {@link StepContext} that basically does nothing.
+ */
+public class NoOpStepContext implements StepContext, Serializable {
+
+ @Override
+ public StateInternals stateInternals() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TimerInternals timerInternals() {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
new file mode 100644
index 0000000..2dae955
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.utils;
+
+import com.google.common.collect.Lists;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.gearpump.translators.TranslationContext;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import org.apache.gearpump.streaming.dsl.api.functions.FoldFunction;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+
+/**
+ * Utility methods for translators.
+ */
+public class TranslatorUtils {
+
+ public static Instant jodaTimeToJava8Time(org.joda.time.Instant time) {
+ return Instant.ofEpochMilli(time.getMillis());
+ }
+
+ public static org.joda.time.Instant java8TimeToJodaTime(Instant time) {
+ return new org.joda.time.Instant(time.toEpochMilli());
+ }
+
+ public static Window boundedWindowToGearpumpWindow(BoundedWindow window) {
+ // Gearpump window upper bound is exclusive
+ Instant end = TranslatorUtils.jodaTimeToJava8Time(window.maxTimestamp().plus(1L));
+ if (window instanceof IntervalWindow) {
+ IntervalWindow intervalWindow = (IntervalWindow) window;
+ Instant start = TranslatorUtils.jodaTimeToJava8Time(intervalWindow.start());
+ return new Window(start, end);
+ } else if (window instanceof GlobalWindow) {
+ return new Window(TranslatorUtils.jodaTimeToJava8Time(BoundedWindow.TIMESTAMP_MIN_VALUE),
+ end);
+ } else {
+ throw new RuntimeException("unknown window " + window.getClass().getName());
+ }
+ }
+
+ public static <InputT> JavaStream<RawUnionValue> withSideInputStream(
+ TranslationContext context,
+ JavaStream<WindowedValue<InputT>> inputStream,
+ Map<String, PCollectionView<?>> tagsToSideInputs) {
+ JavaStream<RawUnionValue> mainStream =
+ inputStream.map(new ToRawUnionValue<>("0"), "map_to_RawUnionValue");
+
+ for (Map.Entry<String, PCollectionView<?>> tagToSideInput: tagsToSideInputs.entrySet()) {
+ JavaStream<WindowedValue<List<?>>> sideInputStream = context.getInputStream(
+ tagToSideInput.getValue());
+ mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
+ tagToSideInput.getKey()), "map_to_RawUnionValue"), 1, "merge_to_MainStream");
+ }
+ return mainStream;
+ }
+
+ public static Map<String, PCollectionView<?>> getTagsToSideInputs(
+ Collection<PCollectionView<?>> sideInputs) {
+ Map<String, PCollectionView<?>> tagsToSideInputs = new HashMap<>();
+ // tag 0 is reserved for main input
+ int tag = 1;
+ for (PCollectionView<?> sideInput: sideInputs) {
+ tagsToSideInputs.put(tag + "", sideInput);
+ tag++;
+ }
+ return tagsToSideInputs;
+ }
+
+ public static JavaStream<List<RawUnionValue>> toList(JavaStream<RawUnionValue> stream) {
+ return stream.fold(new FoldFunction<RawUnionValue, List<RawUnionValue>>() {
+
+ @Override
+ public List<RawUnionValue> init() {
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public List<RawUnionValue> fold(List<RawUnionValue> accumulator,
+ RawUnionValue rawUnionValue) {
+ accumulator.add(rawUnionValue);
+ return accumulator;
+ }
+ }, "fold_to_iterable");
+ }
+
+ /**
+ * Converts @link{RawUnionValue} to @link{WindowedValue}.
+ */
+ public static class FromRawUnionValue<OutputT> extends
+ MapFunction<RawUnionValue, WindowedValue<OutputT>> {
+
+ private static final long serialVersionUID = -4764968219713478955L;
+
+ @Override
+ public WindowedValue<OutputT> map(RawUnionValue value) {
+ return (WindowedValue<OutputT>) value.getValue();
+ }
+ }
+
+ private static class ToRawUnionValue<T> extends
+ MapFunction<WindowedValue<T>, RawUnionValue> {
+
+ private static final long serialVersionUID = 8648852871014813583L;
+ private final String tag;
+
+ ToRawUnionValue(String tag) {
+ this.tag = tag;
+ }
+
+ @Override
+ public RawUnionValue map(WindowedValue<T> windowedValue) {
+ return new RawUnionValue(tag, windowedValue);
+ }
+ }
+
+ /**
+ * This is copied from org.apache.beam.sdk.transforms.join.RawUnionValue.
+ */
+ public static class RawUnionValue {
+ private final String unionTag;
+ private final Object value;
+
+ /**
+ * Constructs a partial union from the given union tag and value.
+ */
+ public RawUnionValue(String unionTag, Object value) {
+ this.unionTag = unionTag;
+ this.value = value;
+ }
+
+ public String getUnionTag() {
+ return unionTag;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return unionTag + ":" + value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ RawUnionValue that = (RawUnionValue) o;
+
+ if (unionTag != that.unionTag) {
+ return false;
+ }
+ return value != null ? value.equals(that.value) : that.value == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = unionTag.hashCode();
+ result = 31 * result + value.hashCode();
+ return result;
+ }
+ }
+
+}
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
new file mode 100644
index 0000000..ab2a6ea
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for translators.
+ */
+package org.apache.beam.runners.gearpump.translators.utils;
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
new file mode 100644
index 0000000..9a01d20
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Test;
+
+/**
+ * Tests for {@link GearpumpRunnerRegistrar}.
+ */
+public class GearpumpRunnerRegistrarTest {
+
+ @Test
+ public void testFullName() {
+ String[] args =
+ new String[] {String.format("--runner=%s", GearpumpRunner.class.getName())};
+ PipelineOptions opts = PipelineOptionsFactory.fromArgs(args).create();
+ assertEquals(opts.getRunner(), GearpumpRunner.class);
+ }
+
+ @Test
+ public void testClassName() {
+ String[] args =
+ new String[] {String.format("--runner=%s", GearpumpRunner.class.getSimpleName())};
+ PipelineOptions opts = PipelineOptionsFactory.fromArgs(args).create();
+ assertEquals(opts.getRunner(), GearpumpRunner.class);
+ }
+
+ @Test
+ public void testOptions() {
+ assertEquals(
+ ImmutableList.of(GearpumpPipelineOptions.class),
+ new GearpumpRunnerRegistrar.Options().getPipelineOptions());
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
new file mode 100644
index 0000000..994856b
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.junit.Test;
+
+/**
+ * Tests for {@link GearpumpPipelineOptions}.
+ */
+public class PipelineOptionsTest {
+
+ @Test
+ public void testIgnoredFieldSerialization() throws IOException {
+ String appName = "forTest";
+ Map<String, String> serializers = Maps.newHashMap();
+ serializers.put("classA", "SerializerA");
+ GearpumpPipelineOptions options = PipelineOptionsFactory.create()
+ .as(GearpumpPipelineOptions.class);
+ Config config = ClusterConfig.master(null);
+ EmbeddedCluster cluster = new EmbeddedCluster(config);
+ options.setSerializers(serializers);
+ options.setApplicationName(appName);
+ options.setEmbeddedCluster(cluster);
+ options.setParallelism(10);
+
+ byte[] serializedOptions = serialize(options);
+ GearpumpPipelineOptions deserializedOptions = new ObjectMapper()
+ .readValue(serializedOptions, PipelineOptions.class).as(GearpumpPipelineOptions.class);
+
+ assertNull(deserializedOptions.getEmbeddedCluster());
+ assertNull(deserializedOptions.getSerializers());
+ assertEquals(10, deserializedOptions.getParallelism());
+ assertEquals(appName, deserializedOptions.getApplicationName());
+ }
+
+ private byte[] serialize(Object obj) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ new ObjectMapper().writeValue(baos, obj);
+ return baos.toByteArray();
+ } catch (Exception e) {
+ throw new RuntimeException("Couldn't serialize PipelineOptions.", e);
+ }
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
new file mode 100644
index 0000000..511eed1
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.junit.Test;
+
+/** Tests for {@link CreateGearpumpPCollectionViewTranslator}. */
+public class CreateGearpumpPCollectionViewTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ CreateGearpumpPCollectionViewTranslator translator =
+ new CreateGearpumpPCollectionViewTranslator();
+
+ CreateStreamingGearpumpView.CreateGearpumpPCollectionView pCollectionView =
+ mock(CreateStreamingGearpumpView.CreateGearpumpPCollectionView.class);
+
+ JavaStream javaStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ PValue mockInput = mock(PValue.class);
+ when(translationContext.getInput()).thenReturn(mockInput);
+ when(translationContext.getInputStream(mockInput)).thenReturn(javaStream);
+
+ PCollectionView view = mock(PCollectionView.class);
+ when(pCollectionView.getView()).thenReturn(view);
+
+ translator.translate(pCollectionView, translationContext);
+ verify(translationContext, times(1)).setOutputStream(view, javaStream);
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
new file mode 100644
index 0000000..1115fad
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link FlattenPCollectionsTranslator}. */
+public class FlattenPCollectionsTranslatorTest {
+
+ private FlattenPCollectionsTranslator translator = new FlattenPCollectionsTranslator();
+ private Flatten.PCollections transform = mock(Flatten.PCollections.class);
+
+ class UnboundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof UnboundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslateWithEmptyCollection() {
+ PCollection mockOutput = mock(PCollection.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ when(translationContext.getInputs()).thenReturn(Collections.EMPTY_MAP);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+ when(translationContext.getPipelineOptions())
+ .thenReturn(PipelineOptionsFactory.as(GearpumpPipelineOptions.class));
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslateWithOneCollection() {
+ JavaStream javaStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag = mock(TupleTag.class);
+ PCollection mockCollection = mock(PCollection.class);
+ inputs.put(tag, mockCollection);
+
+ when(translationContext.getInputs()).thenReturn(inputs);
+ when(translationContext.getInputStream(mockCollection)).thenReturn(javaStream);
+
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext, times(1)).setOutputStream(mockOutput, javaStream);
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testWithMoreThanOneCollections() {
+ String transformName = "transform";
+ when(transform.getName()).thenReturn(transformName);
+
+ JavaStream javaStream1 = mock(JavaStream.class);
+ JavaStream javaStream2 = mock(JavaStream.class);
+ JavaStream mergedStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag1 = mock(TupleTag.class);
+ PCollection mockCollection1 = mock(PCollection.class);
+ inputs.put(tag1, mockCollection1);
+
+ TupleTag tag2 = mock(TupleTag.class);
+ PCollection mockCollection2 = mock(PCollection.class);
+ inputs.put(tag2, mockCollection2);
+
+ PCollection output = mock(PCollection.class);
+
+ when(translationContext.getInputs()).thenReturn(inputs);
+ when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
+ when(translationContext.getInputStream(mockCollection2)).thenReturn(javaStream2);
+ when(javaStream1.merge(javaStream2, 1, transformName)).thenReturn(mergedStream);
+ when(javaStream2.merge(javaStream1, 1, transformName)).thenReturn(mergedStream);
+
+ when(translationContext.getOutput()).thenReturn(output);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).setOutputStream(output, mergedStream);
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testWithDuplicatedCollections() {
+ String transformName = "transform";
+ when(transform.getName()).thenReturn(transformName);
+
+ JavaStream javaStream1 = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag1 = mock(TupleTag.class);
+ PCollection mockCollection1 = mock(PCollection.class);
+ inputs.put(tag1, mockCollection1);
+
+ TupleTag tag2 = mock(TupleTag.class);
+ inputs.put(tag2, mockCollection1);
+
+ when(translationContext.getInputs()).thenReturn(inputs);
+ when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
+ when(translationContext.getPipelineOptions())
+ .thenReturn(PipelineOptionsFactory.as(GearpumpPipelineOptions.class));
+
+ translator.translate(transform, translationContext);
+ verify(javaStream1).map(any(MapFunction.class), eq("dummy"));
+ verify(javaStream1).merge(any(JavaStream.class), eq(1), eq(transformName));
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
new file mode 100644
index 0000000..d5b931b
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator.GearpumpWindowFn;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.gearpump.streaming.dsl.window.api.WindowFunction;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.joda.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/** Tests for {@link GroupByKeyTranslator}. */
+@RunWith(Parameterized.class)
+public class GroupByKeyTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testGearpumpWindowFn() {
+ GearpumpWindowFn windowFn = new GearpumpWindowFn(true);
+ List<BoundedWindow> windows =
+ Lists.newArrayList(
+ new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10)),
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(15)));
+
+ WindowFunction.Context<WindowedValue<String>> context =
+ new WindowFunction.Context<WindowedValue<String>>() {
+ @Override
+ public Instant timestamp() {
+ return Instant.EPOCH;
+ }
+
+ @Override
+ public WindowedValue<String> element() {
+ return WindowedValue.of(
+ "v1", new org.joda.time.Instant(6), windows, PaneInfo.NO_FIRING);
+ }
+ };
+
+ Window[] result = windowFn.apply(context);
+ List<Window> expected = Lists.newArrayList();
+ for (BoundedWindow w : windows) {
+ expected.add(TranslatorUtils.boundedWindowToGearpumpWindow(w));
+ }
+ assertThat(result, equalTo(expected.toArray()));
+ }
+
+ @Parameterized.Parameters(name = "{index}: {0}")
+ public static Iterable<TimestampCombiner> data() {
+ return ImmutableList.of(
+ TimestampCombiner.EARLIEST,
+ TimestampCombiner.LATEST,
+ TimestampCombiner.END_OF_WINDOW);
+ }
+
+ @Parameterized.Parameter(0)
+ public TimestampCombiner timestampCombiner;
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testKeyedByTimestamp() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
+ BoundedWindow window =
+ new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10));
+ GroupByKeyTranslator.KeyedByTimestamp keyedByTimestamp =
+ new GroupByKeyTranslator.KeyedByTimestamp(slidingWindows, timestampCombiner);
+ WindowedValue<KV<String, String>> value =
+ WindowedValue.of(
+ KV.of("key", "val"), org.joda.time.Instant.now(), window, PaneInfo.NO_FIRING);
+ KV<org.joda.time.Instant, WindowedValue<KV<String, String>>> result =
+ keyedByTimestamp.map(value);
+ org.joda.time.Instant time =
+ timestampCombiner.assign(window,
+ slidingWindows.getOutputTime(value.getTimestamp(), window));
+ assertThat(result, equalTo(KV.of(time, value)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testMerge() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
+ GroupByKeyTranslator.Merge merge = new GroupByKeyTranslator.Merge(slidingWindows,
+ timestampCombiner);
+ org.joda.time.Instant key1 = new org.joda.time.Instant(5);
+ WindowedValue<KV<String, String>> value1 =
+ WindowedValue.of(
+ KV.of("key1", "value1"),
+ key1,
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(10)),
+ PaneInfo.NO_FIRING);
+
+ org.joda.time.Instant key2 = new org.joda.time.Instant(10);
+ WindowedValue<KV<String, String>> value2 =
+ WindowedValue.of(
+ KV.of("key2", "value2"),
+ key2,
+ new IntervalWindow(new org.joda.time.Instant(9), new org.joda.time.Instant(14)),
+ PaneInfo.NO_FIRING);
+
+ KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result1 =
+ merge.fold(KV.<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>>of(
+ null, null), KV.of(key1, value1));
+ assertThat(result1.getKey(), equalTo(key1));
+ assertThat(result1.getValue().getValue().getValue(), equalTo(Lists.newArrayList("value1")));
+
+ KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result2 =
+ merge.fold(result1, KV.of(key2, value2));
+ assertThat(result2.getKey(), equalTo(timestampCombiner.combine(key1, key2)));
+ Collection<? extends BoundedWindow> resultWindows = result2.getValue().getWindows();
+ assertThat(resultWindows.size(), equalTo(1));
+ IntervalWindow expectedWindow =
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(14));
+ assertThat(resultWindows.toArray()[0], equalTo(expectedWindow));
+ assertThat(
+ result2.getValue().getValue().getValue(), equalTo(Lists.newArrayList("value1", "value2")));
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
new file mode 100644
index 0000000..20ee1a2
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.io.BoundedSourceWrapper;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link ReadBoundedTranslator}. */
+public class ReadBoundedTranslatorTest {
+
+ class BoundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof BoundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ ReadBoundedTranslator translator = new ReadBoundedTranslator();
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ Read.Bounded transform = mock(Read.Bounded.class);
+ BoundedSource source = mock(BoundedSource.class);
+ when(transform.getSource()).thenReturn(source);
+
+ TranslationContext translationContext = mock(TranslationContext.class);
+ when(translationContext.getPipelineOptions()).thenReturn(options);
+
+ JavaStream stream = mock(JavaStream.class);
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+ when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new BoundedSourceWrapperMatcher()));
+ verify(translationContext).setOutputStream(mockOutput, stream);
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
new file mode 100644
index 0000000..f27b568
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link ReadUnboundedTranslator}. */
+public class ReadUnboundedTranslatorTest {
+
+ class UnboundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof UnboundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ ReadUnboundedTranslator translator = new ReadUnboundedTranslator();
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ Read.Unbounded transform = mock(Read.Unbounded.class);
+ UnboundedSource source = mock(UnboundedSource.class);
+ when(transform.getSource()).thenReturn(source);
+
+ TranslationContext translationContext = mock(TranslationContext.class);
+ when(translationContext.getPipelineOptions()).thenReturn(options);
+
+ JavaStream stream = mock(JavaStream.class);
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+ when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
+ verify(translationContext).setOutputStream(mockOutput, stream);
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
new file mode 100644
index 0000000..06ccaaf
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+
+/** Tests for {@link WindowAssignTranslator}. */
+public class WindowAssignTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsWithSlidingWindow() {
+ WindowFn slidingWindows = SlidingWindows.of(Duration.millis(10)).every(Duration.millis(5));
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(0), new Instant(10)),
+ PaneInfo.NO_FIRING));
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(-5), new Instant(5)),
+ PaneInfo.NO_FIRING));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsWithSessions() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(1), new Instant(11)),
+ PaneInfo.NO_FIRING));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsGlobal() {
+ WindowFn slidingWindows = new GlobalWindows();
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(WindowedValue.timestampedValueInGlobalWindow(value, timestamp));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
new file mode 100644
index 0000000..cc4284f
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.gearpump.DefaultMessage;
+import org.apache.gearpump.Message;
+import org.apache.gearpump.streaming.source.Watermark;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** Tests for {@link GearpumpSource}. */
+public class GearpumpSourceTest {
+ private static final List<TimestampedValue<String>> TEST_VALUES =
+ Lists.newArrayList(
+ TimestampedValue.of("a", BoundedWindow.TIMESTAMP_MIN_VALUE),
+ TimestampedValue.of("b", new org.joda.time.Instant(0)),
+ TimestampedValue.of("c", new org.joda.time.Instant(53)),
+ TimestampedValue.of("d", BoundedWindow.TIMESTAMP_MAX_VALUE)
+ );
+
+ private static class SourceForTest<T> extends GearpumpSource<T> {
+ private ValuesSource<T> valuesSource;
+
+ SourceForTest(PipelineOptions options, ValuesSource<T> valuesSource) {
+ super(options);
+ this.valuesSource = valuesSource;
+ }
+
+ @Override
+ protected Source.Reader<T> createReader(PipelineOptions options) throws IOException {
+ return this.valuesSource.createReader(options, null);
+ }
+ }
+
+ @Test
+ public void testGearpumpSource() {
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ ValuesSource<TimestampedValue<String>> valuesSource =
+ new ValuesSource<>(
+ TEST_VALUES, TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()));
+ SourceForTest<TimestampedValue<String>> sourceForTest =
+ new SourceForTest<>(options, valuesSource);
+ sourceForTest.open(null, Instant.EPOCH);
+
+ for (int i = 0; i < TEST_VALUES.size(); i++) {
+ TimestampedValue<String> value = TEST_VALUES.get(i);
+
+ // Check the watermark first since the Source will advance when it's opened
+ if (i < TEST_VALUES.size() - 1) {
+ Instant expectedWaterMark = TranslatorUtils.jodaTimeToJava8Time(value.getTimestamp());
+ Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
+ } else {
+ Assert.assertEquals(Watermark.MAX(), sourceForTest.getWatermark());
+ }
+
+ Message expectedMsg =
+ new DefaultMessage(
+ WindowedValue.timestampedValueInGlobalWindow(value, value.getTimestamp()),
+ value.getTimestamp().getMillis());
+ Message message = sourceForTest.read();
+ Assert.assertEquals(expectedMsg, message);
+ }
+
+ Assert.assertNull(sourceForTest.read());
+ Assert.assertEquals(Watermark.MAX(), sourceForTest.getWatermark());
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
new file mode 100644
index 0000000..439e1b1
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump.translators.io;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValueFactory;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.GearpumpRunner;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.apache.gearpump.util.Constants;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** Tests for {@link ValuesSource}. */
+public class ValueSoureTest {
+
+ @Test
+ public void testValueSource() {
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ Config config = ClusterConfig.master(null);
+ config =
+ config.withValue(Constants.APPLICATION_TOTAL_RETRIES(), ConfigValueFactory.fromAnyRef(0));
+ EmbeddedCluster cluster = new EmbeddedCluster(config);
+ cluster.start();
+
+ options.setEmbeddedCluster(cluster);
+ options.setRunner(GearpumpRunner.class);
+ options.setParallelism(1);
+ Pipeline p = Pipeline.create(options);
+ List<String> values = Lists.newArrayList("1", "2", "3", "4", "5");
+ ValuesSource<String> source = new ValuesSource<>(values, StringUtf8Coder.of());
+ p.apply(Read.from(source)).apply(ParDo.of(new ResultCollector()));
+
+ p.run().waitUntilFinish();
+ cluster.stop();
+
+ Assert.assertEquals(Sets.newHashSet(values), ResultCollector.RESULTS);
+ }
+
+ private static class ResultCollector extends DoFn<Object, Void> {
+ private static final Set<Object> RESULTS = Collections.synchronizedSet(new HashSet<>());
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ RESULTS.add(c.element());
+ }
+ }
+}
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
new file mode 100644
index 0000000..6ebe59b
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.utils;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+
+import java.time.Instant;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.junit.Test;
+
+/**
+ * Tests for {@link TranslatorUtils}.
+ */
+public class TranslatorUtilsTest {
+
+ private static final List<KV<org.joda.time.Instant, Instant>> TEST_VALUES = Lists.newArrayList(
+ KV.of(new org.joda.time.Instant(0), Instant.EPOCH),
+ KV.of(new org.joda.time.Instant(42), Instant.ofEpochMilli(42)),
+ KV.of(new org.joda.time.Instant(Long.MIN_VALUE), Instant.ofEpochMilli(Long.MIN_VALUE)),
+ KV.of(new org.joda.time.Instant(Long.MAX_VALUE), Instant.ofEpochMilli(Long.MAX_VALUE)));
+
+ @Test
+ public void testJodaTimeAndJava8TimeConversion() {
+ for (KV<org.joda.time.Instant, Instant> kv: TEST_VALUES) {
+ assertThat(TranslatorUtils.jodaTimeToJava8Time(kv.getKey()),
+ equalTo(kv.getValue()));
+ assertThat(TranslatorUtils.java8TimeToJodaTime(kv.getValue()),
+ equalTo(kv.getKey()));
+ }
+ }
+
+ @Test
+ public void testBoundedWindowToGearpumpWindow() {
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(
+ new IntervalWindow(new org.joda.time.Instant(0),
+ new org.joda.time.Instant(Long.MAX_VALUE))),
+ equalTo(Window.apply(Instant.EPOCH, Instant.ofEpochMilli(Long.MAX_VALUE))));
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(
+ new IntervalWindow(new org.joda.time.Instant(Long.MIN_VALUE),
+ new org.joda.time.Instant(Long.MAX_VALUE))),
+ equalTo(Window.apply(Instant.ofEpochMilli(Long.MIN_VALUE),
+ Instant.ofEpochMilli(Long.MAX_VALUE))));
+ BoundedWindow globalWindow = GlobalWindow.INSTANCE;
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(globalWindow),
+ equalTo(Window.apply(Instant.ofEpochMilli(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()),
+ Instant.ofEpochMilli(globalWindow.maxTimestamp().getMillis() + 1))));
+ }
+}
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 9577ceb..2e08181 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -33,7 +33,7 @@
<packaging>jar</packaging>
<properties>
- <dataflow.container_version>beam-master-20170512</dataflow.container_version>
+ <dataflow.container_version>beam-master-20170926</dataflow.container_version>
<dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
<dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
</properties>
@@ -122,6 +122,29 @@
</plugins>
</build>
</profile>
+
+ <!-- this profile enables IO IT benchmarking by disabling unit tests -->
+ <profile>
+ <id>io-it</id>
+ <activation>
+ <property><name>io-it</name></property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire-plugin.version}</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <skipITs>false</skipITs>
+ </properties>
+ </profile>
</profiles>
<build>
@@ -142,6 +165,73 @@
<skip>true</skip>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>org.apache.beam:beam-model-pipeline</include>
+ <include>org.apache.beam:beam-runners-core-construction-java</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <!-- TODO: Once ready, change the following pattern to 'com'
+ only, exclude 'org.apache.beam.**', and remove
+ the second relocation. -->
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <excludes>
+ <!-- com.google.common is too generic, need to exclude guava-testlib -->
+ <exclude>com.google.common.**.testing.*</exclude>
+ </excludes>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.thirdparty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.beam.model</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.model</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.beam.runners.core</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.runners.core</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
@@ -159,12 +249,18 @@
<execution>
<id>validates-runner-tests</id>
<configuration>
+ <!--
+ UsesSplittableParDoWithWindowedSideInputs because of
+ https://issues.apache.org/jira/browse/BEAM-2476
+ -->
<excludedGroups>
+ org.apache.beam.sdk.testing.LargeKeys$Above10MB,
+ org.apache.beam.sdk.testing.UsesAttemptedMetrics,
org.apache.beam.sdk.testing.UsesDistributionMetrics,
org.apache.beam.sdk.testing.UsesGaugeMetrics,
org.apache.beam.sdk.testing.UsesSetState,
org.apache.beam.sdk.testing.UsesMapState,
- org.apache.beam.sdk.testing.UsesSplittableParDo,
+ org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs,
org.apache.beam.sdk.testing.UsesUnboundedPCollections,
org.apache.beam.sdk.testing.UsesTestStream,
</excludedGroups>
@@ -173,70 +269,6 @@
</executions>
</plugin>
- <!-- Ensure that the Maven jar plugin runs before the Maven
- shade plugin by listing the plugin higher within the file. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-and-repackage</id>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadeTestJar>true</shadeTestJar>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- <include>org.apache.beam:beam-runners-core-construction-java</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <!-- TODO: Once ready, change the following pattern to 'com'
- only, exclude 'org.apache.beam.**', and remove
- the second relocation. -->
- <relocation>
- <pattern>com.google.common</pattern>
- <excludes>
- <!-- com.google.common is too generic, need to exclude guava-testlib -->
- <exclude>com.google.common.**.testing.*</exclude>
- </excludes>
- <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.thirdparty</pattern>
- <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.thirdparty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.beam.runners.core</pattern>
- <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.runners.core</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
@@ -248,6 +280,11 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
@@ -258,11 +295,6 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
</dependency>
@@ -444,7 +476,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
deleted file mode 100644
index 572b005..0000000
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.dataflow;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * A primitive {@link PTransform} that implements the {@link Window#into(WindowFn)}
- * {@link PTransform}.
- *
- * <p>For an application of {@link Window#into(WindowFn)} that changes the {@link WindowFn}, applies
- * a primitive {@link PTransform} in the Dataflow service.
- *
- * <p>For an application of {@link Window#into(WindowFn)} that does not change the {@link WindowFn},
- * applies an identity {@link ParDo} and sets the windowing strategy of the output
- * {@link PCollection}.
- *
- * <p>For internal use only.
- *
- * @param <T> the type of input element
- */
-class AssignWindows<T> extends PTransform<PCollection<T>, PCollection<T>> {
- private final Window<T> transform;
-
- /**
- * Builds an instance of this class from the overriden transform.
- */
- @SuppressWarnings("unused") // Used via reflection
- public AssignWindows(Window<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollection<T> expand(PCollection<T> input) {
- WindowingStrategy<?, ?> outputStrategy =
- transform.getOutputStrategyInternal(input.getWindowingStrategy());
- if (transform.getWindowFn() != null) {
- // If the windowFn changed, we create a primitive, and run the AssignWindows operation here.
- return PCollection.<T>createPrimitiveOutputInternal(
- input.getPipeline(), outputStrategy, input.isBounded());
- } else {
- // If the windowFn didn't change, we just run a pass-through transform and then set the
- // new windowing strategy.
- return input.apply("Identity", ParDo.of(new DoFn<T, T>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- c.output(c.element());
- }
- })).setWindowingStrategyInternal(outputStrategy);
- }
- }
-
- @Override
- public void validate(PipelineOptions options) {
- transform.validate(options);
- }
-
- @Override
- protected Coder<?> getDefaultOutputCoder(PCollection<T> input) {
- return input.getCoder();
- }
-
- @Override
- protected String getKindString() {
- return "Window.Into()";
- }
-}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
index 4d9a57f..d7e9d06 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
@@ -19,10 +19,12 @@
import static com.google.common.base.Preconditions.checkState;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -52,6 +54,9 @@
* stateful {@link ParDo} using window-unaware {@link GroupByKeyAndSortValuesOnly} to linearize
* processing per key.
*
+ * <p>For the Fn API, the {@link PTransformOverrideFactory} is only required to perform
+ * per key grouping and expansion.
+ *
* <p>This implementation relies on implementation details of the Dataflow runner, specifically
* standard fusion behavior of {@link ParDo} tranforms following a {@link GroupByKey}.
*/
@@ -65,8 +70,8 @@
PTransformOverrideFactory<
PCollection<KV<K, InputT>>, PCollection<OutputT>,
ParDo.SingleOutput<KV<K, InputT>, OutputT>>
- singleOutputOverrideFactory() {
- return new SingleOutputOverrideFactory<>();
+ singleOutputOverrideFactory(DataflowPipelineOptions options) {
+ return new SingleOutputOverrideFactory<>(isFnApi(options));
}
/**
@@ -77,8 +82,13 @@
PTransformOverrideFactory<
PCollection<KV<K, InputT>>, PCollectionTuple,
ParDo.MultiOutput<KV<K, InputT>, OutputT>>
- multiOutputOverrideFactory() {
- return new MultiOutputOverrideFactory<>();
+ multiOutputOverrideFactory(DataflowPipelineOptions options) {
+ return new MultiOutputOverrideFactory<>(isFnApi(options));
+ }
+
+ private static boolean isFnApi(DataflowPipelineOptions options) {
+ List<String> experiments = options.getExperiments();
+ return experiments != null && experiments.contains("beam_fn_api");
}
private static class SingleOutputOverrideFactory<K, InputT, OutputT>
@@ -86,6 +96,11 @@
PCollection<KV<K, InputT>>, PCollection<OutputT>,
ParDo.SingleOutput<KV<K, InputT>, OutputT>> {
+ private final boolean isFnApi;
+ private SingleOutputOverrideFactory(boolean isFnApi) {
+ this.isFnApi = isFnApi;
+ }
+
@Override
public PTransformReplacement<PCollection<KV<K, InputT>>, PCollection<OutputT>>
getReplacementTransform(
@@ -95,7 +110,7 @@
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new StatefulSingleOutputParDo<>(transform.getTransform()));
+ new StatefulSingleOutputParDo<>(transform.getTransform(), isFnApi));
}
@Override
@@ -104,11 +119,15 @@
return ReplacementOutputs.singleton(outputs, newOutput);
}
}
-
private static class MultiOutputOverrideFactory<K, InputT, OutputT>
implements PTransformOverrideFactory<
PCollection<KV<K, InputT>>, PCollectionTuple, ParDo.MultiOutput<KV<K, InputT>, OutputT>> {
+ private final boolean isFnApi;
+ private MultiOutputOverrideFactory(boolean isFnApi) {
+ this.isFnApi = isFnApi;
+ }
+
@Override
public PTransformReplacement<PCollection<KV<K, InputT>>, PCollectionTuple>
getReplacementTransform(
@@ -118,7 +137,7 @@
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new StatefulMultiOutputParDo<>(transform.getTransform()));
+ new StatefulMultiOutputParDo<>(transform.getTransform(), isFnApi));
}
@Override
@@ -132,9 +151,12 @@
extends PTransform<PCollection<KV<K, InputT>>, PCollection<OutputT>> {
private final ParDo.SingleOutput<KV<K, InputT>, OutputT> originalParDo;
+ private final boolean isFnApi;
- StatefulSingleOutputParDo(ParDo.SingleOutput<KV<K, InputT>, OutputT> originalParDo) {
+ StatefulSingleOutputParDo(ParDo.SingleOutput<KV<K, InputT>, OutputT> originalParDo,
+ boolean isFnApi) {
this.originalParDo = originalParDo;
+ this.isFnApi = isFnApi;
}
ParDo.SingleOutput<KV<K, InputT>, OutputT> getOriginalParDo() {
@@ -145,6 +167,14 @@
public PCollection<OutputT> expand(PCollection<KV<K, InputT>> input) {
DoFn<KV<K, InputT>, OutputT> fn = originalParDo.getFn();
verifyFnIsStateful(fn);
+ DataflowRunner.verifyStateSupported(fn);
+ DataflowRunner.verifyStateSupportForWindowingStrategy(input.getWindowingStrategy());
+
+ if (isFnApi) {
+ return input.apply(GroupByKey.<K, InputT>create())
+ .apply(ParDo.of(new ExpandGbkFn<K, InputT>()))
+ .apply(originalParDo);
+ }
PTransform<
PCollection<? extends KV<K, Iterable<KV<Instant, WindowedValue<KV<K, InputT>>>>>>,
@@ -160,15 +190,26 @@
extends PTransform<PCollection<KV<K, InputT>>, PCollectionTuple> {
private final ParDo.MultiOutput<KV<K, InputT>, OutputT> originalParDo;
+ private final boolean isFnApi;
- StatefulMultiOutputParDo(ParDo.MultiOutput<KV<K, InputT>, OutputT> originalParDo) {
+ StatefulMultiOutputParDo(ParDo.MultiOutput<KV<K, InputT>, OutputT> originalParDo,
+ boolean isFnApi) {
this.originalParDo = originalParDo;
+ this.isFnApi = isFnApi;
}
@Override
public PCollectionTuple expand(PCollection<KV<K, InputT>> input) {
DoFn<KV<K, InputT>, OutputT> fn = originalParDo.getFn();
verifyFnIsStateful(fn);
+ DataflowRunner.verifyStateSupported(fn);
+ DataflowRunner.verifyStateSupportForWindowingStrategy(input.getWindowingStrategy());
+
+ if (isFnApi) {
+ return input.apply(GroupByKey.<K, InputT>create())
+ .apply(ParDo.of(new ExpandGbkFn<K, InputT>()))
+ .apply(originalParDo);
+ }
PTransform<
PCollection<? extends KV<K, Iterable<KV<Instant, WindowedValue<KV<K, InputT>>>>>>,
@@ -243,6 +284,21 @@
}
/**
+ * A key preserving {@link DoFn} that expands the output of a GBK {@code KV<K, Iterable<V>>} into
+ * individual KVs.
+ */
+ static class ExpandGbkFn<K, V>
+ extends DoFn<KV<K, Iterable<V>>, KV<K, V>> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ K k = c.element().getKey();
+ for (V v : c.element().getValue()) {
+ c.output(KV.of(k, v));
+ }
+ }
+ }
+
+ /**
* A key-preserving {@link DoFn} that explodes an iterable that has been grouped by key and
* window.
*/
@@ -283,3 +339,4 @@
ParDo.class.getSimpleName());
}
}
+
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index b4a6e64..9a77b4b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -39,8 +39,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.beam.runners.core.construction.PTransformReplacements;
-import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.dataflow.internal.IsmFormat;
import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecord;
import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecordCoder;
@@ -57,17 +55,11 @@
import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
-import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.View.AsSingleton;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -83,7 +75,6 @@
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -192,12 +183,13 @@
}
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Map<K, V>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsMap(DataflowRunner runner, View.AsMap<K, V> transform) {
+ public BatchViewAsMap(
+ DataflowRunner runner, CreatePCollectionView<KV<K, V>, Map<K, V>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
@@ -207,12 +199,7 @@
private <W extends BoundedWindow> PCollectionView<Map<K, V>>
applyInternal(PCollection<KV<K, V>> input) {
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
try {
- PCollectionView<Map<K, V>> view = PCollectionViews.mapView(
- input, input.getWindowingStrategy(), inputCoder);
return BatchViewAsMultimap.applyForMapLike(runner, input, view, true /* unique keys */);
} catch (NonDeterministicException e) {
runner.recordViewUsesNonDeterministicKeyCoder(this);
@@ -249,19 +236,14 @@
inputCoder.getKeyCoder(),
FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder)));
- TransformedMap<K, WindowedValue<V>, V> defaultValue = new TransformedMap<>(
- WindowedValueToValue.<V>of(),
- ImmutableMap.<K, WindowedValue<V>>of());
-
return BatchViewAsSingleton.<KV<K, V>, TransformedMap<K, WindowedValue<V>, V>,
Map<K, V>,
W> applyForSingleton(
runner,
input,
new ToMapDoFn<K, V, W>(windowCoder),
- true,
- defaultValue,
- finalValueCoder);
+ finalValueCoder,
+ view);
}
}
@@ -680,12 +662,13 @@
}
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Map<K, Iterable<V>>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsMultimap(DataflowRunner runner, View.AsMultimap<K, V> transform) {
+ public BatchViewAsMultimap(
+ DataflowRunner runner, CreatePCollectionView<KV<K, V>, Map<K, Iterable<V>>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
@@ -695,12 +678,7 @@
private <W extends BoundedWindow> PCollectionView<Map<K, Iterable<V>>>
applyInternal(PCollection<KV<K, V>> input) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
try {
- PCollectionView<Map<K, Iterable<V>>> view = PCollectionViews.multimapView(
- input, input.getWindowingStrategy(), inputCoder);
-
return applyForMapLike(runner, input, view, false /* unique keys not expected */);
} catch (NonDeterministicException e) {
runner.recordViewUsesNonDeterministicKeyCoder(this);
@@ -738,16 +716,15 @@
IterableWithWindowedValuesToIterable.<V>of(),
ImmutableMap.<K, Iterable<WindowedValue<V>>>of());
- return BatchViewAsSingleton.<KV<K, V>,
- TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>,
- Map<K, Iterable<V>>,
- W> applyForSingleton(
- runner,
- input,
- new ToMultimapDoFn<K, V, W>(windowCoder),
- true,
- defaultValue,
- finalValueCoder);
+ return BatchViewAsSingleton
+ .<KV<K, V>, TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>,
+ Map<K, Iterable<V>>, W>
+ applyForSingleton(
+ runner,
+ input,
+ new ToMultimapDoFn<K, V, W>(windowCoder),
+ finalValueCoder,
+ view);
}
private static <K, V, W extends BoundedWindow, ViewT> PCollectionView<ViewT> applyForMapLike(
@@ -827,10 +804,9 @@
PCollectionList.of(ImmutableList.of(
perHashWithReifiedWindows, windowMapSizeMetadata, windowMapKeysMetadata));
- return Pipeline.applyTransform(outputs,
- Flatten.<IsmRecord<WindowedValue<V>>>pCollections())
- .apply(CreateDataflowView.<IsmRecord<WindowedValue<V>>,
- ViewT>of(view));
+ Pipeline.applyTransform(outputs, Flatten.<IsmRecord<WindowedValue<V>>>pCollections())
+ .apply(CreateDataflowView.<IsmRecord<WindowedValue<V>>, ViewT>of(view));
+ return view;
}
@Override
@@ -915,14 +891,12 @@
}
private final DataflowRunner runner;
- private final View.AsSingleton<T> transform;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<T> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsSingleton(DataflowRunner runner, View.AsSingleton<T> transform) {
+ public BatchViewAsSingleton(DataflowRunner runner, CreatePCollectionView<T, T> transform) {
this.runner = runner;
- this.transform = transform;
+ this.view = transform.getView();
}
@Override
@@ -935,9 +909,8 @@
runner,
input,
new IsmRecordForSingularValuePerWindowDoFn<T, BoundedWindow>(windowCoder),
- transform.hasDefaultValue(),
- transform.defaultValue(),
- input.getCoder());
+ input.getCoder(),
+ view);
}
static <T, FinalT, ViewT, W extends BoundedWindow> PCollectionView<ViewT>
@@ -946,23 +919,13 @@
PCollection<T> input,
DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
IsmRecord<WindowedValue<FinalT>>> doFn,
- boolean hasDefault,
- FinalT defaultValue,
- Coder<FinalT> defaultValueCoder) {
+ Coder<FinalT> defaultValueCoder,
+ PCollectionView<ViewT> view) {
@SuppressWarnings("unchecked")
Coder<W> windowCoder = (Coder<W>)
input.getWindowingStrategy().getWindowFn().windowCoder();
- @SuppressWarnings({"rawtypes", "unchecked"})
- PCollectionView<ViewT> view =
- (PCollectionView<ViewT>) PCollectionViews.<FinalT, W>singletonView(
- (PCollection) input,
- (WindowingStrategy) input.getWindowingStrategy(),
- hasDefault,
- defaultValue,
- defaultValueCoder);
-
IsmRecordCoder<WindowedValue<FinalT>> ismCoder =
coderForSingleton(windowCoder, defaultValueCoder);
@@ -972,8 +935,9 @@
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<FinalT>>, ViewT>of(view));
+ return view;
}
@Override
@@ -1079,18 +1043,18 @@
}
private final DataflowRunner runner;
+ private final PCollectionView<List<T>> view;
/**
* Builds an instance of this class from the overridden transform.
*/
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsList(DataflowRunner runner, View.AsList<T> transform) {
+ public BatchViewAsList(DataflowRunner runner, CreatePCollectionView<T, List<T>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view = PCollectionViews.listView(
- input, input.getWindowingStrategy(), input.getCoder());
return applyForIterableLike(runner, input, view);
}
@@ -1116,8 +1080,9 @@
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<T>>, ViewT>of(view));
+ return view;
}
PCollection<IsmRecord<WindowedValue<T>>> reifiedPerWindowAndSorted = input
@@ -1126,8 +1091,9 @@
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<T>>, ViewT>of(view));
+ return view;
}
@Override
@@ -1164,18 +1130,17 @@
extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Iterable<T>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsIterable(DataflowRunner runner, View.AsIterable<T> transform) {
+ public BatchViewAsIterable(
+ DataflowRunner runner, CreatePCollectionView<T, Iterable<T>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view = PCollectionViews.iterableView(
- input, input.getWindowingStrategy(), input.getCoder());
return BatchViewAsList.applyForIterableLike(runner, input, view);
}
}
@@ -1293,18 +1258,13 @@
@Override
public PCollection<KV<K1, Iterable<KV<K2, V>>>> expand(PCollection<KV<K1, KV<K2, V>>> input) {
- PCollection<KV<K1, Iterable<KV<K2, V>>>> rval =
- PCollection.<KV<K1, Iterable<KV<K2, V>>>>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- IsBounded.BOUNDED);
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- KvCoder<K1, KV<K2, V>> inputCoder = (KvCoder) input.getCoder();
- rval.setCoder(
- KvCoder.of(inputCoder.getKeyCoder(),
- IterableCoder.of(inputCoder.getValueCoder())));
- return rval;
+ @SuppressWarnings("unchecked")
+ KvCoder<K1, KV<K2, V>> inputCoder = (KvCoder<K1, KV<K2, V>>) input.getCoder();
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getValueCoder())));
}
}
@@ -1377,59 +1337,4 @@
verifyDeterministic(this, "Expected map coder to be deterministic.", originalMapCoder);
}
}
-
- static class BatchCombineGloballyAsSingletonViewFactory<ElemT, ViewT>
- extends SingleInputOutputOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>,
- Combine.GloballyAsSingletonView<ElemT, ViewT>> {
- private final DataflowRunner runner;
-
- BatchCombineGloballyAsSingletonViewFactory(DataflowRunner runner) {
- this.runner = runner;
- }
-
- @Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>>
- getReplacementTransform(
- AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>,
- GloballyAsSingletonView<ElemT, ViewT>>
- transform) {
- GloballyAsSingletonView<ElemT, ViewT> combine = transform.getTransform();
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- new BatchCombineGloballyAsSingletonView<>(
- runner, combine.getCombineFn(), combine.getFanout(), combine.getInsertDefault()));
- }
-
- private static class BatchCombineGloballyAsSingletonView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
- private final DataflowRunner runner;
- private final GlobalCombineFn<? super ElemT, ?, ViewT> combineFn;
- private final int fanout;
- private final boolean insertDefault;
-
- BatchCombineGloballyAsSingletonView(
- DataflowRunner runner,
- GlobalCombineFn<? super ElemT, ?, ViewT> combineFn,
- int fanout,
- boolean insertDefault) {
- this.runner = runner;
- this.combineFn = combineFn;
- this.fanout = fanout;
- this.insertDefault = insertDefault;
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- PCollection<ViewT> combined =
- input.apply(Combine.globally(combineFn).withoutDefaults().withFanout(fanout));
- AsSingleton<ViewT> viewAsSingleton = View.asSingleton();
- if (insertDefault) {
- viewAsSingleton.withDefaultValue(combineFn.defaultValue());
- }
- return combined.apply(new BatchViewAsSingleton<>(runner, viewAsSingleton));
- }
- }
- }
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
index e7542cb..3b01d69 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
@@ -24,7 +24,7 @@
/** A {@link DataflowRunner} marker class for creating a {@link PCollectionView}. */
public class CreateDataflowView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
public static <ElemT, ViewT> CreateDataflowView<ElemT, ViewT> of(PCollectionView<ViewT> view) {
return new CreateDataflowView<>(view);
}
@@ -36,8 +36,9 @@
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return view;
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
}
public PCollectionView<ViewT> getView() {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
index a6a6a43..4c9c493 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
@@ -19,7 +19,9 @@
import static com.google.common.base.MoreObjects.firstNonNull;
+import com.google.api.client.util.ArrayMap;
import com.google.api.services.dataflow.model.JobMetrics;
+import com.google.api.services.dataflow.model.MetricUpdate;
import com.google.auto.value.AutoValue;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableList;
@@ -28,8 +30,9 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import org.apache.beam.runners.core.metrics.MetricFiltering;
-import org.apache.beam.runners.core.metrics.MetricKey;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.metrics.MetricFiltering;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.GaugeResult;
import org.apache.beam.sdk.metrics.MetricName;
@@ -73,34 +76,6 @@
}
/**
- * Build an immutable map that serves as a hash key for a metric update.
- * @return a {@link MetricKey} that can be hashed and used to identify a metric.
- */
- private MetricKey metricHashKey(
- com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
- String fullStepName = metricUpdate.getName().getContext().get("step");
- fullStepName = (dataflowPipelineJob.transformStepNames != null
- ? dataflowPipelineJob.transformStepNames
- .inverse().get(fullStepName).getFullName() : fullStepName);
- return MetricKey.create(
- fullStepName,
- MetricName.named(
- metricUpdate.getName().getContext().get("namespace"),
- metricUpdate.getName().getName()));
- }
-
- /**
- * Check whether a {@link com.google.api.services.dataflow.model.MetricUpdate} is a tentative
- * update or not.
- * @return true if update is tentative, false otherwise
- */
- private boolean isMetricTentative(
- com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
- return (metricUpdate.getName().getContext().containsKey("tentative")
- && Objects.equal(metricUpdate.getName().getContext().get("tentative"), "true"));
- }
-
- /**
* Take a list of metric updates coming from the Dataflow service, and format it into a
* Metrics API MetricQueryResults instance.
* @param metricUpdates
@@ -109,65 +84,8 @@
private MetricQueryResults populateMetricQueryResults(
List<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates,
MetricsFilter filter) {
- // Separate metric updates by name and by tentative/committed.
- HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate>
- tentativeByName = new HashMap<>();
- HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate>
- committedByName = new HashMap<>();
- HashSet<MetricKey> metricHashKeys = new HashSet<>();
-
- // If the Context of the metric update does not have a namespace, then these are not
- // actual metrics counters.
- for (com.google.api.services.dataflow.model.MetricUpdate update : metricUpdates) {
- if (Objects.equal(update.getName().getOrigin(), "user") && isMetricTentative(update)
- && update.getName().getContext().containsKey("namespace")) {
- tentativeByName.put(metricHashKey(update), update);
- metricHashKeys.add(metricHashKey(update));
- } else if (Objects.equal(update.getName().getOrigin(), "user")
- && update.getName().getContext().containsKey("namespace")
- && !isMetricTentative(update)) {
- committedByName.put(metricHashKey(update), update);
- metricHashKeys.add(metricHashKey(update));
- }
- }
- // Create the lists with the metric result information.
- ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder();
- ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults =
- ImmutableList.builder();
- ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder();
- for (MetricKey metricKey : metricHashKeys) {
- if (!MetricFiltering.matches(filter, metricKey)) {
- // Skip unmatched metrics early.
- continue;
- }
-
- // This code is not robust to evolutions in the types of metrics that can be returned, so
- // wrap it in a try-catch and log errors.
- try {
- String metricName = metricKey.metricName().name();
- if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]")
- || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) {
- // Skip distribution metrics, as these are not yet properly supported.
- LOG.warn("Distribution metrics are not yet supported. You can see them in the Dataflow"
- + "User Interface");
- continue;
- }
-
- String namespace = metricKey.metricName().namespace();
- String step = metricKey.stepName();
- Long committed = ((Number) committedByName.get(metricKey).getScalar()).longValue();
- Long attempted = ((Number) tentativeByName.get(metricKey).getScalar()).longValue();
- counterResults.add(
- DataflowMetricResult.create(
- MetricName.named(namespace, metricName), step, committed, attempted));
- } catch (Exception e) {
- LOG.warn("Error handling metric {} for filter {}, skipping result.", metricKey, filter);
- }
- }
- return DataflowMetricQueryResults.create(
- counterResults.build(),
- distributionResults.build(),
- gaugeResults.build());
+ return DataflowMetricQueryResultsFactory.create(dataflowPipelineJob, metricUpdates, filter)
+ .build();
}
private MetricQueryResults queryServiceForMetrics(MetricsFilter filter) {
@@ -206,6 +124,214 @@
return result;
}
+ private static class DataflowMetricResultExtractor {
+ private final ImmutableList.Builder<MetricResult<Long>> counterResults;
+ private final ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults;
+ private final ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults;
+ private final boolean isStreamingJob;
+
+ DataflowMetricResultExtractor(boolean isStreamingJob) {
+ counterResults = ImmutableList.builder();
+ distributionResults = ImmutableList.builder();
+ gaugeResults = ImmutableList.builder();
+ this.isStreamingJob = isStreamingJob;
+ }
+
+ public void addMetricResult(
+ MetricKey metricKey,
+ @Nullable com.google.api.services.dataflow.model.MetricUpdate committed,
+ @Nullable com.google.api.services.dataflow.model.MetricUpdate attempted) {
+ if (committed == null || attempted == null) {
+ LOG.warn(
+ "Metric {} did not have both a committed ({}) and tentative value ({}).",
+ metricKey, committed, attempted);
+ } else if (committed.getDistribution() != null && attempted.getDistribution() != null) {
+ // distribution metric
+ DistributionResult value = getDistributionValue(committed);
+ distributionResults.add(
+ DataflowMetricResult.create(
+ metricKey.metricName(),
+ metricKey.stepName(),
+ isStreamingJob ? null : value, // Committed
+ isStreamingJob ? value : null)); // Attempted
+ /* In Dataflow streaming jobs, only ATTEMPTED metrics are available.
+ * In Dataflow batch jobs, only COMMITTED metrics are available.
+ * Reporting the appropriate metric depending on whether it's a batch/streaming job.
+ */
+ } else if (committed.getScalar() != null && attempted.getScalar() != null) {
+ // counter metric
+ Long value = getCounterValue(committed);
+ counterResults.add(
+ DataflowMetricResult.create(
+ metricKey.metricName(),
+ metricKey.stepName(),
+ isStreamingJob ? null : value, // Committed
+ isStreamingJob ? value : null)); // Attempted
+ /* In Dataflow streaming jobs, only ATTEMPTED metrics are available.
+ * In Dataflow batch jobs, only COMMITTED metrics are available.
+ * Reporting the appropriate metric depending on whether it's a batch/streaming job.
+ */
+ } else {
+ // This is exceptionally unexpected. We expect matching user metrics to only have the
+ // value types provided by the Metrics API.
+ LOG.warn("Unexpected / mismatched metric types."
+ + " Please report JOB ID to Dataflow Support. Metric key: {}."
+ + " Committed / attempted Metric updates: {} / {}",
+ metricKey.toString(), committed.toString(), attempted.toString());
+ }
+ }
+
+ private Long getCounterValue(com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
+ if (metricUpdate.getScalar() == null) {
+ return 0L;
+ }
+ return ((Number) metricUpdate.getScalar()).longValue();
+ }
+
+ private DistributionResult getDistributionValue(
+ com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
+ if (metricUpdate.getDistribution() == null) {
+ return DistributionResult.ZERO;
+ }
+ ArrayMap distributionMap = (ArrayMap) metricUpdate.getDistribution();
+ Long count = ((Number) distributionMap.get("count")).longValue();
+ Long min = ((Number) distributionMap.get("min")).longValue();
+ Long max = ((Number) distributionMap.get("max")).longValue();
+ Long sum = ((Number) distributionMap.get("sum")).longValue();
+ return DistributionResult.create(sum, count, min, max);
+ }
+
+ public Iterable<MetricResult<DistributionResult>> getDistributionResults() {
+ return distributionResults.build();
+ }
+
+ public Iterable<MetricResult<Long>> getCounterResults() {
+ return counterResults.build();
+ }
+
+ public Iterable<MetricResult<GaugeResult>> getGaugeResults() {
+ return gaugeResults.build();
+ }
+ }
+
+ private static class DataflowMetricQueryResultsFactory {
+ private final Iterable<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates;
+ private final MetricsFilter filter;
+ private final HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate>
+ tentativeByName;
+ private final HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate>
+ committedByName;
+ private final HashSet<MetricKey> metricHashKeys;
+ private final DataflowPipelineJob dataflowPipelineJob;
+
+ public static DataflowMetricQueryResultsFactory create(DataflowPipelineJob dataflowPipelineJob,
+ Iterable<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates,
+ MetricsFilter filter) {
+ return new DataflowMetricQueryResultsFactory(dataflowPipelineJob, metricUpdates, filter);
+ }
+
+ private DataflowMetricQueryResultsFactory(DataflowPipelineJob dataflowPipelineJob,
+ Iterable<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates,
+ MetricsFilter filter) {
+ this.dataflowPipelineJob = dataflowPipelineJob;
+ this.metricUpdates = metricUpdates;
+ this.filter = filter;
+
+ tentativeByName = new HashMap<>();
+ committedByName = new HashMap<>();
+ metricHashKeys = new HashSet<>();
+ }
+
+ /**
+ * Check whether a {@link com.google.api.services.dataflow.model.MetricUpdate} is a tentative
+ * update or not.
+ * @return true if update is tentative, false otherwise
+ */
+ private boolean isMetricTentative(
+ com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
+ return (metricUpdate.getName().getContext().containsKey("tentative")
+ && Objects.equal(metricUpdate.getName().getContext().get("tentative"), "true"));
+ }
+
+ /**
+ * Build an {@link MetricKey} that serves as a hash key for a metric update.
+ * @return a {@link MetricKey} that can be hashed and used to identify a metric.
+ */
+ private MetricKey getMetricHashKey(
+ com.google.api.services.dataflow.model.MetricUpdate metricUpdate) {
+ String fullStepName = metricUpdate.getName().getContext().get("step");
+ if (dataflowPipelineJob.transformStepNames == null
+ || !dataflowPipelineJob.transformStepNames.inverse().containsKey(fullStepName)) {
+ // If we can't translate internal step names to user step names, we just skip them
+ // altogether.
+ return null;
+ }
+ fullStepName = dataflowPipelineJob.transformStepNames
+ .inverse().get(fullStepName).getFullName();
+ return MetricKey.create(
+ fullStepName,
+ MetricName.named(
+ metricUpdate.getName().getContext().get("namespace"),
+ metricUpdate.getName().getName()));
+ }
+
+ private void buildMetricsIndex() {
+ // If the Context of the metric update does not have a namespace, then these are not
+ // actual metrics counters.
+ for (com.google.api.services.dataflow.model.MetricUpdate update : metricUpdates) {
+ if (update.getName().getOrigin() != null
+ && (!update.getName().getOrigin().toLowerCase().equals("user")
+ || !update.getName().getContext().containsKey("namespace"))) {
+ // Skip non-user metrics, which should have both a "user" origin and a namespace.
+ continue;
+ }
+
+ MetricKey updateKey = getMetricHashKey(update);
+ if (updateKey == null || !MetricFiltering.matches(filter, updateKey)) {
+ // Skip unmatched metrics early.
+ continue;
+ }
+
+ metricHashKeys.add(updateKey);
+ if (isMetricTentative(update)) {
+ MetricUpdate previousUpdate = tentativeByName.put(updateKey, update);
+ if (previousUpdate != null) {
+ LOG.warn("Metric {} already had a tentative value of {}", updateKey, previousUpdate);
+ }
+ } else {
+ MetricUpdate previousUpdate = committedByName.put(updateKey, update);
+ if (previousUpdate != null) {
+ LOG.warn("Metric {} already had a committed value of {}", updateKey, previousUpdate);
+ }
+ }
+ }
+ }
+
+ public MetricQueryResults build() {
+ buildMetricsIndex();
+
+ DataflowMetricResultExtractor extractor = new DataflowMetricResultExtractor(
+ dataflowPipelineJob.getDataflowOptions().isStreaming());
+ for (MetricKey metricKey : metricHashKeys) {
+ String metricName = metricKey.metricName().name();
+ if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]")
+ || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) {
+ // Skip distribution metrics, as these are not yet properly supported.
+ // TODO: remove this when distributions stop being broken up for the UI.
+ continue;
+ }
+
+ extractor.addMetricResult(metricKey,
+ committedByName.get(metricKey),
+ tentativeByName.get(metricKey));
+ }
+ return DataflowMetricQueryResults.create(
+ extractor.getCounterResults(),
+ extractor.getDistributionResults(),
+ extractor.getGaugeResults());
+ }
+ }
+
@AutoValue
abstract static class DataflowMetricQueryResults implements MetricQueryResults {
public static MetricQueryResults create(
@@ -223,7 +349,9 @@
// and the generated constructor is usable and consistent
public abstract MetricName name();
public abstract String step();
+ @Nullable
public abstract T committed();
+ @Nullable
public abstract T attempted();
public static <T> MetricResult<T> create(MetricName name, String scope,
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index 2d23983..e736373 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -164,6 +164,17 @@
return dataflowOptions.getProject();
}
+ public DataflowPipelineOptions getDataflowOptions() {
+ return dataflowOptions;
+ }
+
+ /**
+ * Get the region this job exists in.
+ */
+ public String getRegion() {
+ return dataflowOptions.getRegion();
+ }
+
/**
* Returns a new {@link DataflowPipelineJob} for the job that replaced this one, if applicable.
*
@@ -340,7 +351,9 @@
getJobId(),
getReplacedByJob().getJobId(),
MonitoringUtil.getJobMonitoringPageURL(
- getReplacedByJob().getProjectId(), getReplacedByJob().getJobId()));
+ getReplacedByJob().getProjectId(),
+ getRegion(),
+ getReplacedByJob().getJobId()));
break;
default:
LOG.info("Job {} failed with status {}.", getJobId(), state);
@@ -418,7 +431,8 @@
"Failed to cancel job in state %s, "
+ "please go to the Developers Console to cancel it manually: %s",
state,
- MonitoringUtil.getJobMonitoringPageURL(getProjectId(), getJobId()));
+ MonitoringUtil.getJobMonitoringPageURL(
+ getProjectId(), getRegion(), getJobId()));
LOG.warn(errorMsg);
throw new IOException(errorMsg, e);
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 840bda8..4f9b939 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -56,7 +56,9 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
-import org.apache.beam.runners.core.construction.WindowingStrategies;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.runners.core.construction.TransformInputs;
+import org.apache.beam.runners.core.construction.WindowingStrategyTranslation;
import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
import org.apache.beam.runners.dataflow.DataflowRunner.CombineGroupedValues;
import org.apache.beam.runners.dataflow.PrimitiveParDoSingleFactory.ParDoSingle;
@@ -65,7 +67,6 @@
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.runners.dataflow.util.CloudObject;
import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.runners.dataflow.util.DoFnInfo;
import org.apache.beam.runners.dataflow.util.OutputReference;
import org.apache.beam.runners.dataflow.util.PropertyNames;
import org.apache.beam.sdk.Pipeline;
@@ -90,6 +91,7 @@
import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.AppliedCombineFn;
+import org.apache.beam.sdk.util.DoFnInfo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.apache.beam.sdk.values.KV;
@@ -124,7 +126,7 @@
private static byte[] serializeWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) {
try {
- return WindowingStrategies.toProto(windowingStrategy).toByteArray();
+ return WindowingStrategyTranslation.toProto(windowingStrategy).toByteArray();
} catch (Exception e) {
throw new RuntimeException(
String.format("Unable to format windowing strategy %s as bytes", windowingStrategy), e);
@@ -343,6 +345,9 @@
workerPool.setPackages(packages);
workerPool.setNumWorkers(options.getNumWorkers());
+ if (options.getLabels() != null) {
+ job.setLabels(options.getLabels());
+ }
if (options.isStreaming()
&& !DataflowRunner.hasExperiment(options, "enable_windmill_service")) {
// Use separate data disk for streaming.
@@ -395,7 +400,9 @@
@Override
public <InputT extends PValue> InputT getInput(PTransform<InputT, ?> transform) {
- return (InputT) Iterables.getOnlyElement(getInputs(transform).values());
+ return (InputT)
+ Iterables.getOnlyElement(
+ TransformInputs.nonAdditionalInputs(getCurrentTransform(transform)));
}
@Override
@@ -431,19 +438,24 @@
transform,
node.getFullName());
LOG.debug("Translating {}", transform);
- currentTransform = node.toAppliedPTransform();
+ currentTransform = node.toAppliedPTransform(getPipeline());
translator.translate(transform, this);
currentTransform = null;
}
@Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {
- producers.put(value, producer.toAppliedPTransform());
LOG.debug("Checking translation of {}", value);
- if (!producer.isCompositeNode()) {
- // Primitive transforms are the only ones assigned step names.
- asOutputReference(value, producer.toAppliedPTransform());
+ // Primitive transforms are the only ones assigned step names.
+ if (producer.getTransform() instanceof CreateDataflowView) {
+ // CreateDataflowView produces a dummy output (as it must be a primitive transform) but
+ // in the Dataflow Job graph produces only the view and not the output PCollection.
+ asOutputReference(
+ ((CreateDataflowView) producer.getTransform()).getView(),
+ producer.toAppliedPTransform(getPipeline()));
+ return;
}
+ asOutputReference(value, producer.toAppliedPTransform(getPipeline()));
}
@Override
@@ -468,51 +480,10 @@
StepTranslator stepContext = new StepTranslator(this, step);
stepContext.addInput(PropertyNames.USER_NAME, getFullName(transform));
stepContext.addDisplayData(step, stepName, transform);
+ LOG.info("Adding {} as step {}", getCurrentTransform(transform).getFullName(), stepName);
return stepContext;
}
- @Override
- public Step addStep(PTransform<?, ? extends PValue> transform, Step original) {
- Step step = original.clone();
- String stepName = step.getName();
- if (stepNames.put(getCurrentTransform(transform), stepName) != null) {
- throw new IllegalArgumentException(transform + " already has a name specified");
- }
-
- Map<String, Object> properties = step.getProperties();
- if (properties != null) {
- @Nullable List<Map<String, Object>> outputInfoList = null;
- try {
- // TODO: This should be done via a Structs accessor.
- @Nullable List<Map<String, Object>> list =
- (List<Map<String, Object>>) properties.get(PropertyNames.OUTPUT_INFO);
- outputInfoList = list;
- } catch (Exception e) {
- throw new RuntimeException("Inconsistent dataflow pipeline translation", e);
- }
- if (outputInfoList != null && outputInfoList.size() > 0) {
- Map<String, Object> firstOutputPort = outputInfoList.get(0);
- @Nullable String name;
- try {
- name = getString(firstOutputPort, PropertyNames.OUTPUT_NAME);
- } catch (Exception e) {
- name = null;
- }
- if (name != null) {
- registerOutputName(getOutput(transform), name);
- }
- }
- }
-
- List<Step> steps = job.getSteps();
- if (steps == null) {
- steps = new LinkedList<>();
- job.setSteps(steps);
- }
- steps.add(step);
- return step;
- }
-
public OutputReference asOutputReference(PValue value, AppliedPTransform<?, ?, ?> producer) {
String stepName = stepNames.get(producer);
checkArgument(stepName != null, "%s doesn't have a name specified", producer);
@@ -607,26 +578,19 @@
}
@Override
- public long addOutput(PValue value) {
- Coder<?> coder;
- if (value instanceof PCollection) {
- coder = ((PCollection<?>) value).getCoder();
- if (value instanceof PCollection) {
- // Wrap the PCollection element Coder inside a WindowedValueCoder.
- coder = WindowedValue.getFullCoder(
- coder,
- ((PCollection<?>) value).getWindowingStrategy().getWindowFn().windowCoder());
- }
- } else {
- // No output coder to encode.
- coder = null;
- }
+ public long addOutput(PCollection<?> value) {
+ translator.producers.put(value, translator.currentTransform);
+ // Wrap the PCollection element Coder inside a WindowedValueCoder.
+ Coder<?> coder =
+ WindowedValue.getFullCoder(
+ value.getCoder(), value.getWindowingStrategy().getWindowFn().windowCoder());
return addOutput(value, coder);
}
@Override
public long addCollectionToSingletonOutput(
- PValue inputValue, PValue outputValue) {
+ PCollection<?> inputValue, PCollectionView<?> outputValue) {
+ translator.producers.put(outputValue, translator.currentTransform);
Coder<?> inputValueCoder =
checkNotNull(translator.outputCoders.get(inputValue));
// The inputValueCoder for the input PCollection should be some
@@ -729,7 +693,7 @@
context.addStep(transform, "CollectionToSingleton");
PCollection<ElemT> input = context.getInput(transform);
stepContext.addInput(PropertyNames.PARALLEL_INPUT, input);
- stepContext.addCollectionToSingletonOutput(input, context.getOutput(transform));
+ stepContext.addCollectionToSingletonOutput(input, transform.getView());
}
});
@@ -832,6 +796,7 @@
context.getPipelineOptions().as(StreamingOptions.class).isStreaming();
boolean disallowCombinerLifting =
!windowingStrategy.getWindowFn().isNonMerging()
+ || !windowingStrategy.getWindowFn().assignsToOneWindow()
|| (isStreaming && !transform.fewKeys())
// TODO: Allow combiner lifting on the non-default trigger, as appropriate.
|| !(windowingStrategy.getTrigger() instanceof DefaultTrigger);
@@ -926,6 +891,45 @@
// IO Translation.
registerTransformTranslator(Read.Bounded.class, new ReadTranslator());
+
+ ///////////////////////////////////////////////////////////////////////////
+ // Splittable DoFn translation.
+
+ registerTransformTranslator(
+ SplittableParDo.ProcessKeyedElements.class,
+ new TransformTranslator<SplittableParDo.ProcessKeyedElements>() {
+ @Override
+ public void translate(
+ SplittableParDo.ProcessKeyedElements transform, TranslationContext context) {
+ translateTyped(transform, context);
+ }
+
+ private <InputT, OutputT, RestrictionT> void translateTyped(
+ SplittableParDo.ProcessKeyedElements<InputT, OutputT, RestrictionT> transform,
+ TranslationContext context) {
+ StepTranslationContext stepContext =
+ context.addStep(transform, "SplittableProcessKeyed");
+
+ translateInputs(
+ stepContext, context.getInput(transform), transform.getSideInputs(), context);
+ BiMap<Long, TupleTag<?>> outputMap =
+ translateOutputs(context.getOutputs(transform), stepContext);
+ stepContext.addInput(
+ PropertyNames.SERIALIZED_FN,
+ byteArrayToJsonString(
+ serializeToByteArray(
+ DoFnInfo.forFn(
+ transform.getFn(),
+ transform.getInputWindowingStrategy(),
+ transform.getSideInputs(),
+ transform.getElementCoder(),
+ outputMap.inverse().get(transform.getMainOutputTag()),
+ outputMap))));
+ stepContext.addInput(
+ PropertyNames.RESTRICTION_CODER,
+ CloudObjects.asCloudObject(transform.getRestrictionCoder()));
+ }
+ });
}
private static void translateInputs(
@@ -972,6 +976,11 @@
fn));
}
+ if (signature.usesState() || signature.usesTimers()) {
+ DataflowRunner.verifyStateSupported(fn);
+ DataflowRunner.verifyStateSupportForWindowingStrategy(windowingStrategy);
+ }
+
stepContext.addInput(PropertyNames.USER_FN, fn.getClass().getName());
stepContext.addInput(
PropertyNames.SERIALIZED_FN,
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 2ef8737..334c8e5 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -49,6 +49,7 @@
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -57,19 +58,24 @@
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.runners.core.construction.DeduplicatedFlattenFactory;
import org.apache.beam.runners.core.construction.EmptyFlattenAsCreateFactory;
import org.apache.beam.runners.core.construction.PTransformMatchers;
import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.PipelineTranslation;
+import org.apache.beam.runners.core.construction.RehydratedComponents;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource;
import org.apache.beam.runners.core.construction.UnconsumedReads;
-import org.apache.beam.runners.dataflow.BatchViewOverrides.BatchCombineGloballyAsSingletonViewFactory;
+import org.apache.beam.runners.core.construction.WriteFilesTranslation;
import org.apache.beam.runners.dataflow.DataflowPipelineTranslator.JobSpecification;
import org.apache.beam.runners.dataflow.StreamingViewOverrides.StreamingCreatePCollectionViewFactory;
import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions;
import org.apache.beam.runners.dataflow.util.DataflowTemplateJob;
import org.apache.beam.runners.dataflow.util.DataflowTransport;
import org.apache.beam.runners.dataflow.util.MonitoringUtil;
@@ -79,15 +85,20 @@
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder;
@@ -101,8 +112,11 @@
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Combine.GroupedValues;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
@@ -112,21 +126,25 @@
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.WithKeys;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.InstanceBuilder;
import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.NameUtils;
-import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.ValueWithRecordId;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.joda.time.DateTimeUtils;
@@ -171,6 +189,12 @@
@VisibleForTesting
static final int GCS_UPLOAD_BUFFER_SIZE_BYTES_DEFAULT = 1024 * 1024;
+ @VisibleForTesting
+ static final String PIPELINE_FILE_NAME = "pipeline.pb";
+
+ @VisibleForTesting
+ static final String STAGED_PIPELINE_METADATA_PROPERTY = "pipeline_url";
+
private final Set<PCollection<?>> pcollectionsRequiringIndexedFormat;
/**
@@ -226,11 +250,15 @@
if (dataflowOptions.getFilesToStage() == null) {
dataflowOptions.setFilesToStage(detectClassPathResourcesToStage(
DataflowRunner.class.getClassLoader()));
- LOG.info("PipelineOptions.filesToStage was not specified. "
- + "Defaulting to files from the classpath: will stage {} files. "
- + "Enable logging at DEBUG level to see which files will be staged.",
- dataflowOptions.getFilesToStage().size());
- LOG.debug("Classpath elements: {}", dataflowOptions.getFilesToStage());
+ if (dataflowOptions.getFilesToStage().isEmpty()) {
+ throw new IllegalArgumentException("No files to stage has been found.");
+ } else {
+ LOG.info("PipelineOptions.filesToStage was not specified. "
+ + "Defaulting to files from the classpath: will stage {} files. "
+ + "Enable logging at DEBUG level to see which files will be staged.",
+ dataflowOptions.getFilesToStage().size());
+ LOG.debug("Classpath elements: {}", dataflowOptions.getFilesToStage());
+ }
}
// Verify jobName according to service requirements, truncating converting to lowercase if
@@ -276,6 +304,12 @@
dataflowOptions.setGcsUploadBufferSizeBytes(GCS_UPLOAD_BUFFER_SIZE_BYTES_DEFAULT);
}
+ DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo();
+ String userAgent = String
+ .format("%s/%s", dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion())
+ .replace(" ", "_");
+ dataflowOptions.setUserAgent(userAgent);
+
return new DataflowRunner(dataflowOptions);
}
@@ -304,7 +338,7 @@
overridesBuilder.add(
PTransformOverride.of(
PTransformMatchers.classEqualTo(PubsubUnboundedSource.class),
- new ReflectiveRootOverrideFactory(StreamingPubsubIORead.class, this)));
+ new StreamingPubsubIOReadOverrideFactory()));
}
if (!hasExperiment(options, "enable_custom_pubsub_sink")) {
overridesBuilder.add(
@@ -312,17 +346,41 @@
PTransformMatchers.classEqualTo(PubsubUnboundedSink.class),
new StreamingPubsubIOWriteOverrideFactory(this)));
}
+ if (hasExperiment(options, "beam_fn_api")) {
+ overridesBuilder.add(
+ PTransformOverride.of(
+ PTransformMatchers.classEqualTo(Create.Values.class),
+ new StreamingFnApiCreateOverrideFactory()));
+ }
overridesBuilder
+ // Support Splittable DoFn for now only in streaming mode.
+ // The order of the following overrides is important because they are applied in order.
+
+ // By default Dataflow runner replaces single-output ParDo with a ParDoSingle override.
+ // However, we want a different expansion for single-output splittable ParDo.
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.splittableParDoSingle(),
+ new ReflectiveOneToOneOverrideFactory(
+ SplittableParDoOverrides.ParDoSingleViaMulti.class, this)))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.splittableParDoMulti(),
+ new SplittableParDoOverrides.SplittableParDoOverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.writeWithRunnerDeterminedSharding(),
+ new StreamingShardedWriteFactory(options)))
.add(
// Streaming Bounded Read is implemented in terms of Streaming Unbounded Read, and
// must precede it
PTransformOverride.of(
PTransformMatchers.classEqualTo(Read.Bounded.class),
- new ReflectiveRootOverrideFactory(StreamingBoundedRead.class, this)))
+ new StreamingBoundedReadOverrideFactory()))
.add(
PTransformOverride.of(
PTransformMatchers.classEqualTo(Read.Unbounded.class),
- new ReflectiveRootOverrideFactory(StreamingUnboundedRead.class, this)))
+ new StreamingUnboundedReadOverrideFactory()))
.add(
PTransformOverride.of(
PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
@@ -333,39 +391,34 @@
.add(
PTransformOverride.of(
PTransformMatchers.stateOrTimerParDoMulti(),
- BatchStatefulParDoOverrides.multiOutputOverrideFactory()))
+ BatchStatefulParDoOverrides.multiOutputOverrideFactory(options)))
.add(
PTransformOverride.of(
PTransformMatchers.stateOrTimerParDoSingle(),
- BatchStatefulParDoOverrides.singleOutputOverrideFactory()))
-
+ BatchStatefulParDoOverrides.singleOutputOverrideFactory(options)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new BatchCombineGloballyAsSingletonViewFactory(this)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMap.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.MapViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsMap.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMultimap.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.MultimapViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsMultimap.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsSingleton.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsList.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.ListViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsList.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsIterable.class, this)));
}
@@ -412,38 +465,6 @@
}
}
- private static class ReflectiveRootOverrideFactory<T>
- implements PTransformOverrideFactory<
- PBegin, PCollection<T>, PTransform<PInput, PCollection<T>>> {
- private final Class<PTransform<PBegin, PCollection<T>>> replacement;
- private final DataflowRunner runner;
-
- private ReflectiveRootOverrideFactory(
- Class<PTransform<PBegin, PCollection<T>>> replacement, DataflowRunner runner) {
- this.replacement = replacement;
- this.runner = runner;
- }
-
- @Override
- public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
- AppliedPTransform<PBegin, PCollection<T>, PTransform<PInput, PCollection<T>>> transform) {
- PTransform<PInput, PCollection<T>> original = transform.getTransform();
- return PTransformReplacement.of(
- transform.getPipeline().begin(),
- InstanceBuilder.ofType(replacement)
- .withArg(DataflowRunner.class, runner)
- .withArg(
- (Class<? super PTransform<PInput, PCollection<T>>>) original.getClass(), original)
- .build());
- }
-
- @Override
- public Map<PValue, ReplacementOutput> mapOutputs(
- Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) {
- return ReplacementOutputs.singleton(outputs, newOutput);
- }
- }
-
private String debuggerMessage(String projectId, String uniquifier) {
return String.format("To debug your job, visit Google Cloud Debugger at: "
+ "https://console.developers.google.com/debug?project=%s&dbgee=%s",
@@ -500,8 +521,12 @@
LOG.info("Executing pipeline on the Dataflow Service, which will have billing implications "
+ "related to Google Compute Engine usage and other Google Cloud Services.");
- List<DataflowPackage> packages = options.getStager().stageFiles();
+ List<DataflowPackage> packages = options.getStager().stageDefaultFiles();
+ byte[] serializedProtoPipeline = PipelineTranslation.toProto(pipeline).toByteArray();
+ LOG.info("Staging pipeline description to {}", options.getStagingLocation());
+ DataflowPackage stagedPipeline =
+ options.getStager().stageToFile(serializedProtoPipeline, PIPELINE_FILE_NAME);
// Set a unique client_request_id in the CreateJob request.
// This is used to ensure idempotence of job creation across retried
@@ -524,14 +549,14 @@
Job newJob = jobSpecification.getJob();
newJob.setClientRequestId(requestId);
- ReleaseInfo releaseInfo = ReleaseInfo.getReleaseInfo();
- String version = releaseInfo.getVersion();
+ DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo();
+ String version = dataflowRunnerInfo.getVersion();
checkState(
!version.equals("${pom.version}"),
"Unable to submit a job to the Dataflow service with unset version ${pom.version}");
System.out.println("Dataflow SDK version: " + version);
- newJob.getEnvironment().setUserAgent((Map) releaseInfo.getProperties());
+ newJob.getEnvironment().setUserAgent((Map) dataflowRunnerInfo.getProperties());
// The Dataflow Service may write to the temporary directory directly, so
// must be verified.
if (!isNullOrEmpty(options.getGcpTempLocation())) {
@@ -546,6 +571,10 @@
String workerHarnessContainerImage = getContainerImageForJob(options);
for (WorkerPool workerPool : newJob.getEnvironment().getWorkerPools()) {
workerPool.setWorkerHarnessContainerImage(workerHarnessContainerImage);
+
+ // https://issues.apache.org/jira/browse/BEAM-3116
+ // workerPool.setMetadata(
+ // ImmutableMap.of(STAGED_PIPELINE_METADATA_PROPERTY, stagedPipeline.getLocation()));
}
newJob.getEnvironment().setVersion(getEnvironmentVersion(options));
@@ -646,7 +675,8 @@
}
LOG.info("To access the Dataflow monitoring console, please navigate to {}",
- MonitoringUtil.getJobMonitoringPageURL(options.getProject(), jobResult.getId()));
+ MonitoringUtil.getJobMonitoringPageURL(
+ options.getProject(), options.getRegion(), jobResult.getId()));
System.out.println("Submitted job: " + jobResult.getId());
LOG.info("To cancel the job using the 'gcloud' tool, run:\n> {}",
@@ -729,7 +759,7 @@
if (!ptransformViewsWithNonDeterministicKeyCoders.isEmpty()) {
final SortedSet<String> ptransformViewNamesWithNonDeterministicKeyCoders = new TreeSet<>();
pipeline.traverseTopologically(
- new PipelineVisitor() {
+ new PipelineVisitor.Defaults() {
@Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {}
@@ -801,6 +831,24 @@
// PubsubIO translations
// ================================================================================
+ private static class StreamingPubsubIOReadOverrideFactory
+ implements PTransformOverrideFactory<
+ PBegin, PCollection<PubsubMessage>, PubsubUnboundedSource> {
+ @Override
+ public PTransformReplacement<PBegin, PCollection<PubsubMessage>> getReplacementTransform(
+ AppliedPTransform<PBegin, PCollection<PubsubMessage>, PubsubUnboundedSource> transform) {
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(), new StreamingPubsubIORead(transform.getTransform()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<PubsubMessage> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+
+
/**
* Suppress application of {@link PubsubUnboundedSource#expand} in streaming mode so that we can
* instead defer to Windmill's implementation.
@@ -809,11 +857,7 @@
extends PTransform<PBegin, PCollection<PubsubMessage>> {
private final PubsubUnboundedSource transform;
- /**
- * Builds an instance of this class from the overridden transform.
- */
- public StreamingPubsubIORead(
- DataflowRunner runner, PubsubUnboundedSource transform) {
+ public StreamingPubsubIORead(PubsubUnboundedSource transform) {
this.transform = transform;
}
@@ -823,9 +867,11 @@
@Override
public PCollection<PubsubMessage> expand(PBegin input) {
- return PCollection.<PubsubMessage>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
- .setCoder(new PubsubMessageWithAttributesCoder());
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ IsBounded.UNBOUNDED,
+ new PubsubMessageWithAttributesCoder());
}
@Override
@@ -982,6 +1028,158 @@
// ================================================================================
/**
+ * A PTranform override factory which maps Create.Values PTransforms for streaming pipelines
+ * into a Dataflow specific variant.
+ */
+ private static class StreamingFnApiCreateOverrideFactory<T>
+ implements PTransformOverrideFactory<PBegin, PCollection<T>, Create.Values<T>> {
+
+ @Override
+ public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) {
+ Create.Values<T> original = transform.getTransform();
+ PCollection<T> output =
+ (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(),
+ new StreamingFnApiCreate<>(original, output));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.Create.Values Create.Values} for the Dataflow runner in
+ * streaming mode over the Fn API.
+ */
+ private static class StreamingFnApiCreate<T> extends PTransform<PBegin, PCollection<T>> {
+ private final Create.Values<T> transform;
+ private final PCollection<T> originalOutput;
+
+ private StreamingFnApiCreate(
+ Create.Values<T> transform,
+ PCollection<T> originalOutput) {
+ this.transform = transform;
+ this.originalOutput = originalOutput;
+ }
+
+ @Override
+ public final PCollection<T> expand(PBegin input) {
+ try {
+ PCollection<T> pc = Pipeline
+ .applyTransform(input, new Impulse(IsBounded.BOUNDED))
+ .apply(ParDo.of(DecodeAndEmitDoFn
+ .fromIterable(transform.getElements(), originalOutput.getCoder())));
+ pc.setCoder(originalOutput.getCoder());
+ return pc;
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to encode elements.", e);
+ }
+ }
+
+ /**
+ * A DoFn which stores encoded versions of elements and a representation of a Coder
+ * capable of decoding those elements.
+ *
+ * <p>TODO: BEAM-2422 - Make this a SplittableDoFn.
+ */
+ private static class DecodeAndEmitDoFn<T> extends DoFn<byte[], T> {
+ public static <T> DecodeAndEmitDoFn<T> fromIterable(Iterable<T> elements, Coder<T> elemCoder)
+ throws IOException {
+ ImmutableList.Builder<byte[]> allElementsBytes = ImmutableList.builder();
+ for (T element : elements) {
+ byte[] bytes = CoderUtils.encodeToByteArray(elemCoder, element);
+ allElementsBytes.add(bytes);
+ }
+ return new DecodeAndEmitDoFn<>(allElementsBytes.build(), elemCoder);
+ }
+
+ private final Collection<byte[]> elements;
+ private final RunnerApi.MessageWithComponents coderSpec;
+
+ // lazily initialized by parsing coderSpec
+ private transient Coder<T> coder;
+ private Coder<T> getCoder() throws IOException {
+ if (coder == null) {
+ coder =
+ (Coder)
+ CoderTranslation.fromProto(
+ coderSpec.getCoder(),
+ RehydratedComponents.forComponents(coderSpec.getComponents()));
+ }
+ return coder;
+ }
+
+ private DecodeAndEmitDoFn(Collection<byte[]> elements, Coder<T> coder) throws IOException {
+ this.elements = elements;
+ this.coderSpec = CoderTranslation.toProto(coder);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) throws IOException {
+ for (byte[] element : elements) {
+ context.output(CoderUtils.decodeFromByteArray(getCoder(), element));
+ }
+ }
+ }
+ }
+
+ /** The Dataflow specific override for the impulse primitive. */
+ private static class Impulse extends PTransform<PBegin, PCollection<byte[]>> {
+ private final IsBounded isBounded;
+
+ private Impulse(IsBounded isBounded) {
+ this.isBounded = isBounded;
+ }
+
+ @Override
+ public PCollection<byte[]> expand(PBegin input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), WindowingStrategy.globalDefault(), isBounded, ByteArrayCoder.of());
+ }
+
+ private static class Translator implements TransformTranslator<Impulse> {
+ @Override
+ public void translate(Impulse transform, TranslationContext context) {
+ if (context.getPipelineOptions().isStreaming()) {
+ StepTranslationContext stepContext = context.addStep(transform, "ParallelRead");
+ stepContext.addInput(PropertyNames.FORMAT, "pubsub");
+ stepContext.addInput(PropertyNames.PUBSUB_SUBSCRIPTION, "_starting_signal/");
+ stepContext.addOutput(context.getOutput(transform));
+ } else {
+ throw new UnsupportedOperationException(
+ "Impulse source for batch pipelines has not been defined.");
+ }
+ }
+ }
+
+ static {
+ DataflowPipelineTranslator.registerTransformTranslator(Impulse.class, new Translator());
+ }
+ }
+
+ private static class StreamingUnboundedReadOverrideFactory<T>
+ implements PTransformOverrideFactory<PBegin, PCollection<T>, Read.Unbounded<T>> {
+ @Override
+ public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PBegin, PCollection<T>, Read.Unbounded<T>> transform) {
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(), new StreamingUnboundedRead<>(transform.getTransform()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+
+ /**
* Specialized implementation for
* {@link org.apache.beam.sdk.io.Read.Unbounded Read.Unbounded} for the
* Dataflow runner in streaming mode.
@@ -992,20 +1190,11 @@
private static class StreamingUnboundedRead<T> extends PTransform<PBegin, PCollection<T>> {
private final UnboundedSource<T, ?> source;
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public StreamingUnboundedRead(DataflowRunner runner, Read.Unbounded<T> transform) {
+ public StreamingUnboundedRead(Read.Unbounded<T> transform) {
this.source = transform.getSource();
}
@Override
- protected Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
- }
-
- @Override
public final PCollection<T> expand(PBegin input) {
source.validate();
@@ -1032,13 +1221,9 @@
@Override
public final PCollection<ValueWithRecordId<T>> expand(PInput input) {
- return PCollection.<ValueWithRecordId<T>>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED);
- }
-
- @Override
- protected Coder<ValueWithRecordId<T>> getDefaultOutputCoder() {
- return ValueWithRecordId.ValueWithRecordIdCoder.of(source.getDefaultOutputCoder());
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED,
+ ValueWithRecordId.ValueWithRecordIdCoder.of(source.getOutputCoder()));
}
@Override
@@ -1102,6 +1287,22 @@
}
}
+ private static class StreamingBoundedReadOverrideFactory<T>
+ implements PTransformOverrideFactory<PBegin, PCollection<T>, Read.Bounded<T>> {
+ @Override
+ public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform) {
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(), new StreamingBoundedRead<>(transform.getTransform()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+
/**
* Specialized implementation for {@link org.apache.beam.sdk.io.Read.Bounded Read.Bounded} for the
* Dataflow runner in streaming mode.
@@ -1109,18 +1310,11 @@
private static class StreamingBoundedRead<T> extends PTransform<PBegin, PCollection<T>> {
private final BoundedSource<T> source;
- /** Builds an instance of this class from the overridden transform. */
- @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public StreamingBoundedRead(DataflowRunner runner, Read.Bounded<T> transform) {
+ public StreamingBoundedRead(Read.Bounded<T> transform) {
this.source = transform.getSource();
}
@Override
- protected Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
- }
-
- @Override
public final PCollection<T> expand(PBegin input) {
source.validate();
@@ -1133,7 +1327,7 @@
* A marker {@link DoFn} for writing the contents of a {@link PCollection} to a streaming
* {@link PCollectionView} backend implementation.
*/
- @Deprecated
+ @Internal
public static class StreamingPCollectionViewWriterFn<T> extends DoFn<Iterable<T>, T> {
private final PCollectionView<?> view;
private final Coder<T> dataCoder;
@@ -1230,15 +1424,19 @@
static class CombineGroupedValues<K, InputT, OutputT>
extends PTransform<PCollection<KV<K, Iterable<InputT>>>, PCollection<KV<K, OutputT>>> {
private final Combine.GroupedValues<K, InputT, OutputT> original;
+ private final Coder<KV<K, OutputT>> outputCoder;
- CombineGroupedValues(GroupedValues<K, InputT, OutputT> original) {
+ CombineGroupedValues(
+ GroupedValues<K, InputT, OutputT> original, Coder<KV<K, OutputT>> outputCoder) {
this.original = original;
+ this.outputCoder = outputCoder;
}
@Override
public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, Iterable<InputT>>> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(),
+ outputCoder);
}
public Combine.GroupedValues<K, InputT, OutputT> getOriginalCombine() {
@@ -1259,7 +1457,9 @@
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new CombineGroupedValues<>(transform.getTransform()));
+ new CombineGroupedValues<>(
+ transform.getTransform(),
+ PTransformReplacements.getSingletonMainOutput(transform).getCoder()));
}
@Override
@@ -1296,6 +1496,66 @@
}
@VisibleForTesting
+ static class StreamingShardedWriteFactory<UserT, DestinationT, OutputT>
+ implements PTransformOverrideFactory<
+ PCollection<UserT>, WriteFilesResult<DestinationT>,
+ WriteFiles<UserT, DestinationT, OutputT>> {
+ // We pick 10 as a a default, as it works well with the default number of workers started
+ // by Dataflow.
+ static final int DEFAULT_NUM_SHARDS = 10;
+ DataflowPipelineWorkerPoolOptions options;
+
+ StreamingShardedWriteFactory(PipelineOptions options) {
+ this.options = options.as(DataflowPipelineWorkerPoolOptions.class);
+ }
+
+ @Override
+ public PTransformReplacement<PCollection<UserT>, WriteFilesResult<DestinationT>>
+ getReplacementTransform(
+ AppliedPTransform<
+ PCollection<UserT>, WriteFilesResult<DestinationT>,
+ WriteFiles<UserT, DestinationT, OutputT>>
+ transform) {
+ // By default, if numShards is not set WriteFiles will produce one file per bundle. In
+ // streaming, there are large numbers of small bundles, resulting in many tiny files.
+ // Instead we pick max workers * 2 to ensure full parallelism, but prevent too-many files.
+ // (current_num_workers * 2 might be a better choice, but that value is not easily available
+ // today).
+ // If the user does not set either numWorkers or maxNumWorkers, default to 10 shards.
+ int numShards;
+ if (options.getMaxNumWorkers() > 0) {
+ numShards = options.getMaxNumWorkers() * 2;
+ } else if (options.getNumWorkers() > 0) {
+ numShards = options.getNumWorkers() * 2;
+ } else {
+ numShards = DEFAULT_NUM_SHARDS;
+ }
+
+ try {
+ List<PCollectionView<?>> sideInputs =
+ WriteFilesTranslation.getDynamicDestinationSideInputs(transform);
+ FileBasedSink sink = WriteFilesTranslation.getSink(transform);
+ WriteFiles<UserT, DestinationT, OutputT> replacement =
+ WriteFiles.to(sink).withSideInputs(sideInputs);
+ if (WriteFilesTranslation.isWindowedWrites(transform)) {
+ replacement = replacement.withWindowedWrites();
+ }
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform),
+ replacement.withNumShards(numShards));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, WriteFilesResult<DestinationT> newOutput) {
+ return Collections.emptyMap();
+ }
+ }
+
+ @VisibleForTesting
static String getContainerImageForJob(DataflowPipelineOptions options) {
String workerHarnessContainerImage = options.getWorkerHarnessContainerImage();
if (!workerHarnessContainerImage.contains("IMAGE")) {
@@ -1308,4 +1568,39 @@
return workerHarnessContainerImage.replace("IMAGE", "beam-java-batch");
}
}
+
+ static void verifyStateSupported(DoFn<?, ?> fn) {
+ DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
+
+ for (DoFnSignature.StateDeclaration stateDecl : signature.stateDeclarations().values()) {
+
+ // https://issues.apache.org/jira/browse/BEAM-1474
+ if (stateDecl.stateType().isSubtypeOf(TypeDescriptor.of(MapState.class))) {
+ throw new UnsupportedOperationException(String.format(
+ "%s does not currently support %s",
+ DataflowRunner.class.getSimpleName(),
+ MapState.class.getSimpleName()
+ ));
+ }
+
+ // https://issues.apache.org/jira/browse/BEAM-1479
+ if (stateDecl.stateType().isSubtypeOf(TypeDescriptor.of(SetState.class))) {
+ throw new UnsupportedOperationException(String.format(
+ "%s does not currently support %s",
+ DataflowRunner.class.getSimpleName(),
+ SetState.class.getSimpleName()
+ ));
+ }
+ }
+ }
+
+ static void verifyStateSupportForWindowingStrategy(WindowingStrategy strategy) {
+ // https://issues.apache.org/jira/browse/BEAM-2507
+ if (!strategy.getWindowFn().isNonMerging()) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s does not currently support state or timers with merging windows",
+ DataflowRunner.class.getSimpleName()));
+ }
+ }
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java
index 12b3f38..69e4f46 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java
@@ -19,23 +19,66 @@
import static com.google.common.base.Preconditions.checkState;
+import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Map;
import java.util.Properties;
+import org.apache.beam.sdk.util.ReleaseInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Populates versioning and other information for {@link DataflowRunner}.
*/
-public final class DataflowRunnerInfo {
+public final class DataflowRunnerInfo extends ReleaseInfo {
private static final Logger LOG = LoggerFactory.getLogger(DataflowRunnerInfo.class);
- private static final String PROPERTIES_PATH =
+ private static final String APACHE_BEAM_DISTRIBUTION_PROPERTIES_PATH =
"/org/apache/beam/runners/dataflow/dataflow.properties";
+ private static final String DATAFLOW_DISTRIBUTION_PROPERTIES_PATH =
+ "/org/apache/beam/runners/dataflow/dataflow-distribution.properties";
+ private static final String FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY =
+ "fnapi.environment.major.version";
+ private static final String LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY =
+ "legacy.environment.major.version";
+ private static final String CONTAINER_VERSION_KEY = "container.version";
private static class LazyInit {
- private static final DataflowRunnerInfo INSTANCE = new DataflowRunnerInfo(PROPERTIES_PATH);
+ private static final DataflowRunnerInfo INSTANCE;
+ static {
+ Properties properties;
+ try {
+ properties = load(DATAFLOW_DISTRIBUTION_PROPERTIES_PATH);
+ if (properties == null) {
+ properties = load(APACHE_BEAM_DISTRIBUTION_PROPERTIES_PATH);
+ }
+ if (properties == null) {
+ // Print a warning if we can not load either the Dataflow distribution properties
+ // or the
+ LOG.warn("Dataflow runner properties resource not found.");
+ properties = new Properties();
+ }
+ } catch (IOException e) {
+ LOG.warn("Error loading Dataflow runner properties resource: ", e);
+ properties = new Properties();
+ }
+
+ // Inherit the name and version from the Apache Beam distribution if this isn't
+ // the Dataflow distribution.
+ if (!properties.containsKey("name")) {
+ properties.setProperty("name", ReleaseInfo.getReleaseInfo().getName());
+ }
+ if (!properties.containsKey("version")) {
+ properties.setProperty("version", ReleaseInfo.getReleaseInfo().getVersion());
+ }
+ copyFromSystemProperties("java.vendor", properties);
+ copyFromSystemProperties("java.version", properties);
+ copyFromSystemProperties("os.arch", properties);
+ copyFromSystemProperties("os.name", properties);
+ copyFromSystemProperties("os.version", properties);
+ INSTANCE = new DataflowRunnerInfo(ImmutableMap.copyOf((Map) properties));
+ }
}
/**
@@ -45,20 +88,12 @@
return LazyInit.INSTANCE;
}
- private Properties properties;
-
- private static final String FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY =
- "fnapi.environment.major.version";
- private static final String LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY =
- "legacy.environment.major.version";
- private static final String CONTAINER_VERSION_KEY = "container.version";
-
/** Provides the legacy environment's major version number. */
public String getLegacyEnvironmentMajorVersion() {
checkState(
properties.containsKey(LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY),
"Unknown legacy environment major version");
- return properties.getProperty(LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY);
+ return properties.get(LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY);
}
/** Provides the FnAPI environment's major version number. */
@@ -66,7 +101,7 @@
checkState(
properties.containsKey(FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY),
"Unknown FnAPI environment major version");
- return properties.getProperty(FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY);
+ return properties.get(FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY);
}
/** Provides the container version that will be used for constructing harness image paths. */
@@ -74,21 +109,33 @@
checkState(
properties.containsKey(CONTAINER_VERSION_KEY),
"Unknown container version");
- return properties.getProperty(CONTAINER_VERSION_KEY);
+ return properties.get(CONTAINER_VERSION_KEY);
}
- private DataflowRunnerInfo(String resourcePath) {
- properties = new Properties();
+ public Map<String, String> getProperties() {
+ return ImmutableMap.copyOf((Map) properties);
+ }
- try (InputStream in = DataflowRunnerInfo.class.getResourceAsStream(PROPERTIES_PATH)) {
+ private final Map<String, String> properties;
+ private DataflowRunnerInfo(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ private static Properties load(String path) throws IOException {
+ Properties properties = new Properties();
+ try (InputStream in = DataflowRunnerInfo.class.getResourceAsStream(path)) {
if (in == null) {
- LOG.warn("Dataflow runner properties resource not found: {}", resourcePath);
- return;
+ return null;
}
-
properties.load(in);
- } catch (IOException e) {
- LOG.warn("Error loading Dataflow runner properties resource: ", e);
+ }
+ return properties;
+ }
+
+ private static void copyFromSystemProperties(String property, Properties properties) {
+ String value = System.getProperty(property);
+ if (value != null) {
+ properties.setProperty(property, value);
}
}
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
index 8611d3c..aaf6ee1 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
@@ -19,10 +19,11 @@
package org.apache.beam.runners.dataflow;
import java.util.List;
+import org.apache.beam.model.pipeline.v1.RunnerApi.DisplayData;
import org.apache.beam.runners.core.construction.ForwardingPTransform;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.DisplayData;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.DoFn;
@@ -49,7 +50,9 @@
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new ParDoSingle<>(transform.getTransform()));
+ new ParDoSingle<>(
+ transform.getTransform(),
+ PTransformReplacements.getSingletonMainOutput(transform).getCoder()));
}
/**
@@ -58,15 +61,18 @@
public static class ParDoSingle<InputT, OutputT>
extends ForwardingPTransform<PCollection<? extends InputT>, PCollection<OutputT>> {
private final ParDo.SingleOutput<InputT, OutputT> original;
+ private final Coder<OutputT> outputCoder;
- private ParDoSingle(ParDo.SingleOutput<InputT, OutputT> original) {
+ private ParDoSingle(SingleOutput<InputT, OutputT> original, Coder<OutputT> outputCoder) {
this.original = original;
+ this.outputCoder = outputCoder;
}
@Override
public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(),
+ outputCoder);
}
public DoFn<InputT, OutputT> getFn() {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
index 30ecbf5..693748a 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
@@ -29,7 +29,7 @@
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.PCollection;
/**
* Translator for the {@code Read} {@code PTransform} for the Dataflow back-end.
@@ -40,8 +40,9 @@
translateReadHelper(transform.getSource(), transform, context);
}
- public static <T> void translateReadHelper(Source<T> source,
- PTransform<?, ? extends PValue> transform,
+ public static <T> void translateReadHelper(
+ Source<T> source,
+ PTransform<?, ? extends PCollection<?>> transform,
TranslationContext context) {
try {
StepTranslationContext stepContext = context.addStep(transform, "ParallelRead");
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/SplittableParDoOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/SplittableParDoOverrides.java
new file mode 100644
index 0000000..7b65950
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/SplittableParDoOverrides.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow;
+
+import java.util.Map;
+import org.apache.beam.runners.core.construction.ForwardingPTransform;
+import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+
+/** Transform overrides for supporting {@link SplittableParDo} in the Dataflow runner. */
+class SplittableParDoOverrides {
+ static class ParDoSingleViaMulti<InputT, OutputT>
+ extends ForwardingPTransform<PCollection<? extends InputT>, PCollection<OutputT>> {
+ private final ParDo.SingleOutput<InputT, OutputT> original;
+
+ public ParDoSingleViaMulti(
+ DataflowRunner ignored, ParDo.SingleOutput<InputT, OutputT> original) {
+ this.original = original;
+ }
+
+ @Override
+ protected PTransform<PCollection<? extends InputT>, PCollection<OutputT>> delegate() {
+ return original;
+ }
+
+ @Override
+ public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
+ TupleTag<OutputT> mainOutput = new TupleTag<>();
+ return input.apply(original.withOutputTags(mainOutput, TupleTagList.empty())).get(mainOutput);
+ }
+ }
+
+ static class SplittableParDoOverrideFactory<InputT, OutputT, RestrictionT>
+ implements PTransformOverrideFactory<
+ PCollection<InputT>, PCollectionTuple, ParDo.MultiOutput<InputT, OutputT>> {
+ @Override
+ public PTransformReplacement<PCollection<InputT>, PCollectionTuple> getReplacementTransform(
+ AppliedPTransform<PCollection<InputT>, PCollectionTuple, ParDo.MultiOutput<InputT, OutputT>>
+ appliedTransform) {
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(appliedTransform),
+ SplittableParDo.forAppliedParDo(appliedTransform));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollectionTuple newOutput) {
+ return ReplacementOutputs.tagged(outputs, newOutput);
+ }
+ }
+}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
index 6c385d7..1853248 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
@@ -42,12 +42,12 @@
class StreamingViewOverrides {
static class StreamingCreatePCollectionViewFactory<ElemT, ViewT>
extends SingleInputOutputOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
@Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>>
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>>
getReplacementTransform(
AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>>
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
transform) {
StreamingCreatePCollectionView<ElemT, ViewT> streamingView =
new StreamingCreatePCollectionView<>(transform.getTransform().getView());
@@ -56,7 +56,7 @@
}
private static class StreamingCreatePCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private final PCollectionView<ViewT> view;
private StreamingCreatePCollectionView(PCollectionView<ViewT> view) {
@@ -64,7 +64,7 @@
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
return input
.apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
.apply(ParDo.of(StreamingPCollectionViewWriterFn.create(view, input.getCoder())))
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
index 23949bd..06ed1e0 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
@@ -17,15 +17,17 @@
*/
package org.apache.beam.runners.dataflow;
-import com.google.api.services.dataflow.model.Step;
import java.util.List;
import java.util.Map;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.runners.dataflow.util.OutputReference;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
@@ -35,7 +37,8 @@
* A {@link TransformTranslator} knows how to translate a particular subclass of {@link PTransform}
* for the Cloud Dataflow service. It does so by mutating the {@link TranslationContext}.
*/
-interface TransformTranslator<TransformT extends PTransform> {
+@Internal
+public interface TransformTranslator<TransformT extends PTransform> {
void translate(TransformT transform, TranslationContext context);
/**
@@ -65,14 +68,6 @@
*/
StepTranslationContext addStep(PTransform<?, ?> transform, String type);
- /**
- * Adds a pre-defined step to the Dataflow workflow. The given PTransform should be consistent
- * with the Step, in terms of input, output and coder types.
- *
- * <p>This is a low-level operation, when using this method it is up to the caller to ensure
- * that names do not collide.
- */
- Step addStep(PTransform<?, ? extends PValue> transform, Step step);
/** Encode a PValue reference as an output reference. */
OutputReference asOutputReference(PValue value, AppliedPTransform<?, ?, ?> producer);
@@ -100,10 +95,11 @@
* Adds an input with the given name to this Dataflow step, coming from the specified input
* PValue.
*
- * <p>The input {@link PValue} must have already been produced by a step earlier in this {@link
- * Pipeline}. If the input value has not yet been produced yet (either by a call to {@link
- * StepTranslationContext#addOutput(PValue)} or within a call to {@link
- * TranslationContext#addStep(PTransform, Step)}), this method will throw an exception.
+ * <p>The input {@link PValue} must have already been produced by a step earlier in this
+ * {@link Pipeline}. If the input value has not yet been produced yet (by a call to either
+ * {@link StepTranslationContext#addOutput(PCollection)} or
+ * {@link StepTranslationContext#addCollectionToSingletonOutput(PCollection, PCollectionView)})
+ * this method will throw an exception.
*/
void addInput(String name, PInput value);
@@ -114,18 +110,18 @@
void addInput(String name, List<? extends Map<String, Object>> elements);
/**
- * Adds an output to this Dataflow step, producing the specified output {@code PValue},
+ * Adds a primitive output to this Dataflow step, producing the specified output {@code PValue},
* including its {@code Coder} if a {@code TypedPValue}. If the {@code PValue} is a {@code
* PCollection}, wraps its coder inside a {@code WindowedValueCoder}. Returns a pipeline level
* unique id.
*/
- long addOutput(PValue value);
+ long addOutput(PCollection<?> value);
/**
* Adds an output to this {@code CollectionToSingleton} Dataflow step, consuming the specified
* input {@code PValue} and producing the specified output {@code PValue}. This step requires
* special treatment for its output encoding. Returns a pipeline level unique id.
*/
- long addCollectionToSingletonOutput(PValue inputValue, PValue outputValue);
+ long addCollectionToSingletonOutput(PCollection<?> inputValue, PCollectionView<?> outputValue);
}
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/CloudDebuggerOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/CloudDebuggerOptions.java
index d1c8e7a..317a30b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/CloudDebuggerOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/CloudDebuggerOptions.java
@@ -23,6 +23,7 @@
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.Hidden;
+import org.apache.beam.sdk.options.PipelineOptions;
/**
* Options for controlling Cloud Debugger.
@@ -30,7 +31,7 @@
@Description("[Experimental] Used to configure the Cloud Debugger")
@Experimental
@Hidden
-public interface CloudDebuggerOptions {
+public interface CloudDebuggerOptions extends PipelineOptions {
/** Whether to enable the Cloud Debugger snapshot agent for the current job. */
@Description("Whether to enable the Cloud Debugger snapshot agent for the current job.")
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index d0ea722..ec108da 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -19,16 +19,14 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.api.services.dataflow.Dataflow;
-import java.util.List;
import java.util.Map;
-import javax.annotation.Nullable;
import org.apache.beam.runners.dataflow.util.DataflowTransport;
import org.apache.beam.runners.dataflow.util.GcsStager;
import org.apache.beam.runners.dataflow.util.Stager;
-import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.DefaultValueFactory;
import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.ExperimentalOptions;
import org.apache.beam.sdk.options.Hidden;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.InstanceBuilder;
@@ -40,24 +38,7 @@
@Description("[Internal] Options used to control execution of the Dataflow SDK for "
+ "debugging and testing purposes.")
@Hidden
-public interface DataflowPipelineDebugOptions extends PipelineOptions {
-
- /**
- * The list of backend experiments to enable.
- *
- * <p>Dataflow provides a number of experimental features that can be enabled
- * with this flag.
- *
- * <p>Please sync with the Dataflow team before enabling any experiments.
- */
- @Description("[Experimental] Dataflow provides a number of experimental features that can "
- + "be enabled with this flag. Please sync with the Dataflow team before enabling any "
- + "experiments.")
- @Experimental
- @Nullable
- List<String> getExperiments();
- void setExperiments(@Nullable List<String> value);
-
+public interface DataflowPipelineDebugOptions extends ExperimentalOptions, PipelineOptions {
/**
* The root URL for the Dataflow API. {@code dataflowEndpoint} can override this value
* if it contains an absolute URL, otherwise {@code apiRootUrl} will be combined with
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 4af420b..091f89b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.dataflow.options;
+import java.util.Map;
import org.apache.beam.runners.dataflow.DataflowRunner;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
@@ -116,6 +117,13 @@
void setRegion(String region);
/**
+ * Labels that will be applied to the billing records for this job.
+ */
+ @Description("Labels that will be applied to the billing records for this job.")
+ Map<String, String> getLabels();
+ void setLabels(Map<String, String> labels);
+
+ /**
* Returns a default staging location under {@link GcpOptions#getGcpTempLocation}.
*/
class StagingLocationFactory implements DefaultValueFactory<String> {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
index 00d2194..2239462 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
@@ -53,6 +53,9 @@
/** Use numWorkers machines. Do not autoscale the worker pool. */
NONE("AUTOSCALING_ALGORITHM_NONE"),
+ /**
+ * @deprecated use {@link #THROUGHPUT_BASED}.
+ */
@Deprecated
BASIC("AUTOSCALING_ALGORITHM_BASIC"),
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptions.java
index a87d688..ef9b6e6 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptions.java
@@ -21,6 +21,7 @@
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.Hidden;
+import org.apache.beam.sdk.options.PipelineOptions;
/**
* Options for controlling profiling of pipeline execution.
@@ -28,7 +29,7 @@
@Description("[Experimental] Used to configure profiling of the Dataflow pipeline")
@Experimental
@Hidden
-public interface DataflowProfilingOptions {
+public interface DataflowProfilingOptions extends PipelineOptions {
@Description("When set to a non-empty value, enables recording profiles and saving them to GCS.\n"
+ "Profiles will continue until the pipeline is stopped or updated without this option.\n")
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java
index fae851c..a419b76 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java
@@ -29,8 +29,14 @@
/**
* Options that are used to control logging configuration on the Dataflow worker.
+ *
+ * @deprecated This interface will no longer be the source of truth for worker logging configuration
+ * once jobs are executed using a dedicated SDK harness instead of user code being co-located
+ * alongside Dataflow worker code. Please set the option below and also the corresponding option
+ * within {@link org.apache.beam.sdk.options.SdkHarnessOptions} to ensure forward compatibility.
*/
@Description("Options that are used to control logging configuration on the Dataflow worker.")
+@Deprecated
public interface DataflowWorkerLoggingOptions extends PipelineOptions {
/**
* The set of log levels that can be used on the Dataflow worker.
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectKinds.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectKinds.java
index 403ade2..58bc446 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectKinds.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectKinds.java
@@ -28,4 +28,5 @@
static final String KIND_PAIR = "kind:pair";
static final String KIND_STREAM = "kind:stream";
static final String KIND_WINDOWED_VALUE = "kind:windowed_value";
+ static final String KIND_BYTES = "kind:bytes";
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java
index 012a669..ad2363d 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java
@@ -279,7 +279,8 @@
@Override
public CloudObject toCloudObject(ByteArrayCoder target) {
return addComponents(
- CloudObject.forClass(target.getClass()), Collections.<Coder<?>>emptyList());
+ CloudObject.forClassName(CloudObjectKinds.KIND_BYTES),
+ Collections.<Coder<?>>emptyList());
}
@Override
@@ -294,7 +295,7 @@
@Override
public String cloudObjectClassName() {
- return CloudObject.forClass(ByteArrayCoder.class).getClassName();
+ return CloudObjectKinds.KIND_BYTES;
}
};
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
index 5d42a5f..ff89933 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
@@ -48,6 +48,7 @@
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.gcp.bigquery.TableDestinationCoder;
+import org.apache.beam.sdk.io.gcp.bigquery.TableDestinationCoderV2;
import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
/**
@@ -97,6 +98,7 @@
RandomAccessDataCoder.class,
StringUtf8Coder.class,
TableDestinationCoder.class,
+ TableDestinationCoderV2.class,
TableRowJsonCoder.class,
TextualIntegerCoder.class,
VarIntCoder.class,
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
deleted file mode 100644
index bd2742f..0000000
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.dataflow.util;
-
-import java.io.Serializable;
-import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * Wrapper class holding the necessary information to serialize a {@link DoFn}.
- *
- * @param <InputT> the type of the (main) input elements of the {@link DoFn}
- * @param <OutputT> the type of the (main) output elements of the {@link DoFn}
- */
-public class DoFnInfo<InputT, OutputT> implements Serializable {
- private final DoFn<InputT, OutputT> doFn;
- private final WindowingStrategy<?, ?> windowingStrategy;
- private final Iterable<PCollectionView<?>> sideInputViews;
- private final Coder<InputT> inputCoder;
- private final long mainOutput;
- private final Map<Long, TupleTag<?>> outputMap;
-
- /**
- * Creates a {@link DoFnInfo} for the given {@link DoFn}.
- */
- public static <InputT, OutputT> DoFnInfo<InputT, OutputT> forFn(
- DoFn<InputT, OutputT> doFn,
- WindowingStrategy<?, ?> windowingStrategy,
- Iterable<PCollectionView<?>> sideInputViews,
- Coder<InputT> inputCoder,
- long mainOutput,
- Map<Long, TupleTag<?>> outputMap) {
- return new DoFnInfo<>(
- doFn, windowingStrategy, sideInputViews, inputCoder, mainOutput, outputMap);
- }
-
- /** TODO: remove this when Dataflow worker uses the DoFn overload. */
- @Deprecated
- @SuppressWarnings("unchecked")
- public static <InputT, OutputT> DoFnInfo<InputT, OutputT> forFn(
- Serializable doFn,
- WindowingStrategy<?, ?> windowingStrategy,
- Iterable<PCollectionView<?>> sideInputViews,
- Coder<InputT> inputCoder,
- long mainOutput,
- Map<Long, TupleTag<?>> outputMap) {
- return forFn(
- (DoFn<InputT, OutputT>) doFn,
- windowingStrategy,
- sideInputViews,
- inputCoder,
- mainOutput,
- outputMap);
- }
-
- public DoFnInfo<InputT, OutputT> withFn(DoFn<InputT, OutputT> newFn) {
- return DoFnInfo.forFn(newFn,
- windowingStrategy,
- sideInputViews,
- inputCoder,
- mainOutput,
- outputMap);
- }
-
- private DoFnInfo(
- DoFn<InputT, OutputT> doFn,
- WindowingStrategy<?, ?> windowingStrategy,
- Iterable<PCollectionView<?>> sideInputViews,
- Coder<InputT> inputCoder,
- long mainOutput,
- Map<Long, TupleTag<?>> outputMap) {
- this.doFn = doFn;
- this.windowingStrategy = windowingStrategy;
- this.sideInputViews = sideInputViews;
- this.inputCoder = inputCoder;
- this.mainOutput = mainOutput;
- this.outputMap = outputMap;
- }
-
- /** TODO: remove this when Dataflow worker uses {@link #getDoFn}. */
- @Deprecated
- public Serializable getFn() {
- return doFn;
- }
-
- /** Returns the embedded function. */
- public DoFn<InputT, OutputT> getDoFn() {
- return doFn;
- }
-
- public WindowingStrategy<?, ?> getWindowingStrategy() {
- return windowingStrategy;
- }
-
- public Iterable<PCollectionView<?>> getSideInputViews() {
- return sideInputViews;
- }
-
- public Coder<InputT> getInputCoder() {
- return inputCoder;
- }
-
- public long getMainOutput() {
- return mainOutput;
- }
-
- public Map<Long, TupleTag<?>> getOutputMap() {
- return outputMap;
- }
-}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java
index d18e306..7ed78e8 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java
@@ -29,9 +29,7 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.MimeTypes;
-/**
- * Utility class for staging files to GCS.
- */
+/** Utility class for staging files to GCS. */
public class GcsStager implements Stager {
private DataflowPipelineOptions options;
@@ -39,32 +37,63 @@
this.options = options;
}
- @SuppressWarnings("unused") // used via reflection
+ @SuppressWarnings("unused") // used via reflection
public static GcsStager fromOptions(PipelineOptions options) {
return new GcsStager(options.as(DataflowPipelineOptions.class));
}
+ /**
+ * Stages {@link DataflowPipelineOptions#getFilesToStage()}, which defaults to every file on the
+ * classpath unless overridden, as well as {@link
+ * DataflowPipelineDebugOptions#getOverrideWindmillBinary()} if specified.
+ *
+ * @see #stageFiles(List)
+ */
@Override
- public List<DataflowPackage> stageFiles() {
+ public List<DataflowPackage> stageDefaultFiles() {
checkNotNull(options.getStagingLocation());
String windmillBinary =
options.as(DataflowPipelineDebugOptions.class).getOverrideWindmillBinary();
+
+ List<String> filesToStage = options.getFilesToStage();
+
if (windmillBinary != null) {
- options.getFilesToStage().add("windmill_main=" + windmillBinary);
+ filesToStage.add("windmill_main=" + windmillBinary);
}
+ return stageFiles(filesToStage);
+ }
+
+ /**
+ * Stages files to {@link DataflowPipelineOptions#getStagingLocation()}, suffixed with their md5
+ * hash to avoid collisions.
+ *
+ * <p>Uses {@link DataflowPipelineOptions#getGcsUploadBufferSizeBytes()}.
+ */
+ @Override
+ public List<DataflowPackage> stageFiles(List<String> filesToStage) {
+ try (PackageUtil packageUtil = PackageUtil.withDefaultThreadPool()) {
+ return packageUtil.stageClasspathElements(
+ filesToStage, options.getStagingLocation(), buildCreateOptions());
+ }
+ }
+
+ @Override
+ public DataflowPackage stageToFile(byte[] bytes, String baseName) {
+ try (PackageUtil packageUtil = PackageUtil.withDefaultThreadPool()) {
+ return packageUtil.stageToFile(
+ bytes, baseName, options.getStagingLocation(), buildCreateOptions());
+ }
+ }
+
+ private GcsCreateOptions buildCreateOptions() {
int uploadSizeBytes = firstNonNull(options.getGcsUploadBufferSizeBytes(), 1024 * 1024);
checkArgument(uploadSizeBytes > 0, "gcsUploadBufferSizeBytes must be > 0");
uploadSizeBytes = Math.min(uploadSizeBytes, 1024 * 1024);
- GcsCreateOptions createOptions = GcsCreateOptions.builder()
+ return GcsCreateOptions.builder()
.setGcsUploadBufferSizeBytes(uploadSizeBytes)
.setMimeType(MimeTypes.BINARY)
.build();
-
- return PackageUtil.stageClasspathElements(
- options.getFilesToStage(),
- options.getStagingLocation(),
- createOptions);
}
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
index 759387c..cf46406 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
@@ -46,7 +46,7 @@
*/
public class MonitoringUtil {
- private static final String GCLOUD_DATAFLOW_PREFIX = "gcloud beta dataflow";
+ private static final String GCLOUD_DATAFLOW_PREFIX = "gcloud dataflow";
private static final String ENDPOINT_OVERRIDE_ENV_VAR =
"CLOUDSDK_API_ENDPOINT_OVERRIDES_DATAFLOW";
@@ -180,14 +180,24 @@
return allMessages;
}
+ /**
+ * @deprecated this method defaults the region to "us-central1". Prefer using the overload with
+ * an explicit regionId parameter.
+ */
+ @Deprecated
public static String getJobMonitoringPageURL(String projectName, String jobId) {
+ return getJobMonitoringPageURL(projectName, "us-central1", jobId);
+ }
+
+ public static String getJobMonitoringPageURL(String projectName, String regionId, String jobId) {
try {
// Project name is allowed in place of the project id: the user will be redirected to a URL
// that has the project name replaced with project id.
return String.format(
- "https://console.developers.google.com/project/%s/dataflow/job/%s",
- URLEncoder.encode(projectName, "UTF-8"),
- URLEncoder.encode(jobId, "UTF-8"));
+ "https://console.cloud.google.com/dataflow/jobsDetail/locations/%s/jobs/%s?project=%s",
+ URLEncoder.encode(regionId, "UTF-8"),
+ URLEncoder.encode(jobId, "UTF-8"),
+ URLEncoder.encode(projectName, "UTF-8"));
} catch (UnsupportedEncodingException e) {
// Should never happen.
throw new AssertionError("UTF-8 encoding is not supported by the environment", e);
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
index 931f7ea..565e965 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
@@ -18,57 +18,76 @@
package org.apache.beam.runners.dataflow.util;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
import com.fasterxml.jackson.core.Base64Variants;
import com.google.api.client.util.BackOff;
import com.google.api.client.util.Sleeper;
import com.google.api.services.dataflow.model.DataflowPackage;
+import com.google.auto.value.AutoValue;
import com.google.cloud.hadoop.util.ApiErrorExtractor;
-import com.google.common.collect.Lists;
+import com.google.common.base.Function;
import com.google.common.hash.Funnels;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
import com.google.common.io.CountingOutputStream;
import com.google.common.io.Files;
+import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.extensions.gcp.storage.GcsCreateOptions;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.CreateOptions;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.util.BackOffAdapter;
import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.ZipFiles;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Helper routines for packages. */
-class PackageUtil {
+@Internal
+class PackageUtil implements Closeable {
+
private static final Logger LOG = LoggerFactory.getLogger(PackageUtil.class);
+
/**
* A reasonable upper bound on the number of jars required to launch a Dataflow job.
*/
private static final int SANE_CLASSPATH_SIZE = 1000;
+ private static final int DEFAULT_THREAD_POOL_SIZE = 32;
+
+ private static final Sleeper DEFAULT_SLEEPER = Sleeper.DEFAULT;
+
+ private static final CreateOptions DEFAULT_CREATE_OPTIONS =
+ GcsCreateOptions.builder()
+ .setGcsUploadBufferSizeBytes(1024 * 1024)
+ .setMimeType(MimeTypes.BINARY)
+ .build();
+
private static final FluentBackoff BACKOFF_FACTORY =
FluentBackoff.DEFAULT.withMaxRetries(4).withInitialBackoff(Duration.standardSeconds(5));
@@ -77,49 +96,27 @@
*/
private static final ApiErrorExtractor ERROR_EXTRACTOR = new ApiErrorExtractor();
- /**
- * Compute and cache the attributes of a classpath element that we will need to stage it.
- *
- * @param source the file or directory to be staged.
- * @param stagingPath The base location for staged classpath elements.
- * @param overridePackageName If non-null, use the given value as the package name
- * instead of generating one automatically.
- * @return a {@link PackageAttributes} that containing metadata about the object to be staged.
- */
- static PackageAttributes createPackageAttributes(File source,
- String stagingPath, @Nullable String overridePackageName) {
- boolean directory = source.isDirectory();
+ private final ListeningExecutorService executorService;
- // Compute size and hash in one pass over file or directory.
- Hasher hasher = Hashing.md5().newHasher();
- OutputStream hashStream = Funnels.asOutputStream(hasher);
- try (CountingOutputStream countingOutputStream = new CountingOutputStream(hashStream)) {
- if (!directory) {
- // Files are staged as-is.
- Files.asByteSource(source).copyTo(countingOutputStream);
- } else {
- // Directories are recursively zipped.
- ZipFiles.zipDirectory(source, countingOutputStream);
- }
- countingOutputStream.flush();
-
- long size = countingOutputStream.getCount();
- String hash = Base64Variants.MODIFIED_FOR_URL.encode(hasher.hash().asBytes());
-
- // Create the DataflowPackage with staging name and location.
- String uniqueName = getUniqueContentName(source, hash);
- String resourcePath = FileSystems.matchNewResource(stagingPath, true)
- .resolve(uniqueName, StandardResolveOptions.RESOLVE_FILE).toString();
- DataflowPackage target = new DataflowPackage();
- target.setName(overridePackageName != null ? overridePackageName : uniqueName);
- target.setLocation(resourcePath);
-
- return new PackageAttributes(size, hash, directory, target, source.getPath());
- } catch (IOException e) {
- throw new RuntimeException("Package setup failure for " + source, e);
- }
+ private PackageUtil(ListeningExecutorService executorService) {
+ this.executorService = executorService;
}
+ public static PackageUtil withDefaultThreadPool() {
+ return PackageUtil.withExecutorService(
+ MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEFAULT_THREAD_POOL_SIZE)));
+ }
+
+ public static PackageUtil withExecutorService(ListeningExecutorService executorService) {
+ return new PackageUtil(executorService);
+ }
+
+ @Override
+ public void close() {
+ executorService.shutdown();
+ }
+
+
/** Utility comparator used in uploading packages efficiently. */
private static class PackageUploadOrder implements Comparator<PackageAttributes> {
@Override
@@ -136,115 +133,179 @@
}
}
- /**
- * Utility function that computes sizes and hashes of packages so that we can validate whether
- * they have already been correctly staged.
- */
- private static List<PackageAttributes> computePackageAttributes(
- Collection<String> classpathElements, final String stagingPath,
- ListeningExecutorService executorService) {
- List<ListenableFuture<PackageAttributes>> futures = new LinkedList<>();
- for (String classpathElement : classpathElements) {
- @Nullable String userPackageName = null;
- if (classpathElement.contains("=")) {
- String[] components = classpathElement.split("=", 2);
- userPackageName = components[0];
- classpathElement = components[1];
- }
- @Nullable final String packageName = userPackageName;
+ /** Asynchronously computes {@link PackageAttributes} for a single staged file. */
+ private ListenableFuture<PackageAttributes> computePackageAttributes(
+ final DataflowPackage source, final String stagingPath) {
- final File file = new File(classpathElement);
- if (!file.exists()) {
- LOG.warn("Skipping non-existent classpath element {} that was specified.",
- classpathElement);
- continue;
- }
-
- ListenableFuture<PackageAttributes> future =
- executorService.submit(new Callable<PackageAttributes>() {
- @Override
- public PackageAttributes call() throws Exception {
- return createPackageAttributes(file, stagingPath, packageName);
+ return executorService.submit(
+ new Callable<PackageAttributes>() {
+ @Override
+ public PackageAttributes call() throws Exception {
+ final File file = new File(source.getLocation());
+ if (!file.exists()) {
+ throw new FileNotFoundException(
+ String.format("Non-existent file to stage: %s", file.getAbsolutePath()));
}
- });
- futures.add(future);
+
+ PackageAttributes attributes = PackageAttributes.forFileToStage(file, stagingPath);
+ if (source.getName() != null) {
+ attributes = attributes.withPackageName(source.getName());
+ }
+ return attributes;
+ }
+ });
+ }
+
+ private boolean alreadyStaged(PackageAttributes attributes) throws IOException {
+ try {
+ long remoteLength =
+ FileSystems.matchSingleFileSpec(attributes.getDestination().getLocation()).sizeBytes();
+ return remoteLength == attributes.getSize();
+ } catch (FileNotFoundException expected) {
+ // If the file doesn't exist, it means we need to upload it.
+ return false;
+ }
+ }
+
+ /** Stages one file ("package") if necessary. */
+ public ListenableFuture<StagingResult> stagePackage(
+ final PackageAttributes attributes,
+ final Sleeper retrySleeper,
+ final CreateOptions createOptions) {
+ return executorService.submit(
+ new Callable<StagingResult>() {
+ @Override
+ public StagingResult call() throws Exception {
+ return stagePackageSynchronously(attributes, retrySleeper, createOptions);
+ }
+ });
+ }
+
+ /** Synchronously stages a package, with retry and backoff for resiliency. */
+ private StagingResult stagePackageSynchronously(
+ PackageAttributes attributes, Sleeper retrySleeper, CreateOptions createOptions)
+ throws IOException, InterruptedException {
+ String sourceDescription = attributes.getSourceDescription();
+ String target = attributes.getDestination().getLocation();
+
+ if (alreadyStaged(attributes)) {
+ LOG.debug("Skipping file already staged: {} at {}", sourceDescription, target);
+ return StagingResult.cached(attributes);
}
try {
- return Futures.allAsList(futures).get();
+ return tryStagePackageWithRetry(attributes, retrySleeper, createOptions);
+ } catch (Exception miscException) {
+ throw new RuntimeException(
+ String.format("Could not stage %s to %s", sourceDescription, target), miscException);
+ }
+ }
+
+ private StagingResult tryStagePackageWithRetry(
+ PackageAttributes attributes, Sleeper retrySleeper, CreateOptions createOptions)
+ throws IOException, InterruptedException {
+ String sourceDescription = attributes.getSourceDescription();
+ String target = attributes.getDestination().getLocation();
+ BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
+
+ while (true) {
+ try {
+ return tryStagePackage(attributes, createOptions);
+ } catch (IOException ioException) {
+
+ if (ERROR_EXTRACTOR.accessDenied(ioException)) {
+ String errorMessage =
+ String.format(
+ "Uploaded failed due to permissions error, will NOT retry staging "
+ + "of %s. Please verify credentials are valid and that you have "
+ + "write access to %s. Stale credentials can be resolved by executing "
+ + "'gcloud auth application-default login'.",
+ sourceDescription, target);
+ LOG.error(errorMessage);
+ throw new IOException(errorMessage, ioException);
+ }
+
+ long sleep = backoff.nextBackOffMillis();
+ if (sleep == BackOff.STOP) {
+ LOG.error(
+ "Upload failed, will NOT retry staging of package: {}",
+ sourceDescription,
+ ioException);
+ throw new RuntimeException("Could not stage %s to %s", ioException);
+ } else {
+ LOG.warn(
+ "Upload attempt failed, sleeping before retrying staging of package: {}",
+ sourceDescription,
+ ioException);
+ retrySleeper.sleep(sleep);
+ }
+ }
+ }
+ }
+
+ private StagingResult tryStagePackage(PackageAttributes attributes, CreateOptions createOptions)
+ throws IOException, InterruptedException {
+ String sourceDescription = attributes.getSourceDescription();
+ String target = attributes.getDestination().getLocation();
+
+ LOG.info("Uploading {} to {}", sourceDescription, target);
+ try (WritableByteChannel writer =
+ FileSystems.create(FileSystems.matchNewResource(target, false), createOptions)) {
+ if (attributes.getBytes() != null) {
+ ByteSource.wrap(attributes.getBytes()).copyTo(Channels.newOutputStream(writer));
+ } else {
+ File sourceFile = attributes.getSource();
+ checkState(
+ sourceFile != null,
+ "Internal inconsistency: we tried to stage something to %s, but neither a source file "
+ + "nor the byte content was specified",
+ target);
+ if (sourceFile.isDirectory()) {
+ ZipFiles.zipDirectory(sourceFile, Channels.newOutputStream(writer));
+ } else {
+ Files.asByteSource(sourceFile).copyTo(Channels.newOutputStream(writer));
+ }
+ }
+ }
+ return StagingResult.uploaded(attributes);
+ }
+
+ /**
+ * Transfers the classpath elements to the staging location using a default {@link Sleeper}.
+ *
+ * @see {@link #stageClasspathElements(Collection, String, Sleeper, CreateOptions)}
+ */
+ List<DataflowPackage> stageClasspathElements(
+ Collection<String> classpathElements, String stagingPath, CreateOptions createOptions) {
+ return stageClasspathElements(classpathElements, stagingPath, DEFAULT_SLEEPER, createOptions);
+ }
+
+ /**
+ * Transfers the classpath elements to the staging location using default settings.
+ *
+ * @see {@link #stageClasspathElements(Collection, String, Sleeper, CreateOptions)}
+ */
+ List<DataflowPackage> stageClasspathElements(
+ Collection<String> classpathElements, String stagingPath) {
+ return stageClasspathElements(
+ classpathElements, stagingPath, DEFAULT_SLEEPER, DEFAULT_CREATE_OPTIONS);
+ }
+
+ public DataflowPackage stageToFile(
+ byte[] bytes, String target, String stagingPath, CreateOptions createOptions) {
+ try {
+ return stagePackage(
+ PackageAttributes.forBytesToStage(bytes, target, stagingPath),
+ DEFAULT_SLEEPER,
+ createOptions)
+ .get()
+ .getPackageAttributes()
+ .getDestination();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- throw new RuntimeException("Interrupted while staging packages", e);
+ throw new RuntimeException("Interrupted while staging pipeline", e);
} catch (ExecutionException e) {
- throw new RuntimeException("Error while staging packages", e.getCause());
- }
- }
-
- private static WritableByteChannel makeWriter(String target, CreateOptions createOptions)
- throws IOException {
- return FileSystems.create(FileSystems.matchNewResource(target, false), createOptions);
- }
-
- /**
- * Utility to verify whether a package has already been staged and, if not, copy it to the
- * staging location.
- */
- private static void stageOnePackage(
- PackageAttributes attributes, AtomicInteger numUploaded, AtomicInteger numCached,
- Sleeper retrySleeper, CreateOptions createOptions) {
- String source = attributes.getSourcePath();
- String target = attributes.getDataflowPackage().getLocation();
-
- // TODO: Should we attempt to detect the Mime type rather than
- // always using MimeTypes.BINARY?
- try {
- try {
- long remoteLength = FileSystems.matchSingleFileSpec(target).sizeBytes();
- if (remoteLength == attributes.getSize()) {
- LOG.debug("Skipping classpath element already staged: {} at {}",
- attributes.getSourcePath(), target);
- numCached.incrementAndGet();
- return;
- }
- } catch (FileNotFoundException expected) {
- // If the file doesn't exist, it means we need to upload it.
- }
-
- // Upload file, retrying on failure.
- BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
- while (true) {
- try {
- LOG.debug("Uploading classpath element {} to {}", source, target);
- try (WritableByteChannel writer = makeWriter(target, createOptions)) {
- copyContent(source, writer);
- }
- numUploaded.incrementAndGet();
- break;
- } catch (IOException e) {
- if (ERROR_EXTRACTOR.accessDenied(e)) {
- String errorMessage = String.format(
- "Uploaded failed due to permissions error, will NOT retry staging "
- + "of classpath %s. Please verify credentials are valid and that you have "
- + "write access to %s. Stale credentials can be resolved by executing "
- + "'gcloud auth application-default login'.", source, target);
- LOG.error(errorMessage);
- throw new IOException(errorMessage, e);
- }
- long sleep = backoff.nextBackOffMillis();
- if (sleep == BackOff.STOP) {
- // Rethrow last error, to be included as a cause in the catch below.
- LOG.error("Upload failed, will NOT retry staging of classpath: {}",
- source, e);
- throw e;
- } else {
- LOG.warn("Upload attempt failed, sleeping before retrying staging of classpath: {}",
- source, e);
- retrySleeper.sleep(sleep);
- }
- }
- }
- } catch (Exception e) {
- throw new RuntimeException("Could not stage classpath element: " + source, e);
+ throw new RuntimeException("Error while staging pipeline", e.getCause());
}
}
@@ -255,22 +316,10 @@
* @param stagingPath The base location to stage the elements to.
* @return A list of cloud workflow packages, each representing a classpath element.
*/
- static List<DataflowPackage> stageClasspathElements(
- Collection<String> classpathElements, String stagingPath, CreateOptions createOptions) {
- ListeningExecutorService executorService =
- MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(32));
- try {
- return stageClasspathElements(classpathElements, stagingPath, Sleeper.DEFAULT,
- executorService, createOptions);
- } finally {
- executorService.shutdown();
- }
- }
-
- // Visible for testing.
- static List<DataflowPackage> stageClasspathElements(
- Collection<String> classpathElements, final String stagingPath,
- final Sleeper retrySleeper, ListeningExecutorService executorService,
+ List<DataflowPackage> stageClasspathElements(
+ Collection<String> classpathElements,
+ final String stagingPath,
+ final Sleeper retrySleeper,
final CreateOptions createOptions) {
LOG.info("Uploading {} files from PipelineOptions.filesToStage to staging location to "
+ "prepare for execution.", classpathElements.size());
@@ -288,45 +337,69 @@
stagingPath != null,
"Can't stage classpath elements because no staging location has been provided");
- // Inline a copy here because the inner code returns an immutable list and we want to mutate it.
- List<PackageAttributes> packageAttributes =
- new LinkedList<>(computePackageAttributes(classpathElements, stagingPath, executorService));
-
- // Compute the returned list of DataflowPackage objects here so that they are returned in the
- // same order as on the classpath.
- List<DataflowPackage> packages = Lists.newArrayListWithExpectedSize(packageAttributes.size());
- for (final PackageAttributes attributes : packageAttributes) {
- packages.add(attributes.getDataflowPackage());
- }
-
- // Order package attributes in descending size order so that we upload the largest files first.
- Collections.sort(packageAttributes, new PackageUploadOrder());
final AtomicInteger numUploaded = new AtomicInteger(0);
final AtomicInteger numCached = new AtomicInteger(0);
+ List<ListenableFuture<DataflowPackage>> destinationPackages = new ArrayList<>();
- List<ListenableFuture<?>> futures = new LinkedList<>();
- for (final PackageAttributes attributes : packageAttributes) {
- futures.add(executorService.submit(new Runnable() {
- @Override
- public void run() {
- stageOnePackage(attributes, numUploaded, numCached, retrySleeper, createOptions);
- }
- }));
+ for (String classpathElement : classpathElements) {
+ DataflowPackage sourcePackage = new DataflowPackage();
+ if (classpathElement.contains("=")) {
+ String[] components = classpathElement.split("=", 2);
+ sourcePackage.setName(components[0]);
+ sourcePackage.setLocation(components[1]);
+ } else {
+ sourcePackage.setName(null);
+ sourcePackage.setLocation(classpathElement);
+ }
+
+ File sourceFile = new File(sourcePackage.getLocation());
+ if (!sourceFile.exists()) {
+ LOG.warn("Skipping non-existent file to stage {}.", sourceFile);
+ continue;
+ }
+
+ // TODO: Java 8 / Guava 23.0: FluentFuture
+ ListenableFuture<StagingResult> stagingResult =
+ Futures.transformAsync(
+ computePackageAttributes(sourcePackage, stagingPath),
+ new AsyncFunction<PackageAttributes, StagingResult>() {
+ @Override
+ public ListenableFuture<StagingResult> apply(
+ final PackageAttributes packageAttributes) throws Exception {
+ return stagePackage(packageAttributes, retrySleeper, createOptions);
+ }
+ });
+
+ ListenableFuture<DataflowPackage> stagedPackage =
+ Futures.transform(
+ stagingResult,
+ new Function<StagingResult, DataflowPackage>() {
+ @Override
+ public DataflowPackage apply(StagingResult stagingResult) {
+ if (stagingResult.alreadyStaged()) {
+ numCached.incrementAndGet();
+ } else {
+ numUploaded.incrementAndGet();
+ }
+ return stagingResult.getPackageAttributes().getDestination();
+ }
+ });
+
+ destinationPackages.add(stagedPackage);
}
+
try {
- Futures.allAsList(futures).get();
+ List<DataflowPackage> stagedPackages = Futures.allAsList(destinationPackages).get();
+ LOG.info(
+ "Staging files complete: {} files cached, {} files newly uploaded",
+ numCached.get(), numUploaded.get());
+ return stagedPackages;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted while staging packages", e);
} catch (ExecutionException e) {
throw new RuntimeException("Error while staging packages", e.getCause());
}
-
- LOG.info(
- "Staging files complete: {} files cached, {} files newly uploaded",
- numCached.get(), numUploaded.get());
-
- return packages;
}
/**
@@ -350,75 +423,113 @@
return fileName + "-" + contentHash + "." + fileExtension;
}
- /**
- * Copies the contents of the classpathElement to the output channel.
- *
- * <p>If the classpathElement is a directory, a Zip stream is constructed on the fly,
- * otherwise the file contents are copied as-is.
- *
- * <p>The output channel is not closed.
- */
- private static void copyContent(String classpathElement, WritableByteChannel outputChannel)
- throws IOException {
- final File classpathElementFile = new File(classpathElement);
- if (classpathElementFile.isDirectory()) {
- ZipFiles.zipDirectory(classpathElementFile, Channels.newOutputStream(outputChannel));
- } else {
- Files.asByteSource(classpathElementFile).copyTo(Channels.newOutputStream(outputChannel));
+ @AutoValue
+ abstract static class StagingResult {
+ abstract PackageAttributes getPackageAttributes();
+
+ abstract boolean alreadyStaged();
+
+ public static StagingResult cached(PackageAttributes attributes) {
+ return new AutoValue_PackageUtil_StagingResult(attributes, true);
+ }
+
+ public static StagingResult uploaded(PackageAttributes attributes) {
+ return new AutoValue_PackageUtil_StagingResult(attributes, false);
}
}
+
/**
* Holds the metadata necessary to stage a file or confirm that a staged file has not changed.
*/
- static class PackageAttributes {
- private final boolean directory;
- private final long size;
- private final String hash;
- private final String sourcePath;
- private DataflowPackage dataflowPackage;
+ @AutoValue
+ abstract static class PackageAttributes {
- public PackageAttributes(long size, String hash, boolean directory,
- DataflowPackage dataflowPackage, String sourcePath) {
- this.size = size;
- this.hash = Objects.requireNonNull(hash, "hash");
- this.directory = directory;
- this.sourcePath = Objects.requireNonNull(sourcePath, "sourcePath");
- this.dataflowPackage = Objects.requireNonNull(dataflowPackage, "dataflowPackage");
+ public static PackageAttributes forFileToStage(File source, String stagingPath)
+ throws IOException {
+
+ // Compute size and hash in one pass over file or directory.
+ long size;
+ String hash;
+ Hasher hasher = Hashing.md5().newHasher();
+ OutputStream hashStream = Funnels.asOutputStream(hasher);
+ try (CountingOutputStream countingOutputStream = new CountingOutputStream(hashStream)) {
+ if (!source.isDirectory()) {
+ // Files are staged as-is.
+ Files.asByteSource(source).copyTo(countingOutputStream);
+ } else {
+ // Directories are recursively zipped.
+ ZipFiles.zipDirectory(source, countingOutputStream);
+ }
+ countingOutputStream.flush();
+
+ size = countingOutputStream.getCount();
+ hash = Base64Variants.MODIFIED_FOR_URL.encode(hasher.hash().asBytes());
+ }
+
+ String uniqueName = getUniqueContentName(source, hash);
+
+ String resourcePath =
+ FileSystems.matchNewResource(stagingPath, true)
+ .resolve(uniqueName, StandardResolveOptions.RESOLVE_FILE)
+ .toString();
+ DataflowPackage target = new DataflowPackage();
+ target.setName(uniqueName);
+ target.setLocation(resourcePath);
+
+ return new AutoValue_PackageUtil_PackageAttributes(source, null, target, size, hash);
}
- /**
- * @return the dataflowPackage
- */
- public DataflowPackage getDataflowPackage() {
- return dataflowPackage;
+ public static PackageAttributes forBytesToStage(
+ byte[] bytes, String targetName, String stagingPath) {
+ Hasher hasher = Hashing.md5().newHasher();
+ String hash = Base64Variants.MODIFIED_FOR_URL.encode(hasher.putBytes(bytes).hash().asBytes());
+ long size = bytes.length;
+
+ String uniqueName = getUniqueContentName(new File(targetName), hash);
+
+ String resourcePath =
+ FileSystems.matchNewResource(stagingPath, true)
+ .resolve(uniqueName, StandardResolveOptions.RESOLVE_FILE)
+ .toString();
+ DataflowPackage target = new DataflowPackage();
+ target.setName(uniqueName);
+ target.setLocation(resourcePath);
+
+ return new AutoValue_PackageUtil_PackageAttributes(null, bytes, target, size, hash);
}
- /**
- * @return the directory
- */
- public boolean isDirectory() {
- return directory;
+ public PackageAttributes withPackageName(String overridePackageName) {
+ DataflowPackage newDestination = new DataflowPackage();
+ newDestination.setName(overridePackageName);
+ newDestination.setLocation(getDestination().getLocation());
+
+ return new AutoValue_PackageUtil_PackageAttributes(
+ getSource(), getBytes(), newDestination, getSize(), getHash());
}
- /**
- * @return the size
- */
- public long getSize() {
- return size;
- }
+ /** @return the file to be uploaded, if any */
+ @Nullable
+ public abstract File getSource();
- /**
- * @return the hash
- */
- public String getHash() {
- return hash;
- }
+ /** @return the bytes to be uploaded, if any */
+ @Nullable
+ public abstract byte[] getBytes();
- /**
- * @return the file to be uploaded
- */
- public String getSourcePath() {
- return sourcePath;
+ /** @return the dataflowPackage */
+ public abstract DataflowPackage getDestination();
+
+ /** @return the size */
+ public abstract long getSize();
+
+ /** @return the hash */
+ public abstract String getHash();
+
+ public String getSourceDescription() {
+ if (getSource() != null) {
+ return getSource().toString();
+ } else {
+ return String.format("<%s bytes, hash %s>", getSize(), getHash());
+ }
}
}
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PropertyNames.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PropertyNames.java
index c8c9903..55e0c4e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PropertyNames.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PropertyNames.java
@@ -21,67 +21,31 @@
* Constant property names used by the SDK in CloudWorkflow specifications.
*/
public class PropertyNames {
- public static final String ALLOWED_ENCODINGS = "allowed_encodings";
- public static final String APPEND_TRAILING_NEWLINES = "append_trailing_newlines";
- public static final String BIGQUERY_CREATE_DISPOSITION = "create_disposition";
- public static final String BIGQUERY_DATASET = "dataset";
- public static final String BIGQUERY_PROJECT = "project";
- public static final String BIGQUERY_SCHEMA = "schema";
- public static final String BIGQUERY_TABLE = "table";
- public static final String BIGQUERY_QUERY = "bigquery_query";
- public static final String BIGQUERY_FLATTEN_RESULTS = "bigquery_flatten_results";
- public static final String BIGQUERY_USE_LEGACY_SQL = "bigquery_use_legacy_sql";
- public static final String BIGQUERY_WRITE_DISPOSITION = "write_disposition";
- public static final String BIGQUERY_EXPORT_FORMAT = "bigquery_export_format";
- public static final String BIGQUERY_EXPORT_SCHEMA = "bigquery_export_schema";
public static final String CO_GBK_RESULT_SCHEMA = "co_gbk_result_schema";
- public static final String COMBINE_FN = "combine_fn";
public static final String COMPONENT_ENCODINGS = "component_encodings";
- public static final String COMPRESSION_TYPE = "compression_type";
public static final String CUSTOM_SOURCE_FORMAT = "custom_source";
- public static final String CONCAT_SOURCE_SOURCES = "sources";
- public static final String CONCAT_SOURCE_BASE_SPECS = "base_specs";
public static final String SOURCE_STEP_INPUT = "custom_source_step_input";
public static final String SOURCE_SPEC = "spec";
public static final String SOURCE_METADATA = "metadata";
public static final String SOURCE_DOES_NOT_NEED_SPLITTING = "does_not_need_splitting";
- public static final String SOURCE_PRODUCES_SORTED_KEYS = "produces_sorted_keys";
public static final String SOURCE_IS_INFINITE = "is_infinite";
public static final String SOURCE_ESTIMATED_SIZE_BYTES = "estimated_size_bytes";
- public static final String ELEMENT = "element";
- public static final String ELEMENTS = "elements";
public static final String ENCODING = "encoding";
- public static final String ENCODING_ID = "encoding_id";
- public static final String END_INDEX = "end_index";
- public static final String END_OFFSET = "end_offset";
- public static final String END_SHUFFLE_POSITION = "end_shuffle_position";
public static final String ENVIRONMENT_VERSION_JOB_TYPE_KEY = "job_type";
public static final String ENVIRONMENT_VERSION_MAJOR_KEY = "major";
- public static final String FILENAME = "filename";
- public static final String FILENAME_PREFIX = "filename_prefix";
- public static final String FILENAME_SUFFIX = "filename_suffix";
- public static final String FILEPATTERN = "filepattern";
- public static final String FOOTER = "footer";
public static final String FORMAT = "format";
- public static final String HEADER = "header";
public static final String INPUTS = "inputs";
- public static final String INPUT_CODER = "input_coder";
- public static final String IS_GENERATED = "is_generated";
public static final String IS_MERGING_WINDOW_FN = "is_merging_window_fn";
public static final String IS_PAIR_LIKE = "is_pair_like";
public static final String IS_STREAM_LIKE = "is_stream_like";
public static final String IS_WRAPPER = "is_wrapper";
public static final String DISALLOW_COMBINER_LIFTING = "disallow_combiner_lifting";
public static final String NON_PARALLEL_INPUTS = "non_parallel_inputs";
- public static final String NUM_SHARD_CODERS = "num_shard_coders";
- public static final String NUM_METADATA_SHARD_CODERS = "num_metadata_shard_coders";
- public static final String NUM_SHARDS = "num_shards";
public static final String OBJECT_TYPE_NAME = "@type";
public static final String OUTPUT = "output";
public static final String OUTPUT_INFO = "output_info";
public static final String OUTPUT_NAME = "output_name";
public static final String PARALLEL_INPUT = "parallel_input";
- public static final String PHASE = "phase";
public static final String PUBSUB_ID_ATTRIBUTE = "pubsub_id_label";
public static final String PUBSUB_SERIALIZED_ATTRIBUTES_FN = "pubsub_serialized_attributes_fn";
public static final String PUBSUB_SUBSCRIPTION = "pubsub_subscription";
@@ -91,22 +55,13 @@
public static final String PUBSUB_TOPIC_OVERRIDE = "pubsub_topic_runtime_override";
public static final String SCALAR_FIELD_NAME = "value";
public static final String SERIALIZED_FN = "serialized_fn";
- public static final String SHARD_NAME_TEMPLATE = "shard_template";
- public static final String SHUFFLE_KIND = "shuffle_kind";
- public static final String SHUFFLE_READER_CONFIG = "shuffle_reader_config";
- public static final String SHUFFLE_WRITER_CONFIG = "shuffle_writer_config";
public static final String SORT_VALUES = "sort_values";
- public static final String START_INDEX = "start_index";
- public static final String START_OFFSET = "start_offset";
- public static final String START_SHUFFLE_POSITION = "start_shuffle_position";
- public static final String STRIP_TRAILING_NEWLINES = "strip_trailing_newlines";
public static final String TUPLE_TAGS = "tuple_tags";
public static final String USE_INDEXED_FORMAT = "use_indexed_format";
public static final String USER_FN = "user_fn";
public static final String USER_NAME = "user_name";
public static final String USES_KEYED_STATE = "uses_keyed_state";
- public static final String VALIDATE_SINK = "validate_sink";
- public static final String VALIDATE_SOURCE = "validate_source";
public static final String VALUE = "value";
public static final String DISPLAY_DATA = "display_data";
+ public static final String RESTRICTION_CODER = "restriction_coder";
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index 5ea9f07..0c08902 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -350,7 +350,7 @@
// Try to double the size of the buffer, if thats not enough, just use the new capacity.
// Note that we use Math.min(long, long) to not cause overflow on the multiplication.
- int newCapacity = (int) Math.min(Integer.MAX_VALUE, buffer.length * 2L);
+ int newCapacity = (int) Math.min(Integer.MAX_VALUE - 8, buffer.length * 2L);
if (newCapacity < minCapacity) {
newCapacity = minCapacity;
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/Stager.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/Stager.java
index 3e3c17f..0b2013e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/Stager.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/Stager.java
@@ -20,10 +20,32 @@
import com.google.api.services.dataflow.model.DataflowPackage;
import java.util.List;
-/**
- * Interface for staging files needed for running a Dataflow pipeline.
- */
+/** Interface for staging files needed for running a Dataflow pipeline. */
public interface Stager {
- /* Stage files and return a list of packages. */
- List<DataflowPackage> stageFiles();
+ /**
+ * Stage default files and return a list of {@link DataflowPackage} objects describing the actual
+ * location at which each file was staged.
+ *
+ * <p>This is required to be identical to calling {@link #stageFiles(List)} with the default set
+ * of files.
+ *
+ * <p>The default is controlled by the implementation of {@link Stager}. The only known
+ * implementation of stager is {@link GcsStager}. See that class for more detail.
+ */
+ List<DataflowPackage> stageDefaultFiles();
+
+ /**
+ * Stage files and return a list of packages {@link DataflowPackage} objects describing th actual
+ * location at which each file was staged.
+ *
+ * <p>The mechanism for staging is owned by the implementation. The only requirement is that the
+ * location specified in the returned {@link DataflowPackage} should, in fact, contain the
+ * contents of the staged file.
+ */
+ List<DataflowPackage> stageFiles(List<String> filesToStage);
+
+ /**
+ * Stage bytes to a target file name wherever this stager stages things.
+ */
+ DataflowPackage stageToFile(byte[] bytes, String baseName);
}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java
index bff379f..172dc6e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.dataflow.util;
+import com.google.common.base.Strings;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
@@ -98,26 +99,19 @@
int hour = Integer.valueOf(matcher.group(4));
int minute = Integer.valueOf(matcher.group(5));
int second = Integer.valueOf(matcher.group(6));
- int millis = 0;
-
- String frac = matcher.group(7);
- if (frac != null) {
- int fracs = Integer.valueOf(frac);
- if (frac.length() == 3) { // millisecond resolution
- millis = fracs;
- } else if (frac.length() == 6) { // microsecond resolution
- millis = fracs / 1000;
- } else if (frac.length() == 9) { // nanosecond resolution
- millis = fracs / 1000000;
- } else {
- return null;
- }
- }
+ int millis = computeMillis(matcher.group(7));
return new DateTime(year, month, day, hour, minute, second, millis,
ISOChronology.getInstanceUTC()).toInstant();
}
+ private static int computeMillis(String frac) {
+ if (frac == null) {
+ return 0;
+ }
+ return Integer.valueOf(frac.length() > 3 ? frac.substring(0, 3) : Strings.padEnd(frac, 3, '0'));
+ }
+
/**
* Converts a {@link ReadableDuration} into a Dataflow API duration string.
*/
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
index d2ab357..db9b224 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
@@ -52,6 +52,7 @@
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -77,17 +78,54 @@
}
@Test
+ public void testFnApiSingleOutputOverrideNonCrashing() throws Exception {
+ DataflowPipelineOptions options = buildPipelineOptions("--experiments=beam_fn_api");
+ options.setRunner(DataflowRunner.class);
+ Pipeline pipeline = Pipeline.create(options);
+
+ DummyStatefulDoFn fn = new DummyStatefulDoFn();
+ pipeline.apply(Create.of(KV.of(1, 2))).apply(ParDo.of(fn));
+
+ DataflowRunner runner = DataflowRunner.fromOptions(options);
+ runner.replaceTransforms(pipeline);
+ assertThat(findBatchStatefulDoFn(pipeline), equalTo((DoFn) fn));
+ }
+
+ @Test
public void testMultiOutputOverrideNonCrashing() throws Exception {
DataflowPipelineOptions options = buildPipelineOptions();
options.setRunner(DataflowRunner.class);
Pipeline pipeline = Pipeline.create(options);
TupleTag<Integer> mainOutputTag = new TupleTag<Integer>() {};
+ TupleTag<Integer> sideOutputTag = new TupleTag<Integer>() {};
DummyStatefulDoFn fn = new DummyStatefulDoFn();
pipeline
.apply(Create.of(KV.of(1, 2)))
- .apply(ParDo.of(fn).withOutputTags(mainOutputTag, TupleTagList.empty()));
+ .apply(ParDo.of(fn).withOutputTags(mainOutputTag, TupleTagList.of(sideOutputTag)));
+
+ DataflowRunner runner = DataflowRunner.fromOptions(options);
+ runner.replaceTransforms(pipeline);
+ assertThat(findBatchStatefulDoFn(pipeline), equalTo((DoFn) fn));
+ }
+
+ @Test
+ @Ignore("TODO: BEAM-2902 Add support for user state in a ParDo.Multi once PTransformMatcher "
+ + "exposes a way to know when the replacement is not required by checking that the "
+ + "preceding ParDos to a GBK are key preserving.")
+ public void testFnApiMultiOutputOverrideNonCrashing() throws Exception {
+ DataflowPipelineOptions options = buildPipelineOptions("--experiments=beam_fn_api");
+ options.setRunner(DataflowRunner.class);
+ Pipeline pipeline = Pipeline.create(options);
+
+ TupleTag<Integer> mainOutputTag = new TupleTag<Integer>() {};
+ TupleTag<Integer> sideOutputTag = new TupleTag<Integer>() {};
+
+ DummyStatefulDoFn fn = new DummyStatefulDoFn();
+ pipeline
+ .apply(Create.of(KV.of(1, 2)))
+ .apply(ParDo.of(fn).withOutputTags(mainOutputTag, TupleTagList.of(sideOutputTag)));
DataflowRunner runner = DataflowRunner.fromOptions(options);
runner.replaceTransforms(pipeline);
@@ -146,7 +184,7 @@
}
}
- private static DataflowPipelineOptions buildPipelineOptions() throws IOException {
+ private static DataflowPipelineOptions buildPipelineOptions(String ... args) throws IOException {
GcsUtil mockGcsUtil = mock(GcsUtil.class);
when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
@Override
@@ -156,11 +194,13 @@
});
when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);
- DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
+ DataflowPipelineOptions options =
+ PipelineOptionsFactory.fromArgs(args).as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setJobName("some-job-name");
options.setProject("some-project");
+ options.setRegion("some-region");
options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
options.setFilesToStage(new LinkedList<String>());
options.setGcsUtil(mockGcsUtil);
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
index 85a0979..05fe687 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
@@ -20,6 +20,7 @@
import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
import static org.apache.beam.sdk.metrics.MetricResultsMatchers.committedMetricsResult;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
@@ -28,20 +29,25 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.api.client.util.ArrayMap;
import com.google.api.services.dataflow.Dataflow;
import com.google.api.services.dataflow.model.Job;
import com.google.api.services.dataflow.model.JobMetrics;
import com.google.api.services.dataflow.model.MetricStructuredName;
import com.google.api.services.dataflow.model.MetricUpdate;
+import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.math.BigDecimal;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
+import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.MetricQueryResults;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -93,6 +99,9 @@
modelJob.setCurrentState(State.RUNNING.toString());
DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
when(job.getState()).thenReturn(State.RUNNING);
job.jobId = JOB_ID;
@@ -113,6 +122,9 @@
modelJob.setCurrentState(State.RUNNING.toString());
DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
when(job.getState()).thenReturn(State.DONE);
job.jobId = JOB_ID;
@@ -129,11 +141,8 @@
verify(dataflowClient, times(1)).getJobMetrics(JOB_ID);
}
- private MetricUpdate makeCounterMetricUpdate(String name, String namespace, String step,
- long scalar, boolean tentative) {
- MetricUpdate update = new MetricUpdate();
- update.setScalar(new BigDecimal(scalar));
-
+ private MetricUpdate setStructuredName(MetricUpdate update, String name, String namespace,
+ String step, boolean tentative) {
MetricStructuredName structuredName = new MetricStructuredName();
structuredName.setName(name);
structuredName.setOrigin("user");
@@ -148,13 +157,42 @@
return update;
}
+ private MetricUpdate makeDistributionMetricUpdate(String name, String namespace, String step,
+ Long sum, Long count, Long min, Long max, boolean tentative) {
+ MetricUpdate update = new MetricUpdate();
+ ArrayMap<String, BigDecimal> distribution = ArrayMap.create();
+ distribution.add("count", new BigDecimal(count));
+ distribution.add("mean", new BigDecimal(sum / count));
+ distribution.add("sum", new BigDecimal(sum));
+ distribution.add("min", new BigDecimal(min));
+ distribution.add("max", new BigDecimal(max));
+ update.setDistribution(distribution);
+ return setStructuredName(update, name, namespace, step, tentative);
+ }
+
+ private MetricUpdate makeCounterMetricUpdate(String name, String namespace, String step,
+ long scalar, boolean tentative) {
+ MetricUpdate update = new MetricUpdate();
+ update.setScalar(new BigDecimal(scalar));
+ return setStructuredName(update, name, namespace, step, tentative);
+
+ }
+
@Test
public void testSingleCounterUpdates() throws IOException {
JobMetrics jobMetrics = new JobMetrics();
DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
when(job.getState()).thenReturn(State.RUNNING);
job.jobId = JOB_ID;
+ AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class);
+ when(myStep.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep, "s2");
+
MetricUpdate update = new MetricUpdate();
long stepValue = 1234L;
update.setScalar(new BigDecimal(stepValue));
@@ -172,9 +210,9 @@
DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
MetricQueryResults result = dataflowMetrics.queryMetrics(null);
assertThat(result.counters(), containsInAnyOrder(
- attemptedMetricsResult("counterNamespace", "counterName", "s2", 1233L)));
+ attemptedMetricsResult("counterNamespace", "counterName", "myStepName", (Long) null)));
assertThat(result.counters(), containsInAnyOrder(
- committedMetricsResult("counterNamespace", "counterName", "s2", 1234L)));
+ committedMetricsResult("counterNamespace", "counterName", "myStepName", 1234L)));
}
@Test
@@ -183,23 +221,101 @@
DataflowClient dataflowClient = mock(DataflowClient.class);
when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
when(job.getState()).thenReturn(State.RUNNING);
job.jobId = JOB_ID;
+ AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class);
+ when(myStep.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep, "s2");
+
// The parser relies on the fact that one tentative and one committed metric update exist in
// the job metrics results.
jobMetrics.setMetrics(ImmutableList.of(
makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false),
- makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1234L, true),
- makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s3", 0L, false),
- makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s3", 0L, true)));
+ makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, true),
+ makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, false),
+ makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, true)));
DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
MetricQueryResults result = dataflowMetrics.queryMetrics(null);
assertThat(result.counters(), containsInAnyOrder(
- attemptedMetricsResult("counterNamespace", "counterName", "s2", 1234L)));
+ attemptedMetricsResult("counterNamespace", "counterName", "myStepName", (Long) null)));
assertThat(result.counters(), containsInAnyOrder(
- committedMetricsResult("counterNamespace", "counterName", "s2", 1233L)));
+ committedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L)));
+ }
+
+ @Test
+ public void testDistributionUpdates() throws IOException {
+ JobMetrics jobMetrics = new JobMetrics();
+ DataflowClient dataflowClient = mock(DataflowClient.class);
+ when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
+ DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
+ when(job.getState()).thenReturn(State.RUNNING);
+ job.jobId = JOB_ID;
+
+ AppliedPTransform<?, ?, ?> myStep2 = mock(AppliedPTransform.class);
+ when(myStep2.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep2, "s2");
+
+ // The parser relies on the fact that one tentative and one committed metric update exist in
+ // the job metrics results.
+ jobMetrics.setMetrics(ImmutableList.of(
+ makeDistributionMetricUpdate("distributionName", "distributionNamespace", "s2",
+ 18L, 2L, 2L, 16L, false),
+ makeDistributionMetricUpdate("distributionName", "distributionNamespace", "s2",
+ 18L, 2L, 2L, 16L, true)));
+
+ DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
+ MetricQueryResults result = dataflowMetrics.queryMetrics(null);
+ assertThat(result.distributions(), contains(
+ attemptedMetricsResult("distributionNamespace", "distributionName", "myStepName",
+ (DistributionResult) null)));
+ assertThat(result.distributions(), contains(
+ committedMetricsResult("distributionNamespace", "distributionName", "myStepName",
+ DistributionResult.create(18, 2, 2, 16))));
+ }
+
+ @Test
+ public void testDistributionUpdatesStreaming() throws IOException {
+ JobMetrics jobMetrics = new JobMetrics();
+ DataflowClient dataflowClient = mock(DataflowClient.class);
+ when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
+ DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(true);
+ when(job.getDataflowOptions()).thenReturn(options);
+ when(job.getState()).thenReturn(State.RUNNING);
+ job.jobId = JOB_ID;
+
+ AppliedPTransform<?, ?, ?> myStep2 = mock(AppliedPTransform.class);
+ when(myStep2.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep2, "s2");
+
+ // The parser relies on the fact that one tentative and one committed metric update exist in
+ // the job metrics results.
+ jobMetrics.setMetrics(ImmutableList.of(
+ makeDistributionMetricUpdate("distributionName", "distributionNamespace", "s2",
+ 18L, 2L, 2L, 16L, false),
+ makeDistributionMetricUpdate("distributionName", "distributionNamespace", "s2",
+ 18L, 2L, 2L, 16L, true)));
+
+ DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
+ MetricQueryResults result = dataflowMetrics.queryMetrics(null);
+ assertThat(result.distributions(), contains(
+ committedMetricsResult("distributionNamespace", "distributionName", "myStepName",
+ (DistributionResult) null)));
+ assertThat(result.distributions(), contains(
+ attemptedMetricsResult("distributionNamespace", "distributionName", "myStepName",
+ DistributionResult.create(18, 2, 2, 16))));
}
@Test
@@ -208,9 +324,72 @@
DataflowClient dataflowClient = mock(DataflowClient.class);
when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(false);
+ when(job.getDataflowOptions()).thenReturn(options);
when(job.getState()).thenReturn(State.RUNNING);
job.jobId = JOB_ID;
+ AppliedPTransform<?, ?, ?> myStep2 = mock(AppliedPTransform.class);
+ when(myStep2.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep2, "s2");
+ AppliedPTransform<?, ?, ?> myStep3 = mock(AppliedPTransform.class);
+ when(myStep3.getFullName()).thenReturn("myStepName3");
+ job.transformStepNames.put(myStep3, "s3");
+ AppliedPTransform<?, ?, ?> myStep4 = mock(AppliedPTransform.class);
+ when(myStep4.getFullName()).thenReturn("myStepName4");
+ job.transformStepNames.put(myStep4, "s4");
+
+
+ // The parser relies on the fact that one tentative and one committed metric update exist in
+ // the job metrics results.
+ jobMetrics.setMetrics(ImmutableList.of(
+ makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false),
+ makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1234L, true),
+ makeCounterMetricUpdate("otherCounter", "otherNamespace", "s3", 12L, false),
+ makeCounterMetricUpdate("otherCounter", "otherNamespace", "s3", 12L, true),
+ makeCounterMetricUpdate("counterName", "otherNamespace", "s4", 1200L, false),
+ makeCounterMetricUpdate("counterName", "otherNamespace", "s4", 1233L, true),
+ // The following counter can not have its name translated thus it won't appear.
+ makeCounterMetricUpdate("lostName", "otherNamespace", "s5", 1200L, false),
+ makeCounterMetricUpdate("lostName", "otherNamespace", "s5", 1200L, true)));
+
+ DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
+ MetricQueryResults result = dataflowMetrics.queryMetrics(null);
+ assertThat(result.counters(), containsInAnyOrder(
+ attemptedMetricsResult("counterNamespace", "counterName", "myStepName", (Long) null),
+ attemptedMetricsResult("otherNamespace", "otherCounter", "myStepName3", (Long) null),
+ attemptedMetricsResult("otherNamespace", "counterName", "myStepName4", (Long) null)));
+ assertThat(result.counters(), containsInAnyOrder(
+ committedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L),
+ committedMetricsResult("otherNamespace", "otherCounter", "myStepName3", 12L),
+ committedMetricsResult("otherNamespace", "counterName", "myStepName4", 1200L)));
+ }
+
+ @Test
+ public void testMultipleCounterUpdatesStreaming() throws IOException {
+ JobMetrics jobMetrics = new JobMetrics();
+ DataflowClient dataflowClient = mock(DataflowClient.class);
+ when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
+ DataflowPipelineJob job = mock(DataflowPipelineJob.class);
+ DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
+ when(options.isStreaming()).thenReturn(true);
+ when(job.getDataflowOptions()).thenReturn(options);
+ when(job.getState()).thenReturn(State.RUNNING);
+ job.jobId = JOB_ID;
+
+ AppliedPTransform<?, ?, ?> myStep2 = mock(AppliedPTransform.class);
+ when(myStep2.getFullName()).thenReturn("myStepName");
+ job.transformStepNames = HashBiMap.create();
+ job.transformStepNames.put(myStep2, "s2");
+ AppliedPTransform<?, ?, ?> myStep3 = mock(AppliedPTransform.class);
+ when(myStep3.getFullName()).thenReturn("myStepName3");
+ job.transformStepNames.put(myStep3, "s3");
+ AppliedPTransform<?, ?, ?> myStep4 = mock(AppliedPTransform.class);
+ when(myStep4.getFullName()).thenReturn("myStepName4");
+ job.transformStepNames.put(myStep4, "s4");
+
// The parser relies on the fact that one tentative and one committed metric update exist in
// the job metrics results.
jobMetrics.setMetrics(ImmutableList.of(
@@ -224,12 +403,12 @@
DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
MetricQueryResults result = dataflowMetrics.queryMetrics(null);
assertThat(result.counters(), containsInAnyOrder(
- attemptedMetricsResult("counterNamespace", "counterName", "s2", 1234L),
- attemptedMetricsResult("otherNamespace", "otherCounter", "s3", 12L),
- attemptedMetricsResult("otherNamespace", "counterName", "s4", 1233L)));
+ committedMetricsResult("counterNamespace", "counterName", "myStepName", (Long) null),
+ committedMetricsResult("otherNamespace", "otherCounter", "myStepName3", (Long) null),
+ committedMetricsResult("otherNamespace", "counterName", "myStepName4", (Long) null)));
assertThat(result.counters(), containsInAnyOrder(
- committedMetricsResult("counterNamespace", "counterName", "s2", 1233L),
- committedMetricsResult("otherNamespace", "otherCounter", "s3", 12L),
- committedMetricsResult("otherNamespace", "counterName", "s4", 1200L)));
+ attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L),
+ attemptedMetricsResult("otherNamespace", "otherCounter", "myStepName3", 12L),
+ attemptedMetricsResult("otherNamespace", "counterName", "myStepName4", 1200L)));
}
}
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 87744f0..81e7a97 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -17,14 +17,15 @@
*/
package org.apache.beam.runners.dataflow;
-import static org.apache.beam.runners.dataflow.util.Structs.addObject;
-import static org.apache.beam.runners.dataflow.util.Structs.getDictionary;
import static org.apache.beam.runners.dataflow.util.Structs.getString;
+import static org.apache.beam.sdk.util.StringUtils.jsonStringToByteArray;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -68,18 +69,21 @@
import org.apache.beam.runners.dataflow.DataflowPipelineTranslator.JobSpecification;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions;
+import org.apache.beam.runners.dataflow.util.CloudObject;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
import org.apache.beam.runners.dataflow.util.OutputReference;
import org.apache.beam.runners.dataflow.util.PropertyNames;
import org.apache.beam.runners.dataflow.util.Structs;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
import org.apache.beam.sdk.extensions.gcp.storage.GcsPathValidator;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.ValueProvider;
@@ -94,7 +98,13 @@
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.DoFnInfo;
import org.apache.beam.sdk.util.GcsUtil;
+import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
@@ -103,6 +113,8 @@
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -188,6 +200,7 @@
options.setGcpCredential(new TestCredential());
options.setJobName("some-job-name");
options.setProject("some-project");
+ options.setRegion("some-region");
options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
options.setFilesToStage(new LinkedList<String>());
options.setDataflowClient(buildMockDataflow(new IsValidCreateRequest()));
@@ -531,57 +544,6 @@
job.getEnvironment().getWorkerPools().get(0).getDiskSizeGb());
}
- @Test
- public void testPredefinedAddStep() throws Exception {
- DataflowPipelineOptions options = buildPipelineOptions();
-
- DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);
- DataflowPipelineTranslator.registerTransformTranslator(
- EmbeddedTransform.class, new EmbeddedTranslator());
-
- // Create a predefined step using another pipeline
- Step predefinedStep = createPredefinedStep();
-
- // Create a pipeline that the predefined step will be embedded into
- Pipeline pipeline = Pipeline.create(options);
- pipeline.apply("ReadMyFile", TextIO.read().from("gs://bucket/in"))
- .apply(ParDo.of(new NoOpFn()))
- .apply(new EmbeddedTransform(predefinedStep.clone()))
- .apply(ParDo.of(new NoOpFn()));
- DataflowRunner runner = DataflowRunner.fromOptions(options);
- runner.replaceTransforms(pipeline);
- Job job =
- translator
- .translate(
- pipeline,
- runner,
- Collections.<DataflowPackage>emptyList())
- .getJob();
- assertAllStepOutputsHaveUniqueIds(job);
-
- List<Step> steps = job.getSteps();
- assertEquals(4, steps.size());
-
- // The input to the embedded step should match the output of the step before
- Map<String, Object> step1Out = getOutputPortReference(steps.get(1));
- Map<String, Object> step2In = getDictionary(
- steps.get(2).getProperties(), PropertyNames.PARALLEL_INPUT);
- assertEquals(step1Out, step2In);
-
- // The output from the embedded step should match the input of the step after
- Map<String, Object> step2Out = getOutputPortReference(steps.get(2));
- Map<String, Object> step3In = getDictionary(
- steps.get(3).getProperties(), PropertyNames.PARALLEL_INPUT);
- assertEquals(step2Out, step3In);
-
- // The step should not have been modified other than remapping the input
- Step predefinedStepClone = predefinedStep.clone();
- Step embeddedStepClone = steps.get(2).clone();
- predefinedStepClone.getProperties().remove(PropertyNames.PARALLEL_INPUT);
- embeddedStepClone.getProperties().remove(PropertyNames.PARALLEL_INPUT);
- assertEquals(predefinedStepClone, embeddedStepClone);
- }
-
/**
* Construct a OutputReference for the output of the step.
*/
@@ -630,47 +592,6 @@
}
/**
- * A placeholder transform that will be used to substitute a predefined Step.
- */
- private static class EmbeddedTransform
- extends PTransform<PCollection<String>, PCollection<String>> {
- private final Step step;
-
- public EmbeddedTransform(Step step) {
- this.step = step;
- }
-
- @Override
- public PCollection<String> expand(PCollection<String> input) {
- return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- input.isBounded());
- }
-
- @Override
- protected Coder<?> getDefaultOutputCoder() {
- return StringUtf8Coder.of();
- }
- }
-
- /**
- * A TransformTranslator that adds the predefined Step using
- * {@link TranslationContext#addStep} and remaps the input port reference.
- */
- private static class EmbeddedTranslator
- implements TransformTranslator<EmbeddedTransform> {
- @Override public void translate(EmbeddedTransform transform, TranslationContext context) {
- PCollection<String> input = context.getInput(transform);
- addObject(
- transform.step.getProperties(),
- PropertyNames.PARALLEL_INPUT,
- context.asOutputReference(input, context.getProducer(input)));
- context.addStep(transform, transform.step);
- }
- }
-
- /**
* A composite transform that returns an output that is unrelated to
* the input.
*/
@@ -685,11 +606,6 @@
// Return a value unrelated to the input.
return input.getPipeline().apply(Create.of(1, 2, 3, 4));
}
-
- @Override
- protected Coder<?> getDefaultOutputCoder() {
- return VarIntCoder.of();
- }
}
/**
@@ -705,11 +621,6 @@
return PDone.in(input.getPipeline());
}
-
- @Override
- protected Coder<?> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
}
/**
@@ -729,10 +640,13 @@
// Fails here when attempting to construct a tuple with an unbound object.
return PCollectionTuple.of(sumTag, sum)
- .and(doneTag, PCollection.<Void>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- input.isBounded()));
+ .and(
+ doneTag,
+ PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ input.isBounded(),
+ VoidCoder.of()));
}
}
@@ -834,7 +748,7 @@
/**
* Test that in translation the name for a collection (in this case just a Create output) is
- * overriden to be what the Dataflow service expects.
+ * overridden to be what the Dataflow service expects.
*/
@Test
public void testNamesOverridden() throws Exception {
@@ -991,6 +905,68 @@
not(equalTo("true")));
}
+ /**
+ * Smoke test to fail fast if translation of a splittable ParDo
+ * in streaming breaks.
+ */
+ @Test
+ public void testStreamingSplittableParDoTranslation() throws Exception {
+ DataflowPipelineOptions options = buildPipelineOptions();
+ DataflowRunner runner = DataflowRunner.fromOptions(options);
+ options.setStreaming(true);
+ DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);
+
+ Pipeline pipeline = Pipeline.create(options);
+
+ PCollection<String> windowedInput = pipeline
+ .apply(Create.of("a"))
+ .apply(Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
+ windowedInput.apply(ParDo.of(new TestSplittableFn()));
+
+ runner.replaceTransforms(pipeline);
+
+ Job job =
+ translator
+ .translate(
+ pipeline,
+ runner,
+ Collections.<DataflowPackage>emptyList())
+ .getJob();
+
+ // The job should contain a SplittableParDo.ProcessKeyedElements step, translated as
+ // "SplittableProcessKeyed".
+
+ List<Step> steps = job.getSteps();
+ Step processKeyedStep = null;
+ for (Step step : steps) {
+ if (step.getKind().equals("SplittableProcessKeyed")) {
+ assertNull(processKeyedStep);
+ processKeyedStep = step;
+ }
+ }
+ assertNotNull(processKeyedStep);
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ DoFnInfo<String, Integer> fnInfo =
+ (DoFnInfo<String, Integer>)
+ SerializableUtils.deserializeFromByteArray(
+ jsonStringToByteArray(
+ Structs.getString(
+ processKeyedStep.getProperties(), PropertyNames.SERIALIZED_FN)),
+ "DoFnInfo");
+ assertThat(fnInfo.getDoFn(), instanceOf(TestSplittableFn.class));
+ assertThat(
+ fnInfo.getWindowingStrategy().getWindowFn(),
+ Matchers.<WindowFn>equalTo(FixedWindows.of(Duration.standardMinutes(1))));
+ Coder<?> restrictionCoder =
+ CloudObjects.coderFromCloudObject(
+ (CloudObject)
+ Structs.getObject(
+ processKeyedStep.getProperties(), PropertyNames.RESTRICTION_CODER));
+
+ assertEquals(SerializableCoder.of(OffsetRange.class), restrictionCoder);
+ }
+
@Test
public void testToSingletonTranslationWithIsmSideInput() throws Exception {
// A "change detector" test that makes sure the translation
@@ -1015,15 +991,15 @@
assertAllStepOutputsHaveUniqueIds(job);
List<Step> steps = job.getSteps();
- assertEquals(5, steps.size());
+ assertEquals(9, steps.size());
@SuppressWarnings("unchecked")
List<Map<String, Object>> toIsmRecordOutputs =
- (List<Map<String, Object>>) steps.get(3).getProperties().get(PropertyNames.OUTPUT_INFO);
+ (List<Map<String, Object>>) steps.get(7).getProperties().get(PropertyNames.OUTPUT_INFO);
assertTrue(
Structs.getBoolean(Iterables.getOnlyElement(toIsmRecordOutputs), "use_indexed_format"));
- Step collectionToSingletonStep = steps.get(4);
+ Step collectionToSingletonStep = steps.get(8);
assertEquals("CollectionToSingleton", collectionToSingletonStep.getKind());
}
@@ -1185,4 +1161,16 @@
assertTrue(String.format("Found duplicate output ids %s", outputIds),
outputIds.size() == 0);
}
+
+ private static class TestSplittableFn extends DoFn<String, Integer> {
+ @ProcessElement
+ public void process(ProcessContext c, OffsetRangeTracker tracker) {
+ // noop
+ }
+
+ @GetInitialRestriction
+ public OffsetRange getInitialRange(String element) {
+ return null;
+ }
+ }
}
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerInfoTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerInfoTest.java
index 3502040..051f37e 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerInfoTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerInfoTest.java
@@ -19,6 +19,7 @@
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -26,6 +27,9 @@
/**
* Tests for {@link DataflowRunnerInfo}.
+ *
+ * <p>Note that tests for checking that the Dataflow distribution correctly loads overridden
+ * properties is contained within the Dataflow distribution.
*/
public class DataflowRunnerInfoTest {
@@ -50,5 +54,10 @@
"container version invalid",
info.getContainerVersion(),
not(containsString("$")));
+
+ for (String property
+ : new String[]{ "java.vendor", "java.version", "os.arch", "os.name", "os.version"}) {
+ assertEquals(System.getProperty(property), info.getProperties().get(property));
+ }
}
}
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index 8f10b18..66cf11d 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -23,6 +23,7 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -44,11 +45,13 @@
import com.google.api.services.dataflow.model.DataflowPackage;
import com.google.api.services.dataflow.model.Job;
import com.google.api.services.dataflow.model.ListJobsResponse;
+import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.channels.FileChannel;
@@ -62,36 +65,62 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.dataflow.DataflowRunner.StreamingShardedWriteFactory;
import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
-import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.extensions.gcp.auth.NoopCredentialFactory;
import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
+import org.apache.beam.sdk.io.DynamicFileDestinations;
+import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.WriteFilesResult;
+import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
import org.apache.beam.sdk.testing.ExpectedLogs;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.GcsUtil;
-import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.hamcrest.Description;
import org.hamcrest.Matchers;
import org.hamcrest.TypeSafeMatcher;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Before;
import org.junit.Rule;
@@ -107,10 +136,13 @@
/**
* Tests for the {@link DataflowRunner}.
+ *
+ * <p>Implements {@link Serializable} because it is caught in closures.
*/
@RunWith(JUnit4.class)
-public class DataflowRunnerTest {
+public class DataflowRunnerTest implements Serializable {
+ private static final String VALID_BUCKET = "valid-bucket";
private static final String VALID_STAGING_BUCKET = "gs://valid-bucket/staging";
private static final String VALID_TEMP_BUCKET = "gs://valid-bucket/temp";
private static final String VALID_PROFILE_BUCKET = "gs://valid-bucket/profiles";
@@ -119,35 +151,56 @@
private static final String PROJECT_ID = "some-project";
private static final String REGION_ID = "some-region-1";
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
- @Rule
- public ExpectedException thrown = ExpectedException.none();
- @Rule
- public ExpectedLogs expectedLogs = ExpectedLogs.none(DataflowRunner.class);
+ @Rule public transient TemporaryFolder tmpFolder = new TemporaryFolder();
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+ @Rule public transient ExpectedLogs expectedLogs = ExpectedLogs.none(DataflowRunner.class);
- private Dataflow.Projects.Locations.Jobs mockJobs;
- private GcsUtil mockGcsUtil;
+ private transient Dataflow.Projects.Locations.Jobs mockJobs;
+ private transient GcsUtil mockGcsUtil;
// Asserts that the given Job has all expected fields set.
private static void assertValidJob(Job job) {
assertNull(job.getId());
assertNull(job.getCurrentState());
assertTrue(Pattern.matches("[a-z]([-a-z0-9]*[a-z0-9])?", job.getName()));
+
+ // https://issues.apache.org/jira/browse/BEAM-3116
+ // for (WorkerPool workerPool : job.getEnvironment().getWorkerPools()) {
+ // assertThat(workerPool.getMetadata(),
+ // hasKey(DataflowRunner.STAGED_PIPELINE_METADATA_PROPERTY));
+ // }
}
@Before
public void setUp() throws IOException {
this.mockGcsUtil = mock(GcsUtil.class);
+
when(mockGcsUtil.create(any(GcsPath.class), anyString()))
- .then(new Answer<SeekableByteChannel>() {
- @Override
- public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
- return FileChannel.open(
- Files.createTempFile("channel-", ".tmp"),
- StandardOpenOption.CREATE, StandardOpenOption.DELETE_ON_CLOSE);
- }
- });
+ .then(
+ new Answer<SeekableByteChannel>() {
+ @Override
+ public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
+ return FileChannel.open(
+ Files.createTempFile("channel-", ".tmp"),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.DELETE_ON_CLOSE);
+ }
+ });
+
+ when(mockGcsUtil.create(any(GcsPath.class), anyString(), anyInt()))
+ .then(
+ new Answer<SeekableByteChannel>() {
+ @Override
+ public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
+ return FileChannel.open(
+ Files.createTempFile("channel-", ".tmp"),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.DELETE_ON_CLOSE);
+ }
+ });
+
when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
@Override
public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
@@ -155,13 +208,36 @@
}
});
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_STAGING_BUCKET))).thenReturn(true);
- when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_STAGING_BUCKET))).thenReturn(true);
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET))).thenReturn(true);
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET + "/staging/"))).
thenReturn(true);
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_PROFILE_BUCKET))).thenReturn(true);
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(NON_EXISTENT_BUCKET))).thenReturn(false);
+ // Let every valid path be matched
+ when(mockGcsUtil.getObjects(anyListOf(GcsPath.class)))
+ .thenAnswer(
+ new Answer<List<GcsUtil.StorageObjectOrIOException>>() {
+ @Override
+ public List<GcsUtil.StorageObjectOrIOException> answer(
+ InvocationOnMock invocationOnMock) throws Throwable {
+
+ List<GcsPath> gcsPaths = (List<GcsPath>) invocationOnMock.getArguments()[0];
+ List<GcsUtil.StorageObjectOrIOException> results = new ArrayList<>();
+
+ for (GcsPath gcsPath : gcsPaths) {
+ if (gcsPath.getBucket().equals(VALID_BUCKET)) {
+ StorageObject resultObject = new StorageObject();
+ resultObject.setBucket(gcsPath.getBucket());
+ resultObject.setName(gcsPath.getObject());
+ results.add(GcsUtil.StorageObjectOrIOException.create(resultObject));
+ }
+ }
+
+ return results;
+ }
+ });
+
// The dataflow pipeline attempts to output to this location.
when(mockGcsUtil.bucketAccessible(GcsPath.fromUri("gs://bucket/object"))).thenReturn(true);
@@ -314,6 +390,18 @@
}
@Test
+ public void testFromOptionsUserAgentFromPipelineInfo() throws Exception {
+ DataflowPipelineOptions options = buildPipelineOptions();
+ DataflowRunner.fromOptions(options);
+
+ String expectedName = DataflowRunnerInfo.getDataflowRunnerInfo().getName().replace(" ", "_");
+ assertThat(options.getUserAgent(), containsString(expectedName));
+
+ String expectedVersion = DataflowRunnerInfo.getDataflowRunnerInfo().getVersion();
+ assertThat(options.getUserAgent(), containsString(expectedVersion));
+ }
+
+ @Test
public void testRun() throws IOException {
DataflowPipelineOptions options = buildPipelineOptions();
Pipeline p = buildDataflowPipeline(options);
@@ -485,14 +573,17 @@
options.setGcpCredential(new TestCredential());
when(mockGcsUtil.create(any(GcsPath.class), anyString(), anyInt()))
- .then(new Answer<SeekableByteChannel>() {
- @Override
- public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
- return FileChannel.open(
- Files.createTempFile("channel-", ".tmp"),
- StandardOpenOption.CREATE, StandardOpenOption.DELETE_ON_CLOSE);
- }
- });
+ .then(
+ new Answer<SeekableByteChannel>() {
+ @Override
+ public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
+ return FileChannel.open(
+ Files.createTempFile("channel-", ".tmp"),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.DELETE_ON_CLOSE);
+ }
+ });
Pipeline p = buildDataflowPipeline(options);
@@ -521,10 +612,10 @@
cloudDataflowDataset,
workflowJob.getEnvironment().getDataset());
assertEquals(
- ReleaseInfo.getReleaseInfo().getName(),
+ DataflowRunnerInfo.getDataflowRunnerInfo().getName(),
workflowJob.getEnvironment().getUserAgent().get("name"));
assertEquals(
- ReleaseInfo.getReleaseInfo().getVersion(),
+ DataflowRunnerInfo.getDataflowRunnerInfo().getVersion(),
workflowJob.getEnvironment().getUserAgent().get("version"));
}
@@ -823,7 +914,6 @@
DataflowRunner.fromOptions(options);
}
-
@Test
public void testValidProfileLocation() throws IOException {
DataflowPipelineOptions options = buildPipelineOptions();
@@ -925,15 +1015,11 @@
@Override
public PCollection<Integer> expand(PCollection<Integer> input) {
- return PCollection.<Integer>createPrimitiveOutputInternal(
+ return PCollection.createPrimitiveOutputInternal(
input.getPipeline(),
WindowingStrategy.globalDefault(),
- input.isBounded());
- }
-
- @Override
- protected Coder<?> getDefaultOutputCoder(PCollection<Integer> input) {
- return input.getCoder();
+ input.isBounded(),
+ input.getCoder());
}
}
@@ -991,6 +1077,71 @@
assertTrue(transform.translated);
}
+ private void verifyMapStateUnsupported(PipelineOptions options) throws Exception {
+ Pipeline p = Pipeline.create(options);
+ p.apply(Create.of(KV.of(13, 42)))
+ .apply(
+ ParDo.of(
+ new DoFn<KV<Integer, Integer>, Void>() {
+ @StateId("fizzle")
+ private final StateSpec<MapState<Void, Void>> voidState = StateSpecs.map();
+
+ @ProcessElement
+ public void process() {}
+ }));
+
+ thrown.expectMessage("MapState");
+ thrown.expect(UnsupportedOperationException.class);
+ p.run();
+ }
+
+ @Test
+ public void testMapStateUnsupportedInBatch() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(false);
+ verifyMapStateUnsupported(options);
+ }
+
+ @Test
+ public void testMapStateUnsupportedInStreaming() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(true);
+ verifyMapStateUnsupported(options);
+ }
+
+ private void verifySetStateUnsupported(PipelineOptions options) throws Exception {
+ Pipeline p = Pipeline.create(options);
+ p.apply(Create.of(KV.of(13, 42)))
+ .apply(
+ ParDo.of(
+ new DoFn<KV<Integer, Integer>, Void>() {
+ @StateId("fizzle")
+ private final StateSpec<SetState<Void>> voidState = StateSpecs.set();
+
+ @ProcessElement
+ public void process() {}
+ }));
+
+ thrown.expectMessage("SetState");
+ thrown.expect(UnsupportedOperationException.class);
+ p.run();
+ }
+
+ @Test
+ public void testSetStateUnsupportedInBatch() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(false);
+ Pipeline p = Pipeline.create(options);
+ verifySetStateUnsupported(options);
+ }
+
+ @Test
+ public void testSetStateUnsupportedInStreaming() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(true);
+ verifySetStateUnsupported(options);
+ }
+
/** Records all the composite transforms visited within the Pipeline. */
private static class CompositeTransformRecorder extends PipelineVisitor.Defaults {
private List<PTransform<?, ?>> transforms = new ArrayList<>();
@@ -1047,8 +1198,8 @@
}
/**
- * Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally
- * when the runner issuccessfully run.
+ * Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally when the
+ * runner is successfully run.
*/
@Test
public void testTemplateRunnerFullCompletion() throws Exception {
@@ -1127,4 +1278,107 @@
assertThat(
getContainerImageForJob(options), equalTo("gcr.io/java/foo"));
}
+
+ @Test
+ public void testStreamingWriteWithNoShardingReturnsNewTransform() {
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ options.as(DataflowPipelineWorkerPoolOptions.class).setMaxNumWorkers(10);
+ testStreamingWriteOverride(options, 20);
+ }
+
+ @Test
+ public void testStreamingWriteWithNoShardingReturnsNewTransformMaxWorkersUnset() {
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ testStreamingWriteOverride(options, StreamingShardedWriteFactory.DEFAULT_NUM_SHARDS);
+ }
+
+ private void verifyMergingStatefulParDoRejected(PipelineOptions options) throws Exception {
+ Pipeline p = Pipeline.create(options);
+
+ p.apply(Create.of(KV.of(13, 42)))
+ .apply(Window.<KV<Integer, Integer>>into(Sessions.withGapDuration(Duration.millis(1))))
+ .apply(ParDo.of(new DoFn<KV<Integer, Integer>, Void>() {
+ @StateId("fizzle")
+ private final StateSpec<ValueState<Void>> voidState = StateSpecs.value();
+
+ @ProcessElement
+ public void process() {}
+ }));
+
+ thrown.expectMessage("merging");
+ thrown.expect(UnsupportedOperationException.class);
+ p.run();
+ }
+
+ @Test
+ public void testMergingStatefulRejectedInStreaming() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(true);
+ verifyMergingStatefulParDoRejected(options);
+ }
+
+ @Test
+ public void testMergingStatefulRejectedInBatch() throws Exception {
+ PipelineOptions options = buildPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(false);
+ verifyMergingStatefulParDoRejected(options);
+ }
+
+ private void testStreamingWriteOverride(PipelineOptions options, int expectedNumShards) {
+ TestPipeline p = TestPipeline.fromOptions(options);
+
+ StreamingShardedWriteFactory<Object, Void, Object> factory =
+ new StreamingShardedWriteFactory<>(p.getOptions());
+ WriteFiles<Object, Void, Object> original = WriteFiles.to(new TestSink(tmpFolder.toString()));
+ PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of()));
+ AppliedPTransform<PCollection<Object>, WriteFilesResult<Void>, WriteFiles<Object, Void, Object>>
+ originalApplication =
+ AppliedPTransform.of(
+ "writefiles",
+ objs.expand(),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ original,
+ p);
+
+ WriteFiles<Object, Void, Object> replacement =
+ (WriteFiles<Object, Void, Object>)
+ factory.getReplacementTransform(originalApplication).getTransform();
+ assertThat(replacement, not(equalTo((Object) original)));
+ assertThat(replacement.getNumShards().get(), equalTo(expectedNumShards));
+ }
+
+ private static class TestSink extends FileBasedSink<Object, Void, Object> {
+ @Override
+ public void validate(PipelineOptions options) {}
+
+ TestSink(String tmpFolder) {
+ super(
+ StaticValueProvider.of(FileSystems.matchNewResource(tmpFolder, true)),
+ DynamicFileDestinations.constant(
+ new FilenamePolicy() {
+ @Override
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("should not be called");
+ }
+
+ @Nullable
+ @Override
+ public ResourceId unwindowedFilename(
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
+ throw new UnsupportedOperationException("should not be called");
+ }
+ },
+ SerializableFunctions.identity()));
+ }
+
+ @Override
+ public WriteOperation<Void, Object> createWriteOperation() {
+ throw new IllegalArgumentException("Should not be used");
+ }
+ }
}
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
index 737b408..c198ebf 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
@@ -26,6 +26,7 @@
import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
@@ -36,7 +37,6 @@
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.joda.time.Duration;
import org.junit.Before;
@@ -105,11 +105,11 @@
new PTransform<PBegin, PCollection<KV<String, Integer>>>() {
@Override
public PCollection<KV<String, Integer>> expand(PBegin input) {
- return PCollection.<KV<String, Integer>>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.UNBOUNDED)
- .setTypeDescriptor(new TypeDescriptor<KV<String, Integer>>() {});
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
}
});
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
index dea96b9..e2e42a6 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
@@ -21,6 +21,9 @@
import org.apache.beam.runners.dataflow.DataflowRunner;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
@@ -33,7 +36,6 @@
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.hamcrest.Matchers;
import org.joda.time.Duration;
@@ -94,11 +96,11 @@
new PTransform<PBegin, PCollection<KV<String, Integer>>>() {
@Override
public PCollection<KV<String, Integer>> expand(PBegin input) {
- return PCollection.<KV<String, Integer>>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.UNBOUNDED)
- .setTypeDescriptor(new TypeDescriptor<KV<String, Integer>>() {});
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
}
})
.apply(view);
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java
index c048776..4991982 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java
@@ -121,7 +121,7 @@
options.setProject(PROJECT_ID);
options.setGcpCredential(new TestCredential());
String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
- assertEquals("gcloud beta dataflow jobs --project=someProject cancel 1234", cancelCommand);
+ assertEquals("gcloud dataflow jobs --project=someProject cancel 1234", cancelCommand);
}
@Test
@@ -135,7 +135,7 @@
String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
assertEquals(
"CLOUDSDK_API_ENDPOINT_OVERRIDES_DATAFLOW=https://dataflow.googleapis.com/v0neverExisted/ "
- + "gcloud beta dataflow jobs --project=someProject cancel 1234",
+ + "gcloud dataflow jobs --project=someProject cancel 1234",
cancelCommand);
}
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index 5d0c0f2..0b94f7c 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -71,6 +71,7 @@
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import javax.annotation.Nullable;
import org.apache.beam.runners.dataflow.util.PackageUtil.PackageAttributes;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.io.FileSystems;
@@ -86,6 +87,7 @@
import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
import org.hamcrest.Matchers;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -97,22 +99,19 @@
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-/** Tests for PackageUtil. */
+/** Tests for {@link PackageUtil}. */
@RunWith(JUnit4.class)
public class PackageUtilTest {
@Rule public ExpectedLogs logged = ExpectedLogs.none(PackageUtil.class);
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
+ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+ @Rule public FastNanoClockAndSleeper fastNanoClockAndSleeper = new FastNanoClockAndSleeper();
- @Rule
- public FastNanoClockAndSleeper fastNanoClockAndSleeper = new FastNanoClockAndSleeper();
-
- @Mock
- GcsUtil mockGcsUtil;
+ @Mock GcsUtil mockGcsUtil;
// 128 bits, base64 encoded is 171 bits, rounds to 22 bytes
private static final String HASH_PATTERN = "[a-zA-Z0-9+-]{22}";
private CreateOptions createOptions;
+ private PackageUtil defaultPackageUtil;
@Before
public void setUp() {
@@ -122,6 +121,12 @@
pipelineOptions.setGcsUtil(mockGcsUtil);
FileSystems.setDefaultPipelineOptions(pipelineOptions);
createOptions = StandardCreateOptions.builder().setMimeType(MimeTypes.BINARY).build();
+ defaultPackageUtil = PackageUtil.withDefaultThreadPool();
+ }
+
+ @After
+ public void teardown() {
+ defaultPackageUtil.close();
}
private File makeFileWithContents(String name, String contents) throws Exception {
@@ -133,8 +138,14 @@
static final GcsPath STAGING_GCS_PATH = GcsPath.fromComponents("somebucket", "base/path/");
static final String STAGING_PATH = STAGING_GCS_PATH.toString();
- private static PackageAttributes makePackageAttributes(File file, String overridePackageName) {
- return PackageUtil.createPackageAttributes(file, STAGING_PATH, overridePackageName);
+
+ private static PackageAttributes makePackageAttributes(
+ File file, @Nullable String overridePackageName) throws IOException {
+ PackageAttributes attributes = PackageUtil.PackageAttributes.forFileToStage(file, STAGING_PATH);
+ if (overridePackageName != null) {
+ attributes = attributes.withPackageName(overridePackageName);
+ }
+ return attributes;
}
@Test
@@ -142,7 +153,7 @@
String contents = "This is a test!";
File tmpFile = makeFileWithContents("file.txt", contents);
PackageAttributes attr = makePackageAttributes(tmpFile, null);
- DataflowPackage target = attr.getDataflowPackage();
+ DataflowPackage target = attr.getDestination();
assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
@@ -152,7 +163,7 @@
@Test
public void testPackageNamingWithFileNoExtension() throws Exception {
File tmpFile = makeFileWithContents("file", "This is a test!");
- DataflowPackage target = makePackageAttributes(tmpFile, null).getDataflowPackage();
+ DataflowPackage target = makePackageAttributes(tmpFile, null).getDestination();
assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN));
assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
@@ -161,7 +172,7 @@
@Test
public void testPackageNamingWithDirectory() throws Exception {
File tmpDirectory = tmpFolder.newFolder("folder");
- DataflowPackage target = makePackageAttributes(tmpDirectory, null).getDataflowPackage();
+ DataflowPackage target = makePackageAttributes(tmpDirectory, null).getDestination();
assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
@@ -171,11 +182,11 @@
public void testPackageNamingWithFilesHavingSameContentsAndSameNames() throws Exception {
File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
makeFileWithContents("folder1/folderA/sameName", "This is a test!");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+ DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDestination();
File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
makeFileWithContents("folder2/folderA/sameName", "This is a test!");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+ DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDestination();
assertEquals(target1.getName(), target2.getName());
assertEquals(target1.getLocation(), target2.getLocation());
@@ -185,11 +196,11 @@
public void testPackageNamingWithFilesHavingSameContentsButDifferentNames() throws Exception {
File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
makeFileWithContents("folder1/folderA/uniqueName1", "This is a test!");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+ DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDestination();
File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
makeFileWithContents("folder2/folderA/uniqueName2", "This is a test!");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+ DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDestination();
assertNotEquals(target1.getName(), target2.getName());
assertNotEquals(target1.getLocation(), target2.getLocation());
@@ -200,11 +211,11 @@
throws Exception {
File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
tmpFolder.newFolder("folder1", "folderA", "uniqueName1");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+ DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDestination();
File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
tmpFolder.newFolder("folder2", "folderA", "uniqueName2");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+ DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDestination();
assertNotEquals(target1.getName(), target2.getName());
assertNotEquals(target1.getLocation(), target2.getLocation());
@@ -224,7 +235,7 @@
classpathElements.add(eltName + '=' + tmpFile.getAbsolutePath());
}
- PackageUtil.stageClasspathElements(classpathElements, STAGING_PATH, createOptions);
+ defaultPackageUtil.stageClasspathElements(classpathElements, STAGING_PATH, createOptions);
logged.verifyWarn("Your classpath contains 1005 elements, which Google Cloud Dataflow");
}
@@ -239,8 +250,9 @@
when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
+ List<DataflowPackage> targets =
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
DataflowPackage target = Iterables.getOnlyElement(targets);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -269,9 +281,11 @@
}
});
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(smallFile.getAbsolutePath(), largeFile.getAbsolutePath()),
- STAGING_PATH, createOptions);
+ List<DataflowPackage> targets =
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(smallFile.getAbsolutePath(), largeFile.getAbsolutePath()),
+ STAGING_PATH,
+ createOptions);
// Verify that the packages are returned small, then large, matching input order even though
// the large file would be uploaded first.
assertThat(targets.get(0).getName(), startsWith("small"));
@@ -292,7 +306,7 @@
new FileNotFoundException("some/path"))));
when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
- PackageUtil.stageClasspathElements(
+ defaultPackageUtil.stageClasspathElements(
ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -320,8 +334,9 @@
new FileNotFoundException("some/path"))));
when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
+ List<DataflowPackage> targets =
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
DataflowPackage target = Iterables.getOnlyElement(targets);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -342,10 +357,12 @@
when(mockGcsUtil.create(any(GcsPath.class), anyString()))
.thenThrow(new IOException("Fake Exception: Upload error"));
- try {
- PackageUtil.stageClasspathElements(
+ try (PackageUtil directPackageUtil =
+ PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
+ directPackageUtil.stageClasspathElements(
ImmutableList.of(tmpFile.getAbsolutePath()),
- STAGING_PATH, fastNanoClockAndSleeper, MoreExecutors.newDirectExecutorService(),
+ STAGING_PATH,
+ fastNanoClockAndSleeper,
createOptions);
} finally {
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -365,10 +382,12 @@
googleJsonResponseException(
HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Permission denied", "Test message")));
- try {
- PackageUtil.stageClasspathElements(
+ try (PackageUtil directPackageUtil =
+ PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
+ directPackageUtil.stageClasspathElements(
ImmutableList.of(tmpFile.getAbsolutePath()),
- STAGING_PATH, fastNanoClockAndSleeper, MoreExecutors.newDirectExecutorService(),
+ STAGING_PATH,
+ fastNanoClockAndSleeper,
createOptions);
fail("Expected RuntimeException");
} catch (RuntimeException e) {
@@ -400,10 +419,13 @@
.thenThrow(new IOException("Fake Exception: 410 Gone")) // First attempt fails
.thenReturn(pipe.sink()); // second attempt succeeds
- try {
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, fastNanoClockAndSleeper,
- MoreExecutors.newDirectExecutorService(), createOptions);
+ try (PackageUtil directPackageUtil =
+ PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) {
+ directPackageUtil.stageClasspathElements(
+ ImmutableList.of(tmpFile.getAbsolutePath()),
+ STAGING_PATH,
+ fastNanoClockAndSleeper,
+ createOptions);
} finally {
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
verify(mockGcsUtil, times(2)).create(any(GcsPath.class), anyString());
@@ -418,8 +440,8 @@
.thenReturn(ImmutableList.of(StorageObjectOrIOException.create(
createStorageObject(STAGING_PATH, tmpFile.length()))));
- PackageUtil.stageClasspathElements(ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH,
- createOptions);
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
verifyNoMoreInteractions(mockGcsUtil);
@@ -438,7 +460,7 @@
createStorageObject(STAGING_PATH, Long.MAX_VALUE))));
when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
- PackageUtil.stageClasspathElements(
+ defaultPackageUtil.stageClasspathElements(
ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -457,9 +479,11 @@
new FileNotFoundException("some/path"))));
when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()), STAGING_PATH,
- createOptions);
+ List<DataflowPackage> targets =
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()),
+ STAGING_PATH,
+ createOptions);
DataflowPackage target = Iterables.getOnlyElement(targets);
verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
@@ -473,10 +497,14 @@
@Test
public void testPackageUploadIsSkippedWithNonExistentResource() throws Exception {
- String nonExistentFile = FileSystems.matchNewResource(tmpFolder.getRoot().getPath(), true)
- .resolve("non-existent-file", StandardResolveOptions.RESOLVE_FILE).toString();
- assertEquals(Collections.EMPTY_LIST, PackageUtil.stageClasspathElements(
- ImmutableList.of(nonExistentFile), STAGING_PATH, createOptions));
+ String nonExistentFile =
+ FileSystems.matchNewResource(tmpFolder.getRoot().getPath(), true)
+ .resolve("non-existent-file", StandardResolveOptions.RESOLVE_FILE)
+ .toString();
+ assertEquals(
+ Collections.EMPTY_LIST,
+ defaultPackageUtil.stageClasspathElements(
+ ImmutableList.of(nonExistentFile), STAGING_PATH, createOptions));
}
/**
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java
index e0785d4..1ac9fab 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java
@@ -47,8 +47,14 @@
assertEquals(new Instant(1), fromCloudTime("1970-01-01T00:00:00.001001Z"));
assertEquals(new Instant(1), fromCloudTime("1970-01-01T00:00:00.001000000Z"));
assertEquals(new Instant(1), fromCloudTime("1970-01-01T00:00:00.001000001Z"));
+ assertEquals(new Instant(0), fromCloudTime("1970-01-01T00:00:00.0Z"));
+ assertEquals(new Instant(0), fromCloudTime("1970-01-01T00:00:00.00Z"));
+ assertEquals(new Instant(420), fromCloudTime("1970-01-01T00:00:00.42Z"));
+ assertEquals(new Instant(300), fromCloudTime("1970-01-01T00:00:00.3Z"));
+ assertEquals(new Instant(20), fromCloudTime("1970-01-01T00:00:00.02Z"));
assertNull(fromCloudTime(""));
assertNull(fromCloudTime("1970-01-01T00:00:00"));
+ assertNull(fromCloudTime("1970-01-01T00:00:00.1e3Z"));
}
@Test
diff --git a/runners/java-fn-execution/pom.xml b/runners/java-fn-execution/pom.xml
new file mode 100644
index 0000000..6ff08b7
--- /dev/null
+++ b/runners/java-fn-execution/pom.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-java-fn-execution</artifactId>
+
+ <name>Apache Beam :: Runners :: Java Fn Execution</name>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <exclude>com.google.guava:guava-jdk5</exclude>
+ <exclude>com.google.protobuf:protobuf-lite</exclude>
+ <exclude>org.apache.beam:beam-sdks-java-core</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/ServerFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/ServerFactory.java
new file mode 100644
index 0000000..918672a
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/ServerFactory.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.fnexecution;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.net.HostAndPort;
+import io.grpc.BindableService;
+import io.grpc.Server;
+import io.grpc.netty.NettyServerBuilder;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import org.apache.beam.harness.channel.SocketAddressFactory;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+
+/**
+ * A {@link Server gRPC server} factory.
+ */
+public abstract class ServerFactory {
+ /**
+ * Create a default {@link ServerFactory}.
+ */
+ public static ServerFactory createDefault() {
+ return new InetSocketAddressServerFactory();
+ }
+
+ /**
+ * Creates an instance of this server using an ephemeral port chosen automatically. The chosen
+ * port is accessible to the caller from the URL set in the input {@link
+ * Endpoints.ApiServiceDescriptor.Builder}.
+ */
+ public abstract Server allocatePortAndCreate(
+ BindableService service, Endpoints.ApiServiceDescriptor.Builder builder) throws IOException;
+
+ /**
+ * Creates an instance of this server at the address specified by the given service descriptor.
+ */
+ public abstract Server create(
+ BindableService service, Endpoints.ApiServiceDescriptor serviceDescriptor) throws IOException;
+
+ /**
+ * Creates a {@link Server gRPC Server} using the default server factory.
+ *
+ * <p>The server is created listening any open port on "localhost".
+ */
+ public static class InetSocketAddressServerFactory extends ServerFactory {
+ private InetSocketAddressServerFactory() {}
+
+ @Override
+ public Server allocatePortAndCreate(
+ BindableService service, Endpoints.ApiServiceDescriptor.Builder apiServiceDescriptor)
+ throws IOException {
+ InetSocketAddress address = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
+ Server server = createServer(service, address);
+ apiServiceDescriptor.setUrl(
+ HostAndPort.fromParts(address.getHostName(), server.getPort()).toString());
+ return server;
+ }
+
+ @Override
+ public Server create(BindableService service, Endpoints.ApiServiceDescriptor serviceDescriptor)
+ throws IOException {
+ SocketAddress socketAddress = SocketAddressFactory.createFrom(serviceDescriptor.getUrl());
+ checkArgument(
+ socketAddress instanceof InetSocketAddress,
+ "%s %s requires a host:port socket address, got %s",
+ getClass().getSimpleName(), ServerFactory.class.getSimpleName(),
+ serviceDescriptor.getUrl());
+ return createServer(service, (InetSocketAddress) socketAddress);
+ }
+
+ private static Server createServer(BindableService service, InetSocketAddress socket)
+ throws IOException {
+ Server server =
+ NettyServerBuilder.forPort(socket.getPort())
+ .addService(service)
+ // Set the message size to max value here. The actual size is governed by the
+ // buffer size in the layers above.
+ .maxMessageSize(Integer.MAX_VALUE)
+ .build();
+ server.start();
+ return server;
+ }
+
+ }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClient.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClient.java
new file mode 100644
index 0000000..8133988
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClient.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import io.grpc.stub.StreamObserver;
+import java.io.Closeable;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A client for the control plane of an SDK harness, which can issue requests to it over the Fn API.
+ *
+ * <p>This class presents a low-level Java API de-inverting the Fn API's gRPC layer.
+ *
+ * <p>The Fn API is inverted so the runner is the server and the SDK harness is the client, for
+ * firewalling reasons (the runner may execute in a more privileged environment forbidding outbound
+ * connections).
+ *
+ * <p>This low-level client is responsible only for correlating requests with responses.
+ */
+class FnApiControlClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(FnApiControlClient.class);
+
+ // All writes to this StreamObserver need to be synchronized.
+ private final StreamObserver<BeamFnApi.InstructionRequest> requestReceiver;
+ private final ResponseStreamObserver responseObserver = new ResponseStreamObserver();
+ private final Map<String, SettableFuture<BeamFnApi.InstructionResponse>> outstandingRequests;
+ private volatile boolean isClosed;
+
+ private FnApiControlClient(StreamObserver<BeamFnApi.InstructionRequest> requestReceiver) {
+ this.requestReceiver = requestReceiver;
+ this.outstandingRequests = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Returns a {@link FnApiControlClient} which will submit its requests to the provided
+ * observer.
+ *
+ * <p>It is the responsibility of the caller to register this object as an observer of incoming
+ * responses (this will generally be done as part of fulfilling the contract of a gRPC service).
+ */
+ public static FnApiControlClient forRequestObserver(
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver) {
+ return new FnApiControlClient(requestObserver);
+ }
+
+ public synchronized ListenableFuture<BeamFnApi.InstructionResponse> handle(
+ BeamFnApi.InstructionRequest request) {
+ LOG.debug("Sending InstructionRequest {}", request);
+ SettableFuture<BeamFnApi.InstructionResponse> resultFuture = SettableFuture.create();
+ outstandingRequests.put(request.getInstructionId(), resultFuture);
+ requestReceiver.onNext(request);
+ return resultFuture;
+ }
+
+ StreamObserver<BeamFnApi.InstructionResponse> asResponseObserver() {
+ return responseObserver;
+ }
+
+ @Override
+ public void close() {
+ closeAndTerminateOutstandingRequests(new IllegalStateException("Runner closed connection"));
+ }
+
+ /** Closes this client and terminates any outstanding requests exceptionally. */
+ private synchronized void closeAndTerminateOutstandingRequests(Throwable cause) {
+ if (isClosed) {
+ return;
+ }
+
+ // Make a copy of the map to make the view of the outstanding requests consistent.
+ Map<String, SettableFuture<BeamFnApi.InstructionResponse>> outstandingRequestsCopy =
+ new ConcurrentHashMap<>(outstandingRequests);
+ outstandingRequests.clear();
+ isClosed = true;
+
+ if (outstandingRequestsCopy.isEmpty()) {
+ requestReceiver.onCompleted();
+ return;
+ }
+ requestReceiver.onError(
+ new StatusRuntimeException(Status.CANCELLED.withDescription(cause.getMessage())));
+
+ LOG.error(
+ "{} closed, clearing outstanding requests {}",
+ FnApiControlClient.class.getSimpleName(),
+ outstandingRequestsCopy);
+ for (SettableFuture<BeamFnApi.InstructionResponse> outstandingRequest :
+ outstandingRequestsCopy.values()) {
+ outstandingRequest.setException(cause);
+ }
+ }
+
+ /**
+ * A private view of this class as a {@link StreamObserver} for connecting as a gRPC listener.
+ */
+ private class ResponseStreamObserver implements StreamObserver<BeamFnApi.InstructionResponse> {
+ /**
+ * Processes an incoming {@link BeamFnApi.InstructionResponse} by correlating it with the
+ * corresponding {@link BeamFnApi.InstructionRequest} and completes the future that was returned
+ * by {@link #handle}.
+ */
+ @Override
+ public void onNext(BeamFnApi.InstructionResponse response) {
+ LOG.debug("Received InstructionResponse {}", response);
+ SettableFuture<BeamFnApi.InstructionResponse> completableFuture =
+ outstandingRequests.remove(response.getInstructionId());
+ if (completableFuture != null) {
+ completableFuture.set(response);
+ }
+ }
+
+ /** */
+ @Override
+ public void onCompleted() {
+ closeAndTerminateOutstandingRequests(
+ new IllegalStateException("SDK harness closed connection"));
+ }
+
+ @Override
+ public void onError(Throwable cause) {
+ LOG.error("{} received error {}", FnApiControlClient.class.getSimpleName(), cause);
+ closeAndTerminateOutstandingRequests(cause);
+ }
+ }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java
new file mode 100644
index 0000000..37fae00
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.BlockingQueue;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A Fn API control service which adds incoming SDK harness connections to a pool. */
+public class FnApiControlClientPoolService extends BeamFnControlGrpc.BeamFnControlImplBase {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FnApiControlClientPoolService.class);
+
+ private final BlockingQueue<FnApiControlClient> clientPool;
+
+ private FnApiControlClientPoolService(BlockingQueue<FnApiControlClient> clientPool) {
+ this.clientPool = clientPool;
+ }
+
+ /**
+ * Creates a new {@link FnApiControlClientPoolService} which will enqueue and vend new SDK harness
+ * connections.
+ */
+ public static FnApiControlClientPoolService offeringClientsToPool(
+ BlockingQueue<FnApiControlClient> clientPool) {
+ return new FnApiControlClientPoolService(clientPool);
+ }
+
+ /**
+ * Called by gRPC for each incoming connection from an SDK harness, and enqueue an available SDK
+ * harness client.
+ *
+ * <p>Note: currently does not distinguish what sort of SDK it is, so a separate instance is
+ * required for each.
+ */
+ @Override
+ public StreamObserver<BeamFnApi.InstructionResponse> control(
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver) {
+ LOGGER.info("Beam Fn Control client connected.");
+ FnApiControlClient newClient = FnApiControlClient.forRequestObserver(requestObserver);
+ try {
+ clientPool.put(newClient);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ return newClient.asResponseObserver();
+ }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java
new file mode 100644
index 0000000..5b47a58
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Function;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import java.io.IOException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.runners.fnexecution.data.FnDataReceiver;
+
+/**
+ * A high-level client for an SDK harness.
+ *
+ * <p>This provides a Java-friendly wrapper around {@link FnApiControlClient} and {@link
+ * FnDataReceiver}, which handle lower-level gRPC message wrangling.
+ */
+public class SdkHarnessClient {
+
+ /**
+ * A supply of unique identifiers, used internally. These must be unique across all Fn API
+ * clients.
+ */
+ public interface IdGenerator {
+ String getId();
+ }
+
+ /** A supply of unique identifiers that are simply incrementing longs. */
+ private static class CountingIdGenerator implements IdGenerator {
+ private final AtomicLong nextId = new AtomicLong(0L);
+
+ @Override
+ public String getId() {
+ return String.valueOf(nextId.incrementAndGet());
+ }
+ }
+
+ /**
+ * An active bundle for a particular {@link
+ * BeamFnApi.ProcessBundleDescriptor}.
+ */
+ @AutoValue
+ public abstract static class ActiveBundle<InputT> {
+ public abstract String getBundleId();
+
+ public abstract Future<BeamFnApi.ProcessBundleResponse> getBundleResponse();
+
+ public abstract FnDataReceiver<InputT> getInputReceiver();
+
+ public static <InputT> ActiveBundle<InputT> create(
+ String bundleId,
+ Future<BeamFnApi.ProcessBundleResponse> response,
+ FnDataReceiver<InputT> dataReceiver) {
+ return new AutoValue_SdkHarnessClient_ActiveBundle(bundleId, response, dataReceiver);
+ }
+ }
+
+ private final IdGenerator idGenerator;
+ private final FnApiControlClient fnApiControlClient;
+
+ private SdkHarnessClient(
+ FnApiControlClient fnApiControlClient,
+ IdGenerator idGenerator) {
+ this.idGenerator = idGenerator;
+ this.fnApiControlClient = fnApiControlClient;
+ }
+
+ /**
+ * Creates a client for a particular SDK harness. It is the responsibility of the caller to ensure
+ * that these correspond to the same SDK harness, so control plane and data plane messages can be
+ * correctly associated.
+ */
+ public static SdkHarnessClient usingFnApiClient(FnApiControlClient fnApiControlClient) {
+ return new SdkHarnessClient(fnApiControlClient, new CountingIdGenerator());
+ }
+
+ public SdkHarnessClient withIdGenerator(IdGenerator idGenerator) {
+ return new SdkHarnessClient(fnApiControlClient, idGenerator);
+ }
+
+ /**
+ * Registers a {@link BeamFnApi.ProcessBundleDescriptor} for future
+ * processing.
+ *
+ * <p>A client may block on the result future, but may also proceed without blocking.
+ */
+ public Future<BeamFnApi.RegisterResponse> register(
+ Iterable<BeamFnApi.ProcessBundleDescriptor> processBundleDescriptors) {
+
+ // TODO: validate that all the necessary data endpoints are known
+
+ ListenableFuture<BeamFnApi.InstructionResponse> genericResponse =
+ fnApiControlClient.handle(
+ BeamFnApi.InstructionRequest.newBuilder()
+ .setInstructionId(idGenerator.getId())
+ .setRegister(
+ BeamFnApi.RegisterRequest.newBuilder()
+ .addAllProcessBundleDescriptor(processBundleDescriptors)
+ .build())
+ .build());
+
+ return Futures.transform(
+ genericResponse,
+ new Function<BeamFnApi.InstructionResponse, BeamFnApi.RegisterResponse>() {
+ @Override
+ public BeamFnApi.RegisterResponse apply(BeamFnApi.InstructionResponse input) {
+ return input.getRegister();
+ }
+ });
+ }
+
+ /**
+ * Start a new bundle for the given {@link
+ * BeamFnApi.ProcessBundleDescriptor} identifier.
+ *
+ * <p>The input channels for the returned {@link ActiveBundle} are derived from the
+ * instructions in the {@link BeamFnApi.ProcessBundleDescriptor}.
+ */
+ public ActiveBundle newBundle(String processBundleDescriptorId) {
+ String bundleId = idGenerator.getId();
+
+ // TODO: acquire an input receiver from appropriate FnDataService
+ FnDataReceiver dataReceiver = new FnDataReceiver() {
+ @Override
+ public void accept(Object input) throws Exception {
+ throw new UnsupportedOperationException("Placeholder FnDataReceiver cannot accept data.");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // noop
+ }
+ };
+
+ ListenableFuture<BeamFnApi.InstructionResponse> genericResponse =
+ fnApiControlClient.handle(
+ BeamFnApi.InstructionRequest.newBuilder()
+ .setProcessBundle(
+ BeamFnApi.ProcessBundleRequest.newBuilder()
+ .setProcessBundleDescriptorReference(processBundleDescriptorId))
+ .build());
+
+ ListenableFuture<BeamFnApi.ProcessBundleResponse> specificResponse =
+ Futures.transform(
+ genericResponse,
+ new Function<BeamFnApi.InstructionResponse, BeamFnApi.ProcessBundleResponse>() {
+ @Override
+ public BeamFnApi.ProcessBundleResponse apply(BeamFnApi.InstructionResponse input) {
+ return input.getProcessBundle();
+ }
+ });
+
+ return ActiveBundle.create(bundleId, specificResponse, dataReceiver);
+ }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/package-info.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/package-info.java
new file mode 100644
index 0000000..791faa2
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for a Beam runner to interact with the Fn API {@link
+ * org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc Control Service} via java abstractions.
+ */
+package org.apache.beam.runners.fnexecution.control;
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/FnDataReceiver.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/FnDataReceiver.java
new file mode 100644
index 0000000..5573d94
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/FnDataReceiver.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.data;
+
+import java.io.Closeable;
+
+/**
+ * A receiver of streamed data.
+ */
+public interface FnDataReceiver<T> extends Closeable {
+ void accept(T input) throws Exception;
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/package-info.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/package-info.java
new file mode 100644
index 0000000..4c0a269
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for a Beam runner to interact with the Fn API {@link
+ * org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc Data Service} via java abstractions.
+ */
+package org.apache.beam.runners.fnexecution.data;
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/package-info.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/package-info.java
new file mode 100644
index 0000000..bc36f5e
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities used by runners to interact with the fn execution components of the Beam Portability
+ * Framework.
+ */
+package org.apache.beam.runners.fnexecution;
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
new file mode 100644
index 0000000..b78e88a
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.fnexecution;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.Uninterruptibles;
+import io.grpc.ManagedChannel;
+import io.grpc.Server;
+import io.grpc.stub.CallStreamObserver;
+import io.grpc.stub.StreamObserver;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.beam.harness.channel.ManagedChannelFactory;
+import org.apache.beam.harness.test.Consumer;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements;
+import org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.junit.Test;
+
+/**
+ * Tests for {@link ServerFactory}.
+ */
+public class ServerFactoryTest {
+
+ private static final BeamFnApi.Elements CLIENT_DATA = BeamFnApi.Elements.newBuilder()
+ .addData(BeamFnApi.Elements.Data.newBuilder().setInstructionReference("1"))
+ .build();
+ private static final BeamFnApi.Elements SERVER_DATA = BeamFnApi.Elements.newBuilder()
+ .addData(BeamFnApi.Elements.Data.newBuilder().setInstructionReference("1"))
+ .build();
+
+ @Test
+ public void testCreatingDefaultServer() throws Exception {
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ runTestUsing(ServerFactory.createDefault(), ManagedChannelFactory.createDefault());
+ HostAndPort hostAndPort = HostAndPort.fromString(apiServiceDescriptor.getUrl());
+ assertThat(hostAndPort.getHost(), anyOf(
+ equalTo(InetAddress.getLoopbackAddress().getHostName()),
+ equalTo(InetAddress.getLoopbackAddress().getHostAddress())));
+ assertThat(hostAndPort.getPort(), allOf(greaterThan(0), lessThan(65536)));
+ }
+
+ private Endpoints.ApiServiceDescriptor runTestUsing(
+ ServerFactory serverFactory, ManagedChannelFactory channelFactory) throws Exception {
+ Endpoints.ApiServiceDescriptor.Builder apiServiceDescriptorBuilder =
+ Endpoints.ApiServiceDescriptor.newBuilder();
+
+ final Collection<Elements> serverElements = new ArrayList<>();
+ final CountDownLatch clientHangedUp = new CountDownLatch(1);
+ CallStreamObserver<Elements> serverInboundObserver =
+ TestStreams.withOnNext(
+ new Consumer<Elements>() {
+ @Override
+ public void accept(Elements item) {
+ serverElements.add(item);
+ }
+ })
+ .withOnCompleted(
+ new Runnable() {
+ @Override
+ public void run() {
+ clientHangedUp.countDown();
+ }
+ })
+ .build();
+ TestDataService service = new TestDataService(serverInboundObserver);
+ Server server = serverFactory.allocatePortAndCreate(service, apiServiceDescriptorBuilder);
+ assertFalse(server.isShutdown());
+
+ ManagedChannel channel = channelFactory.forDescriptor(apiServiceDescriptorBuilder.build());
+ BeamFnDataGrpc.BeamFnDataStub stub = BeamFnDataGrpc.newStub(channel);
+ final Collection<BeamFnApi.Elements> clientElements = new ArrayList<>();
+ final CountDownLatch serverHangedUp = new CountDownLatch(1);
+ CallStreamObserver<BeamFnApi.Elements> clientInboundObserver =
+ TestStreams.withOnNext(
+ new Consumer<Elements>() {
+ @Override
+ public void accept(Elements item) {
+ clientElements.add(item);
+ }
+ })
+ .withOnCompleted(
+ new Runnable() {
+ @Override
+ public void run() {
+ serverHangedUp.countDown();
+ }
+ })
+ .build();
+
+ StreamObserver<Elements> clientOutboundObserver = stub.data(clientInboundObserver);
+ StreamObserver<BeamFnApi.Elements> serverOutboundObserver = service.outboundObservers.take();
+
+ clientOutboundObserver.onNext(CLIENT_DATA);
+ serverOutboundObserver.onNext(SERVER_DATA);
+ clientOutboundObserver.onCompleted();
+ clientHangedUp.await();
+ serverOutboundObserver.onCompleted();
+ serverHangedUp.await();
+
+ assertThat(clientElements, contains(SERVER_DATA));
+ assertThat(serverElements, contains(CLIENT_DATA));
+
+ return apiServiceDescriptorBuilder.build();
+ }
+
+ /** A test gRPC service that uses the provided inbound observer for all clients. */
+ private static class TestDataService extends BeamFnDataGrpc.BeamFnDataImplBase {
+ private final LinkedBlockingQueue<StreamObserver<BeamFnApi.Elements>> outboundObservers;
+ private final StreamObserver<BeamFnApi.Elements> inboundObserver;
+ private TestDataService(StreamObserver<BeamFnApi.Elements> inboundObserver) {
+ this.inboundObserver = inboundObserver;
+ this.outboundObservers = new LinkedBlockingQueue<>();
+ }
+
+ @Override
+ public StreamObserver<BeamFnApi.Elements> data(
+ StreamObserver<BeamFnApi.Elements> outboundObserver) {
+ Uninterruptibles.putUninterruptibly(outboundObservers, outboundObserver);
+ return inboundObserver;
+ }
+ }
+}
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
new file mode 100644
index 0000000..9392ee0
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link FnApiControlClientPoolService}. */
+@RunWith(JUnit4.class)
+public class FnApiControlClientPoolServiceTest {
+
+ // For ease of straight-line testing, we use a LinkedBlockingQueue; in practice a SynchronousQueue
+ // for matching incoming connections and server threads is likely.
+ private final BlockingQueue<FnApiControlClient> pool = new LinkedBlockingQueue<>();
+ private FnApiControlClientPoolService controlService =
+ FnApiControlClientPoolService.offeringClientsToPool(pool);
+
+ @Test
+ public void testIncomingConnection() throws Exception {
+ StreamObserver<BeamFnApi.InstructionRequest> requestObserver = mock(StreamObserver.class);
+ StreamObserver<BeamFnApi.InstructionResponse> responseObserver =
+ controlService.control(requestObserver);
+
+ FnApiControlClient client = pool.take();
+
+ // Check that the client is wired up to the request channel
+ String id = "fakeInstruction";
+ ListenableFuture<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+ verify(requestObserver).onNext(any(BeamFnApi.InstructionRequest.class));
+ assertThat(responseFuture.isDone(), is(false));
+
+ // Check that the response channel really came from the client
+ responseObserver.onNext(
+ BeamFnApi.InstructionResponse.newBuilder().setInstructionId(id).build());
+ responseFuture.get();
+ }
+}
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
new file mode 100644
index 0000000..4732f5e
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isA;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import io.grpc.stub.StreamObserver;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Unit tests for {@link FnApiControlClient}. */
+@RunWith(JUnit4.class)
+public class FnApiControlClientTest {
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Mock public StreamObserver<BeamFnApi.InstructionRequest> mockObserver;
+ private FnApiControlClient client;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ client = FnApiControlClient.forRequestObserver(mockObserver);
+ }
+
+ @Test
+ public void testRequestSent() {
+ String id = "instructionId";
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ verify(mockObserver).onNext(any(BeamFnApi.InstructionRequest.class));
+ }
+
+ @Test
+ public void testRequestSuccess() throws Exception {
+ String id = "successfulInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+ client
+ .asResponseObserver()
+ .onNext(BeamFnApi.InstructionResponse.newBuilder().setInstructionId(id).build());
+
+ BeamFnApi.InstructionResponse response = responseFuture.get();
+
+ assertThat(response.getInstructionId(), equalTo(id));
+ }
+
+ @Test
+ public void testUnknownResponseIgnored() throws Exception {
+ String id = "actualInstruction";
+ String unknownId = "unknownInstruction";
+
+ ListenableFuture<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client
+ .asResponseObserver()
+ .onNext(BeamFnApi.InstructionResponse.newBuilder().setInstructionId(unknownId).build());
+
+ assertThat(responseFuture.isDone(), is(false));
+ assertThat(responseFuture.isCancelled(), is(false));
+ }
+
+ @Test
+ public void testOnCompletedCancelsOutstanding() throws Exception {
+ String id = "clientHangUpInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client.asResponseObserver().onCompleted();
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(IllegalStateException.class));
+ thrown.expectMessage("closed");
+ responseFuture.get();
+ }
+
+ @Test
+ public void testOnErrorCancelsOutstanding() throws Exception {
+ String id = "errorInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ class FrazzleException extends Exception {}
+ client.asResponseObserver().onError(new FrazzleException());
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(FrazzleException.class));
+ responseFuture.get();
+ }
+
+ @Test
+ public void testCloseCancelsOutstanding() throws Exception {
+ String id = "serverCloseInstruction";
+
+ Future<BeamFnApi.InstructionResponse> responseFuture =
+ client.handle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId(id).build());
+
+ client.close();
+
+ thrown.expect(ExecutionException.class);
+ thrown.expectCause(isA(IllegalStateException.class));
+ thrown.expectMessage("closed");
+ responseFuture.get();
+ }
+}
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
new file mode 100644
index 0000000..09437c7
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.control;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.SettableFuture;
+import java.util.concurrent.Future;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Unit tests for {@link SdkHarnessClient}. */
+@RunWith(JUnit4.class)
+public class SdkHarnessClientTest {
+
+ @Mock public FnApiControlClient fnApiControlClient;
+
+ private SdkHarnessClient sdkHarnessClient;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ sdkHarnessClient = SdkHarnessClient.usingFnApiClient(fnApiControlClient);
+ }
+
+ @Test
+ public void testRegisterDoesNotCrash() throws Exception {
+ String descriptorId1 = "descriptor1";
+ String descriptorId2 = "descriptor2";
+
+ SettableFuture<BeamFnApi.InstructionResponse> registerResponseFuture = SettableFuture.create();
+ when(fnApiControlClient.handle(any(BeamFnApi.InstructionRequest.class)))
+ .thenReturn(registerResponseFuture);
+
+ Future<BeamFnApi.RegisterResponse> responseFuture = sdkHarnessClient.register(
+ ImmutableList.of(
+ BeamFnApi.ProcessBundleDescriptor.newBuilder().setId(descriptorId1).build(),
+ BeamFnApi.ProcessBundleDescriptor.newBuilder().setId(descriptorId2).build()));
+
+ // Correlating the RegisterRequest and RegisterResponse is owned by the underlying
+ // FnApiControlClient. The SdkHarnessClient owns just wrapping the request and unwrapping
+ // the response.
+ //
+ // Currently there are no fields so there's nothing to check. This test is formulated
+ // to match the pattern it should have if/when the response is meaningful.
+ BeamFnApi.RegisterResponse response = BeamFnApi.RegisterResponse.getDefaultInstance();
+ registerResponseFuture.set(
+ BeamFnApi.InstructionResponse.newBuilder().setRegister(response).build());
+ responseFuture.get();
+ }
+
+ @Test
+ public void testNewBundleNoDataDoesNotCrash() throws Exception {
+ String descriptorId1 = "descriptor1";
+
+ SettableFuture<BeamFnApi.InstructionResponse> processBundleResponseFuture =
+ SettableFuture.create();
+ when(fnApiControlClient.handle(any(BeamFnApi.InstructionRequest.class)))
+ .thenReturn(processBundleResponseFuture);
+
+ SdkHarnessClient.ActiveBundle activeBundle = sdkHarnessClient.newBundle(descriptorId1);
+
+ // Correlating the ProcessBundleRequest and ProcessBundleReponse is owned by the underlying
+ // FnApiControlClient. The SdkHarnessClient owns just wrapping the request and unwrapping
+ // the response.
+ //
+ // Currently there are no fields so there's nothing to check. This test is formulated
+ // to match the pattern it should have if/when the response is meaningful.
+ BeamFnApi.ProcessBundleResponse response = BeamFnApi.ProcessBundleResponse.getDefaultInstance();
+ processBundleResponseFuture.set(
+ BeamFnApi.InstructionResponse.newBuilder().setProcessBundle(response).build());
+ activeBundle.getBundleResponse().get();
+ }
+}
diff --git a/runners/jstorm/pom.xml b/runners/jstorm/pom.xml
index a433fcb..b6c0b5e 100644
--- a/runners/jstorm/pom.xml
+++ b/runners/jstorm/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/DefaultStepContext.java b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/DefaultStepContext.java
index 9fd584b..3d18697 100644
--- a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/DefaultStepContext.java
+++ b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/DefaultStepContext.java
@@ -19,19 +19,14 @@
import static com.google.common.base.Preconditions.checkNotNull;
-import java.io.IOException;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
/**
* Default StepContext for running DoFn This does not allow accessing state or timer internals.
*/
-class DefaultStepContext implements ExecutionContext.StepContext {
+class DefaultStepContext implements StepContext {
private TimerInternals timerInternals;
@@ -43,34 +38,6 @@
}
@Override
- public String getStepName() {
- return null;
- }
-
- @Override
- public String getTransformName() {
- return null;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> windowedValue) {
-
- }
-
- @Override
- public void noteOutput(TupleTag<?> tupleTag, WindowedValue<?> windowedValue) {
-
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag, Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder, W window, Coder<W> windowCoder)
- throws IOException {
- throw new UnsupportedOperationException("Writing side-input data is not supported.");
- }
-
- @Override
public StateInternals stateInternals() {
return stateInternals;
}
diff --git a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormMetricResults.java b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormMetricResults.java
index 01d4441..79e9919 100644
--- a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormMetricResults.java
+++ b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormMetricResults.java
@@ -30,8 +30,8 @@
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
-import org.apache.beam.runners.core.metrics.MetricFiltering;
-import org.apache.beam.runners.core.metrics.MetricKey;
+import org.apache.beam.runners.core.construction.metrics.MetricFiltering;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.GaugeResult;
import org.apache.beam.sdk.metrics.MetricName;
diff --git a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormPipelineTranslator.java b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormPipelineTranslator.java
index 298ad32..7434364 100644
--- a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormPipelineTranslator.java
+++ b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormPipelineTranslator.java
@@ -121,7 +121,7 @@
@SuppressWarnings("unchecked")
TransformTranslator<T> typedTranslator = (TransformTranslator<T>) translator;
- context.getUserGraphContext().setCurrentTransform(node.toAppliedPTransform());
+ context.getUserGraphContext().setCurrentTransform(node.toAppliedPTransform(getPipeline()));
typedTranslator.translateNode(typedTransform, context);
// Maintain PValue to TupleTag map for side inputs translation.
@@ -137,7 +137,7 @@
@SuppressWarnings("unchecked")
TransformTranslator<T> typedTranslator = (TransformTranslator<T>) translator;
- context.getUserGraphContext().setCurrentTransform(node.toAppliedPTransform());
+ context.getUserGraphContext().setCurrentTransform(node.toAppliedPTransform(getPipeline()));
return typedTranslator.canTranslate(typedTransform, context);
}
diff --git a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormTimerInternals.java b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormTimerInternals.java
index 0e9ee35..cf82a47 100644
--- a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormTimerInternals.java
+++ b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/JStormTimerInternals.java
@@ -47,6 +47,7 @@
setTimer(TimerData.of(timerId, namespace, target, timeDomain));
}
+ /** @deprecated */
@Override
@Deprecated
public void setTimer(TimerData timerData) {
@@ -59,6 +60,7 @@
"Canceling of a timer is not yet supported.");
}
+ /** @deprecated */
@Override
@Deprecated
public void deleteTimer(StateNamespace namespace, String timerId) {
@@ -66,6 +68,7 @@
"Canceling of a timer is not yet supported.");
}
+ /** @deprecated */
@Override
@Deprecated
public void deleteTimer(TimerData timerData) {
diff --git a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/MetricsReporter.java b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/MetricsReporter.java
index 7867a83..1cbd292 100644
--- a/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/MetricsReporter.java
+++ b/runners/jstorm/src/main/java/org/apache/beam/runners/jstorm/translation/MetricsReporter.java
@@ -26,7 +26,7 @@
import com.alibaba.jstorm.metrics.Gauge;
import com.google.common.collect.Maps;
import java.util.Map;
-import org.apache.beam.runners.core.metrics.MetricKey;
+import org.apache.beam.runners.core.construction.metrics.MetricKey;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.GaugeResult;
diff --git a/runners/local-artifact-service-java/pom.xml b/runners/local-artifact-service-java/pom.xml
new file mode 100644
index 0000000..72490cc
--- /dev/null
+++ b/runners/local-artifact-service-java/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-local-artifact-service-java</artifactId>
+ <name>Apache Beam :: Runners :: Java Local Artifact Service</name>
+ <description>The Beam Artifact Service exposes APIs to stage and retrieve
+ artifacts in a manner independent of the underlying storage system, for use
+ by the Beam portability framework. The local implementation uses the local
+ File System as the underlying storage system.</description>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+
+ <!-- Coverage analysis for unit tests. -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-job-management</artifactId>
+ </dependency>
+
+ <!-- build dependencies -->
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
new file mode 100644
index 0000000..a9f595f
--- /dev/null
+++ b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.artifact.local;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.base.Throwables;
+import io.grpc.Status;
+import io.grpc.StatusException;
+import io.grpc.StatusRuntimeException;
+import io.grpc.stub.StreamObserver;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import javax.annotation.Nullable;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** An {@code ArtifactStagingService} which stages files to a local temp directory. */
+public class LocalFileSystemArtifactStagerService
+ extends ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalFileSystemArtifactStagerService.class);
+
+ public static LocalFileSystemArtifactStagerService withRootDirectory(File base) {
+ return new LocalFileSystemArtifactStagerService(base);
+ }
+
+ private final File stagingBase;
+ private final File artifactsBase;
+
+ private LocalFileSystemArtifactStagerService(File stagingBase) {
+ this.stagingBase = stagingBase;
+ if ((stagingBase.mkdirs() || stagingBase.exists()) && stagingBase.canWrite()) {
+ artifactsBase = new File(stagingBase, "artifacts");
+ checkState(
+ (artifactsBase.mkdir() || artifactsBase.exists()) && artifactsBase.canWrite(),
+ "Could not create artifact staging directory at %s",
+ artifactsBase);
+ } else {
+ throw new IllegalStateException(
+ String.format("Could not create staging directory structure at root %s", stagingBase));
+ }
+ }
+
+ @Override
+ public StreamObserver<ArtifactApi.PutArtifactRequest> putArtifact(
+ final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
+ return new CreateAndWriteFileObserver(responseObserver);
+ }
+
+ @Override
+ public void commitManifest(
+ ArtifactApi.CommitManifestRequest request,
+ StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver) {
+ try {
+ commitManifestOrThrow(request, responseObserver);
+ } catch (StatusRuntimeException e) {
+ responseObserver.onError(e);
+ LOG.error("Failed to commit Manifest {}", request.getManifest(), e);
+ } catch (Exception e) {
+ responseObserver.onError(
+ Status.INTERNAL
+ .withCause(e)
+ .withDescription(Throwables.getStackTraceAsString(e))
+ .asRuntimeException());
+ LOG.error("Failed to commit Manifest {}", request.getManifest(), e);
+ }
+ }
+
+ private void commitManifestOrThrow(
+ ArtifactApi.CommitManifestRequest request,
+ StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver)
+ throws IOException {
+ Collection<ArtifactApi.ArtifactMetadata> missing = new ArrayList<>();
+ for (ArtifactApi.ArtifactMetadata artifact : request.getManifest().getArtifactList()) {
+ // TODO: Validate the checksums on the server side, to fail more aggressively if require
+ if (!getArtifactFile(artifact.getName()).exists()) {
+ missing.add(artifact);
+ }
+ }
+ if (!missing.isEmpty()) {
+ throw Status.INVALID_ARGUMENT
+ .withDescription(
+ String.format("Attempted to commit manifest with missing Artifacts: [%s]", missing))
+ .asRuntimeException();
+ }
+ File mf = new File(stagingBase, "MANIFEST");
+ checkState(mf.createNewFile(), "Could not create file to store manifest");
+ try (OutputStream mfOut = new FileOutputStream(mf)) {
+ request.getManifest().writeTo(mfOut);
+ }
+ responseObserver.onNext(
+ ArtifactApi.CommitManifestResponse.newBuilder()
+ .setStagingToken(stagingBase.getCanonicalPath())
+ .build());
+ responseObserver.onCompleted();
+ }
+
+ File getArtifactFile(String artifactName) {
+ return new File(artifactsBase, artifactName);
+ }
+
+ private class CreateAndWriteFileObserver
+ implements StreamObserver<ArtifactApi.PutArtifactRequest> {
+ private final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver;
+ private FileWritingObserver writer;
+
+ private CreateAndWriteFileObserver(
+ StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
+ this.responseObserver = responseObserver;
+ }
+
+ @Override
+ public void onNext(ArtifactApi.PutArtifactRequest value) {
+ try {
+ if (writer == null) {
+ if (!value.getContentCase().equals(ArtifactApi.PutArtifactRequest.ContentCase.METADATA)) {
+ throw Status.INVALID_ARGUMENT
+ .withDescription(
+ String.format(
+ "Expected the first %s to contain the Artifact Name, got %s",
+ ArtifactApi.PutArtifactRequest.class.getSimpleName(),
+ value.getContentCase()))
+ .asRuntimeException();
+ }
+ writer = createFile(value.getMetadata());
+ } else {
+ writer.onNext(value);
+ }
+ } catch (StatusRuntimeException e) {
+ responseObserver.onError(e);
+ } catch (Exception e) {
+ responseObserver.onError(
+ Status.INTERNAL
+ .withCause(e)
+ .withDescription(Throwables.getStackTraceAsString(e))
+ .asRuntimeException());
+ }
+ }
+
+ private FileWritingObserver createFile(ArtifactApi.ArtifactMetadata metadata)
+ throws IOException {
+ File destination = getArtifactFile(metadata.getName());
+ if (!destination.createNewFile()) {
+ throw Status.ALREADY_EXISTS
+ .withDescription(String.format("Artifact with name %s already exists", metadata))
+ .asRuntimeException();
+ }
+ return new FileWritingObserver(
+ destination, new FileOutputStream(destination), responseObserver);
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ if (writer != null) {
+ writer.onError(t);
+ } else {
+ responseObserver.onCompleted();
+ }
+ }
+
+ @Override
+ public void onCompleted() {
+ if (writer != null) {
+ writer.onCompleted();
+ } else {
+ responseObserver.onCompleted();
+ }
+ }
+ }
+
+ private static class FileWritingObserver
+ implements StreamObserver<ArtifactApi.PutArtifactRequest> {
+ private final File destination;
+ private final OutputStream target;
+ private final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver;
+
+ private FileWritingObserver(
+ File destination,
+ OutputStream target,
+ StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
+ this.destination = destination;
+ this.target = target;
+ this.responseObserver = responseObserver;
+ }
+
+ @Override
+ public void onNext(ArtifactApi.PutArtifactRequest value) {
+ try {
+ if (value.getData() == null) {
+ StatusRuntimeException e = Status.INVALID_ARGUMENT.withDescription(String.format(
+ "Expected all chunks in the current stream state to contain data, got %s",
+ value.getContentCase())).asRuntimeException();
+ throw e;
+ }
+ value.getData().getData().writeTo(target);
+ } catch (Exception e) {
+ cleanedUp(e);
+ }
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ if (cleanedUp(null)) {
+ responseObserver.onCompleted();
+ }
+ }
+
+ @Override
+ public void onCompleted() {
+ try {
+ target.close();
+ } catch (IOException e) {
+ LOG.error("Failed to complete writing file {}", destination, e);
+ cleanedUp(e);
+ return;
+ }
+ responseObserver.onNext(ArtifactApi.PutArtifactResponse.getDefaultInstance());
+ responseObserver.onCompleted();
+ }
+
+ /**
+ * Cleans up after the file writing failed exceptionally, due to an error either in the service
+ * or sent from the client.
+ *
+ * @return false if an error was reported, true otherwise
+ */
+ private boolean cleanedUp(@Nullable Throwable whyFailed) {
+ Throwable actual = whyFailed;
+ try {
+ target.close();
+ if (!destination.delete()) {
+ LOG.debug("Couldn't delete failed write at {}", destination);
+ }
+ } catch (IOException e) {
+ if (whyFailed == null) {
+ actual = e;
+ } else {
+ actual.addSuppressed(e);
+ }
+ LOG.error("Failed to clean up after writing file {}", destination, e);
+ }
+ if (actual != null) {
+ if (actual instanceof StatusException || actual instanceof StatusRuntimeException) {
+ responseObserver.onError(actual);
+ } else {
+ Status status =
+ Status.INTERNAL
+ .withCause(actual)
+ .withDescription(Throwables.getStackTraceAsString(actual));
+ responseObserver.onError(status.asException());
+ }
+ }
+ return actual == null;
+ }
+ }
+}
diff --git a/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/package-info.java b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/package-info.java
new file mode 100644
index 0000000..17d0943
--- /dev/null
+++ b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides local implementations of the Artifact API services.
+ */
+package org.apache.beam.artifact.local;
diff --git a/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java b/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
new file mode 100644
index 0000000..d98253b
--- /dev/null
+++ b/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.artifact.local;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.protobuf.ByteString;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
+import io.grpc.internal.ServerImpl;
+import io.grpc.stub.StreamObserver;
+import java.io.File;
+import java.io.FileInputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link LocalFileSystemArtifactStagerService}. */
+@RunWith(JUnit4.class)
+public class LocalFileSystemArtifactStagerServiceTest {
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private ArtifactStagingServiceGrpc.ArtifactStagingServiceStub stub;
+
+ private LocalFileSystemArtifactStagerService stager;
+ private ServerImpl server;
+
+ @Before
+ public void setup() throws Exception {
+ stager = LocalFileSystemArtifactStagerService.withRootDirectory(temporaryFolder.newFolder());
+
+ server =
+ InProcessServerBuilder.forName("fs_stager")
+ .directExecutor()
+ .addService(stager)
+ .build()
+ .start();
+
+ stub =
+ ArtifactStagingServiceGrpc.newStub(
+ InProcessChannelBuilder.forName("fs_stager").usePlaintext(true).build());
+ }
+
+ @After
+ public void teardown() {
+ server.shutdownNow();
+ }
+
+ @Test
+ public void singleDataPutArtifactSucceeds() throws Exception {
+ byte[] data = "foo-bar-baz".getBytes();
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
+
+ String name = "my-artifact";
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
+ .build());
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
+ .build());
+ requestObserver.onCompleted();
+
+ responseObserver.awaitTerminalState();
+
+ File staged = stager.getArtifactFile(name);
+ assertThat(staged.exists(), is(true));
+ ByteBuffer buf = ByteBuffer.allocate(data.length);
+ new FileInputStream(staged).getChannel().read(buf);
+ Assert.assertArrayEquals(data, buf.array());
+ }
+
+ @Test
+ public void multiPartPutArtifactSucceeds() throws Exception {
+ byte[] partOne = "foo-".getBytes();
+ byte[] partTwo = "bar-".getBytes();
+ byte[] partThree = "baz".getBytes();
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
+
+ String name = "my-artifact";
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
+ .build());
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partOne))
+ .build())
+ .build());
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partTwo))
+ .build())
+ .build());
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partThree))
+ .build())
+ .build());
+ requestObserver.onCompleted();
+
+ responseObserver.awaitTerminalState();
+
+ File staged = stager.getArtifactFile(name);
+ assertThat(staged.exists(), is(true));
+ ByteBuffer buf = ByteBuffer.allocate("foo-bar-baz".length());
+ new FileInputStream(staged).getChannel().read(buf);
+ Assert.assertArrayEquals("foo-bar-baz".getBytes(), buf.array());
+ }
+
+ @Test
+ public void putArtifactBeforeNameFails() {
+ byte[] data = "foo-".getBytes();
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
+
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
+ .build());
+
+ responseObserver.awaitTerminalState();
+
+ assertThat(responseObserver.error, Matchers.not(Matchers.nullValue()));
+ }
+
+ @Test
+ public void putArtifactWithNoContentFails() {
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
+
+ requestObserver.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(ArtifactApi.ArtifactChunk.getDefaultInstance())
+ .build());
+
+ responseObserver.awaitTerminalState();
+
+ assertThat(responseObserver.error, Matchers.not(Matchers.nullValue()));
+ }
+
+ @Test
+ public void commitManifestWithAllArtifactsSucceeds() {
+ ArtifactApi.ArtifactMetadata firstArtifact =
+ stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
+ ArtifactApi.ArtifactMetadata secondArtifact =
+ stageBytes("second-artifact", "spam, ham, eggs".getBytes());
+
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder()
+ .addArtifact(firstArtifact)
+ .addArtifact(secondArtifact)
+ .build();
+
+ RecordingStreamObserver<ArtifactApi.CommitManifestResponse> commitResponseObserver =
+ new RecordingStreamObserver<>();
+ stub.commitManifest(
+ ArtifactApi.CommitManifestRequest.newBuilder().setManifest(manifest).build(),
+ commitResponseObserver);
+
+ commitResponseObserver.awaitTerminalState();
+
+ assertThat(commitResponseObserver.completed, is(true));
+ assertThat(commitResponseObserver.responses, Matchers.hasSize(1));
+ ArtifactApi.CommitManifestResponse commitResponse = commitResponseObserver.responses.get(0);
+ assertThat(commitResponse.getStagingToken(), Matchers.not(Matchers.nullValue()));
+ }
+
+ @Test
+ public void commitManifestWithMissingArtifactFails() {
+ ArtifactApi.ArtifactMetadata firstArtifact =
+ stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
+ ArtifactApi.ArtifactMetadata absentArtifact =
+ ArtifactApi.ArtifactMetadata.newBuilder().setName("absent").build();
+
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder()
+ .addArtifact(firstArtifact)
+ .addArtifact(absentArtifact)
+ .build();
+
+ RecordingStreamObserver<ArtifactApi.CommitManifestResponse> commitResponseObserver =
+ new RecordingStreamObserver<>();
+ stub.commitManifest(
+ ArtifactApi.CommitManifestRequest.newBuilder().setManifest(manifest).build(),
+ commitResponseObserver);
+
+ commitResponseObserver.awaitTerminalState();
+
+ assertThat(commitResponseObserver.error, Matchers.not(Matchers.nullValue()));
+ }
+
+ private ArtifactApi.ArtifactMetadata stageBytes(String name, byte[] bytes) {
+ StreamObserver<ArtifactApi.PutArtifactRequest> requests =
+ stub.putArtifact(new RecordingStreamObserver<ArtifactApi.PutArtifactResponse>());
+ requests.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
+ .build());
+ requests.onNext(
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(bytes)).build())
+ .build());
+ requests.onCompleted();
+ return ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build();
+ }
+
+ private static class RecordingStreamObserver<T> implements StreamObserver<T> {
+ private List<T> responses = new ArrayList<>();
+ @Nullable private Throwable error = null;
+ private boolean completed = false;
+
+ @Override
+ public void onNext(T value) {
+ failIfTerminal();
+ responses.add(value);
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ failIfTerminal();
+ error = t;
+ }
+
+ @Override
+ public void onCompleted() {
+ failIfTerminal();
+ completed = true;
+ }
+
+ private boolean isTerminal() {
+ return error != null || completed;
+ }
+
+ private void failIfTerminal() {
+ if (isTerminal()) {
+ Assert.fail(
+ String.format(
+ "Should have terminated after entering a terminal state: completed %s, error %s",
+ completed, error));
+ }
+ }
+
+ void awaitTerminalState() {
+ while (!isTerminal()) {
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
+ }
+ }
+ }
+}
diff --git a/runners/pom.xml b/runners/pom.xml
index 36b8f22..4791847 100644
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -35,12 +35,16 @@
<modules>
<module>core-construction-java</module>
<module>core-java</module>
+ <module>java-fn-execution</module>
+ <module>local-artifact-service-java</module>
+ <module>reference</module>
<module>direct-java</module>
<module>jstorm</module>
<module>flink</module>
<module>google-cloud-dataflow-java</module>
<module>spark</module>
<module>apex</module>
+ <module>gcp</module>
</modules>
<profiles>
@@ -55,6 +59,15 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <modules>
+ <module>gearpump</module>
+ </modules>
+ </profile>
</profiles>
<build>
diff --git a/runners/reference/job-server/pom.xml b/runners/reference/job-server/pom.xml
new file mode 100644
index 0000000..fb0f170
--- /dev/null
+++ b/runners/reference/job-server/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-reference-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-reference-job-orchestrator</artifactId>
+
+ <name>Apache Beam :: Runners :: Reference :: Job Orchestrator</name>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-job-management</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java
new file mode 100644
index 0000000..298f532
--- /dev/null
+++ b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.reference.job;
+
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import java.io.IOException;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A program that runs a {@link ReferenceRunnerJobService}. */
+public class ReferenceRunnerJobServer {
+ private static final Logger LOG = LoggerFactory.getLogger(ReferenceRunnerJobService.class);
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ ServerConfiguration configuration = new ServerConfiguration();
+ CmdLineParser parser = new CmdLineParser(configuration);
+ try {
+ parser.parseArgument(args);
+ } catch (CmdLineException e) {
+ System.err.println(e);
+ printUsage(parser);
+ return;
+ }
+ runServer(configuration);
+ }
+
+ private static void printUsage(CmdLineParser parser) {
+ System.err.println(
+ String.format(
+ "Usage: java %s arguments...", ReferenceRunnerJobService.class.getSimpleName()));
+ parser.printUsage(System.err);
+ System.err.println();
+ }
+
+ private static void runServer(ServerConfiguration configuration)
+ throws IOException, InterruptedException {
+ ReferenceRunnerJobService service = ReferenceRunnerJobService.create();
+ Server server = ServerBuilder.forPort(configuration.port).addService(service).build();
+ server.start();
+ System.out.println(
+ String.format(
+ "Started %s on port %s",
+ ReferenceRunnerJobService.class.getSimpleName(), configuration.port));
+ server.awaitTermination();
+ System.out.println("Server shut down, exiting");
+ }
+
+ private static class ServerConfiguration {
+ @Option(
+ name = "-p",
+ aliases = {"--port"},
+ required = true,
+ usage = "The local port to expose the server on"
+ )
+ private int port = -1;
+ }
+}
diff --git a/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
new file mode 100644
index 0000000..ded09ea
--- /dev/null
+++ b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.reference.job;
+
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import org.apache.beam.model.jobmanagement.v1.JobApi;
+import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.RunJobRequest;
+import org.apache.beam.model.jobmanagement.v1.JobServiceGrpc.JobServiceImplBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** The ReferenceRunner uses the portability framework to execute a Pipeline on a single machine. */
+public class ReferenceRunnerJobService extends JobServiceImplBase {
+ private static final Logger LOG = LoggerFactory.getLogger(ReferenceRunnerJobService.class);
+
+ public static ReferenceRunnerJobService create() {
+ return new ReferenceRunnerJobService();
+ }
+
+ private ReferenceRunnerJobService() {}
+
+ @Override
+ public void prepare(
+ JobApi.PrepareJobRequest request,
+ StreamObserver<JobApi.PrepareJobResponse> responseObserver) {
+ LOG.trace("{} {}", PrepareJobResponse.class.getSimpleName(), request);
+ System.err.println("Preparation Job Blah");
+ responseObserver.onError(Status.UNIMPLEMENTED.asException());
+ }
+
+ @Override
+ public void run(
+ JobApi.RunJobRequest request, StreamObserver<JobApi.RunJobResponse> responseObserver) {
+ LOG.trace("{} {}", RunJobRequest.class.getSimpleName(), request);
+ System.err.println("Run Job Blah");
+ responseObserver.onError(Status.UNIMPLEMENTED.asException());
+ }
+
+ @Override
+ public void getState(
+ GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) {
+ LOG.trace("{} {}", GetJobStateRequest.class.getSimpleName(), request);
+ responseObserver.onError(
+ Status.NOT_FOUND
+ .withDescription(String.format("Unknown Job ID %s", request.getJobId()))
+ .asException());
+ }
+
+ @Override
+ public void cancel(CancelJobRequest request, StreamObserver<CancelJobResponse> responseObserver) {
+ LOG.trace("{} {}", CancelJobRequest.class.getSimpleName(), request);
+ responseObserver.onError(
+ Status.NOT_FOUND
+ .withDescription(String.format("Unknown Job ID %s", request.getJobId()))
+ .asException());
+ }
+}
diff --git a/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/package-info.java b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/package-info.java
new file mode 100644
index 0000000..b6022d9
--- /dev/null
+++ b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * An execution engine for Beam Pipelines that uses the Java Runner harness and the Fn API to
+ * execute.
+ */
+package org.apache.beam.runners.reference.job;
diff --git a/runners/reference/job-server/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java b/runners/reference/job-server/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java
new file mode 100644
index 0000000..16cde11
--- /dev/null
+++ b/runners/reference/job-server/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.reference.job;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link ReferenceRunnerJobService}.
+ */
+@RunWith(JUnit4.class)
+public class ReferenceRunnerJobServiceTest {
+ @Test
+ public void testPrepareJob() {
+ // TODO: Implement when PrepareJob is implemented.
+ }
+}
diff --git a/runners/reference/pom.xml b/runners/reference/pom.xml
new file mode 100644
index 0000000..0c7f939
--- /dev/null
+++ b/runners/reference/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-runners-reference-parent</artifactId>
+
+ <name>Apache Beam :: Runners :: Reference</name>
+ <description>A Pipeline Runner which executes on the local machine using the
+ Beam portability framework to execute an arbitrary Pipeline.</description>
+
+ <packaging>pom</packaging>
+ <modules>
+ <module>job-server</module>
+ </modules>
+</project>
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 2c8372b..0ba6125 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -34,10 +34,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <spark.version>1.6.3</spark.version>
- <hadoop.version>2.2.0</hadoop.version>
<kafka.version>0.9.0.1</kafka.version>
- <jackson.version>2.4.4</jackson.version>
<dropwizard.metrics.version>3.1.2</dropwizard.metrics.version>
</properties>
@@ -77,11 +74,10 @@
org.apache.beam.runners.spark.UsesCheckpointRecovery
</groups>
<excludedGroups>
- org.apache.beam.sdk.testing.UsesStatefulParDo,
- org.apache.beam.sdk.testing.UsesTimersInParDo,
org.apache.beam.sdk.testing.UsesSplittableParDo,
org.apache.beam.sdk.testing.UsesCommittedMetrics,
- org.apache.beam.sdk.testing.UsesTestStream
+ org.apache.beam.sdk.testing.UsesTestStream,
+ org.apache.beam.sdk.testing.UsesCustomWindowMerging
</excludedGroups>
<parallel>none</parallel>
<forkCount>1</forkCount>
@@ -105,6 +101,27 @@
<threadCount>4</threadCount>
</configuration>
</execution>
+ <execution>
+ <id>streaming-tests</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <groups>
+ org.apache.beam.runners.spark.StreamingTest
+ </groups>
+ <systemPropertyVariables>
+ <beamTestPipelineOptions>
+ [
+ "--runner=TestSparkRunner",
+ "--forceStreaming=true",
+ "--enableSparkMetricSinks=true"
+ ]
+ </beamTestPipelineOptions>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
</executions>
</plugin>
</plugins>
@@ -116,31 +133,33 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-network-common_2.10</artifactId>
- <version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <!-- Fix build on JDK-9 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -164,22 +183,12 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -197,6 +206,11 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -218,6 +232,10 @@
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<exclusions>
<!-- Use Hadoop/Spark's backend logger instead of jdk14 for tests -->
@@ -315,7 +333,14 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -358,27 +383,6 @@
</systemPropertyVariables>
</configuration>
</execution>
- <execution>
- <id>streaming-tests</id>
- <phase>test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <groups>
- org.apache.beam.runners.spark.StreamingTest
- </groups>
- <systemPropertyVariables>
- <beamTestPipelineOptions>
- [
- "--runner=TestSparkRunner",
- "--forceStreaming=true",
- "--enableSparkMetricSinks=true"
- ]
- </beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java
index d75c955..6972acb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java
@@ -35,8 +35,7 @@
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
-import org.apache.commons.lang3.text.WordUtils;
-
+import org.apache.commons.text.WordUtils;
/**
* Pipeline visitor for translating a Beam pipeline into equivalent Spark operations.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 8c02f0f..98ca1be 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -26,6 +26,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
import org.apache.beam.runners.spark.io.CreateStream;
import org.apache.beam.runners.spark.metrics.AggregatorMetricSource;
@@ -39,7 +40,7 @@
import org.apache.beam.runners.spark.translation.TransformTranslator;
import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir;
import org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarksListener;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarkAdvancingStreamingListener;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.io.Read;
@@ -170,7 +171,8 @@
}
// register Watermarks listener to broadcast the advanced WMs.
- jssc.addStreamingListener(new JavaStreamingListenerWrapper(new WatermarksListener(jssc)));
+ jssc.addStreamingListener(
+ new JavaStreamingListenerWrapper(new WatermarkAdvancingStreamingListener()));
// The reason we call initAccumulators here even though it is called in
// SparkRunnerStreamingContextFactory is because the factory is not called when resuming
@@ -359,10 +361,12 @@
protected boolean shouldDefer(TransformHierarchy.Node node) {
// if the input is not a PCollection, or it is but with non merging windows, don't defer.
- if (node.getInputs().size() != 1) {
+ Collection<PValue> nonAdditionalInputs =
+ TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
+ if (nonAdditionalInputs.size() != 1) {
return false;
}
- PValue input = Iterables.getOnlyElement(node.getInputs().values());
+ PValue input = Iterables.getOnlyElement(nonAdditionalInputs);
if (!(input instanceof PCollection)
|| ((PCollection) input).getWindowingStrategy().getWindowFn().isNonMerging()) {
return false;
@@ -404,7 +408,7 @@
@SuppressWarnings("unchecked")
TransformEvaluator<TransformT> evaluator = translate(node, transform, transformClass);
LOG.info("Evaluating {}", transform);
- AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+ AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform(getPipeline());
ctxt.setCurrentTransform(appliedTransform);
evaluator.evaluate(transform, ctxt);
ctxt.setCurrentTransform(null);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index eccee57..a13a3b1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -169,7 +169,7 @@
result.waitUntilFinish(Duration.millis(batchDurationMillis));
do {
SparkTimerInternals sparkTimerInternals =
- SparkTimerInternals.global(GlobalWatermarkHolder.get());
+ SparkTimerInternals.global(GlobalWatermarkHolder.get(batchDurationMillis));
sparkTimerInternals.advanceWatermark();
globalWatermark = sparkTimerInternals.currentInputWatermarkTime();
// let another batch-interval period of execution, just to reason about WM propagation.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
index 27f2ec8..a9f2c445 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
@@ -19,18 +19,11 @@
package org.apache.beam.runners.spark.aggregators;
import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.Combine;
/**
@@ -52,17 +45,6 @@
}
/**
- * Constructs a new named aggregators instance that contains a mapping from the specified
- * `named` to the associated initial state.
- *
- * @param name Name of aggregator.
- * @param state Associated State.
- */
- public NamedAggregators(String name, State<?, ?, ?> state) {
- this.mNamedAggregators.put(name, state);
- }
-
- /**
* @param name Name of aggregator to retrieve.
* @param typeClass Type class to cast the value to.
* @param <T> Type to be returned.
@@ -152,79 +134,4 @@
Combine.CombineFn<InputT, InterT, OutputT> getCombineFn();
}
- /**
- * @param <InputT> Input data type
- * @param <InterT> Intermediate data type (useful for averages)
- * @param <OutputT> Output data type
- */
- public static class CombineFunctionState<InputT, InterT, OutputT>
- implements State<InputT, InterT, OutputT> {
-
- private Combine.CombineFn<InputT, InterT, OutputT> combineFn;
- private Coder<InputT> inCoder;
- private SparkRuntimeContext ctxt;
- private transient InterT state;
-
- public CombineFunctionState(
- Combine.CombineFn<InputT, InterT, OutputT> combineFn,
- Coder<InputT> inCoder,
- SparkRuntimeContext ctxt) {
- this.combineFn = combineFn;
- this.inCoder = inCoder;
- this.ctxt = ctxt;
- this.state = combineFn.createAccumulator();
- }
-
- @Override
- public void update(InputT element) {
- combineFn.addInput(state, element);
- }
-
- @Override
- public State<InputT, InterT, OutputT> merge(State<InputT, InterT, OutputT> other) {
- this.state = combineFn.mergeAccumulators(ImmutableList.of(current(), other.current()));
- return this;
- }
-
- @Override
- public InterT current() {
- return state;
- }
-
- @Override
- public OutputT render() {
- return combineFn.extractOutput(state);
- }
-
- @Override
- public Combine.CombineFn<InputT, InterT, OutputT> getCombineFn() {
- return combineFn;
- }
-
- private void writeObject(ObjectOutputStream oos) throws IOException {
- oos.writeObject(ctxt);
- oos.writeObject(combineFn);
- oos.writeObject(inCoder);
- try {
- combineFn.getAccumulatorCoder(ctxt.getCoderRegistry(), inCoder)
- .encode(state, oos);
- } catch (CannotProvideCoderException e) {
- throw new IllegalStateException("Could not determine coder for accumulator", e);
- }
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException {
- ctxt = (SparkRuntimeContext) ois.readObject();
- combineFn = (Combine.CombineFn<InputT, InterT, OutputT>) ois.readObject();
- inCoder = (Coder<InputT>) ois.readObject();
- try {
- state = combineFn.getAccumulatorCoder(ctxt.getCoderRegistry(), inCoder)
- .decode(ois);
- } catch (CannotProvideCoderException e) {
- throw new IllegalStateException("Could not determine coder for accumulator", e);
- }
- }
- }
-
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
index fdcea99..4c73d95 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
@@ -27,7 +27,6 @@
import java.util.List;
import java.util.Queue;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -42,34 +41,34 @@
/**
* Create an input stream from Queue. For SparkRunner tests only.
*
- * <p>To properly compose a stream of micro-batches with their Watermarks, please keep in mind
- * that eventually there a two queues here - one for batches and another for Watermarks.
+ * <p>To properly compose a stream of micro-batches with their Watermarks, please keep in mind that
+ * eventually there a two queues here - one for batches and another for Watermarks.
*
- * <p>While both queues advance according to Spark's batch-interval, there is a slight difference
- * in how data is pushed into the stream compared to the advancement of Watermarks since Watermarks
+ * <p>While both queues advance according to Spark's batch-interval, there is a slight difference in
+ * how data is pushed into the stream compared to the advancement of Watermarks since Watermarks
* advance onBatchCompleted hook call so if you'd want to set the watermark advance for a specific
- * batch it should be called before that batch.
- * Also keep in mind that being a queue that is polled per batch interval, if there is a need to
- * "hold" the same Watermark without advancing it it should be stated explicitly or the Watermark
- * will advance as soon as it can (in the next batch completed hook).
+ * batch it should be called before that batch. Also keep in mind that being a queue that is polled
+ * per batch interval, if there is a need to "hold" the same Watermark without advancing it it
+ * should be stated explicitly or the Watermark will advance as soon as it can (in the next batch
+ * completed hook).
*
* <p>Example 1:
*
- * {@code
- * CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
- * .nextBatch(
- * TimestampedValue.of("foo", endOfGlobalWindow),
- * TimestampedValue.of("bar", endOfGlobalWindow))
- * .advanceNextBatchWatermarkToInfinity();
- * }
- * The first batch will see the default start-of-time WM of
- * {@link BoundedWindow#TIMESTAMP_MIN_VALUE} and any following batch will see
- * the end-of-time WM {@link BoundedWindow#TIMESTAMP_MAX_VALUE}.
+ * <pre>{@code
+ * CreateStream.of(StringUtf8Coder.of(), batchDuration)
+ * .nextBatch(
+ * TimestampedValue.of("foo", endOfGlobalWindow),
+ * TimestampedValue.of("bar", endOfGlobalWindow))
+ * .advanceNextBatchWatermarkToInfinity();
+ * }</pre>
+ * The first batch will see the default start-of-time WM of {@link
+ * BoundedWindow#TIMESTAMP_MIN_VALUE} and any following batch will see the end-of-time WM {@link
+ * BoundedWindow#TIMESTAMP_MAX_VALUE}.
*
* <p>Example 2:
*
- * {@code
- * CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ * <pre>{@code
+ * CreateStream.of(VarIntCoder.of(), batchDuration)
* .nextBatch(
* TimestampedValue.of(1, instant))
* .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(20)))
@@ -78,32 +77,59 @@
* .nextBatch(
* TimestampedValue.of(3, instant))
* .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(30)))
- * }
- * The first batch will see the start-of-time WM and the second will see the advanced (+20 min.) WM.
- * The third WM will see the WM advanced to +30 min, because this is the next advancement of the WM
- * regardless of where it ws called in the construction of CreateStream.
- * //TODO: write a proper Builder enforcing all those rules mentioned.
- * @param <T> stream type.
+ * }</pre>
+ *
+ * <p>
+ * The first batch will see the start-of-time WM and the second will see the advanced (+20 min.)
+ * WM. The third WM will see the WM advanced to +30 min, because this is the next advancement
+ * of the WM regardless of where it ws called in the construction of CreateStream.
+ * </p>
+ *
+ * @param <T> The type of the element in this stream.
*/
+//TODO: write a proper Builder enforcing all those rules mentioned.
public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
- private final Duration batchInterval;
+ private final Duration batchDuration;
private final Queue<Iterable<TimestampedValue<T>>> batches = new LinkedList<>();
private final Deque<SparkWatermarks> times = new LinkedList<>();
private final Coder<T> coder;
private Instant initialSystemTime;
+ private final boolean forceWatermarkSync;
private Instant lowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE; //for test purposes.
- private CreateStream(Duration batchInterval, Instant initialSystemTime, Coder<T> coder) {
- this.batchInterval = batchInterval;
+ private CreateStream(Duration batchDuration,
+ Instant initialSystemTime,
+ Coder<T> coder,
+ boolean forceWatermarkSync) {
+ this.batchDuration = batchDuration;
this.initialSystemTime = initialSystemTime;
this.coder = coder;
+ this.forceWatermarkSync = forceWatermarkSync;
}
- /** Set the batch interval for the stream. */
- public static <T> CreateStream<T> of(Coder<T> coder, Duration batchInterval) {
- return new CreateStream<>(batchInterval, new Instant(0), coder);
+ /**
+ * Creates a new Spark based stream intended for test purposes.
+ *
+ * @param batchDuration the batch duration (interval) to be used for creating this stream.
+ * @param coder the coder to be used for this stream.
+ * @param forceWatermarkSync whether this stream should be synced with the advancement of the
+ * watermark maintained by the
+ * {@link org.apache.beam.runners.spark.util.GlobalWatermarkHolder}.
+ */
+ public static <T> CreateStream<T> of(Coder<T> coder,
+ Duration batchDuration,
+ boolean forceWatermarkSync) {
+ return new CreateStream<>(batchDuration, new Instant(0), coder, forceWatermarkSync);
+ }
+
+ /**
+ * Creates a new Spark based stream without forced watermark sync, intended for test purposes.
+ * See also {@link CreateStream#of(Coder, Duration, boolean)}.
+ */
+ public static <T> CreateStream<T> of(Coder<T> coder, Duration batchDuration) {
+ return of(coder, batchDuration, true);
}
/**
@@ -113,8 +139,7 @@
@SafeVarargs
public final CreateStream<T> nextBatch(TimestampedValue<T>... batchElements) {
// validate timestamps if timestamped elements.
- for (TimestampedValue<T> element: batchElements) {
- TimestampedValue timestampedValue = (TimestampedValue) element;
+ for (final TimestampedValue<T> timestampedValue: batchElements) {
checkArgument(
timestampedValue.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
"Elements must have timestamps before %s. Got: %s",
@@ -178,7 +203,7 @@
// advance the system time.
Instant currentSynchronizedProcessingTime = times.peekLast() == null ? initialSystemTime
: times.peekLast().getSynchronizedProcessingTime();
- Instant nextSynchronizedProcessingTime = currentSynchronizedProcessingTime.plus(batchInterval);
+ Instant nextSynchronizedProcessingTime = currentSynchronizedProcessingTime.plus(batchDuration);
checkArgument(
nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
"Synchronized processing time must always advance.");
@@ -187,6 +212,10 @@
return this;
}
+ public long getBatchDuration() {
+ return batchDuration.getMillis();
+ }
+
/** Get the underlying queue representing the mock stream of micro-batches. */
public Queue<Iterable<TimestampedValue<T>>> getBatches() {
return batches;
@@ -200,14 +229,16 @@
return times;
}
- @Override
- public PCollection<T> expand(PBegin input) {
- return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED);
+ public boolean isForceWatermarkSync() {
+ return forceWatermarkSync;
}
@Override
- protected Coder<T> getDefaultOutputCoder() throws CannotProvideCoderException {
- return coder;
+ public PCollection<T> expand(PBegin input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ coder);
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
index 3b48caf..ae873a3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
@@ -140,8 +140,8 @@
}
@Override
- public Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
+ public Coder<T> getOutputCoder() {
+ return source.getOutputCoder();
}
public Coder<CheckpointMarkT> getCheckpointMarkCoder() {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java
index 20aca5f..b7000b4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java
@@ -20,8 +20,8 @@
import static com.google.common.base.Preconditions.checkArgument;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.spark.api.java.JavaSparkContext$;
@@ -58,7 +58,7 @@
private static final Logger LOG = LoggerFactory.getLogger(SourceDStream.class);
private final UnboundedSource<T, CheckpointMarkT> unboundedSource;
- private final SparkRuntimeContext runtimeContext;
+ private final SerializablePipelineOptions options;
private final Duration boundReadDuration;
// Reader cache interval to expire readers if they haven't been accessed in the last microbatch.
// The reason we expire readers is that upon executor death/addition source split ownership can be
@@ -81,20 +81,20 @@
SourceDStream(
StreamingContext ssc,
UnboundedSource<T, CheckpointMarkT> unboundedSource,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
Long boundMaxRecords) {
super(ssc, JavaSparkContext$.MODULE$.<scala.Tuple2<Source<T>, CheckpointMarkT>>fakeClassTag());
this.unboundedSource = unboundedSource;
- this.runtimeContext = runtimeContext;
+ this.options = options;
- SparkPipelineOptions options = runtimeContext.getPipelineOptions().as(
+ SparkPipelineOptions sparkOptions = options.get().as(
SparkPipelineOptions.class);
// Reader cache expiration interval. 50% of batch interval is added to accommodate latency.
- this.readerCacheInterval = 1.5 * options.getBatchIntervalMillis();
+ this.readerCacheInterval = 1.5 * sparkOptions.getBatchIntervalMillis();
- this.boundReadDuration = boundReadDuration(options.getReadTimePercentage(),
- options.getMinReadTimeMillis());
+ this.boundReadDuration = boundReadDuration(sparkOptions.getReadTimePercentage(),
+ sparkOptions.getMinReadTimeMillis());
// set initial parallelism once.
this.initialParallelism = ssc().sparkContext().defaultParallelism();
checkArgument(this.initialParallelism > 0, "Number of partitions must be greater than zero.");
@@ -104,7 +104,7 @@
try {
this.numPartitions =
createMicrobatchSource()
- .split(options)
+ .split(sparkOptions)
.size();
} catch (Exception e) {
throw new RuntimeException(e);
@@ -116,7 +116,7 @@
RDD<scala.Tuple2<Source<T>, CheckpointMarkT>> rdd =
new SourceRDD.Unbounded<>(
ssc().sparkContext(),
- runtimeContext,
+ options,
createMicrobatchSource(),
numPartitions);
return scala.Option.apply(rdd);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
index 01cc176..a225e0f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
@@ -28,9 +28,9 @@
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
@@ -66,7 +66,7 @@
private static final Logger LOG = LoggerFactory.getLogger(SourceRDD.Bounded.class);
private final BoundedSource<T> source;
- private final SparkRuntimeContext runtimeContext;
+ private final SerializablePipelineOptions options;
private final int numPartitions;
private final String stepName;
private final Accumulator<MetricsContainerStepMap> metricsAccum;
@@ -79,11 +79,11 @@
public Bounded(
SparkContext sc,
BoundedSource<T> source,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
String stepName) {
super(sc, NIL, JavaSparkContext$.MODULE$.<WindowedValue<T>>fakeClassTag());
this.source = source;
- this.runtimeContext = runtimeContext;
+ this.options = options;
// the input parallelism is determined by Spark's scheduler backend.
// when running on YARN/SparkDeploy it's the result of max(totalCores, 2).
// when running on Mesos it's 8.
@@ -103,14 +103,14 @@
long desiredSizeBytes = DEFAULT_BUNDLE_SIZE;
try {
desiredSizeBytes = source.getEstimatedSizeBytes(
- runtimeContext.getPipelineOptions()) / numPartitions;
+ options.get()) / numPartitions;
} catch (Exception e) {
LOG.warn("Failed to get estimated bundle size for source {}, using default bundle "
+ "size of {} bytes.", source, DEFAULT_BUNDLE_SIZE);
}
try {
List<? extends Source<T>> partitionedSources = source.split(desiredSizeBytes,
- runtimeContext.getPipelineOptions());
+ options.get());
Partition[] partitions = new SourcePartition[partitionedSources.size()];
for (int i = 0; i < partitionedSources.size(); i++) {
partitions[i] = new SourcePartition<>(id(), i, partitionedSources.get(i));
@@ -125,7 +125,7 @@
private BoundedSource.BoundedReader<T> createReader(SourcePartition<T> partition) {
try {
return ((BoundedSource<T>) partition.source).createReader(
- runtimeContext.getPipelineOptions());
+ options.get());
} catch (IOException e) {
throw new RuntimeException("Failed to create reader from a BoundedSource.", e);
}
@@ -293,7 +293,7 @@
UnboundedSource.CheckpointMark> extends RDD<scala.Tuple2<Source<T>, CheckpointMarkT>> {
private final MicrobatchSource<T, CheckpointMarkT> microbatchSource;
- private final SparkRuntimeContext runtimeContext;
+ private final SerializablePipelineOptions options;
private final Partitioner partitioner;
// to satisfy Scala API.
@@ -302,12 +302,12 @@
.asScalaBuffer(Collections.<Dependency<?>>emptyList()).toList();
public Unbounded(SparkContext sc,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
MicrobatchSource<T, CheckpointMarkT> microbatchSource,
int initialNumPartitions) {
super(sc, NIL,
JavaSparkContext$.MODULE$.<scala.Tuple2<Source<T>, CheckpointMarkT>>fakeClassTag());
- this.runtimeContext = runtimeContext;
+ this.options = options;
this.microbatchSource = microbatchSource;
this.partitioner = new HashPartitioner(initialNumPartitions);
}
@@ -316,7 +316,7 @@
public Partition[] getPartitions() {
try {
final List<? extends Source<T>> partitionedSources =
- microbatchSource.split(runtimeContext.getPipelineOptions());
+ microbatchSource.split(options.get());
final Partition[] partitions = new CheckpointableSourcePartition[partitionedSources.size()];
for (int i = 0; i < partitionedSources.size(); i++) {
partitions[i] =
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index 7106c73..26af0c0 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -22,12 +22,12 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
import org.apache.beam.runners.spark.stateful.StateSpecFunctions;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.runners.spark.translation.streaming.UnboundedDataset;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
@@ -80,11 +80,11 @@
public static <T, CheckpointMarkT extends CheckpointMark> UnboundedDataset<T> read(
JavaStreamingContext jssc,
- SparkRuntimeContext rc,
+ SerializablePipelineOptions rc,
UnboundedSource<T, CheckpointMarkT> source,
String stepName) {
- SparkPipelineOptions options = rc.getPipelineOptions().as(SparkPipelineOptions.class);
+ SparkPipelineOptions options = rc.get().as(SparkPipelineOptions.class);
Long maxRecordsPerBatch = options.getMaxRecordsPerBatch();
SourceDStream<T, CheckpointMarkT> sourceDStream =
new SourceDStream<>(jssc.ssc(), source, rc, maxRecordsPerBatch);
@@ -116,7 +116,7 @@
// output the actual (deserialized) stream.
WindowedValue.FullWindowedValueCoder<T> coder =
WindowedValue.FullWindowedValueCoder.of(
- source.getDefaultOutputCoder(),
+ source.getOutputCoder(),
GlobalWindow.Coder.INSTANCE);
JavaDStream<WindowedValue<T>> readUnboundedStream =
mapWithStateDStream
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 815b6ba..1fb8700 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -17,12 +17,17 @@
*/
package org.apache.beam.runners.spark.stateful;
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Lists;
import com.google.common.collect.Table;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
import org.apache.beam.runners.core.LateDataUtils;
@@ -31,14 +36,14 @@
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.metrics.CounterCell;
import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.coders.CoderHelpers;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.runners.spark.translation.TranslationUtils;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
import org.apache.beam.runners.spark.util.ByteArray;
@@ -46,7 +51,9 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -54,13 +61,14 @@
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
-import org.apache.spark.Partitioner;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext$;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.Function2;
import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.Time;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.dstream.DStream;
@@ -68,61 +76,481 @@
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Function1;
import scala.Option;
import scala.Tuple2;
import scala.Tuple3;
+import scala.collection.Iterator;
import scala.collection.Seq;
-import scala.reflect.ClassTag;
import scala.runtime.AbstractFunction1;
/**
- * An implementation of {@link GroupAlsoByWindow}
- * logic for grouping by windows and controlling trigger firings and pane accumulation.
+ * An implementation of {@link GroupAlsoByWindow} logic for grouping by windows and controlling
+ * trigger firings and pane accumulation.
*
* <p>This implementation is a composite of Spark transformations revolving around state management
- * using Spark's
- * {@link PairDStreamFunctions#updateStateByKey(Function1, Partitioner, boolean, ClassTag)}
- * to update state with new data and timers.
+ * using Spark's {@link PairDStreamFunctions#updateStateByKey(scala.Function1,
+ * org.apache.spark.Partitioner, boolean, scala.reflect.ClassTag)} to update state with new data and
+ * timers.
*
- * <p>Using updateStateByKey allows to scan through the entire state visiting not just the
- * updated state (new values for key) but also check if timers are ready to fire.
- * Since updateStateByKey bounds the types of state and output to be the same,
- * a (state, output) tuple is used, filtering the state (and output if no firing)
- * in the following steps.
+ * <p>Using updateStateByKey allows to scan through the entire state visiting not just the updated
+ * state (new values for key) but also check if timers are ready to fire. Since updateStateByKey
+ * bounds the types of state and output to be the same, a (state, output) tuple is used, filtering
+ * the state (and output if no firing) in the following steps.
*/
-public class SparkGroupAlsoByWindowViaWindowSet {
- private static final Logger LOG = LoggerFactory.getLogger(
- SparkGroupAlsoByWindowViaWindowSet.class);
+public class SparkGroupAlsoByWindowViaWindowSet implements Serializable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SparkGroupAlsoByWindowViaWindowSet.class);
- /**
- * A helper class that is essentially a {@link Serializable} {@link AbstractFunction1}.
- */
- private abstract static class SerializableFunction1<T1, T2>
- extends AbstractFunction1<T1, T2> implements Serializable {
+ private static class StateAndTimers implements Serializable {
+ //Serializable state for internals (namespace to state tag to coded value).
+ private final Table<String, String, byte[]> state;
+ private final Collection<byte[]> serTimers;
+
+ private StateAndTimers(
+ final Table<String, String, byte[]> state, final Collection<byte[]> timers) {
+ this.state = state;
+ this.serTimers = timers;
+ }
+
+ public Table<String, String, byte[]> getState() {
+ return state;
+ }
+
+ Collection<byte[]> getTimers() {
+ return serTimers;
+ }
}
- public static <K, InputT, W extends BoundedWindow>
- JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> groupAlsoByWindow(
- JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> inputDStream,
+ private static class OutputWindowedValueHolder<K, V>
+ implements OutputWindowedValue<KV<K, Iterable<V>>> {
+ private final List<WindowedValue<KV<K, Iterable<V>>>> windowedValues = new ArrayList<>();
+
+ @Override
+ public void outputWindowedValue(
+ final KV<K, Iterable<V>> output,
+ final Instant timestamp,
+ final Collection<? extends BoundedWindow> windows,
+ final PaneInfo pane) {
+ windowedValues.add(WindowedValue.of(output, timestamp, windows, pane));
+ }
+
+ private List<WindowedValue<KV<K, Iterable<V>>>> getWindowedValues() {
+ return windowedValues;
+ }
+
+ @Override
+ public <AdditionalOutputT> void outputWindowedValue(
+ final TupleTag<AdditionalOutputT> tag,
+ final AdditionalOutputT output,
+ final Instant timestamp,
+ final Collection<? extends BoundedWindow> windows,
+ final PaneInfo pane) {
+ throw new UnsupportedOperationException(
+ "Tagged outputs are not allowed in GroupAlsoByWindow.");
+ }
+ }
+
+ private static class UpdateStateByKeyFunction<K, InputT, W extends BoundedWindow>
+ extends AbstractFunction1<
+ Iterator<
+ Tuple3<
+ /*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
+ Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>,
+ Iterator<
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>
+ implements Serializable {
+
+ private class UpdateStateByKeyOutputIterator
+ extends AbstractIterator<
+ Tuple2<
+ /*K*/ ByteArray,
+ Tuple2<StateAndTimers, /*WV<KV<K, KV<Long(Time),Itr<I>>>>*/ List<byte[]>>>> {
+
+ private final Iterator<
+ Tuple3<ByteArray, Seq<byte[]>, Option<Tuple2<StateAndTimers, List<byte[]>>>>>
+ input;
+ private final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn;
+ private final CounterCell droppedDueToLateness;
+
+ private SparkStateInternals<K> processPreviousState(
+ final Option<Tuple2<StateAndTimers, List<byte[]>>> prevStateAndTimersOpt,
+ final K key,
+ final SparkTimerInternals timerInternals) {
+
+ final SparkStateInternals<K> stateInternals;
+
+ if (prevStateAndTimersOpt.isEmpty()) {
+ // no previous state.
+ stateInternals = SparkStateInternals.forKey(key);
+ } else {
+ // with pre-existing state.
+ final StateAndTimers prevStateAndTimers = prevStateAndTimersOpt.get()._1();
+ // get state(internals) per key.
+ stateInternals = SparkStateInternals.forKeyAndState(key, prevStateAndTimers.getState());
+
+ timerInternals.addTimers(
+ SparkTimerInternals.deserializeTimers(
+ prevStateAndTimers.getTimers(), timerDataCoder));
+ }
+
+ return stateInternals;
+ }
+
+ UpdateStateByKeyOutputIterator(
+ final Iterator<
+ Tuple3<ByteArray, Seq<byte[]>, Option<Tuple2<StateAndTimers, List<byte[]>>>>>
+ input,
+ final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn,
+ final CounterCell droppedDueToLateness) {
+ this.input = input;
+ this.reduceFn = reduceFn;
+ this.droppedDueToLateness = droppedDueToLateness;
+ }
+
+ /**
+ * Retrieves the timers that are eligible for processing by {@link
+ * org.apache.beam.runners.core.ReduceFnRunner}.
+ *
+ * @return A collection of timers that are eligible for processing. For a {@link
+ * TimeDomain#EVENT_TIME} timer, this implies that the watermark has passed the timer's
+ * timestamp. For other <code>TimeDomain</code>s (e.g., {@link
+ * TimeDomain#PROCESSING_TIME}), a timer is always considered eligible for processing (no
+ * restrictions).
+ */
+ private Collection<TimerInternals.TimerData> filterTimersEligibleForProcessing(
+ final Collection<TimerInternals.TimerData> timers, final Instant inputWatermark) {
+ final Predicate<TimerInternals.TimerData> eligibleForProcessing =
+ new Predicate<TimerInternals.TimerData>() {
+
+ @Override
+ public boolean apply(final TimerInternals.TimerData timer) {
+ return !timer.getDomain().equals(TimeDomain.EVENT_TIME)
+ || inputWatermark.isAfter(timer.getTimestamp());
+ }
+ };
+
+ return FluentIterable.from(timers).filter(eligibleForProcessing).toSet();
+ }
+
+
+ @Override
+ protected Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>
+ computeNext() {
+ // input iterator is a Spark partition (~bundle), containing keys and their
+ // (possibly) previous-state and (possibly) new data.
+ while (input.hasNext()) {
+
+ // for each element in the partition:
+ final Tuple3<ByteArray, Seq<byte[]>, Option<Tuple2<StateAndTimers, List<byte[]>>>> next =
+ input.next();
+
+ final ByteArray encodedKey = next._1();
+ final Seq<byte[]> encodedKeyedElements = next._2();
+ final Option<Tuple2<StateAndTimers, List<byte[]>>> prevStateAndTimersOpt = next._3();
+
+ final K key = CoderHelpers.fromByteArray(encodedKey.getValue(), keyCoder);
+
+ final Map<Integer, GlobalWatermarkHolder.SparkWatermarks> watermarks =
+ GlobalWatermarkHolder.get(getBatchDuration(options));
+
+ final SparkTimerInternals timerInternals =
+ SparkTimerInternals.forStreamFromSources(sourceIds, watermarks);
+
+ final SparkStateInternals<K> stateInternals =
+ processPreviousState(prevStateAndTimersOpt, key, timerInternals);
+
+ final ExecutableTriggerStateMachine triggerStateMachine =
+ ExecutableTriggerStateMachine.create(
+ TriggerStateMachines.stateMachineForTrigger(
+ TriggerTranslation.toProto(windowingStrategy.getTrigger())));
+
+ final OutputWindowedValueHolder<K, InputT> outputHolder =
+ new OutputWindowedValueHolder<>();
+
+ final ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
+ new ReduceFnRunner<>(
+ key,
+ windowingStrategy,
+ triggerStateMachine,
+ stateInternals,
+ timerInternals,
+ outputHolder,
+ new UnsupportedSideInputReader("GroupAlsoByWindow"),
+ reduceFn,
+ options.get());
+
+ if (!encodedKeyedElements.isEmpty()) {
+ // new input for key.
+ try {
+ final KV<Long, Iterable<WindowedValue<InputT>>> keyedElements =
+ CoderHelpers.fromByteArray(
+ encodedKeyedElements.head(), KvCoder.of(VarLongCoder.of(), itrWvCoder));
+
+ final Long rddTimestamp = keyedElements.getKey();
+
+ LOG.debug(
+ logPrefix + ": processing RDD with timestamp: {}, watermarks: {}",
+ rddTimestamp,
+ watermarks);
+
+ final Iterable<WindowedValue<InputT>> elements = keyedElements.getValue();
+
+ LOG.trace(logPrefix + ": input elements: {}", elements);
+
+ // Incoming expired windows are filtered based on
+ // timerInternals.currentInputWatermarkTime() and the configured allowed
+ // lateness. Note that this is done prior to calling
+ // timerInternals.advanceWatermark so essentially the inputWatermark is
+ // the highWatermark of the previous batch and the lowWatermark of the
+ // current batch.
+ // The highWatermark of the current batch will only affect filtering
+ // as of the next batch.
+ final Iterable<WindowedValue<InputT>> nonExpiredElements =
+ Lists.newArrayList(
+ LateDataUtils.dropExpiredWindows(
+ key, elements, timerInternals, windowingStrategy, droppedDueToLateness));
+
+ LOG.trace(logPrefix + ": non expired input elements: {}", nonExpiredElements);
+
+ reduceFnRunner.processElements(nonExpiredElements);
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to process element with ReduceFnRunner", e);
+ }
+ } else if (stateInternals.getState().isEmpty()) {
+ // no input and no state -> GC evict now.
+ continue;
+ }
+ try {
+ // advance the watermark to HWM to fire by timers.
+ LOG.debug(
+ logPrefix + ": timerInternals before advance are {}",
+ timerInternals.toString());
+
+ // store the highWatermark as the new inputWatermark to calculate triggers
+ timerInternals.advanceWatermark();
+
+ final Collection<TimerInternals.TimerData> timersEligibleForProcessing =
+ filterTimersEligibleForProcessing(
+ timerInternals.getTimers(), timerInternals.currentInputWatermarkTime());
+
+ LOG.debug(
+ logPrefix + ": timers eligible for processing are {}", timersEligibleForProcessing);
+
+ // Note that at this point, the watermark has already advanced since
+ // timerInternals.advanceWatermark() has been called and the highWatermark
+ // is now stored as the new inputWatermark, according to which triggers are
+ // calculated.
+ // Note 2: The implicit contract between the runner and reduceFnRunner is that
+ // event_time based triggers are only delivered if the watermark has passed their
+ // timestamp.
+ // Note 3: Timer cleanups are performed by the GC timer scheduled by reduceFnRunner as
+ // part of processing timers.
+ // Note 4: Even if a given timer is deemed eligible for processing, it does not
+ // necessarily mean that it will actually fire (firing is determined by the trigger
+ // itself, not the TimerInternals/TimerData objects).
+ reduceFnRunner.onTimers(timersEligibleForProcessing);
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to process ReduceFnRunner onTimer.", e);
+ }
+ // this is mostly symbolic since actual persist is done by emitting output.
+ reduceFnRunner.persist();
+ // obtain output, if fired.
+ final List<WindowedValue<KV<K, Iterable<InputT>>>> outputs =
+ outputHolder.getWindowedValues();
+
+ if (!outputs.isEmpty() || !stateInternals.getState().isEmpty()) {
+ // empty outputs are filtered later using DStream filtering
+ final StateAndTimers updated =
+ new StateAndTimers(
+ stateInternals.getState(),
+ SparkTimerInternals.serializeTimers(
+ timerInternals.getTimers(), timerDataCoder));
+
+ /*
+ Not something we want to happen in production, but is very helpful
+ when debugging - TRACE.
+ */
+ LOG.trace(
+ logPrefix + ": output elements are {}", Joiner.on(", ").join(outputs));
+
+ // persist Spark's state by outputting.
+ final List<byte[]> serOutput = CoderHelpers.toByteArrays(outputs, wvKvIterCoder);
+ return new Tuple2<>(encodedKey, new Tuple2<>(updated, serOutput));
+ }
+ // an empty state with no output, can be evicted completely - do nothing.
+ }
+ return endOfData();
+ }
+ }
+
+ private final FullWindowedValueCoder<InputT> wvCoder;
+ private final Coder<K> keyCoder;
+ private final List<Integer> sourceIds;
+ private final TimerInternals.TimerDataCoder timerDataCoder;
+ private final WindowingStrategy<?, W> windowingStrategy;
+ private final SerializablePipelineOptions options;
+ private final IterableCoder<WindowedValue<InputT>> itrWvCoder;
+ private final String logPrefix;
+ private final Coder<WindowedValue<KV<K, Iterable<InputT>>>> wvKvIterCoder;
+
+ UpdateStateByKeyFunction(
+ final List<Integer> sourceIds,
+ final WindowingStrategy<?, W> windowingStrategy,
+ final FullWindowedValueCoder<InputT> wvCoder,
+ final Coder<K> keyCoder,
+ final SerializablePipelineOptions options,
+ final String logPrefix) {
+ this.wvCoder = wvCoder;
+ this.keyCoder = keyCoder;
+ this.sourceIds = sourceIds;
+ this.timerDataCoder = timerDataCoderOf(windowingStrategy);
+ this.windowingStrategy = windowingStrategy;
+ this.options = options;
+ this.itrWvCoder = IterableCoder.of(wvCoder);
+ this.logPrefix = logPrefix;
+ this.wvKvIterCoder =
+ windowedValueKeyValueCoderOf(
+ keyCoder,
+ wvCoder.getValueCoder(),
+ ((FullWindowedValueCoder<InputT>) wvCoder).getWindowCoder());
+ }
+
+ @Override
+ public Iterator<
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
+ apply(
+ final Iterator<
+ Tuple3<
+ /*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
+ Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>
+ input) {
+ //--- ACTUAL STATEFUL OPERATION:
+ //
+ // Input Iterator: the partition (~bundle) of a co-grouping of the input
+ // and the previous state (if exists).
+ //
+ // Output Iterator: the output key, and the updated state.
+ //
+ // possible input scenarios for (K, Seq, Option<S>):
+ // (1) Option<S>.isEmpty: new data with no previous state.
+ // (2) Seq.isEmpty: no new data, but evaluating previous state (timer-like behaviour).
+ // (3) Seq.nonEmpty && Option<S>.isDefined: new data with previous state.
+
+ final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn =
+ SystemReduceFn.buffering(wvCoder.getValueCoder());
+
+ final MetricsContainerImpl cellProvider = new MetricsContainerImpl("cellProvider");
+
+ final CounterCell droppedDueToClosedWindow =
+ cellProvider.getCounter(
+ MetricName.named(
+ SparkGroupAlsoByWindowViaWindowSet.class,
+ GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER));
+
+ final CounterCell droppedDueToLateness =
+ cellProvider.getCounter(
+ MetricName.named(
+ SparkGroupAlsoByWindowViaWindowSet.class,
+ GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_LATENESS_COUNTER));
+
+ // log if there's something to log.
+ final long lateDropped = droppedDueToLateness.getCumulative();
+ if (lateDropped > 0) {
+ LOG.info(String.format("Dropped %d elements due to lateness.", lateDropped));
+ droppedDueToLateness.inc(-droppedDueToLateness.getCumulative());
+ }
+ final long closedWindowDropped = droppedDueToClosedWindow.getCumulative();
+ if (closedWindowDropped > 0) {
+ LOG.info(String.format("Dropped %d elements due to closed window.", closedWindowDropped));
+ droppedDueToClosedWindow.inc(-droppedDueToClosedWindow.getCumulative());
+ }
+
+ return scala.collection.JavaConversions.asScalaIterator(
+ new UpdateStateByKeyOutputIterator(input, reduceFn, droppedDueToLateness));
+ }
+ }
+
+ private static <K, InputT>
+ FullWindowedValueCoder<KV<K, Iterable<InputT>>> windowedValueKeyValueCoderOf(
final Coder<K> keyCoder,
- final Coder<WindowedValue<InputT>> wvCoder,
- final WindowingStrategy<?, W> windowingStrategy,
- final SparkRuntimeContext runtimeContext,
- final List<Integer> sourceIds) {
+ final Coder<InputT> iCoder,
+ final Coder<? extends BoundedWindow> wCoder) {
+ return FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder);
+ }
- final IterableCoder<WindowedValue<InputT>> itrWvCoder = IterableCoder.of(wvCoder);
- final Coder<InputT> iCoder = ((FullWindowedValueCoder<InputT>) wvCoder).getValueCoder();
- final Coder<? extends BoundedWindow> wCoder =
- ((FullWindowedValueCoder<InputT>) wvCoder).getWindowCoder();
- final Coder<WindowedValue<KV<K, Iterable<InputT>>>> wvKvIterCoder =
- FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder);
- final TimerInternals.TimerDataCoder timerDataCoder =
- TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
+ private static <W extends BoundedWindow> TimerInternals.TimerDataCoder timerDataCoderOf(
+ final WindowingStrategy<?, W> windowingStrategy) {
+ return TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
+ }
- long checkpointDurationMillis =
- runtimeContext.getPipelineOptions().as(SparkPipelineOptions.class)
- .getCheckpointDurationMillis();
+ private static void
+ checkpointIfNeeded(
+ final DStream<Tuple2<ByteArray, Tuple2<StateAndTimers, List<byte[]>>>> firedStream,
+ final SerializablePipelineOptions options) {
+
+ final Long checkpointDurationMillis = getBatchDuration(options);
+
+ if (checkpointDurationMillis > 0) {
+ firedStream.checkpoint(new Duration(checkpointDurationMillis));
+ }
+ }
+
+ private static Long getBatchDuration(final SerializablePipelineOptions options) {
+ return options.get().as(SparkPipelineOptions.class).getCheckpointDurationMillis();
+ }
+
+ private static <K, InputT> JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> stripStateValues(
+ final DStream<Tuple2<ByteArray, Tuple2<StateAndTimers, List<byte[]>>>> firedStream,
+ final Coder<K> keyCoder,
+ final FullWindowedValueCoder<InputT> wvCoder) {
+
+ return JavaPairDStream.fromPairDStream(
+ firedStream,
+ JavaSparkContext$.MODULE$.<ByteArray>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag())
+ .filter(
+ new Function<
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>,
+ Boolean>() {
+ @Override
+ public Boolean call(
+ final Tuple2<
+ /*K*/ ByteArray,
+ Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>
+ t2)
+ throws Exception {
+ // filter output if defined.
+ return !t2._2()._2().isEmpty();
+ }
+ })
+ .flatMap(
+ new FlatMapFunction<
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>,
+ WindowedValue<KV<K, Iterable<InputT>>>>() {
+
+ private final FullWindowedValueCoder<KV<K, Iterable<InputT>>>
+ windowedValueKeyValueCoder =
+ windowedValueKeyValueCoderOf(
+ keyCoder, wvCoder.getValueCoder(), wvCoder.getWindowCoder());
+
+ @Override
+ public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
+ final Tuple2<
+ /*K*/ ByteArray,
+ Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>
+ t2)
+ throws Exception {
+ // drop the state since it is already persisted at this point.
+ // return in serialized form.
+ return CoderHelpers.fromByteArrays(t2._2()._2(), windowedValueKeyValueCoder);
+ }
+ });
+ }
+
+ private static <K, InputT> PairDStreamFunctions<ByteArray, byte[]> buildPairDStream(
+ final JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> inputDStream,
+ final Coder<K> keyCoder,
+ final Coder<WindowedValue<InputT>> wvCoder) {
// we have to switch to Scala API to avoid Optional in the Java API, see: SPARK-4819.
// we also have a broader API for Scala (access to the actual key and entire iterator).
@@ -135,17 +563,19 @@
//---- Iterable: Itr
//---- AccumT: A
//---- InputT: I
- DStream<Tuple2</*K*/ ByteArray, /*Itr<WV<I>>*/ byte[]>> pairDStream =
+ final DStream<Tuple2<ByteArray, byte[]>> tupleDStream =
inputDStream
.transformToPair(
- new Function<
- JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>>,
+ new Function2<
+ JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>>, Time,
JavaPairRDD<ByteArray, byte[]>>() {
+
// we use mapPartitions with the RDD API because its the only available API
// that allows to preserve partitioning.
@Override
public JavaPairRDD<ByteArray, byte[]> call(
- JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> rdd)
+ final JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> rdd,
+ final Time time)
throws Exception {
return rdd.mapPartitions(
TranslationUtils.functionToFlatMapFunction(
@@ -156,273 +586,70 @@
TranslationUtils
.<K, Iterable<WindowedValue<InputT>>>toPairFlatMapFunction(),
true)
+ .mapValues(
+ new Function<
+ Iterable<WindowedValue<InputT>>,
+ KV<Long, Iterable<WindowedValue<InputT>>>>() {
+
+ @Override
+ public KV<Long, Iterable<WindowedValue<InputT>>> call(
+ final Iterable<WindowedValue<InputT>> values) throws Exception {
+ // add the batch timestamp for visibility (e.g., debugging)
+ return KV.of(time.milliseconds(), values);
+ }
+ })
// move to bytes representation and use coders for deserialization
// because of checkpointing.
.mapPartitionsToPair(
TranslationUtils.pairFunctionToPairFlatMapFunction(
- CoderHelpers.toByteFunction(keyCoder, itrWvCoder)),
+ CoderHelpers.toByteFunction(
+ keyCoder,
+ KvCoder.of(VarLongCoder.of(), IterableCoder.of(wvCoder)))),
true);
}
})
.dstream();
- PairDStreamFunctions<ByteArray, byte[]> pairDStreamFunctions =
- DStream.toPairDStreamFunctions(
- pairDStream,
+ return DStream.toPairDStreamFunctions(
+ tupleDStream,
JavaSparkContext$.MODULE$.<ByteArray>fakeClassTag(),
JavaSparkContext$.MODULE$.<byte[]>fakeClassTag(),
null);
- int defaultNumPartitions = pairDStreamFunctions.defaultPartitioner$default$1();
- Partitioner partitioner = pairDStreamFunctions.defaultPartitioner(defaultNumPartitions);
+ }
+
+ public static <K, InputT, W extends BoundedWindow>
+ JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> groupAlsoByWindow(
+ final JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> inputDStream,
+ final Coder<K> keyCoder,
+ final Coder<WindowedValue<InputT>> wvCoder,
+ final WindowingStrategy<?, W> windowingStrategy,
+ final SerializablePipelineOptions options,
+ final List<Integer> sourceIds,
+ final String transformFullName) {
+
+ final PairDStreamFunctions<ByteArray, byte[]> pairDStream =
+ buildPairDStream(inputDStream, keyCoder, wvCoder);
// use updateStateByKey to scan through the state and update elements and timers.
- DStream<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
- firedStream = pairDStreamFunctions.updateStateByKey(
- new SerializableFunction1<
- scala.collection.Iterator<Tuple3</*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
- Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>,
- scala.collection.Iterator<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>() {
+ final UpdateStateByKeyFunction<K, InputT, W> updateFunc =
+ new UpdateStateByKeyFunction<>(
+ sourceIds,
+ windowingStrategy,
+ (FullWindowedValueCoder<InputT>) wvCoder, keyCoder, options, transformFullName
+ );
- @Override
- public scala.collection.Iterator<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>> apply(
- final scala.collection.Iterator<Tuple3</*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
- Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>> iter) {
- //--- ACTUAL STATEFUL OPERATION:
- //
- // Input Iterator: the partition (~bundle) of a cogrouping of the input
- // and the previous state (if exists).
- //
- // Output Iterator: the output key, and the updated state.
- //
- // possible input scenarios for (K, Seq, Option<S>):
- // (1) Option<S>.isEmpty: new data with no previous state.
- // (2) Seq.isEmpty: no new data, but evaluating previous state (timer-like behaviour).
- // (3) Seq.nonEmpty && Option<S>.isDefined: new data with previous state.
-
- final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn =
- SystemReduceFn.buffering(
- ((FullWindowedValueCoder<InputT>) wvCoder).getValueCoder());
- final OutputWindowedValueHolder<K, InputT> outputHolder =
- new OutputWindowedValueHolder<>();
- // use in memory Aggregators since Spark Accumulators are not resilient
- // in stateful operators, once done with this partition.
- final MetricsContainerImpl cellProvider = new MetricsContainerImpl("cellProvider");
- final CounterCell droppedDueToClosedWindow = cellProvider.getCounter(
- MetricName.named(SparkGroupAlsoByWindowViaWindowSet.class,
- GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER));
- final CounterCell droppedDueToLateness = cellProvider.getCounter(
- MetricName.named(SparkGroupAlsoByWindowViaWindowSet.class,
- GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_LATENESS_COUNTER));
-
- AbstractIterator<
+ final DStream<
Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
- outIter = new AbstractIterator<Tuple2</*K*/ ByteArray,
- Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>() {
- @Override
- protected Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> computeNext() {
- // input iterator is a Spark partition (~bundle), containing keys and their
- // (possibly) previous-state and (possibly) new data.
- while (iter.hasNext()) {
- // for each element in the partition:
- Tuple3<ByteArray, Seq<byte[]>,
- Option<Tuple2<StateAndTimers, List<byte[]>>>> next = iter.next();
- ByteArray encodedKey = next._1();
- K key = CoderHelpers.fromByteArray(encodedKey.getValue(), keyCoder);
+ firedStream =
+ pairDStream.updateStateByKey(
+ updateFunc,
+ pairDStream.defaultPartitioner(pairDStream.defaultPartitioner$default$1()),
+ true,
+ JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag());
- Seq<byte[]> seq = next._2();
-
- Option<Tuple2<StateAndTimers,
- List<byte[]>>> prevStateAndTimersOpt = next._3();
-
- SparkStateInternals<K> stateInternals;
- SparkTimerInternals timerInternals = SparkTimerInternals.forStreamFromSources(
- sourceIds, GlobalWatermarkHolder.get());
- // get state(internals) per key.
- if (prevStateAndTimersOpt.isEmpty()) {
- // no previous state.
- stateInternals = SparkStateInternals.forKey(key);
- } else {
- // with pre-existing state.
- StateAndTimers prevStateAndTimers = prevStateAndTimersOpt.get()._1();
- stateInternals = SparkStateInternals.forKeyAndState(key,
- prevStateAndTimers.getState());
- Collection<byte[]> serTimers = prevStateAndTimers.getTimers();
- timerInternals.addTimers(
- SparkTimerInternals.deserializeTimers(serTimers, timerDataCoder));
- }
-
- ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
- new ReduceFnRunner<>(
- key,
- windowingStrategy,
- ExecutableTriggerStateMachine.create(
- TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger()))),
- stateInternals,
- timerInternals,
- outputHolder,
- new UnsupportedSideInputReader("GroupAlsoByWindow"),
- reduceFn,
- runtimeContext.getPipelineOptions());
-
- outputHolder.clear(); // clear before potential use.
- if (!seq.isEmpty()) {
- // new input for key.
- try {
- Iterable<WindowedValue<InputT>> elementsIterable =
- CoderHelpers.fromByteArray(seq.head(), itrWvCoder);
- Iterable<WindowedValue<InputT>> validElements =
- LateDataUtils
- .dropExpiredWindows(
- key,
- elementsIterable,
- timerInternals,
- windowingStrategy,
- droppedDueToLateness);
- reduceFnRunner.processElements(validElements);
- } catch (Exception e) {
- throw new RuntimeException(
- "Failed to process element with ReduceFnRunner", e);
- }
- } else if (stateInternals.getState().isEmpty()) {
- // no input and no state -> GC evict now.
- continue;
- }
- try {
- // advance the watermark to HWM to fire by timers.
- timerInternals.advanceWatermark();
- // call on timers that are ready.
- reduceFnRunner.onTimers(timerInternals.getTimersReadyToProcess());
- } catch (Exception e) {
- throw new RuntimeException(
- "Failed to process ReduceFnRunner onTimer.", e);
- }
- // this is mostly symbolic since actual persist is done by emitting output.
- reduceFnRunner.persist();
- // obtain output, if fired.
- List<WindowedValue<KV<K, Iterable<InputT>>>> outputs = outputHolder.get();
- if (!outputs.isEmpty() || !stateInternals.getState().isEmpty()) {
- StateAndTimers updated = new StateAndTimers(stateInternals.getState(),
- SparkTimerInternals.serializeTimers(
- timerInternals.getTimers(), timerDataCoder));
- // persist Spark's state by outputting.
- List<byte[]> serOutput = CoderHelpers.toByteArrays(outputs, wvKvIterCoder);
- return new Tuple2<>(encodedKey, new Tuple2<>(updated, serOutput));
- }
- // an empty state with no output, can be evicted completely - do nothing.
- }
- return endOfData();
- }
- };
-
- // log if there's something to log.
- long lateDropped = droppedDueToLateness.getCumulative();
- if (lateDropped > 0) {
- LOG.info(String.format("Dropped %d elements due to lateness.", lateDropped));
- droppedDueToLateness.inc(-droppedDueToLateness.getCumulative());
- }
- long closedWindowDropped = droppedDueToClosedWindow.getCumulative();
- if (closedWindowDropped > 0) {
- LOG.info(String.format("Dropped %d elements due to closed window.", closedWindowDropped));
- droppedDueToClosedWindow.inc(-droppedDueToClosedWindow.getCumulative());
- }
-
- return scala.collection.JavaConversions.asScalaIterator(outIter);
- }
- }, partitioner, true,
- JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag());
-
- if (checkpointDurationMillis > 0) {
- firedStream.checkpoint(new Duration(checkpointDurationMillis));
- }
-
- // go back to Java now.
- JavaPairDStream</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>
- javaFiredStream = JavaPairDStream.fromPairDStream(
- firedStream,
- JavaSparkContext$.MODULE$.<ByteArray>fakeClassTag(),
- JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag());
+ checkpointIfNeeded(firedStream, options);
// filter state-only output (nothing to fire) and remove the state from the output.
- return javaFiredStream.filter(
- new Function<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>, Boolean>() {
- @Override
- public Boolean call(
- Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> t2) throws Exception {
- // filter output if defined.
- return !t2._2()._2().isEmpty();
- }
- })
- .flatMap(
- new FlatMapFunction<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>,
- WindowedValue<KV<K, Iterable<InputT>>>>() {
- @Override
- public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
- Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
- /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> t2) throws Exception {
- // drop the state since it is already persisted at this point.
- // return in serialized form.
- return CoderHelpers.fromByteArrays(t2._2()._2(), wvKvIterCoder);
- }
- });
- }
-
- private static class StateAndTimers {
- //Serializable state for internals (namespace to state tag to coded value).
- private final Table<String, String, byte[]> state;
- private final Collection<byte[]> serTimers;
-
- private StateAndTimers(
- Table<String, String, byte[]> state, Collection<byte[]> timers) {
- this.state = state;
- this.serTimers = timers;
- }
-
- public Table<String, String, byte[]> getState() {
- return state;
- }
-
- public Collection<byte[]> getTimers() {
- return serTimers;
- }
- }
-
- private static class OutputWindowedValueHolder<K, V>
- implements OutputWindowedValue<KV<K, Iterable<V>>> {
- private List<WindowedValue<KV<K, Iterable<V>>>> windowedValues = new ArrayList<>();
-
- @Override
- public void outputWindowedValue(
- KV<K, Iterable<V>> output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- windowedValues.add(WindowedValue.of(output, timestamp, windows, pane));
- }
-
- private List<WindowedValue<KV<K, Iterable<V>>>> get() {
- return windowedValues;
- }
-
- private void clear() {
- windowedValues.clear();
- }
-
- @Override
- public <AdditionalOutputT> void outputWindowedValue(
- TupleTag<AdditionalOutputT> tag,
- AdditionalOutputT output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- throw new UnsupportedOperationException(
- "Tagged outputs are not allowed in GroupAlsoByWindow.");
- }
+ return stripStateValues(firedStream, keyCoder, (FullWindowedValueCoder<InputT>) wvCoder);
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
index 107915f..4fd8146 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -23,7 +23,6 @@
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,7 +33,6 @@
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.spark.broadcast.Broadcast;
import org.joda.time.Instant;
@@ -58,10 +56,10 @@
/** Build the {@link TimerInternals} according to the feeding streams. */
public static SparkTimerInternals forStreamFromSources(
List<Integer> sourceIds,
- @Nullable Broadcast<Map<Integer, SparkWatermarks>> broadcast) {
- // if broadcast is invalid for the specific ids, use defaults.
- if (broadcast == null || broadcast.getValue().isEmpty()
- || Collections.disjoint(sourceIds, broadcast.getValue().keySet())) {
+ Map<Integer, SparkWatermarks> watermarks) {
+ // if watermarks are invalid for the specific ids, use defaults.
+ if (watermarks == null || watermarks.isEmpty()
+ || Collections.disjoint(sourceIds, watermarks.keySet())) {
return new SparkTimerInternals(
BoundedWindow.TIMESTAMP_MIN_VALUE, BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(0));
}
@@ -71,7 +69,7 @@
// synchronized processing time should clearly be synchronized.
Instant synchronizedProcessingTime = null;
for (Integer sourceId: sourceIds) {
- SparkWatermarks sparkWatermarks = broadcast.getValue().get(sourceId);
+ SparkWatermarks sparkWatermarks = watermarks.get(sourceId);
if (sparkWatermarks != null) {
// keep slowest WMs.
slowestLowWatermark = slowestLowWatermark.isBefore(sparkWatermarks.getLowWatermark())
@@ -94,30 +92,15 @@
}
/** Build a global {@link TimerInternals} for all feeding streams.*/
- public static SparkTimerInternals global(
- @Nullable Broadcast<Map<Integer, SparkWatermarks>> broadcast) {
- return broadcast == null ? forStreamFromSources(Collections.<Integer>emptyList(), null)
- : forStreamFromSources(Lists.newArrayList(broadcast.getValue().keySet()), broadcast);
+ public static SparkTimerInternals global(Map<Integer, SparkWatermarks> watermarks) {
+ return watermarks == null ? forStreamFromSources(Collections.<Integer>emptyList(), null)
+ : forStreamFromSources(Lists.newArrayList(watermarks.keySet()), watermarks);
}
Collection<TimerData> getTimers() {
return timers;
}
- /** This should only be called after processing the element. */
- Collection<TimerData> getTimersReadyToProcess() {
- Set<TimerData> toFire = Sets.newHashSet();
- Iterator<TimerData> iterator = timers.iterator();
- while (iterator.hasNext()) {
- TimerData timer = iterator.next();
- if (timer.getTimestamp().isBefore(inputWatermark)) {
- toFire.add(timer);
- iterator.remove();
- }
- }
- return toFire;
- }
-
void addTimers(Iterable<TimerData> timers) {
for (TimerData timer: timers) {
this.timers.add(timer);
@@ -190,4 +173,10 @@
return CoderHelpers.fromByteArrays(serTimers, timerDataCoder);
}
+ @Override
+ public String toString() {
+ return "SparkTimerInternals{" + "highWatermark=" + highWatermark
+ + ", synchronizedProcessingTime=" + synchronizedProcessingTime + ", timers=" + timers
+ + ", inputWatermark=" + inputWatermark + '}';
+ }
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index 549bd30..ca54715 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -27,12 +27,12 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.io.EmptyCheckpointMark;
import org.apache.beam.runners.spark.io.MicrobatchSource;
import org.apache.beam.runners.spark.io.SparkUnboundedSource.Metadata;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
@@ -91,7 +91,7 @@
*
* <p>See also <a href="https://issues.apache.org/jira/browse/SPARK-4819">SPARK-4819</a>.</p>
*
- * @param runtimeContext A serializable {@link SparkRuntimeContext}.
+ * @param options A serializable {@link SerializablePipelineOptions}.
* @param <T> The type of the input stream elements.
* @param <CheckpointMarkT> The type of the {@link UnboundedSource.CheckpointMark}.
* @return The appropriate {@link org.apache.spark.streaming.StateSpec} function.
@@ -99,7 +99,7 @@
public static <T, CheckpointMarkT extends UnboundedSource.CheckpointMark>
scala.Function3<Source<T>, scala.Option<CheckpointMarkT>, State<Tuple2<byte[], Instant>>,
Tuple2<Iterable<byte[]>, Metadata>> mapSourceFunction(
- final SparkRuntimeContext runtimeContext, final String stepName) {
+ final SerializablePipelineOptions options, final String stepName) {
return new SerializableFunction3<Source<T>, Option<CheckpointMarkT>,
State<Tuple2<byte[], Instant>>, Tuple2<Iterable<byte[]>, Metadata>>() {
@@ -151,7 +151,7 @@
try {
microbatchReader =
(MicrobatchSource.Reader)
- microbatchSource.getOrCreateReader(runtimeContext.getPipelineOptions(),
+ microbatchSource.getOrCreateReader(options.get(),
checkpointMark);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -161,7 +161,7 @@
final List<byte[]> readValues = new ArrayList<>();
WindowedValue.FullWindowedValueCoder<T> coder =
WindowedValue.FullWindowedValueCoder.of(
- source.getDefaultOutputCoder(),
+ source.getOutputCoder(),
GlobalWindow.Coder.INSTANCE);
try {
// measure how long a read takes per-partition.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
index 652c753..7c38348 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
@@ -98,9 +98,20 @@
}
@Override
- public void cache(String storageLevel) {
- // populate the rdd if needed
- getRDD().persist(StorageLevel.fromString(storageLevel));
+ @SuppressWarnings("unchecked")
+ public void cache(String storageLevel, Coder<?> coder) {
+ StorageLevel level = StorageLevel.fromString(storageLevel);
+ if (TranslationUtils.avoidRddSerialization(level)) {
+ // if it is memory only reduce the overhead of moving to bytes
+ this.rdd = getRDD().persist(level);
+ } else {
+ // Caching can cause Serialization, we need to code to bytes
+ // more details in https://issues.apache.org/jira/browse/BEAM-2669
+ Coder<WindowedValue<T>> windowedValueCoder = (Coder<WindowedValue<T>>) coder;
+ this.rdd = getRDD().map(CoderHelpers.toByteFunction(windowedValueCoder))
+ .persist(level)
+ .map(CoderHelpers.fromByteFunction(windowedValueCoder));
+ }
}
@Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/Dataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/Dataset.java
index b5d550e..b361756 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/Dataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/Dataset.java
@@ -19,6 +19,7 @@
package org.apache.beam.runners.spark.translation;
import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
/**
@@ -26,7 +27,7 @@
*/
public interface Dataset extends Serializable {
- void cache(String storageLevel);
+ void cache(String storageLevel, Coder<?> coder);
void action();
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 8102926..10526f9 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,6 +26,8 @@
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.sdk.Pipeline;
@@ -33,6 +35,7 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
@@ -49,7 +52,6 @@
public class EvaluationContext {
private final JavaSparkContext jsc;
private JavaStreamingContext jssc;
- private final SparkRuntimeContext runtime;
private final Pipeline pipeline;
private final Map<PValue, Dataset> datasets = new LinkedHashMap<>();
private final Map<PValue, Dataset> pcollections = new LinkedHashMap<>();
@@ -59,12 +61,13 @@
private final SparkPCollectionView pviews = new SparkPCollectionView();
private final Map<PCollection, Long> cacheCandidates = new HashMap<>();
private final PipelineOptions options;
+ private final SerializablePipelineOptions serializableOptions;
public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options) {
this.jsc = jsc;
this.pipeline = pipeline;
this.options = options;
- this.runtime = new SparkRuntimeContext(pipeline, options);
+ this.serializableOptions = new SerializablePipelineOptions(options);
}
public EvaluationContext(
@@ -89,8 +92,8 @@
return options;
}
- public SparkRuntimeContext getRuntimeContext() {
- return runtime;
+ public SerializablePipelineOptions getSerializableOptions() {
+ return serializableOptions;
}
public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
@@ -103,7 +106,8 @@
public <T extends PValue> T getInput(PTransform<T, ?> transform) {
@SuppressWarnings("unchecked")
- T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
+ T input =
+ (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
return input;
}
@@ -134,18 +138,30 @@
return false;
}
- public void putDataset(PTransform<?, ? extends PValue> transform, Dataset dataset) {
- putDataset(getOutput(transform), dataset);
+ public void putDataset(PTransform<?, ? extends PValue> transform, Dataset dataset,
+ boolean forceCache) {
+ putDataset(getOutput(transform), dataset, forceCache);
}
- public void putDataset(PValue pvalue, Dataset dataset) {
+
+ public void putDataset(PTransform<?, ? extends PValue> transform, Dataset dataset) {
+ putDataset(transform, dataset, false);
+ }
+
+ public void putDataset(PValue pvalue, Dataset dataset, boolean forceCache) {
try {
dataset.setName(pvalue.getName());
} catch (IllegalStateException e) {
// name not set, ignore
}
- if (shouldCache(pvalue)) {
- dataset.cache(storageLevel());
+ if (forceCache || shouldCache(pvalue)) {
+ // we cache only PCollection
+ if (pvalue instanceof PCollection) {
+ Coder<?> coder = ((PCollection<?>) pvalue).getCoder();
+ Coder<? extends BoundedWindow> wCoder =
+ ((PCollection<?>) pvalue).getWindowingStrategy().getWindowFn().windowCoder();
+ dataset.cache(storageLevel(), WindowedValue.getFullCoder(coder, wCoder));
+ }
}
datasets.put(pvalue, dataset);
leaves.add(dataset);
@@ -251,8 +267,8 @@
return boundedDataset.getValues(pcollection);
}
- private String storageLevel() {
- return runtime.getPipelineOptions().as(SparkPipelineOptions.class).getStorageLevel();
+ public String storageLevel() {
+ return serializableOptions.get().as(SparkPipelineOptions.class).getStorageLevel();
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
index 4a66541..7299583 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
@@ -24,14 +24,22 @@
import com.google.common.collect.Multimap;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.InMemoryStateInternals;
+import org.apache.beam.runners.core.InMemoryTimerInternals;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
+import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
-import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
import org.apache.beam.runners.spark.util.SparkSideInputReader;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
@@ -51,41 +59,45 @@
public class MultiDoFnFunction<InputT, OutputT>
implements PairFlatMapFunction<Iterator<WindowedValue<InputT>>, TupleTag<?>, WindowedValue<?>> {
- private final Accumulator<NamedAggregators> aggAccum;
private final Accumulator<MetricsContainerStepMap> metricsAccum;
private final String stepName;
private final DoFn<InputT, OutputT> doFn;
- private final SparkRuntimeContext runtimeContext;
+ private final SerializablePipelineOptions options;
private final TupleTag<OutputT> mainOutputTag;
+ private final List<TupleTag<?>> additionalOutputTags;
private final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs;
private final WindowingStrategy<?, ?> windowingStrategy;
+ private final boolean stateful;
/**
- * @param aggAccum The Spark {@link Accumulator} that backs the Beam Aggregators.
* @param metricsAccum The Spark {@link Accumulator} that backs the Beam metrics.
* @param doFn The {@link DoFn} to be wrapped.
- * @param runtimeContext The {@link SparkRuntimeContext}.
+ * @param options The {@link SerializablePipelineOptions}.
* @param mainOutputTag The main output {@link TupleTag}.
+ * @param additionalOutputTags Additional {@link TupleTag output tags}.
* @param sideInputs Side inputs used in this {@link DoFn}.
* @param windowingStrategy Input {@link WindowingStrategy}.
+ * @param stateful Stateful {@link DoFn}.
*/
public MultiDoFnFunction(
- Accumulator<NamedAggregators> aggAccum,
Accumulator<MetricsContainerStepMap> metricsAccum,
String stepName,
DoFn<InputT, OutputT> doFn,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
TupleTag<OutputT> mainOutputTag,
+ List<TupleTag<?>> additionalOutputTags,
Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs,
- WindowingStrategy<?, ?> windowingStrategy) {
- this.aggAccum = aggAccum;
+ WindowingStrategy<?, ?> windowingStrategy,
+ boolean stateful) {
this.metricsAccum = metricsAccum;
this.stepName = stepName;
this.doFn = doFn;
- this.runtimeContext = runtimeContext;
+ this.options = options;
this.mainOutputTag = mainOutputTag;
+ this.additionalOutputTags = additionalOutputTags;
this.sideInputs = sideInputs;
this.windowingStrategy = windowingStrategy;
+ this.stateful = stateful;
}
@Override
@@ -94,28 +106,108 @@
DoFnOutputManager outputManager = new DoFnOutputManager();
- DoFnRunner<InputT, OutputT> doFnRunner =
+ final InMemoryTimerInternals timerInternals;
+ final StepContext context;
+ // Now only implements the StatefulParDo in Batch mode.
+ if (stateful) {
+ Object key = null;
+ if (iter.hasNext()) {
+ WindowedValue<InputT> currentValue = iter.next();
+ key = ((KV) currentValue.getValue()).getKey();
+ iter = Iterators.concat(Iterators.singletonIterator(currentValue), iter);
+ }
+ final InMemoryStateInternals<?> stateInternals = InMemoryStateInternals.forKey(key);
+ timerInternals = new InMemoryTimerInternals();
+ context = new StepContext(){
+ @Override
+ public StateInternals stateInternals() {
+ return stateInternals;
+ }
+
+ @Override
+ public TimerInternals timerInternals() {
+ return timerInternals;
+ }
+ };
+ } else {
+ timerInternals = null;
+ context = new SparkProcessContext.NoOpStepContext();
+ }
+
+ final DoFnRunner<InputT, OutputT> doFnRunner =
DoFnRunners.simpleRunner(
- runtimeContext.getPipelineOptions(),
+ options.get(),
doFn,
new SparkSideInputReader(sideInputs),
outputManager,
mainOutputTag,
- Collections.<TupleTag<?>>emptyList(),
- new SparkProcessContext.NoOpStepContext(),
+ additionalOutputTags,
+ context,
windowingStrategy);
DoFnRunnerWithMetrics<InputT, OutputT> doFnRunnerWithMetrics =
new DoFnRunnerWithMetrics<>(stepName, doFnRunner, metricsAccum);
- return new SparkProcessContext<>(doFn, doFnRunnerWithMetrics, outputManager)
- .processPartition(iter);
+ return new SparkProcessContext<>(
+ doFn, doFnRunnerWithMetrics, outputManager,
+ stateful ? new TimerDataIterator(timerInternals) :
+ Collections.<TimerInternals.TimerData>emptyIterator()).processPartition(iter);
+ }
+
+ private static class TimerDataIterator implements Iterator<TimerInternals.TimerData> {
+
+ private InMemoryTimerInternals timerInternals;
+ private boolean hasAdvance;
+ private TimerInternals.TimerData timerData;
+
+ TimerDataIterator(InMemoryTimerInternals timerInternals) {
+ this.timerInternals = timerInternals;
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ // Advance
+ if (!hasAdvance) {
+ try {
+ // Finish any pending windows by advancing the input watermark to infinity.
+ timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ // Finally, advance the processing time to infinity to fire any timers.
+ timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ timerInternals.advanceSynchronizedProcessingTime(
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ hasAdvance = true;
+ }
+
+ // Get timer data
+ return (timerData = timerInternals.removeNextEventTimer()) != null
+ || (timerData = timerInternals.removeNextProcessingTimer()) != null
+ || (timerData = timerInternals.removeNextSynchronizedProcessingTimer()) != null;
+ }
+
+ @Override
+ public TimerInternals.TimerData next() {
+ if (timerData == null) {
+ throw new NoSuchElementException();
+ } else {
+ return timerData;
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("TimerDataIterator not support remove!");
+ }
+
}
private class DoFnOutputManager
implements SparkProcessContext.SparkOutputManager<Tuple2<TupleTag<?>, WindowedValue<?>>> {
- private final Multimap<TupleTag<?>, WindowedValue<?>> outputs = LinkedListMultimap.create();;
+ private final Multimap<TupleTag<?>, WindowedValue<?>> outputs = LinkedListMultimap.create();
@Override
public void clear() {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
index 315f7fb..d8d71ff 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
@@ -30,6 +30,7 @@
import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.SideInputReader;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
import org.apache.beam.runners.spark.util.SparkSideInputReader;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -48,16 +49,16 @@
* {@link org.apache.beam.sdk.transforms.Combine.CombineFn}.
*/
public class SparkAbstractCombineFn implements Serializable {
- protected final SparkRuntimeContext runtimeContext;
+ protected final SerializablePipelineOptions options;
protected final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs;
protected final WindowingStrategy<?, BoundedWindow> windowingStrategy;
public SparkAbstractCombineFn(
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs,
WindowingStrategy<?, ?> windowingStrategy) {
- this.runtimeContext = runtimeContext;
+ this.options = options;
this.sideInputs = sideInputs;
this.windowingStrategy = (WindowingStrategy<?, BoundedWindow>) windowingStrategy;
}
@@ -71,7 +72,7 @@
private transient SparkCombineContext combineContext;
protected SparkCombineContext ctxtForInput(WindowedValue<?> input) {
if (combineContext == null) {
- combineContext = new SparkCombineContext(runtimeContext.getPipelineOptions(),
+ combineContext = new SparkCombineContext(options.get(),
new SparkSideInputReader(sideInputs));
}
return combineContext.forInput(input);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
index cdeddad..0132de3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
@@ -23,7 +23,6 @@
import org.apache.beam.runners.spark.coders.BeamSparkRunnerRegistrator;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.serializer.KryoSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,7 +95,6 @@
conf.setAppName(contextOptions.getAppName());
// register immutable collections serializers because the SDK uses them.
conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName());
- conf.set("spark.serializer", KryoSerializer.class.getName());
return new JavaSparkContext(conf);
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
index d0e9038..81416a3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
@@ -25,6 +25,7 @@
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
import org.apache.beam.sdk.transforms.CombineWithContext;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -49,10 +50,10 @@
public SparkGlobalCombineFn(
CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs,
WindowingStrategy<?, ?> windowingStrategy) {
- super(runtimeContext, sideInputs, windowingStrategy);
+ super(options, sideInputs, windowingStrategy);
this.combineFn = combineFn;
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
index be02335..fcf438c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -30,7 +30,8 @@
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
@@ -55,18 +56,18 @@
private final WindowingStrategy<?, W> windowingStrategy;
private final StateInternalsFactory<K> stateInternalsFactory;
private final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn;
- private final SparkRuntimeContext runtimeContext;
+ private final SerializablePipelineOptions options;
public SparkGroupAlsoByWindowViaOutputBufferFn(
WindowingStrategy<?, W> windowingStrategy,
StateInternalsFactory<K> stateInternalsFactory,
SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
Accumulator<NamedAggregators> accumulator) {
this.windowingStrategy = windowingStrategy;
this.stateInternalsFactory = stateInternalsFactory;
this.reduceFn = reduceFn;
- this.runtimeContext = runtimeContext;
+ this.options = options;
}
@Override
@@ -92,13 +93,13 @@
windowingStrategy,
ExecutableTriggerStateMachine.create(
TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger()))),
+ TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
stateInternals,
timerInternals,
outputter,
new UnsupportedSideInputReader("GroupAlsoByWindow"),
reduceFn,
- runtimeContext.getPipelineOptions());
+ options.get());
// Process the grouped values.
reduceFnRunner.processElements(values);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
index 7ac8e7d..55392e9 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
@@ -25,6 +25,7 @@
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
import org.apache.beam.sdk.transforms.CombineWithContext;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -49,10 +50,10 @@
public SparkKeyedCombineFn(
CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn,
- SparkRuntimeContext runtimeContext,
+ SerializablePipelineOptions options,
Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs,
WindowingStrategy<?, ?> windowingStrategy) {
- super(runtimeContext, sideInputs, windowingStrategy);
+ super(options, sideInputs, windowingStrategy);
this.combineFn = combineFn;
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index ffe343b..729eb1c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -18,21 +18,22 @@
package org.apache.beam.runners.spark.translation;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Lists;
-import java.io.IOException;
import java.util.Iterator;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateNamespace;
+import org.apache.beam.runners.core.StateNamespaces;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
/**
@@ -43,15 +44,18 @@
private final DoFn<FnInputT, FnOutputT> doFn;
private final DoFnRunner<FnInputT, FnOutputT> doFnRunner;
private final SparkOutputManager<OutputT> outputManager;
+ private Iterator<TimerInternals.TimerData> timerDataIterator;
SparkProcessContext(
DoFn<FnInputT, FnOutputT> doFn,
DoFnRunner<FnInputT, FnOutputT> doFnRunner,
- SparkOutputManager<OutputT> outputManager) {
+ SparkOutputManager<OutputT> outputManager,
+ Iterator<TimerInternals.TimerData> timerDataIterator) {
this.doFn = doFn;
this.doFnRunner = doFnRunner;
this.outputManager = outputManager;
+ this.timerDataIterator = timerDataIterator;
}
Iterable<OutputT> processPartition(
@@ -99,29 +103,6 @@
}
static class NoOpStepContext implements StepContext {
- @Override
- public String getStepName() {
- return null;
- }
-
- @Override
- public String getTransformName() {
- return null;
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) { }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) { }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window,
- Coder<W> windowCoder) throws IOException { }
@Override
public StateInternals stateInternals() {
@@ -164,6 +145,10 @@
// grab the next element and process it.
doFnRunner.processElement(inputIterator.next());
outputIterator = getOutputIterator();
+ } else if (timerDataIterator.hasNext()) {
+ clearOutput();
+ fireTimer(timerDataIterator.next());
+ outputIterator = getOutputIterator();
} else {
// no more input to consume, but finishBundle can produce more output
if (!calledFinish) {
@@ -179,5 +164,14 @@
}
}
}
+
+ private void fireTimer(
+ TimerInternals.TimerData timer) {
+ StateNamespace namespace = timer.getNamespace();
+ checkArgument(namespace instanceof StateNamespaces.WindowNamespace);
+ BoundedWindow window = ((StateNamespaces.WindowNamespace) namespace).getWindow();
+ doFnRunner.onTimer(timer.getTimerId(), window, timer.getTimestamp(), timer.getDomain());
+ }
+
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
deleted file mode 100644
index f3fe99c..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.IOException;
-import java.io.Serializable;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.common.ReflectHelpers;
-
-/**
- * The SparkRuntimeContext allows us to define useful features on the client side before our
- * data flow program is launched.
- */
-public class SparkRuntimeContext implements Serializable {
- private final String serializedPipelineOptions;
- private transient CoderRegistry coderRegistry;
-
- SparkRuntimeContext(Pipeline pipeline, PipelineOptions options) {
- this.serializedPipelineOptions = serializePipelineOptions(options);
- }
-
- /**
- * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
- * for user specified configuration injection into the ObjectMapper. This supports user custom
- * types on {@link PipelineOptions}.
- */
- private static ObjectMapper createMapper() {
- return new ObjectMapper().registerModules(
- ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
- }
-
- private String serializePipelineOptions(PipelineOptions pipelineOptions) {
- try {
- return createMapper().writeValueAsString(pipelineOptions);
- } catch (JsonProcessingException e) {
- throw new IllegalStateException("Failed to serialize the pipeline options.", e);
- }
- }
-
- private static PipelineOptions deserializePipelineOptions(String serializedPipelineOptions) {
- try {
- return createMapper().readValue(serializedPipelineOptions, PipelineOptions.class);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to deserialize the pipeline options.", e);
- }
- }
-
- public PipelineOptions getPipelineOptions() {
- return PipelineOptionsHolder.getOrInit(serializedPipelineOptions);
- }
-
- public CoderRegistry getCoderRegistry() {
- if (coderRegistry == null) {
- coderRegistry = CoderRegistry.createDefault();
- }
- return coderRegistry;
- }
-
- private static class PipelineOptionsHolder {
- // on executors, this should deserialize once.
- private static transient volatile PipelineOptions pipelineOptions = null;
-
- static PipelineOptions getOrInit(String serializedPipelineOptions) {
- if (pipelineOptions == null) {
- synchronized (PipelineOptionsHolder.class) {
- if (pipelineOptions == null) {
- pipelineOptions = deserializePipelineOptions(serializedPipelineOptions);
- }
- }
- // Register standard FileSystems.
- FileSystems.setDefaultPipelineOptions(pipelineOptions);
- }
- return pipelineOptions;
- }
- }
-}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
deleted file mode 100644
index 0ecfa75..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * Get RDD storage level for the input PCollection (mostly used for testing purpose).
- */
-public final class StorageLevelPTransform extends PTransform<PCollection<?>, PCollection<String>> {
-
- @Override
- public PCollection<String> expand(PCollection<?> input) {
- return PCollection.createPrimitiveOutputInternal(input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.BOUNDED);
- }
-
- @Override
- public Coder getDefaultOutputCoder() {
- return StringUtf8Coder.of();
- }
-
-}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index b2ed3a9..7cb8628 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -20,14 +20,15 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.runners.spark.translation.TranslationUtils.avoidRddSerialization;
import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectSplittable;
-import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectStateAndTimers;
import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.Collection;
-import java.util.Collections;
+import java.util.Iterator;
import java.util.Map;
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
@@ -40,7 +41,6 @@
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.CombineWithContext;
@@ -52,6 +52,8 @@
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Reshuffle;
import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -68,7 +70,7 @@
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
-
+import org.apache.spark.storage.StorageLevel;
/**
* Supports translation between a Beam transform, and Spark's operations on RDDs.
@@ -143,7 +145,7 @@
windowingStrategy,
new TranslationUtils.InMemoryStateInternalsFactory<K>(),
SystemReduceFn.<K, V, W>buffering(coder.getValueCoder()),
- context.getRuntimeContext(),
+ context.getSerializableOptions(),
accum));
context.putDataset(transform, new BoundedDataset<>(groupedAlsoByWindow));
@@ -168,7 +170,7 @@
(CombineWithContext.CombineFnWithContext<InputT, ?, OutputT>)
CombineFnUtil.toFnWithContext(transform.getFn());
final SparkKeyedCombineFn<K, InputT, ?, OutputT> sparkCombineFn =
- new SparkKeyedCombineFn<>(combineFn, context.getRuntimeContext(),
+ new SparkKeyedCombineFn<>(combineFn, context.getSerializableOptions(),
TranslationUtils.getSideInputs(transform.getSideInputs(), context),
context.getInput(transform).getWindowingStrategy());
@@ -219,18 +221,18 @@
final WindowedValue.FullWindowedValueCoder<OutputT> wvoCoder =
WindowedValue.FullWindowedValueCoder.of(oCoder,
windowingStrategy.getWindowFn().windowCoder());
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
final boolean hasDefault = transform.isInsertDefault();
final SparkGlobalCombineFn<InputT, AccumT, OutputT> sparkCombineFn =
new SparkGlobalCombineFn<>(
combineFn,
- runtimeContext,
+ context.getSerializableOptions(),
TranslationUtils.getSideInputs(transform.getSideInputs(), context),
windowingStrategy);
final Coder<AccumT> aCoder;
try {
- aCoder = combineFn.getAccumulatorCoder(runtimeContext.getCoderRegistry(), iCoder);
+ aCoder = combineFn.getAccumulatorCoder(
+ context.getPipeline().getCoderRegistry(), iCoder);
} catch (CannotProvideCoderException e) {
throw new IllegalStateException("Could not determine coder for accumulator", e);
}
@@ -292,16 +294,16 @@
(CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
CombineFnUtil.toFnWithContext(transform.getFn());
final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs =
TranslationUtils.getSideInputs(transform.getSideInputs(), context);
final SparkKeyedCombineFn<K, InputT, AccumT, OutputT> sparkCombineFn =
- new SparkKeyedCombineFn<>(combineFn, runtimeContext, sideInputs, windowingStrategy);
+ new SparkKeyedCombineFn<>(
+ combineFn, context.getSerializableOptions(), sideInputs, windowingStrategy);
final Coder<AccumT> vaCoder;
try {
vaCoder =
combineFn.getAccumulatorCoder(
- runtimeContext.getCoderRegistry(), inputCoder.getValueCoder());
+ context.getPipeline().getCoderRegistry(), inputCoder.getValueCoder());
} catch (CannotProvideCoderException e) {
throw new IllegalStateException("Could not determine coder for accumulator", e);
}
@@ -347,44 +349,71 @@
private static <InputT, OutputT> TransformEvaluator<ParDo.MultiOutput<InputT, OutputT>> parDo() {
return new TransformEvaluator<ParDo.MultiOutput<InputT, OutputT>>() {
@Override
+ @SuppressWarnings("unchecked")
public void evaluate(
ParDo.MultiOutput<InputT, OutputT> transform, EvaluationContext context) {
String stepName = context.getCurrentTransform().getFullName();
DoFn<InputT, OutputT> doFn = transform.getFn();
rejectSplittable(doFn);
- rejectStateAndTimers(doFn);
- @SuppressWarnings("unchecked")
JavaRDD<WindowedValue<InputT>> inRDD =
((BoundedDataset<InputT>) context.borrowDataset(transform)).getRDD();
WindowingStrategy<?, ?> windowingStrategy =
context.getInput(transform).getWindowingStrategy();
- Accumulator<NamedAggregators> aggAccum = AggregatorsAccumulator.getInstance();
Accumulator<MetricsContainerStepMap> metricsAccum = MetricsAccumulator.getInstance();
- JavaPairRDD<TupleTag<?>, WindowedValue<?>> all =
- inRDD.mapPartitionsToPair(
- new MultiDoFnFunction<>(
- aggAccum,
- metricsAccum,
- stepName,
- doFn,
- context.getRuntimeContext(),
- transform.getMainOutputTag(),
- TranslationUtils.getSideInputs(transform.getSideInputs(), context),
- windowingStrategy));
+
+ JavaPairRDD<TupleTag<?>, WindowedValue<?>> all;
+
+ DoFnSignature signature = DoFnSignatures.getSignature(transform.getFn().getClass());
+ boolean stateful = signature.stateDeclarations().size() > 0
+ || signature.timerDeclarations().size() > 0;
+
+ MultiDoFnFunction<InputT, OutputT> multiDoFnFunction = new MultiDoFnFunction<>(
+ metricsAccum,
+ stepName,
+ doFn,
+ context.getSerializableOptions(),
+ transform.getMainOutputTag(),
+ transform.getAdditionalOutputTags().getAll(),
+ TranslationUtils.getSideInputs(transform.getSideInputs(), context),
+ windowingStrategy,
+ stateful);
+
+ if (stateful) {
+ // Based on the fact that the signature is stateful, DoFnSignatures ensures
+ // that it is also keyed
+ all = statefulParDoTransform(
+ (KvCoder) context.getInput(transform).getCoder(),
+ windowingStrategy.getWindowFn().windowCoder(),
+ (JavaRDD) inRDD,
+ (MultiDoFnFunction) multiDoFnFunction);
+ } else {
+ all = inRDD.mapPartitionsToPair(multiDoFnFunction);
+ }
+
Map<TupleTag<?>, PValue> outputs = context.getOutputs(transform);
if (outputs.size() > 1) {
- // cache the RDD if we're going to filter it more than once.
- all.cache();
+ StorageLevel level = StorageLevel.fromString(context.storageLevel());
+ if (avoidRddSerialization(level)) {
+ // if it is memory only reduce the overhead of moving to bytes
+ all = all.persist(level);
+ } else {
+ // Caching can cause Serialization, we need to code to bytes
+ // more details in https://issues.apache.org/jira/browse/BEAM-2669
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap =
+ TranslationUtils.getTupleTagCoders(outputs);
+ all = all
+ .mapToPair(TranslationUtils.getTupleTagEncodeFunction(coderMap))
+ .persist(level)
+ .mapToPair(TranslationUtils.getTupleTagDecodeFunction(coderMap));
+ }
}
for (Map.Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
- @SuppressWarnings("unchecked")
JavaPairRDD<TupleTag<?>, WindowedValue<?>> filtered =
all.filter(new TranslationUtils.TupleTagFilter(output.getKey()));
- @SuppressWarnings("unchecked")
// Object is the best we can do since different outputs can have different tags
JavaRDD<WindowedValue<Object>> values =
(JavaRDD<WindowedValue<Object>>) (JavaRDD<?>) filtered.values();
- context.putDataset(output.getValue(), new BoundedDataset<>(values));
+ context.putDataset(output.getValue(), new BoundedDataset<>(values), false);
}
}
@@ -395,18 +424,50 @@
};
}
+ private static <K, V, OutputT> JavaPairRDD<TupleTag<?>, WindowedValue<?>> statefulParDoTransform(
+ KvCoder<K, V> kvCoder,
+ Coder<? extends BoundedWindow> windowCoder,
+ JavaRDD<WindowedValue<KV<K, V>>> kvInRDD,
+ MultiDoFnFunction<KV<K, V>, OutputT> doFnFunction) {
+ Coder<K> keyCoder = kvCoder.getKeyCoder();
+
+ final WindowedValue.WindowedValueCoder<V> wvCoder = WindowedValue.FullWindowedValueCoder.of(
+ kvCoder.getValueCoder(), windowCoder);
+
+ JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupRDD =
+ GroupCombineFunctions.groupByKeyOnly(kvInRDD, keyCoder, wvCoder);
+
+ return groupRDD.map(new Function<
+ WindowedValue<KV<K, Iterable<WindowedValue<V>>>>, Iterator<WindowedValue<KV<K, V>>>>() {
+ @Override
+ public Iterator<WindowedValue<KV<K, V>>> call(
+ WindowedValue<KV<K, Iterable<WindowedValue<V>>>> input) throws Exception {
+ final K key = input.getValue().getKey();
+ Iterable<WindowedValue<V>> value = input.getValue().getValue();
+ return FluentIterable.from(value).transform(
+ new com.google.common.base.Function<WindowedValue<V>, WindowedValue<KV<K, V>>>() {
+ @Override
+ public WindowedValue<KV<K, V>> apply(WindowedValue<V> windowedValue) {
+ return windowedValue.withValue(KV.of(key, windowedValue.getValue()));
+ }
+ }).iterator();
+ }
+ }).flatMapToPair(doFnFunction);
+ }
+
private static <T> TransformEvaluator<Read.Bounded<T>> readBounded() {
return new TransformEvaluator<Read.Bounded<T>>() {
@Override
public void evaluate(Read.Bounded<T> transform, EvaluationContext context) {
String stepName = context.getCurrentTransform().getFullName();
final JavaSparkContext jsc = context.getSparkContext();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
// create an RDD from a BoundedSource.
- JavaRDD<WindowedValue<T>> input = new SourceRDD.Bounded<>(
- jsc.sc(), transform.getSource(), runtimeContext, stepName).toJavaRDD();
+ JavaRDD<WindowedValue<T>> input =
+ new SourceRDD.Bounded<>(
+ jsc.sc(), transform.getSource(), context.getSerializableOptions(), stepName)
+ .toJavaRDD();
// cache to avoid re-evaluation of the source by Spark's lazy DAG evaluation.
- context.putDataset(transform, new BoundedDataset<>(input.cache()));
+ context.putDataset(transform, new BoundedDataset<>(input), true);
}
@Override
@@ -457,50 +518,6 @@
};
}
- private static <T> TransformEvaluator<View.AsSingleton<T>> viewAsSingleton() {
- return new TransformEvaluator<View.AsSingleton<T>>() {
- @Override
- public void evaluate(View.AsSingleton<T> transform, EvaluationContext context) {
- Iterable<? extends WindowedValue<?>> iter =
- context.getWindowedValues(context.getInput(transform));
- PCollectionView<T> output = context.getOutput(transform);
- Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
-
- @SuppressWarnings("unchecked")
- Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;
-
- context.putPView(output, iterCast, coderInternal);
- }
-
- @Override
- public String toNativeString() {
- return "collect()";
- }
- };
- }
-
- private static <T> TransformEvaluator<View.AsIterable<T>> viewAsIter() {
- return new TransformEvaluator<View.AsIterable<T>>() {
- @Override
- public void evaluate(View.AsIterable<T> transform, EvaluationContext context) {
- Iterable<? extends WindowedValue<?>> iter =
- context.getWindowedValues(context.getInput(transform));
- PCollectionView<Iterable<T>> output = context.getOutput(transform);
- Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
-
- @SuppressWarnings("unchecked")
- Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;
-
- context.putPView(output, iterCast, coderInternal);
- }
-
- @Override
- public String toNativeString() {
- return "collect()";
- }
- };
- }
-
private static <ReadT, WriteT> TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>
createPCollView() {
return new TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>() {
@@ -509,7 +526,7 @@
EvaluationContext context) {
Iterable<? extends WindowedValue<?>> iter =
context.getWindowedValues(context.getInput(transform));
- PCollectionView<WriteT> output = context.getOutput(transform);
+ PCollectionView<WriteT> output = transform.getView();
Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
@SuppressWarnings("unchecked")
@@ -525,32 +542,6 @@
};
}
- private static TransformEvaluator<StorageLevelPTransform> storageLevel() {
- return new TransformEvaluator<StorageLevelPTransform>() {
- @Override
- public void evaluate(StorageLevelPTransform transform, EvaluationContext context) {
- JavaRDD rdd = ((BoundedDataset) (context).borrowDataset(transform)).getRDD();
- JavaSparkContext javaSparkContext = context.getSparkContext();
-
- WindowedValue.ValueOnlyWindowedValueCoder<String> windowCoder =
- WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
- JavaRDD output =
- javaSparkContext.parallelize(
- CoderHelpers.toByteArrays(
- Collections.singletonList(rdd.getStorageLevel().description()),
- StringUtf8Coder.of()))
- .map(CoderHelpers.fromByteFunction(windowCoder));
-
- context.putDataset(transform, new BoundedDataset<String>(output));
- }
-
- @Override
- public String toNativeString() {
- return "sparkContext.parallelize(rdd.getStorageLevel().description())";
- }
- };
- }
-
private static <K, V, W extends BoundedWindow> TransformEvaluator<Reshuffle<K, V>> reshuffle() {
return new TransformEvaluator<Reshuffle<K, V>>() {
@Override public void evaluate(Reshuffle<K, V> transform, EvaluationContext context) {
@@ -594,13 +585,11 @@
EVALUATORS.put(Combine.PerKey.class, combinePerKey());
EVALUATORS.put(Flatten.PCollections.class, flattenPColl());
EVALUATORS.put(Create.Values.class, create());
- EVALUATORS.put(View.AsSingleton.class, viewAsSingleton());
- EVALUATORS.put(View.AsIterable.class, viewAsIter());
+// EVALUATORS.put(View.AsSingleton.class, viewAsSingleton());
+// EVALUATORS.put(View.AsIterable.class, viewAsIter());
EVALUATORS.put(View.CreatePCollectionView.class, createPCollView());
EVALUATORS.put(Window.Assign.class, window());
EVALUATORS.put(Reshuffle.class, reshuffle());
- // mostly test evaluators
- EVALUATORS.put(StorageLevelPTransform.class, storageLevel());
}
/**
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
index 993062c..90f5ee3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -22,6 +22,7 @@
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import java.io.Serializable;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -29,7 +30,9 @@
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateInternalsFactory;
import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
@@ -39,7 +42,9 @@
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.spark.api.java.JavaSparkContext;
@@ -48,6 +53,7 @@
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
+import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import scala.Tuple2;
@@ -413,4 +419,76 @@
}
};
}
+
+ /**
+ * Utility to get mapping between TupleTag and a coder.
+ * @param outputs - A map of tuple tags and pcollections
+ * @return mapping between TupleTag and a coder
+ */
+ public static Map<TupleTag<?>, Coder<WindowedValue<?>>> getTupleTagCoders(
+ Map<TupleTag<?>, PValue> outputs) {
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap = new HashMap<>(outputs.size());
+
+ for (Map.Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
+ // we get the first PValue as all of them are fro the same type.
+ PCollection<?> pCollection = (PCollection<?>) output.getValue();
+ Coder<?> coder = pCollection.getCoder();
+ Coder<? extends BoundedWindow> wCoder =
+ pCollection.getWindowingStrategy().getWindowFn().windowCoder();
+ @SuppressWarnings("unchecked")
+ Coder<WindowedValue<?>> windowedValueCoder =
+ (Coder<WindowedValue<?>>) (Coder<?>) WindowedValue.getFullCoder(coder, wCoder);
+ coderMap.put(output.getKey(), windowedValueCoder);
+ }
+ return coderMap;
+ }
+
+ /**
+ * Returns a pair function to convert value to bytes via coder.
+ * @param coderMap - mapping between TupleTag and a coder
+ * @return a pair function to convert value to bytes via coder
+ */
+ public static PairFunction<Tuple2<TupleTag<?>, WindowedValue<?>>, TupleTag<?>, byte[]>
+ getTupleTagEncodeFunction(final Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap) {
+ return new PairFunction<Tuple2<TupleTag<?>, WindowedValue<?>>, TupleTag<?>, byte[]>() {
+
+ @Override public Tuple2<TupleTag<?>, byte[]>
+ call(Tuple2<TupleTag<?>, WindowedValue<?>> tuple2) throws Exception {
+ TupleTag<?> tupleTag = tuple2._1;
+ WindowedValue<?> windowedValue = tuple2._2;
+ return new Tuple2<TupleTag<?>, byte[]>
+ (tupleTag, CoderHelpers.toByteArray(windowedValue, coderMap.get(tupleTag)));
+ }
+ };
+ }
+
+ /**
+ * Returns a pair function to convert bytes to value via coder.
+ * @param coderMap - mapping between TupleTag and a coder
+ * @return a pair function to convert bytes to value via coder
+ * */
+ public static PairFunction<Tuple2<TupleTag<?>, byte[]>, TupleTag<?>, WindowedValue<?>>
+ getTupleTagDecodeFunction(final Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap) {
+ return new PairFunction<Tuple2<TupleTag<?>, byte[]>, TupleTag<?>, WindowedValue<?>>() {
+
+ @Override public Tuple2<TupleTag<?>, WindowedValue<?>>
+ call(Tuple2<TupleTag<?>, byte[]> tuple2) throws Exception {
+ TupleTag<?> tupleTag = tuple2._1;
+ byte[] windowedByteValue = tuple2._2;
+ return new Tuple2<TupleTag<?>, WindowedValue<?>>
+ (tupleTag, CoderHelpers.fromByteArray(windowedByteValue, coderMap.get(tupleTag)));
+ }
+ };
+ }
+
+ /**
+ * checking if we can avoid Serialization - relevant to RDDs. DStreams are memory ser in spark.
+ * @param level StorageLevel required
+ * @return
+ */
+ public static boolean avoidRddSerialization(StorageLevel level) {
+ return level.equals(StorageLevel.MEMORY_ONLY()) || level.equals(StorageLevel.MEMORY_ONLY_2());
+ }
+
+
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index acb4a02..ea26007 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -32,9 +32,8 @@
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import javax.annotation.Nonnull;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
-import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
-import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.io.ConsoleIO;
import org.apache.beam.runners.spark.io.CreateStream;
@@ -50,7 +49,6 @@
import org.apache.beam.runners.spark.translation.SparkKeyedCombineFn;
import org.apache.beam.runners.spark.translation.SparkPCollectionView;
import org.apache.beam.runners.spark.translation.SparkPipelineTranslator;
-import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.runners.spark.translation.TransformEvaluator;
import org.apache.beam.runners.spark.translation.TranslationUtils;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
@@ -84,6 +82,7 @@
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.JavaSparkContext$;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaInputDStream;
@@ -125,7 +124,7 @@
transform,
SparkUnboundedSource.read(
context.getStreamingContext(),
- context.getRuntimeContext(),
+ context.getSerializableOptions(),
transform.getSource(),
stepName));
}
@@ -141,18 +140,41 @@
return new TransformEvaluator<CreateStream<T>>() {
@Override
public void evaluate(CreateStream<T> transform, EvaluationContext context) {
- Coder<T> coder = context.getOutput(transform).getCoder();
- JavaStreamingContext jssc = context.getStreamingContext();
- Queue<Iterable<TimestampedValue<T>>> values = transform.getBatches();
- WindowedValue.FullWindowedValueCoder<T> windowCoder =
+
+ final Queue<JavaRDD<WindowedValue<T>>> rddQueue =
+ buildRdds(
+ transform.getBatches(),
+ context.getStreamingContext(),
+ context.getOutput(transform).getCoder());
+
+ final JavaInputDStream<WindowedValue<T>> javaInputDStream =
+ buildInputStream(rddQueue, transform, context);
+
+ final UnboundedDataset<T> unboundedDataset =
+ new UnboundedDataset<>(
+ javaInputDStream, Collections.singletonList(javaInputDStream.inputDStream().id()));
+
+ // add pre-baked Watermarks for the pre-baked batches.
+ GlobalWatermarkHolder.addAll(
+ ImmutableMap.of(unboundedDataset.getStreamSources().get(0), transform.getTimes()));
+
+ context.putDataset(transform, unboundedDataset);
+ }
+
+ private Queue<JavaRDD<WindowedValue<T>>> buildRdds(
+ Queue<Iterable<TimestampedValue<T>>> batches, JavaStreamingContext jssc, Coder<T> coder) {
+
+ final WindowedValue.FullWindowedValueCoder<T> windowCoder =
WindowedValue.FullWindowedValueCoder.of(coder, GlobalWindow.Coder.INSTANCE);
- // create the DStream from queue.
- Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
- for (Iterable<TimestampedValue<T>> tv : values) {
- Iterable<WindowedValue<T>> windowedValues =
+
+ final Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
+
+ for (final Iterable<TimestampedValue<T>> timestampedValues : batches) {
+ final Iterable<WindowedValue<T>> windowedValues =
Iterables.transform(
- tv,
+ timestampedValues,
new com.google.common.base.Function<TimestampedValue<T>, WindowedValue<T>>() {
+
@Override
public WindowedValue<T> apply(@Nonnull TimestampedValue<T> timestampedValue) {
return WindowedValue.of(
@@ -161,22 +183,28 @@
GlobalWindow.INSTANCE,
PaneInfo.NO_FIRING);
}
- });
- JavaRDD<WindowedValue<T>> rdd =
+ });
+
+ final JavaRDD<WindowedValue<T>> rdd =
jssc.sparkContext()
.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder))
.map(CoderHelpers.fromByteFunction(windowCoder));
+
rddQueue.offer(rdd);
}
+ return rddQueue;
+ }
- JavaInputDStream<WindowedValue<T>> inputDStream = jssc.queueStream(rddQueue, true);
- UnboundedDataset<T> unboundedDataset = new UnboundedDataset<T>(
- inputDStream, Collections.singletonList(inputDStream.inputDStream().id()));
- // add pre-baked Watermarks for the pre-baked batches.
- Queue<GlobalWatermarkHolder.SparkWatermarks> times = transform.getTimes();
- GlobalWatermarkHolder.addAll(
- ImmutableMap.of(unboundedDataset.getStreamSources().get(0), times));
- context.putDataset(transform, unboundedDataset);
+ private JavaInputDStream<WindowedValue<T>> buildInputStream(
+ Queue<JavaRDD<WindowedValue<T>>> rddQueue,
+ CreateStream<T> transform,
+ EvaluationContext context) {
+ return transform.isForceWatermarkSync()
+ ? new JavaInputDStream<>(
+ new WatermarkSyncedDStream<>(
+ rddQueue, transform.getBatchDuration(), context.getStreamingContext().ssc()),
+ JavaSparkContext$.MODULE$.<WindowedValue<T>>fakeClassTag())
+ : context.getStreamingContext().queueStream(rddQueue, true);
}
@Override
@@ -273,7 +301,6 @@
JavaDStream<WindowedValue<KV<K, V>>> dStream = inputDataset.getDStream();
@SuppressWarnings("unchecked")
final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
@SuppressWarnings("unchecked")
final WindowingStrategy<?, W> windowingStrategy =
(WindowingStrategy<?, W>) context.getInput(transform).getWindowingStrategy();
@@ -303,8 +330,9 @@
coder.getKeyCoder(),
wvCoder,
windowingStrategy,
- runtimeContext,
- streamSources);
+ context.getSerializableOptions(),
+ streamSources,
+ context.getCurrentTransform().getFullName());
context.putDataset(transform, new UnboundedDataset<>(outStream, streamSources));
}
@@ -336,7 +364,7 @@
((UnboundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform));
JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> dStream = unboundedDataset.getDStream();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
+ final SerializablePipelineOptions options = context.getSerializableOptions();
final SparkPCollectionView pviews = context.getPViews();
JavaDStream<WindowedValue<KV<K, OutputT>>> outStream = dStream.transform(
@@ -347,7 +375,7 @@
call(JavaRDD<WindowedValue<KV<K, Iterable<InputT>>>> rdd)
throws Exception {
SparkKeyedCombineFn<K, InputT, ?, OutputT> combineFnWithContext =
- new SparkKeyedCombineFn<>(fn, runtimeContext,
+ new SparkKeyedCombineFn<>(fn, options,
TranslationUtils.getSideInputs(transform.getSideInputs(),
new JavaSparkContext(rdd.context()), pviews),
windowingStrategy);
@@ -374,7 +402,7 @@
final DoFn<InputT, OutputT> doFn = transform.getFn();
rejectSplittable(doFn);
rejectStateAndTimers(doFn);
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
+ final SerializablePipelineOptions options = context.getSerializableOptions();
final SparkPCollectionView pviews = context.getPViews();
final WindowingStrategy<?, ?> windowingStrategy =
context.getInput(transform).getWindowingStrategy();
@@ -393,8 +421,6 @@
@Override
public JavaPairRDD<TupleTag<?>, WindowedValue<?>> call(
JavaRDD<WindowedValue<InputT>> rdd) throws Exception {
- final Accumulator<NamedAggregators> aggAccum =
- AggregatorsAccumulator.getInstance();
final Accumulator<MetricsContainerStepMap> metricsAccum =
MetricsAccumulator.getInstance();
final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>>
@@ -405,21 +431,30 @@
pviews);
return rdd.mapPartitionsToPair(
new MultiDoFnFunction<>(
- aggAccum,
metricsAccum,
stepName,
doFn,
- runtimeContext,
+ options,
transform.getMainOutputTag(),
+ transform.getAdditionalOutputTags().getAll(),
sideInputs,
- windowingStrategy));
+ windowingStrategy,
+ false));
}
});
+
Map<TupleTag<?>, PValue> outputs = context.getOutputs(transform);
if (outputs.size() > 1) {
- // cache the DStream if we're going to filter it more than once.
- all.cache();
+ // Caching can cause Serialization, we need to code to bytes
+ // more details in https://issues.apache.org/jira/browse/BEAM-2669
+ Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap =
+ TranslationUtils.getTupleTagCoders(outputs);
+ all = all
+ .mapToPair(TranslationUtils.getTupleTagEncodeFunction(coderMap))
+ .cache()
+ .mapToPair(TranslationUtils.getTupleTagDecodeFunction(coderMap));
}
+
for (Map.Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
@SuppressWarnings("unchecked")
JavaPairDStream<TupleTag<?>, WindowedValue<?>> filtered =
@@ -431,7 +466,8 @@
(JavaDStream<?>) TranslationUtils.dStreamValues(filtered);
context.putDataset(
output.getValue(),
- new UnboundedDataset<>(values, unboundedDataset.getStreamSources()));
+ new UnboundedDataset<>(values, unboundedDataset.getStreamSources()),
+ false);
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index ccdaf11..df927af 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -20,11 +20,15 @@
import java.util.ArrayList;
import java.util.List;
+
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.translation.Dataset;
import org.apache.beam.runners.spark.translation.TranslationUtils;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.VoidFunction;
+import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,7 +41,7 @@
private static final Logger LOG = LoggerFactory.getLogger(UnboundedDataset.class);
- private final JavaDStream<WindowedValue<T>> dStream;
+ private JavaDStream<WindowedValue<T>> dStream;
// points to the input streams that created this UnboundedDataset,
// should be greater > 1 in case of Flatten for example.
// when using GlobalWatermarkHolder this information helps to take only the relevant watermarks
@@ -57,15 +61,22 @@
return streamSources;
}
- public void cache() {
- dStream.cache();
- }
-
@Override
- public void cache(String storageLevel) {
+ @SuppressWarnings("unchecked")
+ public void cache(String storageLevel, Coder<?> coder) {
// we "force" MEMORY storage level in streaming
- LOG.warn("Provided StorageLevel ignored for stream, using default level");
- cache();
+ if (!StorageLevel.fromString(storageLevel).equals(StorageLevel.MEMORY_ONLY_SER())) {
+ LOG.warn("Provided StorageLevel: {} is ignored for streams, using the default level: {}",
+ storageLevel,
+ StorageLevel.MEMORY_ONLY_SER());
+ }
+ // Caching can cause Serialization, we need to code to bytes
+ // more details in https://issues.apache.org/jira/browse/BEAM-2669
+ Coder<WindowedValue<T>> wc = (Coder<WindowedValue<T>>) coder;
+ this.dStream = dStream.map(CoderHelpers.toByteFunction(wc))
+ .cache()
+ .map(CoderHelpers.fromByteFunction(wc));
+
}
@Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java
new file mode 100644
index 0000000..e2a7b44
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.translation.streaming;
+
+import static com.google.common.base.Preconditions.checkState;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.Uninterruptibles;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext$;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.streaming.StreamingContext;
+import org.apache.spark.streaming.Time;
+import org.apache.spark.streaming.dstream.InputDStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An {@link InputDStream} that keeps track of the {@link GlobalWatermarkHolder} status and only
+ * generates RDDs when they are in sync. If an RDD for time <code>CURRENT_BATCH_TIME</code> is
+ * requested, this input source will wait until the time of the batch which set the watermark has
+ * caught up and the following holds:
+ *
+ * {@code
+ * CURRENT_BATCH_TIME - TIME_OF_BATCH_WHICH_SET_THE_WATERMARK <= BATCH_DURATION
+ * }
+ *
+ * <p>In other words, this input source will stall and will NOT generate RDDs when the watermark is
+ * too far behind. Once the watermark has caught up with the current batch time, an RDD will be
+ * generated and emitted downstream.
+ *
+ * <p>NOTE: This input source is intended for test-use only, where one needs to be able to simulate
+ * non-trivial scenarios under a deterministic execution even at the cost incorporating test-only
+ * code. Unlike tests, in production <code>InputDStream</code>s will not be synchronous with the
+ * watermark, and the watermark is allowed to lag behind in a non-deterministic manner (since at
+ * this point in time we are reluctant to apply complex and possibly overly synchronous mechanisms
+ * at large scale).
+ *
+ * <p>See also <a href="https://issues.apache.org/jira/browse/BEAM-2671">BEAM-2671</a>, <a
+ * href="https://issues.apache.org/jira/browse/BEAM-2789">BEAM-2789</a>.
+ */
+class WatermarkSyncedDStream<T> extends InputDStream<WindowedValue<T>> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WatermarkSyncedDStream.class.getCanonicalName() + "#compute");
+
+ private static final int SLEEP_DURATION_MILLIS = 10;
+
+ private final Queue<JavaRDD<WindowedValue<T>>> rdds;
+ private final Long batchDuration;
+ private volatile boolean isFirst = true;
+
+ public WatermarkSyncedDStream(final Queue<JavaRDD<WindowedValue<T>>> rdds,
+ final Long batchDuration,
+ final StreamingContext ssc) {
+ super(ssc, JavaSparkContext$.MODULE$.<WindowedValue<T>>fakeClassTag());
+ this.rdds = rdds;
+ this.batchDuration = batchDuration;
+ }
+
+ private void awaitWatermarkSyncWith(final long batchTime) {
+ while (!isFirstBatch() && watermarkOutOfSync(batchTime)) {
+ Uninterruptibles.sleepUninterruptibly(SLEEP_DURATION_MILLIS, TimeUnit.MILLISECONDS);
+ }
+
+ checkState(
+ isFirstBatch() || watermarkIsOneBatchBehind(batchTime),
+ String.format(
+ "Watermark batch time:[%d] should be exactly one batch behind current batch time:[%d]",
+ GlobalWatermarkHolder.getLastWatermarkedBatchTime(), batchTime));
+ }
+
+ private boolean watermarkOutOfSync(final long batchTime) {
+ return batchTime - GlobalWatermarkHolder.getLastWatermarkedBatchTime() > batchDuration;
+ }
+
+ private boolean isFirstBatch() {
+ return isFirst;
+ }
+
+ private RDD<WindowedValue<T>> generateRdd() {
+ return rdds.size() > 0
+ ? rdds.poll().rdd()
+ : ssc().sparkContext().emptyRDD(JavaSparkContext$.MODULE$.<WindowedValue<T>>fakeClassTag());
+ }
+
+ private boolean watermarkIsOneBatchBehind(final long batchTime) {
+ return GlobalWatermarkHolder.getLastWatermarkedBatchTime() == batchTime - batchDuration;
+ }
+
+ @Override
+ public scala.Option<RDD<WindowedValue<T>>> compute(final Time validTime) {
+ final long batchTime = validTime.milliseconds();
+
+ LOG.trace("BEFORE waiting for watermark sync, "
+ + "LastWatermarkedBatchTime: {}, current batch time: {}",
+ GlobalWatermarkHolder.getLastWatermarkedBatchTime(),
+ batchTime);
+
+ final Stopwatch stopwatch = Stopwatch.createStarted();
+
+ awaitWatermarkSyncWith(batchTime);
+
+ stopwatch.stop();
+
+ LOG.info("Waited {} millis for watermarks to sync up with the current batch ({})",
+ stopwatch.elapsed(TimeUnit.MILLISECONDS),
+ batchTime);
+
+ LOG.info("Watermarks are now: {}", GlobalWatermarkHolder.get(batchDuration));
+
+ LOG.trace("AFTER waiting for watermark sync, "
+ + "LastWatermarkedBatchTime: {}, current batch time: {}",
+ GlobalWatermarkHolder.getLastWatermarkedBatchTime(),
+ batchTime);
+
+ final RDD<WindowedValue<T>> rdd = generateRdd();
+ isFirst = false;
+ return scala.Option.apply(rdd);
+ }
+
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void stop() {
+
+ }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
index 8b384d8..8ad3ca4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
@@ -21,31 +21,53 @@
import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Maps;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.SparkEnv;
import org.apache.spark.broadcast.Broadcast;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.apache.spark.storage.BlockId;
+import org.apache.spark.storage.BlockManager;
+import org.apache.spark.storage.BlockResult;
+import org.apache.spark.storage.BlockStore;
+import org.apache.spark.storage.StorageLevel;
+import org.apache.spark.streaming.api.java.JavaBatchInfo;
import org.apache.spark.streaming.api.java.JavaStreamingListener;
import org.apache.spark.streaming.api.java.JavaStreamingListenerBatchCompleted;
import org.joda.time.Instant;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
/**
- * A {@link Broadcast} variable to hold the global watermarks for a micro-batch.
+ * A {@link BlockStore} variable to hold the global watermarks for a micro-batch.
*
* <p>For each source, holds a queue for the watermarks of each micro-batch that was read,
* and advances the watermarks according to the queue (first-in-first-out).
*/
public class GlobalWatermarkHolder {
- // the broadcast is broadcasted to the workers.
- private static volatile Broadcast<Map<Integer, SparkWatermarks>> broadcast = null;
- // this should only live in the driver so transient.
- private static final transient Map<Integer, Queue<SparkWatermarks>> sourceTimes = new HashMap<>();
+
+ private static final Logger LOG = LoggerFactory.getLogger(GlobalWatermarkHolder.class);
+
+ private static final Map<Integer, Queue<SparkWatermarks>> sourceTimes = new HashMap<>();
+ private static final BlockId WATERMARKS_BLOCK_ID = BlockId.apply("broadcast_0WATERMARKS");
+
+ // a local copy of the watermarks is stored on the driver node so that it can be
+ // accessed in test mode instead of fetching blocks remotely
+ private static volatile Map<Integer, SparkWatermarks> driverNodeWatermarks = null;
+
+ private static volatile LoadingCache<String, Map<Integer, SparkWatermarks>> watermarkCache = null;
+ private static volatile long lastWatermarkedBatchTime = 0;
public static void add(int sourceId, SparkWatermarks sparkWatermarks) {
Queue<SparkWatermarks> timesQueue = sourceTimes.get(sourceId);
@@ -67,80 +89,216 @@
}
}
+ public static long getLastWatermarkedBatchTime() {
+ return lastWatermarkedBatchTime;
+ }
+
/**
* Returns the {@link Broadcast} containing the {@link SparkWatermarks} mapped
* to their sources.
*/
- public static Broadcast<Map<Integer, SparkWatermarks>> get() {
- return broadcast;
+ @SuppressWarnings("unchecked")
+ public static Map<Integer, SparkWatermarks> get(Long cacheInterval) {
+ if (canBypassRemoteWatermarkFetching()) {
+ /*
+ driverNodeWatermarks != null =>
+ => advance() was called
+ => WatermarkAdvancingStreamingListener#onBatchCompleted() was called
+ => we are currently running on the driver node
+ => we can get the watermarks from the driver local copy instead of fetching their block
+ remotely using block manger
+ /------------------------------------------------------------------------------------------/
+ In test mode, the system is running inside a single JVM, and thus both driver and executors
+ "canBypassWatermarkBlockFetching" by using the static driverNodeWatermarks copy.
+ This allows tests to avoid the asynchronous nature of using the BlockManager directly.
+ */
+ return getLocalWatermarkCopy();
+ } else {
+ if (watermarkCache == null) {
+ watermarkCache = createWatermarkCache(cacheInterval);
+ }
+ try {
+ return watermarkCache.get("SINGLETON");
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static boolean canBypassRemoteWatermarkFetching() {
+ return driverNodeWatermarks != null;
+ }
+
+ private static synchronized LoadingCache<String, Map<Integer, SparkWatermarks>>
+ createWatermarkCache(final Long batchDuration) {
+ return CacheBuilder.newBuilder()
+ // expire watermarks every half batch duration to ensure they update in every batch.
+ .expireAfterWrite(batchDuration / 2, TimeUnit.MILLISECONDS)
+ .build(new WatermarksLoader());
}
/**
* Advances the watermarks to the next-in-line watermarks.
* SparkWatermarks are monotonically increasing.
*/
- public static void advance(JavaSparkContext jsc) {
- synchronized (GlobalWatermarkHolder.class){
- if (sourceTimes.isEmpty()) {
- return;
+ public static void advance(final String batchId) {
+ synchronized (GlobalWatermarkHolder.class) {
+ final BlockManager blockManager = SparkEnv.get().blockManager();
+ final Map<Integer, SparkWatermarks> newWatermarks = computeNewWatermarks(blockManager);
+
+ if (!newWatermarks.isEmpty()) {
+ writeRemoteWatermarkBlock(newWatermarks, blockManager);
+ writeLocalWatermarkCopy(newWatermarks);
+ } else {
+ LOG.info("No new watermarks could be computed upon completion of batch: {}", batchId);
+ }
+ }
+ }
+
+ private static void writeLocalWatermarkCopy(Map<Integer, SparkWatermarks> newWatermarks) {
+ driverNodeWatermarks = newWatermarks;
+ }
+
+ private static Map<Integer, SparkWatermarks> getLocalWatermarkCopy() {
+ return driverNodeWatermarks;
+ }
+
+ /** See {@link GlobalWatermarkHolder#advance(String)}. */
+ public static void advance() {
+ advance("N/A");
+ }
+
+ /**
+ * Computes the next watermark values per source id.
+ *
+ * @return The new watermarks values or null if no source has reported its progress.
+ */
+ private static Map<Integer, SparkWatermarks> computeNewWatermarks(BlockManager blockManager) {
+
+ if (sourceTimes.isEmpty()) {
+ return new HashMap<>();
+ }
+
+ // update all sources' watermarks into the new broadcast.
+ final Map<Integer, SparkWatermarks> newValues = new HashMap<>();
+
+ for (final Map.Entry<Integer, Queue<SparkWatermarks>> watermarkInfo: sourceTimes.entrySet()) {
+
+ if (watermarkInfo.getValue().isEmpty()) {
+ continue;
}
- // update all sources' watermarks into the new broadcast.
- Map<Integer, SparkWatermarks> newBroadcast = new HashMap<>();
+ final Integer sourceId = watermarkInfo.getKey();
- for (Map.Entry<Integer, Queue<SparkWatermarks>> en: sourceTimes.entrySet()) {
- if (en.getValue().isEmpty()) {
- continue;
- }
- Integer sourceId = en.getKey();
- Queue<SparkWatermarks> timesQueue = en.getValue();
+ // current state, if exists.
+ Instant currentLowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant currentHighWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant currentSynchronizedProcessingTime = BoundedWindow.TIMESTAMP_MIN_VALUE;
- // current state, if exists.
- Instant currentLowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
- Instant currentHighWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
- Instant currentSynchronizedProcessingTime = BoundedWindow.TIMESTAMP_MIN_VALUE;
- if (broadcast != null && broadcast.getValue().containsKey(sourceId)) {
- SparkWatermarks currentTimes = broadcast.getValue().get(sourceId);
- currentLowWatermark = currentTimes.getLowWatermark();
- currentHighWatermark = currentTimes.getHighWatermark();
- currentSynchronizedProcessingTime = currentTimes.getSynchronizedProcessingTime();
- }
+ final Map<Integer, SparkWatermarks> currentWatermarks = initWatermarks(blockManager);
- SparkWatermarks next = timesQueue.poll();
- // advance watermarks monotonically.
- Instant nextLowWatermark = next.getLowWatermark().isAfter(currentLowWatermark)
- ? next.getLowWatermark() : currentLowWatermark;
- Instant nextHighWatermark = next.getHighWatermark().isAfter(currentHighWatermark)
- ? next.getHighWatermark() : currentHighWatermark;
- Instant nextSynchronizedProcessingTime = next.getSynchronizedProcessingTime();
- checkState(!nextLowWatermark.isAfter(nextHighWatermark),
- String.format(
- "Low watermark %s cannot be later then high watermark %s",
- nextLowWatermark, nextHighWatermark));
- checkState(nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
- "Synchronized processing time must advance.");
- newBroadcast.put(
- sourceId,
- new SparkWatermarks(
- nextLowWatermark, nextHighWatermark, nextSynchronizedProcessingTime));
+ if (currentWatermarks.containsKey(sourceId)) {
+ final SparkWatermarks currentTimes = currentWatermarks.get(sourceId);
+ currentLowWatermark = currentTimes.getLowWatermark();
+ currentHighWatermark = currentTimes.getHighWatermark();
+ currentSynchronizedProcessingTime = currentTimes.getSynchronizedProcessingTime();
}
- // update the watermarks broadcast only if something has changed.
- if (!newBroadcast.isEmpty()) {
- if (broadcast != null) {
- // for now this is blocking, we could make this asynchronous
- // but it could slow down WM propagation.
- broadcast.destroy();
- }
- broadcast = jsc.broadcast(newBroadcast);
- }
+ final Queue<SparkWatermarks> timesQueue = watermarkInfo.getValue();
+ final SparkWatermarks next = timesQueue.poll();
+
+ // advance watermarks monotonically.
+
+ final Instant nextLowWatermark =
+ next.getLowWatermark().isAfter(currentLowWatermark)
+ ? next.getLowWatermark()
+ : currentLowWatermark;
+
+ final Instant nextHighWatermark =
+ next.getHighWatermark().isAfter(currentHighWatermark)
+ ? next.getHighWatermark()
+ : currentHighWatermark;
+
+ final Instant nextSynchronizedProcessingTime = next.getSynchronizedProcessingTime();
+
+ checkState(
+ !nextLowWatermark.isAfter(nextHighWatermark),
+ String.format(
+ "Low watermark %s cannot be later then high watermark %s",
+ nextLowWatermark, nextHighWatermark));
+
+ checkState(
+ nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
+ "Synchronized processing time must advance.");
+
+ newValues.put(
+ sourceId,
+ new SparkWatermarks(
+ nextLowWatermark, nextHighWatermark, nextSynchronizedProcessingTime));
+ }
+
+ return newValues;
+ }
+
+ private static void writeRemoteWatermarkBlock(
+ final Map<Integer, SparkWatermarks> newWatermarks, final BlockManager blockManager) {
+ blockManager.removeBlock(WATERMARKS_BLOCK_ID, true);
+ // if an executor tries to fetch the watermark block here, it will fail to do so since
+ // the watermark block has just been removed, but the new copy has not been put yet.
+ blockManager.putSingle(WATERMARKS_BLOCK_ID, newWatermarks, StorageLevel.MEMORY_ONLY(), true);
+ // if an executor tries to fetch the watermark block here, it still may fail to do so since
+ // the put operation might not have been executed yet
+ // see also https://issues.apache.org/jira/browse/BEAM-2789
+ LOG.info("Put new watermark block: {}", newWatermarks);
+ }
+
+ private static Map<Integer, SparkWatermarks> initWatermarks(final BlockManager blockManager) {
+
+ final Map<Integer, SparkWatermarks> watermarks = fetchSparkWatermarks(blockManager);
+
+ if (watermarks == null) {
+ final HashMap<Integer, SparkWatermarks> empty = Maps.newHashMap();
+ blockManager.putSingle(
+ WATERMARKS_BLOCK_ID,
+ empty,
+ StorageLevel.MEMORY_ONLY(),
+ true);
+ return empty;
+ } else {
+ return watermarks;
+ }
+ }
+
+ private static Map<Integer, SparkWatermarks> fetchSparkWatermarks(BlockManager blockManager) {
+ final Option<BlockResult> blockResultOption = blockManager.getRemote(WATERMARKS_BLOCK_ID);
+ if (blockResultOption.isDefined()) {
+ return (Map<Integer, SparkWatermarks>) blockResultOption.get().data().next();
+ } else {
+ return null;
+ }
+ }
+
+ private static class WatermarksLoader extends CacheLoader<String, Map<Integer, SparkWatermarks>> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<Integer, SparkWatermarks> load(@Nonnull String key) throws Exception {
+ final BlockManager blockManager = SparkEnv.get().blockManager();
+ final Map<Integer, SparkWatermarks> watermarks = fetchSparkWatermarks(blockManager);
+ return watermarks != null ? watermarks : Maps.<Integer, SparkWatermarks>newHashMap();
}
}
@VisibleForTesting
public static synchronized void clear() {
sourceTimes.clear();
- broadcast = null;
+ lastWatermarkedBatchTime = 0;
+ writeLocalWatermarkCopy(null);
+ final SparkEnv sparkEnv = SparkEnv.get();
+ if (sparkEnv != null) {
+ final BlockManager blockManager = sparkEnv.blockManager();
+ blockManager.removeBlock(WATERMARKS_BLOCK_ID, true);
+ }
}
/**
@@ -184,16 +342,33 @@
}
/** Advance the WMs onBatchCompleted event. */
- public static class WatermarksListener extends JavaStreamingListener {
- private final JavaStreamingContext jssc;
+ public static class WatermarkAdvancingStreamingListener extends JavaStreamingListener {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WatermarkAdvancingStreamingListener.class);
- public WatermarksListener(JavaStreamingContext jssc) {
- this.jssc = jssc;
+ private long timeOf(JavaBatchInfo info) {
+ return info.batchTime().milliseconds();
+ }
+
+ private long laterOf(long t1, long t2) {
+ return Math.max(t1, t2);
}
@Override
public void onBatchCompleted(JavaStreamingListenerBatchCompleted batchCompleted) {
- GlobalWatermarkHolder.advance(jssc.sparkContext());
+
+ final long currentBatchTime = timeOf(batchCompleted.batchInfo());
+
+ GlobalWatermarkHolder.advance(Long.toString(currentBatchTime));
+
+ // make sure to update the last watermarked batch time AFTER the watermarks have already
+ // been updated (i.e., after the call to GlobalWatermarkHolder.advance(...))
+ // in addition, the watermark's block in the BlockManager is updated in an asynchronous manner
+ lastWatermarkedBatchTime =
+ laterOf(lastWatermarkedBatchTime, currentBatchTime);
+
+ LOG.info("Batch with timestamp: {} has completed, watermarks have been updated.",
+ lastWatermarkedBatchTime);
}
}
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
deleted file mode 100644
index 299f5ba..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.util;
-
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * A {@link PTransform} wrapping another transform.
- */
-public class SinglePrimitiveOutputPTransform<T> extends PTransform<PInput, PCollection<T>> {
- private PTransform<PInput, PCollection<T>> transform;
-
- public SinglePrimitiveOutputPTransform(PTransform<PInput, PCollection<T>> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollection<T> expand(PInput input) {
- try {
- PCollection<T> collection = PCollection.<T>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
- collection.setCoder(transform.getDefaultOutputCoder(input, collection));
- return collection;
- } catch (CannotProvideCoderException e) {
- throw new IllegalArgumentException(
- "Unable to infer a coder and no Coder was specified. "
- + "Please set a coder by invoking Create.withCoder() explicitly.",
- e);
- }
- }
-}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
index 47a6e3f..1708123 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
@@ -65,17 +65,17 @@
instant.plus(Duration.millis(5)),
instant.plus(Duration.millis(5)),
instant));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
// low < high.
GlobalWatermarkHolder.add(1,
new SparkWatermarks(
instant.plus(Duration.millis(10)),
instant.plus(Duration.millis(15)),
instant.plus(Duration.millis(100))));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
// assert watermarks in Broadcast.
- SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get().getValue().get(1);
+ SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get(0L).get(1);
assertThat(currentWatermarks.getLowWatermark(), equalTo(instant.plus(Duration.millis(10))));
assertThat(currentWatermarks.getHighWatermark(), equalTo(instant.plus(Duration.millis(15))));
assertThat(currentWatermarks.getSynchronizedProcessingTime(),
@@ -93,7 +93,7 @@
instant.plus(Duration.millis(25)),
instant.plus(Duration.millis(20)),
instant.plus(Duration.millis(200))));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
}
@Test
@@ -106,7 +106,7 @@
instant.plus(Duration.millis(5)),
instant.plus(Duration.millis(10)),
instant));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Synchronized processing time must advance.");
@@ -117,7 +117,7 @@
instant.plus(Duration.millis(5)),
instant.plus(Duration.millis(10)),
instant));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
}
@Test
@@ -136,15 +136,15 @@
instant.plus(Duration.millis(6)),
instant));
- GlobalWatermarkHolder.advance(jsc);
+ GlobalWatermarkHolder.advance();
// assert watermarks for source 1.
- SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get().getValue().get(1);
+ SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get(0L).get(1);
assertThat(watermarksForSource1.getLowWatermark(), equalTo(instant.plus(Duration.millis(5))));
assertThat(watermarksForSource1.getHighWatermark(), equalTo(instant.plus(Duration.millis(10))));
// assert watermarks for source 2.
- SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get().getValue().get(2);
+ SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get(0L).get(2);
assertThat(watermarksForSource2.getLowWatermark(), equalTo(instant.plus(Duration.millis(3))));
assertThat(watermarksForSource2.getHighWatermark(), equalTo(instant.plus(Duration.millis(6))));
}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
index cfbad01..a5455da 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
@@ -73,8 +73,10 @@
}
private PTransform<PBegin, PCollection<String>> getValues(final SparkPipelineOptions options) {
+ final boolean doNotSyncWithWatermark = false;
return options.isStreaming()
- ? CreateStream.of(StringUtf8Coder.of(), Duration.millis(1)).nextBatch("one", "two")
+ ? CreateStream.of(StringUtf8Coder.of(), Duration.millis(1), doNotSyncWithWatermark)
+ .nextBatch("one", "two")
: Create.of("one", "two");
}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
index 9009751..49e36ca 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
@@ -52,7 +52,6 @@
import org.joda.time.Duration;
import org.junit.Test;
-
/**
* Test {@link SparkRunnerDebugger} with different pipelines.
*/
@@ -85,17 +84,19 @@
.apply(MapElements.via(new WordCount.FormatAsTextFn()))
.apply(TextIO.write().to("!!PLACEHOLDER-OUTPUT-DIR!!").withNumShards(3).withSuffix(".txt"));
- final String expectedPipeline = "sparkContext.parallelize(Arrays.asList(...))\n"
- + "_.mapPartitions(new org.apache.beam.runners.spark.examples.WordCount$ExtractWordsFn())\n"
- + "_.mapPartitions(new org.apache.beam.sdk.transforms.Count$PerElement$1())\n"
- + "_.combineByKey(..., new org.apache.beam.sdk.transforms.Count$CountFn(), ...)\n"
- + "_.groupByKey()\n"
- + "_.map(new org.apache.beam.sdk.transforms.Sum$SumLongFn())\n"
- + "_.mapPartitions(new org.apache.beam.runners.spark"
- + ".SparkRunnerDebuggerTest$PlusOne())\n"
- + "sparkContext.union(...)\n"
- + "_.mapPartitions(new org.apache.beam.runners.spark.examples.WordCount$FormatAsTextFn())\n"
- + "_.<org.apache.beam.sdk.io.AutoValue_TextIO_Write>";
+ final String expectedPipeline =
+ "sparkContext.parallelize(Arrays.asList(...))\n"
+ + "_.mapPartitions("
+ + "new org.apache.beam.runners.spark.examples.WordCount$ExtractWordsFn())\n"
+ + "_.mapPartitions(new org.apache.beam.sdk.transforms.Contextful())\n"
+ + "_.combineByKey(..., new org.apache.beam.sdk.transforms.Count$CountFn(), ...)\n"
+ + "_.groupByKey()\n"
+ + "_.map(new org.apache.beam.sdk.transforms.Sum$SumLongFn())\n"
+ + "_.mapPartitions(new org.apache.beam.sdk.transforms.Contextful())\n"
+ + "sparkContext.union(...)\n"
+ + "_.mapPartitions("
+ + "new org.apache.beam.sdk.transforms.Contextful())\n"
+ + "_.<org.apache.beam.sdk.io.TextIO$Write>";
SparkRunnerDebugger.DebugSparkPipelineResult result =
(SparkRunnerDebugger.DebugSparkPipelineResult) pipeline.run();
@@ -139,11 +140,11 @@
+ "_.map(new org.apache.beam.sdk.transforms.windowing.FixedWindows())\n"
+ "_.mapPartitions(new org.apache.beam.runners.spark."
+ "SparkRunnerDebuggerTest$FormatKVFn())\n"
- + "_.mapPartitions(new org.apache.beam.sdk.transforms.Distinct$2())\n"
+ + "_.mapPartitions(new org.apache.beam.sdk.transforms.Contextful())\n"
+ "_.groupByKey()\n"
+ "_.map(new org.apache.beam.sdk.transforms.Combine$IterableCombineFn())\n"
- + "_.mapPartitions(new org.apache.beam.sdk.transforms.Keys$1())\n"
- + "_.mapPartitions(new org.apache.beam.sdk.transforms.WithKeys$2())\n"
+ + "_.mapPartitions(new org.apache.beam.sdk.transforms.Distinct$3())\n"
+ + "_.mapPartitions(new org.apache.beam.sdk.transforms.Contextful())\n"
+ "_.<org.apache.beam.sdk.io.kafka.AutoValue_KafkaIO_Write>";
SparkRunnerDebugger.DebugSparkPipelineResult result =
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/stateful/SparkStateInternalsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/stateful/SparkStateInternalsTest.java
new file mode 100644
index 0000000..b4597f9
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/stateful/SparkStateInternalsTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.stateful;
+
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsTest;
+import org.junit.Ignore;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link SparkStateInternals}. This is based on {@link StateInternalsTest}.
+ * Ignore set and map tests.
+ */
+@RunWith(JUnit4.class)
+public class SparkStateInternalsTest extends StateInternalsTest {
+
+ @Override
+ protected StateInternals createStateInternals() {
+ return SparkStateInternals.forKey("dummyKey");
+ }
+
+ @Override
+ @Ignore
+ public void testSet() {}
+
+ @Override
+ @Ignore
+ public void testSetIsEmpty() {}
+
+ @Override
+ @Ignore
+ public void testMergeSetIntoSource() {}
+
+ @Override
+ @Ignore
+ public void testMergeSetIntoNewNamespace() {}
+
+ @Override
+ @Ignore
+ public void testMap() {}
+
+ @Override
+ @Ignore
+ public void testSetReadable() {}
+
+ @Override
+ @Ignore
+ public void testMapReadable() {}
+
+}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
deleted file mode 100644
index e8f578a..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation;
-
-import static org.junit.Assert.assertEquals;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.auto.service.AutoService;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.CrashingRunner;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link SparkRuntimeContext}.
- */
-@RunWith(JUnit4.class)
-public class SparkRuntimeContextTest {
- /** PipelineOptions used to test auto registration of Jackson modules. */
- public interface JacksonIncompatibleOptions extends PipelineOptions {
- JacksonIncompatible getJacksonIncompatible();
- void setJacksonIncompatible(JacksonIncompatible value);
- }
-
- /** A Jackson {@link Module} to test auto-registration of modules. */
- @AutoService(Module.class)
- public static class RegisteredTestModule extends SimpleModule {
- public RegisteredTestModule() {
- super("RegisteredTestModule");
- setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
- }
- }
-
- /** A class which Jackson does not know how to serialize/deserialize. */
- public static class JacksonIncompatible {
- private final String value;
- public JacksonIncompatible(String value) {
- this.value = value;
- }
- }
-
- /** A Jackson mixin used to add annotations to other classes. */
- @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
- @JsonSerialize(using = JacksonIncompatibleSerializer.class)
- public static final class JacksonIncompatibleMixin {}
-
- /** A Jackson deserializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleDeserializer extends
- JsonDeserializer<JacksonIncompatible> {
-
- @Override
- public JacksonIncompatible deserialize(JsonParser jsonParser,
- DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- return new JacksonIncompatible(jsonParser.readValueAs(String.class));
- }
- }
-
- /** A Jackson serializer for {@link JacksonIncompatible}. */
- public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
-
- @Override
- public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
- jsonGenerator.writeString(jacksonIncompatible.value);
- }
- }
-
- @Test
- public void testSerializingPipelineOptionsWithCustomUserType() throws Exception {
- PipelineOptions options = PipelineOptionsFactory.fromArgs("--jacksonIncompatible=\"testValue\"")
- .as(JacksonIncompatibleOptions.class);
- options.setRunner(CrashingRunner.class);
- Pipeline p = Pipeline.create(options);
- SparkRuntimeContext context = new SparkRuntimeContext(p, options);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {
- outputStream.writeObject(context);
- }
- try (ObjectInputStream inputStream =
- new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
- SparkRuntimeContext copy = (SparkRuntimeContext) inputStream.readObject();
- assertEquals("testValue",
- copy.getPipelineOptions().as(JacksonIncompatibleOptions.class)
- .getJacksonIncompatible().value);
- }
- }
-}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
deleted file mode 100644
index 8f2e681..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation;
-
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-
-
-/**
- * Test the RDD storage level defined by user.
- */
-public class StorageLevelTest {
-
- private static String beamTestPipelineOptions;
-
- @Rule
- public final TestPipeline pipeline = TestPipeline.create();
-
- @BeforeClass
- public static void init() {
- beamTestPipelineOptions =
- System.getProperty(TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS);
-
- System.setProperty(
- TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS,
- beamTestPipelineOptions.replace("]", ", \"--storageLevel=DISK_ONLY\"]"));
- }
-
- @AfterClass
- public static void teardown() {
- System.setProperty(
- TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS,
- beamTestPipelineOptions);
- }
-
- @Test
- public void test() throws Exception {
- PCollection<String> pCollection = pipeline.apply(Create.of("foo"));
-
- // by default, the Spark runner doesn't cache the RDD if it accessed only one time.
- // So, to "force" the caching of the RDD, we have to call the RDD at least two time.
- // That's why we are using Count fn on the PCollection.
- pCollection.apply(Count.<String>globally());
-
- PCollection<String> output = pCollection.apply(new StorageLevelPTransform());
-
- PAssert.thatSingleton(output).isEqualTo("Disk Serialized 1x Replicated");
-
- pipeline.run();
- }
-
-}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
index 770e0c0..a432fda 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -163,16 +163,16 @@
public void testDiscardingMode() throws IOException {
CreateStream<String> source =
CreateStream.of(StringUtf8Coder.of(), batchDuration())
- .nextBatch(
- TimestampedValue.of("firstPane", new Instant(100)),
- TimestampedValue.of("alsoFirstPane", new Instant(200)))
- .advanceWatermarkForNextBatch(new Instant(1001L))
- .nextBatch(
- TimestampedValue.of("onTimePane", new Instant(500)))
- .advanceNextBatchWatermarkToInfinity()
- .nextBatch(
- TimestampedValue.of("finalLatePane", new Instant(750)),
- TimestampedValue.of("alsoFinalLatePane", new Instant(250)));
+ .nextBatch(
+ TimestampedValue.of("firstPane", new Instant(100)),
+ TimestampedValue.of("alsoFirstPane", new Instant(200)))
+ .advanceWatermarkForNextBatch(new Instant(1001L))
+ .nextBatch(
+ TimestampedValue.of("onTimePane", new Instant(500)))
+ .advanceNextBatchWatermarkToInfinity()
+ .nextBatch(
+ TimestampedValue.of("finalLatePane", new Instant(750)),
+ TimestampedValue.of("alsoFinalLatePane", new Instant(250)));
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
@@ -212,12 +212,13 @@
Instant lateElementTimestamp = new Instant(-1_000_000);
CreateStream<String> source =
CreateStream.of(StringUtf8Coder.of(), batchDuration())
- .emptyBatch()
- .advanceWatermarkForNextBatch(new Instant(0))
- .nextBatch(
- TimestampedValue.of("late", lateElementTimestamp),
- TimestampedValue.of("onTime", new Instant(100)))
- .advanceNextBatchWatermarkToInfinity();
+ .emptyBatch()
+ .advanceWatermarkForNextBatch(new Instant(0))
+ .emptyBatch()
+ .nextBatch(
+ TimestampedValue.of("late", lateElementTimestamp),
+ TimestampedValue.of("onTime", new Instant(100)))
+ .advanceNextBatchWatermarkToInfinity();
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 33a636a..e8a5951 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -148,6 +148,12 @@
}
@Override
+ public void enterPipeline(Pipeline p) {
+ super.enterPipeline(p);
+ evaluator.enterPipeline(p);
+ }
+
+ @Override
public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
return evaluator.enterCompositeTransform(node);
}
@@ -156,7 +162,7 @@
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
PTransform transform = node.getTransform();
if (transform.getClass() == transformClassToAssert) {
- AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+ AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform(getPipeline());
ctxt.setCurrentTransform(appliedTransform);
//noinspection unchecked
Dataset dataset = ctxt.borrowDataset((PTransform<? extends PValue, ?>) transform);
@@ -166,6 +172,12 @@
evaluator.visitPrimitiveTransform(node);
}
}
+
+ @Override
+ public void leavePipeline(Pipeline p) {
+ super.leavePipeline(p);
+ evaluator.leavePipeline(p);
+ }
}
}
diff --git a/runners/spark/src/test/resources/log4j.properties b/runners/spark/src/test/resources/log4j.properties
index 66e83c8..010c7df 100644
--- a/runners/spark/src/test/resources/log4j.properties
+++ b/runners/spark/src/test/resources/log4j.properties
@@ -24,7 +24,16 @@
log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
log4j.appender.testlogger.target = System.err
log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
-log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+log4j.appender.testlogger.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
# TestSparkRunner prints general information abut test pipelines execution.
log4j.logger.org.apache.beam.runners.spark.TestSparkRunner=INFO
+
+# in case of an emergency - uncomment (or better yet, stay calm and uncomment).
+#log4j.logger.org.apache.beam=TRACE
+#log4j.logger.org.apache.beam.sdk.Pipeline=INFO
+#log4j.logger.org.apache.beam.sdk.coders=INFO
+#log4j.logger.org.apache.beam.sdk.runners.TransformHierarchy=ERROR
+#log4j.logger.org.apache.beam.runners.spark.SparkRunner$Evaluator=ERROR
+#log4j.logger.org.apache.beam.runners.spark.translation.streaming.WatermarkSyncedDStream#compute=INFO
+#log4j.logger.org.apache.beam.runners.spark.translation.streaming.WatermarkSyncedDStream=ERROR
diff --git a/sdks/CONTAINERS.md b/sdks/CONTAINERS.md
new file mode 100644
index 0000000..2df8901
--- /dev/null
+++ b/sdks/CONTAINERS.md
@@ -0,0 +1,162 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Docker containers
+
+The Beam [portability effort](https://s.apache.org/beam-fn-api) aims to make it possible
+for any SDK to work with any runner. One aspect of the effort is the isolation of the SDK
+and user code execution environment from the runner execution environment using
+[docker](https://www.docker.com/), as defined in the portability
+[container contract](https://s.apache.org/beam-fn-api-container-contract).
+
+This document describes how to build and push container images to that end. The push
+step generally requires an account with a public docker registry, such
+as [bintray.io](bintray.io) or
+[Google Container Registry](https://cloud.google.com/container-registry). These
+instructions assume familiarity with docker and a bintray account under the
+current username with a docker repository named "apache".
+
+## How to build container images
+
+**Prerequisites**: install [docker](https://www.docker.com/) on your
+platform. You can verify that it works by running `docker images` or any other
+docker command.
+
+Run Maven with the `build-containers` profile:
+
+```
+$ pwd
+[...]/beam
+$ mvn clean install -DskipTests -Pbuild-containers
+[...]
+[INFO] --- dockerfile-maven-plugin:1.3.5:build (default) @ beam-sdks-python-container ---
+[INFO] Using Google application default credentials
+[INFO] loaded credentials for user account with clientId=[...].apps.googleusercontent.com
+[INFO] Building Docker context /Users/herohde/go/src/github.com/apache/beam/sdks/python/container
+[INFO]
+[INFO] Image will be built as herohde-docker-apache.bintray.io/beam/python:latest
+[INFO]
+[INFO] Step 1/4 : FROM python:2
+[INFO] Pulling from library/python
+[INFO] Digest: sha256:181ee8edfd9d44323c82dcba0b187f1ee2eb3d4a11c8398fc06952ed5f9ef32c
+[INFO] Status: Image is up to date for python:2
+[INFO] ---> b1d5c2d7dda8
+[INFO] Step 2/4 : MAINTAINER "Apache Beam <dev@beam.apache.org>"
+[INFO] ---> Running in f1bc3c4943b3
+[INFO] ---> 9867b512e47e
+[INFO] Removing intermediate container f1bc3c4943b3
+[INFO] Step 3/4 : ADD target/linux_amd64/boot /opt/apache/beam/
+[INFO] ---> 5cb81c3d2d90
+[INFO] Removing intermediate container 4a41ad80005a
+[INFO] Step 4/4 : ENTRYPOINT /opt/apache/beam/boot
+[INFO] ---> Running in 40f5b945afe7
+[INFO] ---> c8bf712741c8
+[INFO] Removing intermediate container 40f5b945afe7
+[INFO] Successfully built c8bf712741c8
+[INFO] Successfully tagged herohde-docker-apache.bintray.io/beam/python:latest
+[INFO]
+[INFO] Detected build of image with id c8bf712741c8
+[INFO] Building jar: /Users/herohde/go/src/github.com/apache/beam/sdks/python/container/target/beam-sdks-python-container-2.3.0-SNAPSHOT-docker-info.jar
+[INFO] Successfully built herohde-docker-apache.bintray.io/beam/python:latest
+[INFO]
+[...]
+```
+
+Note that the container images include built content, including the Go boot
+code, so you should build from the top level directory unless you're familiar
+with Maven.
+
+**(Optional)** When built, you can see, inspect and run them locally:
+
+```
+$ docker images
+REPOSITORY TAG IMAGE ID CREATED SIZE
+herohde-docker-apache.bintray.io/beam/python latest c8bf712741c8 About an hour ago 690MB
+herohde-docker-apache.bintray.io/beam/java latest 33efc0947952 About an hour ago 773MB
+[...]
+```
+
+Despite the names, these container images live only on your local machine.
+While we will re-use the same tag "latest" for each build, the
+images IDs will change.
+
+**(Optional)**: the default setting for `docker-repository-root` specifies the above bintray
+location. You can override it by adding:
+
+```
+-Ddocker-repository-root=<location>
+```
+
+Similarly, if you want to specify a specific tag instead of "latest", such as a "2.3.0"
+version, you can do so by adding:
+
+```
+-Ddockerfile.tag=<tag>
+```
+
+## How to push container images
+
+**Preprequisites**: obtain a docker registry account and ensure docker can push images to it,
+usually by doing `docker login` with the appropriate information. The image you want
+to push must also be present in the local docker image repository.
+
+For the Python SDK harness container image, run:
+
+```
+$ docker push $USER-docker-apache.bintray.io/beam/python:latest
+The push refers to a repository [herohde-docker-apache.bintray.io/beam/python]
+f2a8798331f5: Pushed
+6b200cb2b684: Layer already exists
+bf56c6510f38: Layer already exists
+7890d67efa6f: Layer already exists
+b456afdc9996: Layer already exists
+d752a0310ee4: Layer already exists
+db64edce4b5b: Layer already exists
+d5d60fc34309: Layer already exists
+c01c63c6823d: Layer already exists
+latest: digest: sha256:58da4d9173a29622f0572cfa22dfeafc45e6750dde4beab57a47a9d1d17d601b size: 2222
+
+```
+
+Similarly for the Java SDK harness container image. If you want to push the same image
+to multiple registries, you can retagging the image using `docker tag` and push.
+
+**(Optional)** On any machine, you can now pull the pushed container image:
+
+```
+$ docker pull $USER-docker-apache.bintray.io/beam/python:latest
+latest: Pulling from beam/python
+85b1f47fba49: Pull complete
+5409e9a7fa9e: Pull complete
+661393707836: Pull complete
+1bb98c08d57e: Pull complete
+c842a08369e2: Pull complete
+310408aa843f: Pull complete
+d6a27cfc2cf1: Pull complete
+7a24cf0c9043: Pull complete
+290b127dfe35: Pull complete
+Digest: sha256:58da4d9173a29622f0572cfa22dfeafc45e6750dde4beab57a47a9d1d17d601b
+Status: Downloaded newer image for herohde-docker-apache.bintray.io/beam/python:latest
+$ docker images
+REPOSITORY TAG IMAGE ID CREATED SIZE
+herohde-docker-apache.bintray.io/beam/python latest c8bf712741c8 2 hours ago 690MB
+[...]
+```
+
+Note that the image IDs and digests match their local counterparts.
diff --git a/sdks/common/fn-api/pom.xml b/sdks/common/fn-api/pom.xml
deleted file mode 100644
index ded9559..0000000
--- a/sdks/common/fn-api/pom.xml
+++ /dev/null
@@ -1,104 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <packaging>jar</packaging>
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-fn-api</artifactId>
- <name>Apache Beam :: SDKs :: Common :: Fn API</name>
- <description>This artifact generates the stub bindings.</description>
-
- <build>
- <resources>
- <resource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>${project.build.directory}/original_sources_to_package</directory>
- </resource>
- </resources>
-
- <plugins>
- <!-- Skip the checkstyle plugin on generated code -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <!-- Skip the findbugs plugin on generated code -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>compile-custom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-stub</artifactId>
- </dependency>
- </dependencies>
-</project>
diff --git a/sdks/common/fn-api/src/main/proto/beam_fn_api.proto b/sdks/common/fn-api/src/main/proto/beam_fn_api.proto
deleted file mode 100644
index 79e1872..0000000
--- a/sdks/common/fn-api/src/main/proto/beam_fn_api.proto
+++ /dev/null
@@ -1,761 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Fn API and boostrapping.
- *
- * TODO: Usage of plural names in lists looks awkward in Java
- * e.g. getOutputsMap, addCodersBuilder
- *
- * TODO: gRPC / proto field names conflict with generated code
- * e.g. "class" in java, "output" in python
- */
-
-syntax = "proto3";
-
-/* TODO: Consider consolidating common components in another package
- * and lanaguage namespaces for re-use with Runner Api.
- */
-
-package org.apache.beam.fn.v1;
-
-option java_package = "org.apache.beam.fn.v1";
-option java_outer_classname = "BeamFnApi";
-
-import "google/protobuf/any.proto";
-import "google/protobuf/timestamp.proto";
-
-/*
- * Constructs that define the pipeline shape.
- *
- * These are mostly unstable due to the missing pieces to be shared with
- * the Runner Api like windowing strategy, display data, .... There are still
- * some modelling questions related to whether a side input is modelled
- * as another field on a PrimitiveTransform or as part of inputs and we
- * still are missing things like the CompositeTransform.
- */
-
-// A representation of an input or output definition on a primitive transform.
-// Stable
-message Target {
- // A repeated list of target definitions.
- message List {
- repeated Target target = 1;
- }
-
- // (Required) The id of the PrimitiveTransform which is the target.
- string primitive_transform_reference = 1;
-
- // (Required) The local name of an input or output defined on the primitive
- // transform.
- string name = 2;
-}
-
-// Information defining a PCollection
-message PCollection {
- // (Required) A reference to a coder.
- string coder_reference = 1;
-
- // TODO: Windowing strategy, ...
-}
-
-// A primitive transform within Apache Beam.
-message PrimitiveTransform {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) A function spec that is used by this primitive
- // transform to process data.
- FunctionSpec function_spec = 2;
-
- // A map of distinct input names to target definitions.
- // For example, in CoGbk this represents the tag name associated with each
- // distinct input name and a list of primitive transforms that are associated
- // with the specified input.
- map<string, Target.List> inputs = 3;
-
- // A map from local output name to PCollection definitions. For example, in
- // DoFn this represents the tag name associated with each distinct output.
- map<string, PCollection> outputs = 4;
-
- // TODO: Should we model side inputs as a special type of input for a
- // primitive transform or should it be modeled as the relationship that
- // the predecessor input will be a view primitive transform.
- // A map of from side input names to side inputs.
- map<string, SideInput> side_inputs = 5;
-
- // The user name of this step.
- // TODO: This should really be in display data and not at this level
- string step_name = 6;
-}
-
-/*
- * User Definable Functions
- *
- * This is still unstable mainly due to how we model the side input.
- */
-
-// Defines the common elements of user-definable functions, to allow the SDK to
-// express the information the runner needs to execute work.
-// Stable
-message FunctionSpec {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) A globally unique name representing this user definable
- // function.
- //
- // User definable functions use the urn encodings registered such that another
- // may implement the user definable function within another language.
- //
- // For example:
- // urn:org.apache.beam:coder:kv:1.0
- string urn = 2;
-
- // (Required) Reference to specification of execution environment required to
- // invoke this function.
- string environment_reference = 3;
-
- // Data used to parameterize this function. Depending on the urn, this may be
- // optional or required.
- google.protobuf.Any data = 4;
-}
-
-message SideInput {
- // TODO: Coder?
-
- // For RunnerAPI.
- Target input = 1;
-
- // For FnAPI.
- FunctionSpec view_fn = 2;
-}
-
-// Defines how to encode values into byte streams and decode values from byte
-// streams. A coder can be parameterized by additional properties which may or
-// may not be language agnostic.
-//
-// Coders using the urn:org.apache.beam:coder namespace must have their
-// encodings registered such that another may implement the encoding within
-// another language.
-//
-// For example:
-// urn:org.apache.beam:coder:kv:1.0
-// urn:org.apache.beam:coder:iterable:1.0
-// Stable
-message Coder {
- // TODO: This looks weird when compared to the other function specs
- // which use URN to differentiate themselves. Should "Coder" be embedded
- // inside the FunctionSpec data block.
-
- // The data associated with this coder used to reconstruct it.
- FunctionSpec function_spec = 1;
-
- // A list of component coder references.
- //
- // For a key-value coder, there must be exactly two component coder references
- // where the first reference represents the key coder and the second reference
- // is the value coder.
- //
- // For an iterable coder, there must be exactly one component coder reference
- // representing the value coder.
- //
- // TODO: Perhaps this is redundant with the data of the FunctionSpec
- // for known coders?
- repeated string component_coder_reference = 2;
-}
-
-// A descriptor for connecting to a remote port using the Beam Fn Data API.
-// Allows for communication between two environments (for example between the
-// runner and the SDK).
-// Stable
-message RemoteGrpcPort {
- // (Required) An API descriptor which describes where to
- // connect to including any authentication that is required.
- ApiServiceDescriptor api_service_descriptor = 1;
-}
-
-/*
- * Control Plane API
- *
- * Progress reporting and splitting still need further vetting. Also, this may change
- * with the addition of new types of instructions/responses related to metrics.
- */
-
-// An API that describes the work that a SDK harness is meant to do.
-// Stable
-service BeamFnControl {
- // Instructions sent by the runner to the SDK requesting different types
- // of work.
- rpc Control(
- // A stream of responses to instructions the SDK was asked to be performed.
- stream InstructionResponse
- ) returns (
- // A stream of instructions requested of the SDK to be performed.
- stream InstructionRequest
- ) {}
-}
-
-// A request sent by a runner which it the SDK is asked to fulfill.
-// Stable
-message InstructionRequest {
- // (Required) An unique identifier provided by the runner which represents
- // this requests execution. The InstructionResponse MUST have the matching id.
- string instruction_id = 1;
-
- // (Required) A request that the SDK Harness needs to interpret.
- oneof request {
- RegisterRequest register = 1000;
- ProcessBundleRequest process_bundle = 1001;
- ProcessBundleProgressRequest process_bundle_progress = 1002;
- ProcessBundleSplitRequest process_bundle_split = 1003;
- }
-}
-
-// The response for an associated request the SDK had been asked to fulfill.
-// Stable
-message InstructionResponse {
- // (Required) A reference provided by the runner which represents a requests
- // execution. The InstructionResponse MUST have the matching id when
- // responding to the runner.
- string instruction_id = 1;
-
- // If this is specified, then this instruction has failed.
- // A human readable string representing the reason as to why processing has
- // failed.
- string error = 2;
-
- // If the instruction did not fail, it is required to return an equivalent
- // response type depending on the request this matches.
- oneof response {
- RegisterResponse register = 1000;
- ProcessBundleResponse process_bundle = 1001;
- ProcessBundleProgressResponse process_bundle_progress = 1002;
- ProcessBundleSplitResponse process_bundle_split = 1003;
- }
-}
-
-// A list of objects which can be referred to by the runner in
-// future requests.
-// Stable
-message RegisterRequest {
- // (Optional) The set of descriptors used to process bundles.
- repeated ProcessBundleDescriptor process_bundle_descriptor = 1;
-}
-
-// Stable
-message RegisterResponse {
-}
-
-// A descriptor of references used when processing a bundle.
-// Stable
-message ProcessBundleDescriptor {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) A list of primitive transforms that should
- // be used to construct the bundle processing graph.
- repeated PrimitiveTransform primitive_transform = 2;
-
- // (Required) The set of all coders referenced in this bundle.
- repeated Coder coders = 4;
-}
-
-// A request to process a given bundle.
-// Stable
-message ProcessBundleRequest {
- // (Required) A reference to the process bundle descriptor that must be
- // instantiated and executed by the SDK harness.
- string process_bundle_descriptor_reference = 1;
-
- // (Optional) A list of cache tokens that can be used by an SDK to cache
- // data looked up using the State API across multiple bundles.
- repeated CacheToken cache_tokens = 2;
-}
-
-// Stable
-message ProcessBundleResponse {
-}
-
-message ProcessBundleProgressRequest {
- // (Required) A reference to an active process bundle request with the given
- // instruction id.
- string instruction_reference = 1;
-}
-
-message ProcessBundleProgressResponse {
- // (Required) The finished amount of work. A monotonically increasing
- // unitless measure of work finished.
- double finished_work = 1;
-
- // (Required) The known amount of backlog for the process bundle request.
- // Computed as:
- // (estimated known work - finish work) / finished work
- double backlog = 2;
-}
-
-message ProcessBundleSplitRequest {
- // (Required) A reference to an active process bundle request with the given
- // instruction id.
- string instruction_reference = 1;
-
- // (Required) The fraction of work (when compared to the known amount of work)
- // the process bundle request should try to split at.
- double fraction = 2;
-}
-
-// urn:org.apache.beam:restriction:element-count:1.0
-message ElementCountRestriction {
- // A restriction representing the number of elements that should be processed.
- // Effectively the range [0, count]
- int64 count = 1;
-}
-
-// urn:org.apache.beam:restriction:element-count-skip:1.0
-message ElementCountSkipRestriction {
- // A restriction representing the number of elements that should be skipped.
- // Effectively the range (count, infinity]
- int64 count = 1;
-}
-
-// Each primitive transform that is splittable is defined by a restriction
-// it is currently processing. During splitting, that currently active
-// restriction (R_initial) is split into 2 components:
-// * a restriction (R_done) representing all elements that will be fully
-// processed
-// * a restriction (R_todo) representing all elements that will not be fully
-// processed
-//
-// where:
-// R_initial = R_done ⋃ R_todo
-message PrimitiveTransformSplit {
- // (Required) A reference to a primitive transform with the given id that
- // is part of the active process bundle request with the given instruction
- // id.
- string primitive_transform_reference = 1;
-
- // (Required) A function specification describing the restriction
- // that has been completed by the primitive transform.
- //
- // For example, a remote GRPC source will have a specific urn and data
- // block containing an ElementCountRestriction.
- FunctionSpec completed_restriction = 2;
-
- // (Required) A function specification describing the restriction
- // representing the remainder of work for the primitive transform.
- //
- // FOr example, a remote GRPC source will have a specific urn and data
- // block contain an ElemntCountSkipRestriction.
- FunctionSpec remaining_restriction = 3;
-}
-
-message ProcessBundleSplitResponse {
- // (Optional) A set of split responses for a currently active work item.
- //
- // If primitive transform B is a descendant of primitive transform A and both
- // A and B report a split. Then B's restriction is reported as an element
- // restriction pair and thus the fully reported restriction is:
- // R = A_done
- // ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ A_todo
- // If there is a decendant of B named C, then C would similarly report a
- // set of element pair restrictions.
- //
- // This restriction is processed and completed by the currently active process
- // bundle request:
- // A_done ⋃ (A_boundary ⋂ B_done)
- // and these restrictions will be processed by future process bundle requests:
- // A_boundary â‹‚ B_todo (passed to SDF B directly)
- // A_todo (passed to SDF A directly)
-
- // If primitive transform B and C are siblings and descendants of A and A, B,
- // and C report a split. Then B and C's restrictions are relative to A's.
- // R = A_done
- // ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ (A_boundary ⋂ C_todo)
- // ⋃ A_todo
- // If there is no descendant of B or C also reporting a split, than
- // B_boundary = ∅ and C_boundary = ∅
- //
- // This restriction is processed and completed by the currently active process
- // bundle request:
- // A_done ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ C_done)
- // and these restrictions will be processed by future process bundle requests:
- // A_boundary â‹‚ B_todo (passed to SDF B directly)
- // A_boundary â‹‚ C_todo (passed to SDF C directly)
- // A_todo (passed to SDF A directly)
- //
- // Note that descendants splits should only be reported if it is inexpensive
- // to compute the boundary restriction intersected with descendants splits.
- // Also note, that the boundary restriction may represent a set of elements
- // produced by a parent primitive transform which can not be split at each
- // element or that there are intermediate unsplittable primitive transforms
- // between an ancestor splittable function and a descendant splittable
- // function which may have more than one output per element. Finally note
- // that the descendant splits should only be reported if the split
- // information is relatively compact.
- repeated PrimitiveTransformSplit splits = 1;
-}
-
-/*
- * Data Plane API
- */
-
-// Messages used to represent logical byte streams.
-// Stable
-message Elements {
- // Represents multiple encoded elements in nested context for a given named
- // instruction and target.
- message Data {
- // (Required) A reference to an active instruction request with the given
- // instruction id.
- string instruction_reference = 1;
-
- // (Required) A definition representing a consumer or producer of this data.
- // If received by a harness, this represents the consumer within that
- // harness that should consume these bytes. If sent by a harness, this
- // represents the producer of these bytes.
- //
- // Note that a single element may span multiple Data messages.
- //
- // Note that a sending/receiving pair should share the same target
- // identifier.
- Target target = 2;
-
- // (Optional) Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // concatenated together.
- //
- // An empty data block represents the end of stream for the given
- // instruction and target.
- bytes data = 3;
- }
-
- // (Required) A list containing parts of logical byte streams.
- repeated Data data = 1;
-}
-
-// Stable
-service BeamFnData {
- // Used to send data between harnesses.
- rpc Data(
- // A stream of data representing input.
- stream Elements
- ) returns (
- // A stream of data representing output.
- stream Elements
- ) {}
-}
-
-/*
- * State API
- */
-
-message StateRequest {
- // (Required) An unique identifier provided by the SDK which represents this
- // requests execution. The StateResponse corresponding with this request
- // will have the matching id.
- string id = 1;
-
- // (Required) The associated instruction id of the work that is currently
- // being processed. This allows for the runner to associate any modifications
- // to state to be committed with the appropriate work execution.
- string instruction_reference = 2;
-
- // (Required) The state key this request is for.
- StateKey state_key = 3;
-
- // (Required) The action to take on this request.
- oneof request {
- // A request to get state.
- StateGetRequest get = 1000;
-
- // A request to append to state.
- StateAppendRequest append = 1001;
-
- // A request to clear state.
- StateClearRequest clear = 1002;
- }
-}
-
-message StateResponse {
- // (Required) A reference provided by the SDK which represents a requests
- // execution. The StateResponse must have the matching id when responding
- // to the SDK.
- string id = 1;
-
- // (Optional) If this is specified, then the state request has failed.
- // A human readable string representing the reason as to why the request
- // failed.
- string error = 2;
-
- // A corresponding response matching the request will be populated.
- oneof response {
- // A response to getting state.
- StateGetResponse get = 1000;
-
- // A response to appending to state.
- StateAppendResponse append = 1001;
-
- // A response to clearing state.
- StateClearResponse clear = 1002;
- }
-}
-
-service BeamFnState {
- // Used to get/append/clear state stored by the runner on behalf of the SDK.
- rpc State(
- // A stream of state instructions requested of the runner.
- stream StateRequest
- ) returns (
- // A stream of responses to state instructions the runner was asked to be
- // performed.
- stream StateResponse
- ) {}
-}
-
-message CacheToken {
- // (Required) Represents the function spec and tag associated with this state
- // key.
- //
- // By combining the function_spec_reference with the tag representing:
- // * the input, we refer to the iterable portion of a large GBK
- // * the side input, we refer to the side input
- // * the user state, we refer to user state
- Target target = 1;
-
- // (Required) An opaque identifier.
- bytes token = 2;
-}
-
-message StateKey {
- // (Required) Represents the function spec and tag associated with this state
- // key.
- //
- // By combining the function_spec_reference with the tag representing:
- // * the input, we refer to fetching the iterable portion of a large GBK
- // * the side input, we refer to fetching the side input
- // * the user state, we refer to fetching user state
- Target target = 1;
-
- // (Required) The bytes of the window which this state request is for encoded
- // in the nested context.
- bytes window = 2;
-
- // (Required) The user key encoded in the nested context.
- bytes key = 3;
-}
-
-// A logical byte stream which can be continued using the state API.
-message ContinuableStream {
- // (Optional) If specified, represents a token which can be used with the
- // state API to get the next chunk of this logical byte stream. The end of
- // the logical byte stream is signalled by this field being unset.
- bytes continuation_token = 1;
-
- // Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // concatenated together.
- bytes data = 2;
-}
-
-// A request to get state.
-message StateGetRequest {
- // (Optional) If specified, signals to the runner that the response
- // should resume from the following continuation token.
- //
- // If unspecified, signals to the runner that the response should start
- // from the beginning of the logical continuable stream.
- bytes continuation_token = 1;
-}
-
-// A response to get state.
-message StateGetResponse {
- // (Required) The response containing a continuable logical byte stream.
- ContinuableStream stream = 1;
-}
-
-// A request to append state.
-message StateAppendRequest {
- // Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // multiple append requests are concatenated together.
- bytes data = 1;
-}
-
-// A response to append state.
-message StateAppendResponse {
-}
-
-// A request to clear state.
-message StateClearRequest {
-}
-
-// A response to clear state.
-message StateClearResponse {
-}
-
-/*
- * Logging API
- *
- * This is very stable. There can be some changes to how we define a LogEntry,
- * to increase/decrease the severity types, the way we format an exception/stack
- * trace, or the log site.
- */
-
-// A log entry
-message LogEntry {
- // A list of log entries, enables buffering and batching of multiple
- // log messages using the logging API.
- message List {
- // (Required) One or or more log messages.
- repeated LogEntry log_entries = 1;
- }
-
- // The severity of the event described in a log entry, expressed as one of the
- // severity levels listed below. For your reference, the levels are
- // assigned the listed numeric values. The effect of using numeric values
- // other than those listed is undefined.
- //
- // If you are writing log entries, you should map other severity encodings to
- // one of these standard levels. For example, you might map all of
- // Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`.
- //
- // This list is intentionally not comprehensive; the intent is to provide a
- // common set of "good enough" severity levels so that logging front ends
- // can provide filtering and searching across log types. Users of the API are
- // free not to use all severity levels in their log messages.
- enum Severity {
- // Trace level information, also the default log level unless
- // another severity is specified.
- TRACE = 0;
- // Debugging information.
- DEBUG = 10;
- // Normal events.
- INFO = 20;
- // Normal but significant events, such as start up, shut down, or
- // configuration.
- NOTICE = 30;
- // Warning events might cause problems.
- WARN = 40;
- // Error events are likely to cause problems.
- ERROR = 50;
- // Critical events cause severe problems or brief outages and may
- // indicate that a person must take action.
- CRITICAL = 60;
- }
-
- // (Required) The severity of the log statement.
- Severity severity = 1;
-
- // (Required) The time at which this log statement occurred.
- google.protobuf.Timestamp timestamp = 2;
-
- // (Required) A human readable message.
- string message = 3;
-
- // (Optional) An optional trace of the functions involved. For example, in
- // Java this can include multiple causes and multiple suppressed exceptions.
- string trace = 4;
-
- // (Optional) A reference to the instruction this log statement is associated
- // with.
- string instruction_reference = 5;
-
- // (Optional) A reference to the primitive transform this log statement is
- // associated with.
- string primitive_transform_reference = 6;
-
- // (Optional) Human-readable name of the function or method being invoked,
- // with optional context such as the class or package name. The format can
- // vary by language. For example:
- // qual.if.ied.Class.method (Java)
- // dir/package.func (Go)
- // module.function (Python)
- // file.cc:382 (C++)
- string log_location = 7;
-
- // (Optional) The name of the thread this log statement is associated with.
- string thread = 8;
-}
-
-message LogControl {
-}
-
-// Stable
-service BeamFnLogging {
- // Allows for the SDK to emit log entries which the runner can
- // associate with the active job.
- rpc Logging(
- // A stream of log entries batched into lists emitted by the SDK harness.
- stream LogEntry.List
- ) returns (
- // A stream of log control messages used to configure the SDK.
- stream LogControl
- ) {}
-}
-
-/*
- * Environment types
- */
-message ApiServiceDescriptor {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) The URL to connect to.
- string url = 2;
-
- // (Optional) The method for authentication. If unspecified, access to the
- // url is already being performed in a trusted context (e.g. localhost,
- // private network).
- oneof authentication {
- OAuth2ClientCredentialsGrant oauth2_client_credentials_grant = 3;
- }
-}
-
-message OAuth2ClientCredentialsGrant {
- // (Required) The URL to submit a "client_credentials" grant type request for
- // an OAuth access token which will be used as a bearer token for requests.
- string url = 1;
-}
-
-// A Docker container configuration for launching the SDK harness to execute
-// user specified functions.
-message DockerContainer {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) The Docker container URI
- // For example "dataflow.gcr.io/v1beta3/java-batch:1.5.1"
- string uri = 2;
-
- // (Optional) Docker registry specification.
- // If unspecified, the uri is expected to be able to be fetched without
- // requiring additional configuration by a runner.
- string registry_reference = 3;
-}
-
diff --git a/sdks/common/pom.xml b/sdks/common/pom.xml
deleted file mode 100644
index c621ed5..0000000
--- a/sdks/common/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-parent</artifactId>
-
- <packaging>pom</packaging>
-
- <name>Apache Beam :: SDKs :: Common</name>
-
- <modules>
- <module>fn-api</module>
- <module>runner-api</module>
- </modules>
-</project>
diff --git a/sdks/common/runner-api/pom.xml b/sdks/common/runner-api/pom.xml
deleted file mode 100644
index f5536a7..0000000
--- a/sdks/common/runner-api/pom.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <packaging>jar</packaging>
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-runner-api</artifactId>
- <name>Apache Beam :: SDKs :: Common :: Runner API</name>
- <description>This artifact generates the stub bindings.</description>
-
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>${project.build.directory}/original_sources_to_package</directory>
- </resource>
- </resources>
-
- <plugins>
- <!-- Skip the checkstyle plugin on generated code -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <!-- Skip the findbugs plugin on generated code -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- </dependencies>
-</project>
diff --git a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
deleted file mode 100644
index bf4df2a..0000000
--- a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
+++ /dev/null
@@ -1,711 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Runner API, which is the runner-independent,
- * SDK-independent definition of the Beam model.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.runner_api.v1;
-
-option java_package = "org.apache.beam.sdk.common.runner.v1";
-option java_outer_classname = "RunnerApi";
-
-import "google/protobuf/any.proto";
-
-// A set of mappings from id to message. This is included as an optional field
-// on any proto message that may contain references needing resolution.
-message Components {
- // (Required) A map from pipeline-scoped id to PTransform.
- map<string, PTransform> transforms = 1;
-
- // (Required) A map from pipeline-scoped id to PCollection.
- map<string, PCollection> pcollections = 2;
-
- // (Required) A map from pipeline-scoped id to WindowingStrategy.
- map<string, WindowingStrategy> windowing_strategies = 3;
-
- // (Required) A map from pipeline-scoped id to Coder.
- map<string, Coder> coders = 4;
-
- // (Required) A map from pipeline-scoped id to Environment.
- map<string, Environment> environments = 5;
-}
-
-// A disjoint union of all the things that may contain references
-// that require Components to resolve.
-message MessageWithComponents {
-
- // (Optional) The by-reference components of the root message,
- // enabling a standalone message.
- //
- // If this is absent, it is expected that there are no
- // references.
- Components components = 1;
-
- // (Required) The root message that may contain pointers
- // that should be resolved by looking inside components.
- oneof root {
- Coder coder = 2;
- CombinePayload combine_payload = 3;
- SdkFunctionSpec sdk_function_spec = 4;
- ParDoPayload par_do_payload = 6;
- PTransform ptransform = 7;
- PCollection pcollection = 8;
- ReadPayload read_payload = 9;
- SideInput side_input = 11;
- WindowIntoPayload window_into_payload = 12;
- WindowingStrategy windowing_strategy = 13;
- FunctionSpec function_spec = 14;
- }
-}
-
-// A Pipeline is a hierarchical graph of PTransforms, linked
-// by PCollections.
-//
-// This is represented by a number of by-reference maps to nodes,
-// PCollections, SDK environments, UDF, etc., for
-// supporting compact reuse and arbitrary graph structure.
-//
-// All of the keys in the maps here are arbitrary strings that are only
-// required to be internally consistent within this proto message.
-message Pipeline {
-
- // (Required) The coders, UDFs, graph nodes, etc, that make up
- // this pipeline.
- Components components = 1;
-
- // (Required) The ids of all PTransforms that are not contained within another PTransform
- repeated string root_transform_ids = 2;
-
- // (Optional) Static display data for the pipeline. If there is none,
- // it may be omitted.
- DisplayData display_data = 3;
-}
-
-// An applied PTransform! This does not contain the graph data, but only the
-// fields specific to a graph node that is a Runner API transform
-// between PCollections.
-message PTransform {
-
- // (Required) A unique name for the application node.
- //
- // Ideally, this should be stable over multiple evolutions of a pipeline
- // for the purposes of logging and associating pipeline state with a node,
- // etc.
- //
- // If it is not stable, then the runner decides what will happen. But, most
- // importantly, it must always be here and be unique, even if it is
- // autogenerated.
- string unique_name = 5;
-
- // (Optional) A URN and payload that, together, fully defined the semantics
- // of this transform.
- //
- // If absent, this must be an "anonymous" composite transform.
- //
- // For primitive transform in the Runner API, this is required, and the
- // payloads are well-defined messages. When the URN indicates ParDo it
- // is a ParDoPayload, and so on.
- //
- // TODO: document the standardized URNs and payloads
- // TODO: separate standardized payloads into a separate proto file
- //
- // For some special composite transforms, the payload is also officially
- // defined:
- //
- // - when the URN is "urn:beam:transforms:combine" it is a CombinePayload
- //
- FunctionSpec spec = 1;
-
- // (Optional) if this node is a composite, a list of the ids of
- // transforms that it contains.
- repeated string subtransforms = 2;
-
- // (Required) A map from local names of inputs (unique only with this map, and
- // likely embedded in the transform payload and serialized user code) to
- // PCollection ids.
- //
- // The payload for this transform may clarify the relationship of these
- // inputs. For example:
- //
- // - for a Flatten transform they are merged
- // - for a ParDo transform, some may be side inputs
- //
- // All inputs are recorded here so that the topological ordering of
- // the graph is consistent whether or not the payload is understood.
- //
- map<string, string> inputs = 3;
-
- // (Required) A map from local names of outputs (unique only within this map,
- // and likely embedded in the transform payload and serialized user code)
- // to PCollection ids.
- //
- // The URN or payload for this transform node may clarify the type and
- // relationship of these outputs. For example:
- //
- // - for a ParDo transform, these are tags on PCollections, which will be
- // embedded in the DoFn.
- //
- map<string, string> outputs = 4;
-
- // (Optional) Static display data for this PTransform application. If
- // there is none, or it is not relevant (such as use by the Fn API)
- // then it may be omitted.
- DisplayData display_data = 6;
-}
-
-// A PCollection!
-message PCollection {
-
- // (Required) A unique name for the PCollection.
- //
- // Ideally, this should be stable over multiple evolutions of a pipeline
- // for the purposes of logging and associating pipeline state with a node,
- // etc.
- //
- // If it is not stable, then the runner decides what will happen. But, most
- // importantly, it must always be here, even if it is autogenerated.
- string unique_name = 1;
-
- // (Required) The id of the Coder for this PCollection.
- string coder_id = 2;
-
- // (Required) Whether this PCollection is bounded or unbounded
- IsBounded is_bounded = 3;
-
- // (Required) The id of the windowing strategy for this PCollection.
- string windowing_strategy_id = 4;
-
- // (Optional) Static display data for this PTransform application. If
- // there is none, or it is not relevant (such as use by the Fn API)
- // then it may be omitted.
- DisplayData display_data = 5;
-}
-
-// The payload for the primitive ParDo transform.
-message ParDoPayload {
-
- // (Required) The SdkFunctionSpec of the DoFn.
- SdkFunctionSpec do_fn = 1;
-
- // (Required) Additional pieces of context the DoFn may require that
- // are not otherwise represented in the payload.
- // (may force runners to execute the ParDo differently)
- repeated Parameter parameters = 2;
-
- // (Optional) A mapping of local input names to side inputs, describing
- // the expected access pattern.
- map<string, SideInput> side_inputs = 3;
-
- // (Optional) A mapping of local state names to state specifications.
- map<string, StateSpec> state_specs = 4;
-
- // (Optional) A mapping of local timer names to timer specifications.
- map<string, TimerSpec> timer_specs = 5;
-}
-
-// Parameters that a UDF might require.
-//
-// The details of how a runner sends these parameters to the SDK harness
-// are the subject of the Fn API.
-//
-// The details of how an SDK harness delivers them to the UDF is entirely
-// up to the SDK. (for some SDKs there may be parameters that are not
-// represented here if the runner doesn't need to do anything)
-//
-// Here, the parameters are simply indicators to the runner that they
-// need to run the function a particular way.
-//
-// TODO: the evolution of the Fn API will influence what needs explicit
-// representation here
-message Parameter {
- Type type = 1;
-
- enum Type {
- WINDOW = 0;
- PIPELINE_OPTIONS = 1;
- RESTRICTION_TRACKER = 2;
- }
-}
-
-message StateSpec {
- // TODO: AST for state spec
-}
-
-message TimerSpec {
- // TODO: AST for timer spec
-}
-
-enum IsBounded {
- BOUNDED = 0;
- UNBOUNDED = 1;
-}
-
-// The payload for the primitive Read transform.
-message ReadPayload {
-
- // (Required) The SdkFunctionSpec of the source for this Read.
- SdkFunctionSpec source = 1;
-
- // (Required) Whether the source is bounded or unbounded
- IsBounded is_bounded = 2;
-
- // TODO: full audit of fields required by runners as opposed to SDK harness
-}
-
-// The payload for the WindowInto transform.
-message WindowIntoPayload {
-
- // (Required) The SdkFunctionSpec of the WindowFn.
- SdkFunctionSpec window_fn = 1;
-}
-
-// The payload for the special-but-not-primitive Combine transform.
-message CombinePayload {
-
- // (Required) The SdkFunctionSpec of the CombineFn.
- SdkFunctionSpec combine_fn = 1;
-
- // (Required) A reference to the Coder to use for accumulators of the CombineFn
- string accumulator_coder_id = 2;
-
- // (Required) Additional pieces of context the DoFn may require that
- // are not otherwise represented in the payload.
- // (may force runners to execute the ParDo differently)
- repeated Parameter parameters = 3;
-
- // (Optional) A mapping of local input names to side inputs, describing
- // the expected access pattern.
- map<string, SideInput> side_inputs = 4;
-}
-
-// A coder, the binary format for serialization and deserialization of data in
-// a pipeline.
-message Coder {
-
- // (Required) A specification for the coder, as a URN plus parameters. This
- // may be a cross-language agreed-upon format, or it may be a "custom coder"
- // that can only be used by a particular SDK. It does not include component
- // coders, as it is beneficial for these to be comprehensible to a runner
- // regardless of whether the binary format is agree-upon.
- SdkFunctionSpec spec = 1;
-
- // (Optional) If this coder is parametric, such as ListCoder(VarIntCoder),
- // this is a list of the components. In order for encodings to be identical,
- // the SdkFunctionSpec and all components must be identical, recursively.
- repeated string component_coder_ids = 2;
-}
-
-// A windowing strategy describes the window function, triggering, allowed
-// lateness, and accumulation mode for a PCollection.
-//
-// TODO: consider inlining field on PCollection
-message WindowingStrategy {
-
- // (Required) The SdkFunctionSpec of the UDF that assigns windows,
- // merges windows, and shifts timestamps before they are
- // combined according to the OutputTime.
- SdkFunctionSpec window_fn = 1;
-
- // (Required) Whether or not the window fn is merging.
- //
- // This knowledge is required for many optimizations.
- MergeStatus merge_status = 2;
-
- // (Required) The coder for the windows of this PCollection.
- string window_coder_id = 3;
-
- // (Required) The trigger to use when grouping this PCollection.
- Trigger trigger = 4;
-
- // (Required) The accumulation mode indicates whether new panes are a full
- // replacement for prior panes or whether they are deltas to be combined
- // with other panes (the combine should correspond to whatever the upstream
- // grouping transform is).
- AccumulationMode accumulation_mode = 5;
-
- // (Required) The OutputTime specifies, for a grouping transform, how to
- // compute the aggregate timestamp. The window_fn will first possibly shift
- // it later, then the OutputTime takes the max, min, or ignores it and takes
- // the end of window.
- //
- // This is actually only for input to grouping transforms, but since they
- // may be introduced in runner-specific ways, it is carried along with the
- // windowing strategy.
- OutputTime output_time = 6;
-
- // (Required) Indicate when output should be omitted upon window expiration.
- ClosingBehavior closing_behavior = 7;
-
- // (Required) The duration, in milliseconds, beyond the end of a window at
- // which the window becomes droppable.
- int64 allowed_lateness = 8;
-}
-
-// Whether or not a PCollection's WindowFn is non-merging, merging, or
-// merging-but-already-merged, in which case a subsequent GroupByKey is almost
-// always going to do something the user does not want
-enum MergeStatus {
- // The WindowFn does not require merging.
- // Examples: global window, FixedWindows, SlidingWindows
- NON_MERGING = 0;
-
- // The WindowFn is merging and the PCollection has not had merging
- // performed.
- // Example: Sessions prior to a GroupByKey
- NEEDS_MERGE = 1;
-
- // The WindowFn is merging and the PCollection has had merging occur
- // already.
- // Example: Sessions after a GroupByKey
- ALREADY_MERGED = 2;
-}
-
-// Whether or not subsequent outputs of aggregations should be entire
-// replacement values or just the aggregation of inputs received since
-// the prior output.
-enum AccumulationMode {
-
- // The aggregation is discarded when it is output
- DISCARDING = 0;
-
- // The aggregation is accumulated across outputs
- ACCUMULATING = 1;
-}
-
-// Controls whether or not an aggregating transform should output data
-// when a window expires.
-enum ClosingBehavior {
-
- // Emit output when a window expires, whether or not there has been
- // any new data since the last output.
- EMIT_ALWAYS = 0;
-
- // Only emit output when new data has arrives since the last output
- EMIT_IF_NONEMPTY = 1;
-}
-
-// When a number of windowed, timestamped inputs are aggregated, the timestamp
-// for the resulting output.
-enum OutputTime {
- // The output has the timestamp of the end of the window.
- END_OF_WINDOW = 0;
-
- // The output has the latest timestamp of the input elements since
- // the last output.
- LATEST_IN_PANE = 1;
-
- // The output has the earliest timestamp of the input elements since
- // the last output.
- EARLIEST_IN_PANE = 2;
-}
-
-// The different time domains in the Beam model.
-enum TimeDomain {
-
- // Event time is time from the perspective of the data
- EVENT_TIME = 0;
-
- // Processing time is time from the perspective of the
- // execution of your pipeline
- PROCESSING_TIME = 1;
-
- // Synchronized processing time is the minimum of the
- // processing time of all pending elements.
- //
- // The "processing time" of an element refers to
- // the local processing time at which it was emitted
- SYNCHRONIZED_PROCESSING_TIME = 2;
-}
-
-// A small DSL for expressing when to emit new aggregations
-// from a GroupByKey or CombinePerKey
-//
-// A trigger is described in terms of when it is _ready_ to permit output.
-message Trigger {
-
- // Ready when all subtriggers are ready.
- message AfterAll {
- repeated Trigger subtriggers = 1;
- }
-
- // Ready when any subtrigger is ready.
- message AfterAny {
- repeated Trigger subtriggers = 1;
- }
-
- // Starting with the first subtrigger, ready when the _current_ subtrigger
- // is ready. After output, advances the current trigger by one.
- message AfterEach {
- repeated Trigger subtriggers = 1;
- }
-
- // Ready after the input watermark is past the end of the window.
- //
- // May have implicitly-repeated subtriggers for early and late firings.
- // When the end of the window is reached, the trigger transitions between
- // the subtriggers.
- message AfterEndOfWindow {
-
- // (Optional) A trigger governing output prior to the end of the window.
- Trigger early_firings = 1;
-
- // (Optional) A trigger governing output after the end of the window.
- Trigger late_firings = 2;
- }
-
- // After input arrives, ready when the specified delay has passed.
- message AfterProcessingTime {
-
- // (Required) The transforms to apply to an arriving element's timestamp,
- // in order
- repeated TimestampTransform timestamp_transforms = 1;
- }
-
- // Ready whenever upstream processing time has all caught up with
- // the arrival time of an input element
- message AfterSynchronizedProcessingTime {
- }
-
- // The default trigger. Equivalent to Repeat { AfterEndOfWindow } but
- // specially denoted to indicate the user did not alter the triggering.
- message Default {
- }
-
- // Ready whenever the requisite number of input elements have arrived
- message ElementCount {
- int32 element_count = 1;
- }
-
- // Never ready. There will only be an ON_TIME output and a final
- // output at window expiration.
- message Never {
- }
-
- // Always ready. This can also be expressed as ElementCount(1) but
- // is more explicit.
- message Always {
- }
-
- // Ready whenever either of its subtriggers are ready, but finishes output
- // when the finally subtrigger fires.
- message OrFinally {
-
- // (Required) Trigger governing main output; may fire repeatedly.
- Trigger main = 1;
-
- // (Required) Trigger governing termination of output.
- Trigger finally = 2;
- }
-
- // Ready whenever the subtrigger is ready; resets state when the subtrigger
- // completes.
- message Repeat {
- // (Require) Trigger that is run repeatedly.
- Trigger subtrigger = 1;
- }
-
- // The full disjoint union of possible triggers.
- oneof trigger {
- AfterAll after_all = 1;
- AfterAny after_any = 2;
- AfterEach after_each = 3;
- AfterEndOfWindow after_end_of_window = 4;
- AfterProcessingTime after_processing_time = 5;
- AfterSynchronizedProcessingTime after_synchronized_processing_time = 6;
- Always always = 12;
- Default default = 7;
- ElementCount element_count = 8;
- Never never = 9;
- OrFinally or_finally = 10;
- Repeat repeat = 11;
- }
-}
-
-// A specification for a transformation on a timestamp.
-//
-// Primarily used by AfterProcessingTime triggers to transform
-// the arrival time of input to a target time for firing.
-message TimestampTransform {
- oneof timestamp_transform {
- Delay delay = 1;
- AlignTo align_to = 2;
- }
-
- message Delay {
- // (Required) The delay, in milliseconds.
- int64 delay_millis = 1;
- }
-
- message AlignTo {
- // (Required) A duration to which delays should be quantized
- // in milliseconds.
- int64 period = 3;
-
- // (Required) An offset from 0 for the quantization specified by
- // alignment_size, in milliseconds
- int64 offset = 4;
- }
-}
-
-// A specification for how to "side input" a PCollection.
-message SideInput {
- // (Required) URN of the access pattern required by the `view_fn` to present
- // the desired SDK-specific interface to a UDF.
- //
- // This access pattern defines the SDK harness <-> Runner Harness RPC
- // interface for accessing a side input.
- //
- // The only access pattern intended for Beam, because of its superior
- // performance possibilities, is "urn:beam:sideinput:multimap" (or some such
- // URN)
- FunctionSpec access_pattern = 1;
-
- // (Required) The SdkFunctionSpec of the UDF that adapts a particular
- // access_pattern to a user-facing view type.
- //
- // For example, View.asSingleton() may include a `view_fn` that adapts a
- // specially-designed multimap to a single value per window.
- SdkFunctionSpec view_fn = 2;
-
- // (Required) The SdkFunctionSpec of the UDF that maps a main input window
- // to a side input window.
- //
- // For example, when the main input is in fixed windows of one hour, this
- // can specify that the side input should be accessed according to the day
- // in which that hour falls.
- SdkFunctionSpec window_mapping_fn = 3;
-}
-
-// An environment for executing UDFs. Generally an SDK container URL, but
-// there can be many for a single SDK, for example to provide dependency
-// isolation.
-message Environment {
-
- // (Required) The URL of a container
- //
- // TODO: reconcile with Fn API's DockerContainer structure by
- // adding adequate metadata to know how to interpret the container
- string url = 1;
-}
-
-// A specification of a user defined function.
-//
-message SdkFunctionSpec {
-
- // (Required) A full specification of this function.
- FunctionSpec spec = 1;
-
- // (Required) Reference to an execution environment capable of
- // invoking this function.
- string environment_id = 2;
-}
-
-// A URN along with a parameter object whose schema is determined by the
-// URN.
-//
-// This structure is reused in two distinct, but compatible, ways:
-//
-// 1. This can be a specification of the function over PCollections
-// that a PTransform computes.
-// 2. This can be a specification of a user-defined function, possibly
-// SDK-specific. (external to this message must be adequate context
-// to indicate the environment in which the UDF can be understood).
-//
-// Though not explicit in this proto, there are two possibilities
-// for the relationship of a runner to this specification that
-// one should bear in mind:
-//
-// 1. The runner understands the URN. For example, it might be
-// a well-known URN like "urn:beam:transform:Top" or
-// "urn:beam:windowfn:FixedWindows" with
-// an agreed-upon payload (e.g. a number or duration,
-// respectively).
-// 2. The runner does not understand the URN. It might be an
-// SDK specific URN such as "urn:beam:dofn:javasdk:1.0"
-// that indicates to the SDK what the payload is,
-// such as a serialized Java DoFn from a particular
-// version of the Beam Java SDK. The payload will often
-// then be an opaque message such as bytes in a
-// language-specific serialization format.
-message FunctionSpec {
-
- // (Required) A URN that describes the accompanying payload.
- // For any URN that is not recognized (by whomever is inspecting
- // it) the parameter payload should be treated as opaque and
- // passed as-is.
- string urn = 1;
-
- // (Optional) The data specifying any parameters to the URN. If
- // the URN does not require any arguments, this may be omitted.
- google.protobuf.Any parameter = 2;
-}
-
-// TODO: transfer javadoc here
-message DisplayData {
-
- // (Required) The list of display data.
- repeated Item items = 1;
-
- // A complete identifier for a DisplayData.Item
- message Identifier {
-
- // (Required) The transform originating this display data.
- string transform_id = 1;
-
- // (Optional) The URN indicating the type of the originating transform,
- // if there is one.
- string transform_urn = 2;
-
- string key = 3;
- }
-
- // A single item of display data.
- message Item {
- // (Required)
- Identifier id = 1;
-
- // (Required)
- Type type = 2;
-
- // (Required)
- google.protobuf.Any value = 3;
-
- // (Optional)
- google.protobuf.Any short_value = 4;
-
- // (Optional)
- string label = 5;
-
- // (Optional)
- string link_url = 6;
- }
-
- enum Type {
- STRING = 0;
- INTEGER = 1;
- FLOAT = 2;
- BOOLEAN = 3;
- TIMESTAMP = 4;
- DURATION = 5;
- JAVA_CLASS = 6;
- }
-}
diff --git a/sdks/go/BUILD.md b/sdks/go/BUILD.md
new file mode 100644
index 0000000..1bbfdf0
--- /dev/null
+++ b/sdks/go/BUILD.md
@@ -0,0 +1,63 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Go build
+
+This document describes the [Go](golang.org) code layout and build integration
+with Maven. The setup is non-trivial, because the Go toolchain expects a
+certain layout and Maven support is limited.
+
+Goals:
+
+ 1. Go code can be built and tested using Maven w/o special requirements.
+ 1. Go tools such as `go build`, `go test` and `go generate` work as usual.
+ 1. Go code can be pulled with `go get` from `github.com/apache/beam` for users.
+ 1. Go programs can used in docker container images.
+
+In short, the goals are to make both worlds work well.
+
+### Maven integration
+
+The Go toolchain expects the package name to match the directory structure,
+which in turn must be rooted in `github.com/apache/beam` for `go get` to work.
+This directory prefix is beyond the repo itself and we must copy the Go source
+code into such a layout to invoke the tool chain. We use a single directory
+`sdks/go` for all shared library code and export it as a zip file during the
+build process to be used by various tools, such as `sdks/java/container`.
+This scheme balances the convenience of combined Go setup with the desire
+for a unified layout across languages. Python seems to do the same.
+
+The container build adds a small twist to the build integration, because
+container images use linux/amd64 but the development setup might not. We
+therefore additionally cross-compile Go binaries for inclusion into container
+images where needed, generally placed in `target/linux_amd64`.
+
+### Go development setup
+
+Developers must clone their git repository into:
+```
+$GOPATH/src/github.com/apache
+
+```
+to match the package structure expected by the code imports. Go users can just
+`go get` the code directly. For example:
+```
+go get github.com/apache/beam/sdks/go/...
+```
+Developers must invoke Go for cross-compilation manually, if desired.
diff --git a/sdks/go/cmd/beamctl/cmd/artifact.go b/sdks/go/cmd/beamctl/cmd/artifact.go
new file mode 100644
index 0000000..eaaaa9a
--- /dev/null
+++ b/sdks/go/cmd/beamctl/cmd/artifact.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "path/filepath"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/artifact"
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/spf13/cobra"
+)
+
+var (
+ artifactCmd = &cobra.Command{
+ Use: "artifact",
+ Short: "Artifact commands",
+ }
+
+ stageCmd = &cobra.Command{
+ Use: "stage",
+ Short: "Stage local files as artifacts",
+ RunE: stageFn,
+ Args: cobra.MinimumNArgs(1),
+ }
+
+ listCmd = &cobra.Command{
+ Use: "list",
+ Short: "List artifacts",
+ RunE: listFn,
+ Args: cobra.NoArgs,
+ }
+)
+
+func init() {
+ artifactCmd.AddCommand(stageCmd, listCmd)
+}
+
+func stageFn(cmd *cobra.Command, args []string) error {
+ ctx, cc, err := dial()
+ if err != nil {
+ return err
+ }
+ defer cc.Close()
+
+ // (1) Use flat filename as key.
+
+ var files []artifact.KeyedFile
+ for _, arg := range args {
+ files = append(files, artifact.KeyedFile{Key: filepath.Base(arg), Filename: arg})
+ }
+
+ // (2) Stage files in parallel, commit and print out token
+
+ client := pb.NewArtifactStagingServiceClient(cc)
+ list, err := artifact.MultiStage(ctx, client, 10, files)
+ if err != nil {
+ return err
+ }
+ token, err := artifact.Commit(ctx, client, list)
+ if err != nil {
+ return err
+ }
+
+ cmd.Println(token)
+ return nil
+}
+
+func listFn(cmd *cobra.Command, args []string) error {
+ ctx, cc, err := dial()
+ if err != nil {
+ return err
+ }
+ defer cc.Close()
+
+ client := pb.NewArtifactRetrievalServiceClient(cc)
+ md, err := client.GetManifest(ctx, &pb.GetManifestRequest{})
+ if err != nil {
+ return err
+ }
+
+ for _, a := range md.GetManifest().GetArtifact() {
+ cmd.Println(a.Name)
+ }
+ return nil
+}
diff --git a/sdks/go/cmd/beamctl/cmd/root.go b/sdks/go/cmd/beamctl/cmd/root.go
new file mode 100644
index 0000000..a4e7945
--- /dev/null
+++ b/sdks/go/cmd/beamctl/cmd/root.go
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cmd contains the commands for beamctl.
+package cmd
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "github.com/spf13/cobra"
+ "google.golang.org/grpc"
+)
+
+var (
+ // RootCmd is the root for beamctl commands.
+ RootCmd = &cobra.Command{
+ Use: "beamctl",
+ Short: "Apache Beam command line client",
+ }
+
+ id string
+ endpoint string
+)
+
+func init() {
+ RootCmd.AddCommand(artifactCmd)
+ RootCmd.PersistentFlags().StringVarP(&endpoint, "endpoint", "e", "", "Server endpoint, such as localhost:123")
+ RootCmd.PersistentFlags().StringVarP(&id, "id", "i", "", "Client ID")
+}
+
+// dial connects via gRPC to the given endpoint and returns the connection
+// and the context to use.
+func dial() (context.Context, *grpc.ClientConn, error) {
+ if endpoint == "" {
+ return nil, nil, errors.New("endpoint not defined")
+ }
+
+ ctx := grpcx.WriteWorkerID(context.Background(), id)
+ cc, err := grpcx.Dial(ctx, endpoint, time.Minute)
+ return ctx, cc, err
+}
diff --git a/sdks/go/cmd/beamctl/main.go b/sdks/go/cmd/beamctl/main.go
new file mode 100644
index 0000000..7d6ae8a
--- /dev/null
+++ b/sdks/go/cmd/beamctl/main.go
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// beamctl is a command line client for the Apache Beam portability services.
+package main
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/apache/beam/sdks/go/cmd/beamctl/cmd"
+)
+
+func main() {
+ if err := cmd.RootCmd.Execute(); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+}
diff --git a/sdks/go/descriptor.xml b/sdks/go/descriptor.xml
new file mode 100644
index 0000000..15ec4e8
--- /dev/null
+++ b/sdks/go/descriptor.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>pkg-sources</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>pkg</directory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
new file mode 100644
index 0000000..dede7a5
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gcsproxy
+
+import (
+ "fmt"
+ "io"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
+ "github.com/golang/protobuf/proto"
+ "golang.org/x/net/context"
+ "google.golang.org/api/storage/v1"
+)
+
+// RetrievalServer is a artifact retrieval server backed by Google
+// Cloud Storage (GCS). It serves a single manifest and ignores
+// the worker id. The server performs no caching or pre-fetching.
+type RetrievalServer struct {
+ md *pb.Manifest
+ blobs map[string]string
+}
+
+// ReadProxyManifest reads and parses the proxy manifest from GCS.
+func ReadProxyManifest(ctx context.Context, object string) (*pb.ProxyManifest, error) {
+ bucket, obj, err := gcsx.ParseObject(object)
+ if err != nil {
+ return nil, fmt.Errorf("invalid manifest object %v: %v", object, err)
+ }
+
+ cl, err := gcsx.NewClient(ctx, storage.DevstorageReadOnlyScope)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create GCS client: %v", err)
+ }
+ content, err := gcsx.ReadObject(cl, bucket, obj)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read manifest %v: %v", object, err)
+ }
+ var md pb.ProxyManifest
+ if err := proto.Unmarshal(content, &md); err != nil {
+ return nil, fmt.Errorf("invalid manifest %v: %v", object, err)
+ }
+ return &md, nil
+}
+
+// NewRetrievalServer creates a artifact retrieval server for the
+// given manifest. It requires that the locations are in GCS.
+func NewRetrievalServer(md *pb.ProxyManifest) (*RetrievalServer, error) {
+ if err := validate(md); err != nil {
+ return nil, err
+ }
+
+ blobs := make(map[string]string)
+ for _, l := range md.GetLocation() {
+ if _, _, err := gcsx.ParseObject(l.GetUri()); err != nil {
+ return nil, fmt.Errorf("location %v is not a GCS object: %v", l.GetUri(), err)
+ }
+ blobs[l.GetName()] = l.GetUri()
+ }
+ return &RetrievalServer{md: md.GetManifest(), blobs: blobs}, nil
+}
+
+// GetManifest returns the manifest for all artifacts.
+func (s *RetrievalServer) GetManifest(ctx context.Context, req *pb.GetManifestRequest) (*pb.GetManifestResponse, error) {
+ return &pb.GetManifestResponse{Manifest: s.md}, nil
+}
+
+// GetArtifact returns a given artifact.
+func (s *RetrievalServer) GetArtifact(req *pb.GetArtifactRequest, stream pb.ArtifactRetrievalService_GetArtifactServer) error {
+ key := req.GetName()
+ blob, ok := s.blobs[key]
+ if !ok {
+ return fmt.Errorf("artifact %v not found", key)
+ }
+
+ bucket, object := parseObject(blob)
+
+ client, err := gcsx.NewClient(stream.Context(), storage.DevstorageReadOnlyScope)
+ if err != nil {
+ return fmt.Errorf("Failed to create client for %v: %v", key, err)
+ }
+
+ // Stream artifact in up to 1MB chunks.
+
+ resp, err := client.Objects.Get(bucket, object).Download()
+ if err != nil {
+ return fmt.Errorf("Failed to read object for %v: %v", key, err)
+ }
+ defer resp.Body.Close()
+
+ data := make([]byte, 1<<20)
+ for {
+ n, err := resp.Body.Read(data)
+ if n > 0 {
+ if err := stream.Send(&pb.ArtifactChunk{Data: data[:n]}); err != nil {
+ return fmt.Errorf("chunk send failed: %v", err)
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("failed to read from %v: %v", blob, err)
+ }
+ }
+ return nil
+}
+
+func validate(md *pb.ProxyManifest) error {
+ keys := make(map[string]bool)
+ for _, a := range md.GetManifest().GetArtifact() {
+ if _, seen := keys[a.Name]; seen {
+ return fmt.Errorf("multiple artifact with name %v", a.Name)
+ }
+ keys[a.Name] = true
+ }
+ for _, l := range md.GetLocation() {
+ fresh, seen := keys[l.Name]
+ if !seen {
+ return fmt.Errorf("no artifact named %v for location %v", l.Name, l.Uri)
+ }
+ if !fresh {
+ return fmt.Errorf("multiple locations for %v:%v", l.Name, l.Uri)
+ }
+ keys[l.Name] = false
+ }
+
+ for key, fresh := range keys {
+ if fresh {
+ return fmt.Errorf("no location for %v", key)
+ }
+ }
+ return nil
+}
+
+func parseObject(blob string) (string, string) {
+ bucket, object, err := gcsx.ParseObject(blob)
+ if err != nil {
+ panic(err)
+ }
+ return bucket, object
+}
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/staging.go b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
new file mode 100644
index 0000000..6e62bb9
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gcsproxy contains artifact staging and retrieval servers backed by GCS.
+package gcsproxy
+
+import (
+ "bytes"
+ "crypto/md5"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "hash"
+ "path"
+ "sync"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
+ "github.com/golang/protobuf/proto"
+ "golang.org/x/net/context"
+ "google.golang.org/api/storage/v1"
+)
+
+// StagingServer is a artifact staging server backed by Google Cloud Storage
+// (GCS). It commits a single manifest and ignores the staging id.
+type StagingServer struct {
+ manifest string
+ bucket, root string
+ blobs map[string]staged // guarded by mu
+ mu sync.Mutex
+}
+
+type staged struct {
+ object, hash string
+}
+
+// NewStagingServer creates a artifact staging server for the given manifest.
+// It requires that the manifest is in GCS and will stage the supplied
+// artifacts next to it.
+func NewStagingServer(manifest string) (*StagingServer, error) {
+ bucket, object, err := gcsx.ParseObject(manifest)
+ if err != nil {
+ return nil, fmt.Errorf("invalid manifest location: %v", err)
+ }
+ root := path.Join(path.Dir(object), "blobs")
+
+ return &StagingServer{
+ manifest: object,
+ bucket: bucket,
+ root: root,
+ blobs: make(map[string]staged),
+ }, nil
+}
+
+// CommitManifest commits the given artifact manifest to GCS.
+func (s *StagingServer) CommitManifest(ctx context.Context, req *pb.CommitManifestRequest) (*pb.CommitManifestResponse, error) {
+ manifest := req.GetManifest()
+
+ s.mu.Lock()
+ loc, err := matchLocations(manifest.GetArtifact(), s.blobs)
+ if err != nil {
+ s.mu.Unlock()
+ return nil, err
+ }
+ s.mu.Unlock()
+
+ data, err := proto.Marshal(&pb.ProxyManifest{Manifest: manifest, Location: loc})
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal proxy manifest: %v", err)
+ }
+
+ cl, err := gcsx.NewClient(ctx, storage.DevstorageReadWriteScope)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create GCS client: %v", err)
+ }
+ if err := gcsx.WriteObject(cl, s.bucket, s.manifest, bytes.NewReader(data)); err != nil {
+ return nil, fmt.Errorf("failed to write manifest: %v", err)
+ }
+
+ // Commit returns the location of the manifest as the token, which can
+ // then be used to configure the retrieval proxy. It is redundant right
+ // now, but would be needed for a staging server that serves multiple
+ // jobs. Such a server would also use the ID sent with each request.
+
+ return &pb.CommitManifestResponse{StagingToken: gcsx.MakeObject(s.bucket, s.manifest)}, nil
+}
+
+// matchLocations ensures that all artifacts have been staged and have valid
+// content. It is fine for staged artifacts to not appear in the manifest.
+func matchLocations(artifacts []*pb.ArtifactMetadata, blobs map[string]staged) ([]*pb.ProxyManifest_Location, error) {
+ var loc []*pb.ProxyManifest_Location
+ for _, a := range artifacts {
+ info, ok := blobs[a.Name]
+ if !ok {
+ return nil, fmt.Errorf("artifact %v not staged", a.Name)
+ }
+ if a.Md5 == "" {
+ a.Md5 = info.hash
+ }
+ if info.hash != a.Md5 {
+ return nil, fmt.Errorf("staged artifact for %v has invalid MD5: %v, want %v", a.Name, info.hash, a.Md5)
+ }
+
+ loc = append(loc, &pb.ProxyManifest_Location{Name: a.Name, Uri: info.object})
+ }
+ return loc, nil
+}
+
+// PutArtifact stores the given artifact in GCS.
+func (s *StagingServer) PutArtifact(ps pb.ArtifactStagingService_PutArtifactServer) error {
+ // Read header
+
+ header, err := ps.Recv()
+ if err != nil {
+ return fmt.Errorf("failed to receive header: %v", err)
+ }
+ md := header.GetMetadata()
+ if md == nil {
+ return fmt.Errorf("expected header as first message: %v", header)
+ }
+ object := path.Join(s.root, md.Name)
+
+ // Stream content to GCS. We don't have to worry about partial
+ // or abandoned writes, because object writes are atomic.
+
+ cl, err := gcsx.NewClient(ps.Context(), storage.DevstorageReadWriteScope)
+ if err != nil {
+ return fmt.Errorf("failed to create GCS client: %v", err)
+ }
+
+ r := &reader{md5W: md5.New(), stream: ps}
+ if err := gcsx.WriteObject(cl, s.bucket, object, r); err != nil {
+ return fmt.Errorf("failed to stage artifact %v: %v", md.Name, err)
+ }
+ hash := r.MD5()
+ if md.Md5 != "" && md.Md5 != hash {
+ return fmt.Errorf("invalid MD5 for artifact %v: %v want %v", md.Name, hash, md.Md5)
+ }
+
+ s.mu.Lock()
+ s.blobs[md.Name] = staged{object: gcsx.MakeObject(s.bucket, object), hash: hash}
+ s.mu.Unlock()
+
+ return ps.SendAndClose(&pb.PutArtifactResponse{})
+}
+
+// reader is an adapter between the artifact stream and the GCS stream reader.
+// It also computes the MD5 of the content.
+type reader struct {
+ md5W hash.Hash
+ buf []byte
+ stream pb.ArtifactStagingService_PutArtifactServer
+}
+
+func (r *reader) Read(buf []byte) (int, error) {
+ if len(r.buf) == 0 {
+ // Buffer empty. Read from upload stream.
+
+ msg, err := r.stream.Recv()
+ if err != nil {
+ return 0, err // EOF or real error
+ }
+
+ r.buf = msg.GetData().GetData()
+ if len(r.buf) == 0 {
+ return 0, errors.New("empty chunk")
+ }
+ }
+
+ // Copy out bytes from non-empty buffer.
+
+ n := len(r.buf)
+ if n > len(buf) {
+ n = len(buf)
+ }
+ for i := 0; i < n; i++ {
+ buf[i] = r.buf[i]
+ }
+ if _, err := r.md5W.Write(r.buf[:n]); err != nil {
+ panic(err) // cannot fail
+ }
+ r.buf = r.buf[n:]
+ return n, nil
+}
+
+func (r *reader) MD5() string {
+ return base64.StdEncoding.EncodeToString(r.md5W.Sum(nil))
+}
diff --git a/sdks/go/pkg/beam/artifact/materialize.go b/sdks/go/pkg/beam/artifact/materialize.go
new file mode 100644
index 0000000..fb93669
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/materialize.go
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package artifact contains utilities for staging and retrieving artifacts.
+package artifact
+
+import (
+ "bufio"
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "fmt"
+ "io"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/errorx"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+)
+
+// Materialize is a convenience helper for ensuring that all artifacts are
+// present and uncorrupted. It interprets each artifact name as a relative
+// path under the dest directory. It does not retrieve valid artifacts already
+// present.
+func Materialize(ctx context.Context, endpoint string, dest string) ([]*pb.ArtifactMetadata, error) {
+ cc, err := grpcx.Dial(ctx, endpoint, 2*time.Minute)
+ if err != nil {
+ return nil, err
+ }
+ defer cc.Close()
+
+ client := pb.NewArtifactRetrievalServiceClient(cc)
+
+ m, err := client.GetManifest(ctx, &pb.GetManifestRequest{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to get manifest: %v", err)
+ }
+ md := m.GetManifest().GetArtifact()
+ return md, MultiRetrieve(ctx, client, 10, md, dest)
+}
+
+// MultiRetrieve retrieves multiple artifacts concurrently, using at most 'cpus'
+// goroutines. It retries each artifact a few times. Convenience wrapper.
+func MultiRetrieve(ctx context.Context, client pb.ArtifactRetrievalServiceClient, cpus int, list []*pb.ArtifactMetadata, dest string) error {
+ if len(list) == 0 {
+ return nil
+ }
+ if cpus < 1 {
+ cpus = 1
+ }
+ if len(list) < cpus {
+ cpus = len(list)
+ }
+
+ q := slice2queue(list)
+ var permErr errorx.GuardedError
+
+ var wg sync.WaitGroup
+ for i := 0; i < cpus; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for a := range q {
+ if permErr.Error() != nil {
+ continue
+ }
+
+ const attempts = 3
+
+ var failures []string
+ for {
+ err := Retrieve(ctx, client, a, dest)
+ if err == nil || permErr.Error() != nil {
+ break // done or give up
+ }
+ failures = append(failures, err.Error())
+ if len(failures) > attempts {
+ permErr.TrySetError(fmt.Errorf("failed to retrieve %v in %v attempts: %v", a.Name, attempts, strings.Join(failures, "; ")))
+ break // give up
+ }
+ time.Sleep(time.Duration(rand.Intn(5)+1) * time.Second)
+ }
+ }
+ }()
+ }
+ wg.Wait()
+
+ return permErr.Error()
+}
+
+// Retrieve checks whether the given artifact is already successfully
+// retrieved. If not, it retrieves into the dest directory. It overwrites any
+// previous retrieval attempt and may leave a corrupt/partial local file on
+// failure.
+func Retrieve(ctx context.Context, client pb.ArtifactRetrievalServiceClient, a *pb.ArtifactMetadata, dest string) error {
+ filename := filepath.Join(dest, filepath.FromSlash(a.Name))
+
+ _, err := os.Stat(filename)
+ if err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to stat %v: %v", filename, err)
+ }
+ if err == nil {
+ // File already exists. Validate or delete.
+
+ hash, err := computeMD5(filename)
+ if err == nil && a.Md5 == hash {
+ // NOTE(herohde) 10/5/2017: We ignore permissions here, because
+ // they may differ from the requested permissions due to umask
+ // settings on unix systems (which we in turn want to respect).
+ // We have no good way to know what to expect and thus assume
+ // any permissions are fine.
+ return nil
+ }
+
+ if err2 := os.Remove(filename); err2 != nil {
+ return fmt.Errorf("failed to both validate %v and delete: %v (remove: %v)", filename, err, err2)
+ } // else: successfully deleted bad file.
+ } // else: file does not exist.
+
+ if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
+ return err
+ }
+ return retrieve(ctx, client, a, filename)
+}
+
+// retrieve retrieves the given artifact and stores it as the given filename.
+// It validates that the given MD5 matches the content and fails otherwise.
+// It expects the file to not exist, but does not clean up on failure and
+// may leave a corrupt file.
+func retrieve(ctx context.Context, client pb.ArtifactRetrievalServiceClient, a *pb.ArtifactMetadata, filename string) error {
+ stream, err := client.GetArtifact(ctx, &pb.GetArtifactRequest{Name: a.Name})
+ if err != nil {
+ return err
+ }
+
+ fd, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(a.Permissions))
+ if err != nil {
+ return err
+ }
+ w := bufio.NewWriter(fd)
+
+ hash, err := retrieveChunks(stream, w)
+ if err != nil {
+ fd.Close() // drop any buffered content
+ return fmt.Errorf("failed to retrieve chunk for %v: %v", filename, err)
+ }
+ if err := w.Flush(); err != nil {
+ fd.Close()
+ return fmt.Errorf("failed to flush chunks for %v: %v", filename, err)
+ }
+ if err := fd.Close(); err != nil {
+ return err
+ }
+
+ if hash != a.Md5 {
+ return fmt.Errorf("bad MD5 for %v: %v, want %v", filename, hash, a.Md5)
+ }
+ return nil
+}
+
+func retrieveChunks(stream pb.ArtifactRetrievalService_GetArtifactClient, w io.Writer) (string, error) {
+ md5W := md5.New()
+ for {
+ chunk, err := stream.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return "", err
+ }
+
+ if _, err := md5W.Write(chunk.Data); err != nil {
+ panic(err) // cannot fail
+ }
+ if _, err := w.Write(chunk.Data); err != nil {
+ return "", fmt.Errorf("chunk write failed: %v", err)
+ }
+ }
+ return base64.StdEncoding.EncodeToString(md5W.Sum(nil)), nil
+}
+
+func computeMD5(filename string) (string, error) {
+ fd, err := os.Open(filename)
+ if err != nil {
+ return "", err
+ }
+ defer fd.Close()
+
+ md5W := md5.New()
+ data := make([]byte, 1<<20)
+ for {
+ n, err := fd.Read(data)
+ if n > 0 {
+ if _, err := md5W.Write(data[:n]); err != nil {
+ panic(err) // cannot fail
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return "", err
+ }
+ }
+ return base64.StdEncoding.EncodeToString(md5W.Sum(nil)), nil
+}
+
+func slice2queue(list []*pb.ArtifactMetadata) chan *pb.ArtifactMetadata {
+ q := make(chan *pb.ArtifactMetadata, len(list))
+ for _, elm := range list {
+ q <- elm
+ }
+ close(q)
+ return q
+}
+
+func queue2slice(q chan *pb.ArtifactMetadata) []*pb.ArtifactMetadata {
+ var ret []*pb.ArtifactMetadata
+ for elm := range q {
+ ret = append(ret, elm)
+ }
+ return ret
+}
diff --git a/sdks/go/pkg/beam/artifact/materialize_test.go b/sdks/go/pkg/beam/artifact/materialize_test.go
new file mode 100644
index 0000000..144d803
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/materialize_test.go
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "google.golang.org/grpc"
+)
+
+// TestRetrieve tests that we can successfully retrieve fresh files.
+func TestRetrieve(t *testing.T) {
+ cc := startServer(t)
+ defer cc.Close()
+
+ ctx := grpcx.WriteWorkerID(context.Background(), "idA")
+ keys := []string{"foo", "bar", "baz/baz/baz"}
+ artifacts := populate(ctx, cc, t, keys, 300)
+
+ dst := makeTempDir(t)
+ defer os.RemoveAll(dst)
+
+ client := pb.NewArtifactRetrievalServiceClient(cc)
+ for _, a := range artifacts {
+ filename := makeFilename(dst, a.Name)
+ if err := Retrieve(ctx, client, a, dst); err != nil {
+ t.Errorf("failed to retrieve %v: %v", a.Name, err)
+ continue
+ }
+ verifyMD5(t, filename, a.Md5)
+ }
+}
+
+// TestMultiRetrieve tests that we can successfully retrieve fresh files
+// concurrently.
+func TestMultiRetrieve(t *testing.T) {
+ cc := startServer(t)
+ defer cc.Close()
+
+ ctx := grpcx.WriteWorkerID(context.Background(), "idB")
+ keys := []string{"1", "2", "3", "4", "a/5", "a/6", "a/7", "a/8", "a/a/9", "a/a/10", "a/b/11", "a/b/12"}
+ artifacts := populate(ctx, cc, t, keys, 300)
+
+ dst := makeTempDir(t)
+ defer os.RemoveAll(dst)
+
+ client := pb.NewArtifactRetrievalServiceClient(cc)
+ if err := MultiRetrieve(ctx, client, 10, artifacts, dst); err != nil {
+ t.Errorf("failed to retrieve: %v", err)
+ }
+
+ for _, a := range artifacts {
+ verifyMD5(t, makeFilename(dst, a.Name), a.Md5)
+ }
+}
+
+// TestDirtyRetrieve tests that we can successfully retrieve files in a
+// dirty setup with correct and incorrect pre-existing files.
+func TestDirtyRetrieve(t *testing.T) {
+ cc := startServer(t)
+ defer cc.Close()
+
+ ctx := grpcx.WriteWorkerID(context.Background(), "idC")
+ scl := pb.NewArtifactStagingServiceClient(cc)
+
+ list := []*pb.ArtifactMetadata{
+ stage(ctx, scl, t, "good", 500, 100),
+ stage(ctx, scl, t, "bad", 500, 100),
+ }
+ if _, err := Commit(ctx, scl, list); err != nil {
+ t.Fatalf("failed to commit: %v", err)
+ }
+
+ // Kill good file in server by re-staging conflicting content. That ensures
+ // we don't retrieve it.
+ stage(ctx, scl, t, "good", 100, 100)
+
+ dst := makeTempDir(t)
+ defer os.RemoveAll(dst)
+
+ good := filepath.Join(dst, "good")
+ bad := filepath.Join(dst, "bad")
+
+ makeTempFile(t, good, 500) // correct content. Do nothing.
+ makeTempFile(t, bad, 367) // invalid content. Delete and retrieve.
+
+ rcl := pb.NewArtifactRetrievalServiceClient(cc)
+ if err := MultiRetrieve(ctx, rcl, 2, list, dst); err != nil {
+ t.Fatalf("failed to get retrieve: %v", err)
+ }
+
+ verifyMD5(t, good, list[0].Md5)
+ verifyMD5(t, bad, list[1].Md5)
+}
+
+// populate stages a set of artifacts with the given keys, each with
+// slightly different sizes and chucksizes.
+func populate(ctx context.Context, cc *grpc.ClientConn, t *testing.T, keys []string, size int) []*pb.ArtifactMetadata {
+ scl := pb.NewArtifactStagingServiceClient(cc)
+
+ var artifacts []*pb.ArtifactMetadata
+ for i, key := range keys {
+ a := stage(ctx, scl, t, key, size+7*i, 97+i)
+ artifacts = append(artifacts, a)
+ }
+ if _, err := Commit(ctx, scl, artifacts); err != nil {
+ t.Fatalf("failed to commit manifest: %v", err)
+ }
+ return artifacts
+}
+
+// stage stages an artifact with the given key, size and chuck size. The content is
+// always 'z's.
+func stage(ctx context.Context, scl pb.ArtifactStagingServiceClient, t *testing.T, key string, size, chunkSize int) *pb.ArtifactMetadata {
+ data := make([]byte, size)
+ for i := 0; i < size; i++ {
+ data[i] = 'z'
+ }
+
+ md5W := md5.New()
+ md5W.Write(data)
+ hash := base64.StdEncoding.EncodeToString(md5W.Sum(nil))
+ md := makeArtifact(key, hash)
+
+ stream, err := scl.PutArtifact(ctx)
+ if err != nil {
+ t.Fatalf("put failed: %v", err)
+ }
+ header := &pb.PutArtifactRequest{
+ Content: &pb.PutArtifactRequest_Metadata{
+ Metadata: md,
+ },
+ }
+ if err := stream.Send(header); err != nil {
+ t.Fatalf("send header failed: %v", err)
+ }
+
+ for i := 0; i < size; i += chunkSize {
+ end := i + chunkSize
+ if size < end {
+ end = size
+ }
+
+ chunk := &pb.PutArtifactRequest{
+ Content: &pb.PutArtifactRequest_Data{
+ Data: &pb.ArtifactChunk{
+ Data: data[i:end],
+ },
+ },
+ }
+ if err := stream.Send(chunk); err != nil {
+ t.Fatalf("send chunk[%v:%v] failed: %v", i, end, err)
+ }
+ }
+ if _, err := stream.CloseAndRecv(); err != nil {
+ t.Fatalf("close failed: %v", err)
+ }
+ return md
+}
+
+func verifyMD5(t *testing.T, filename, hash string) {
+ actual, err := computeMD5(filename)
+ if err != nil {
+ t.Errorf("failed to compute hash for %v: %v", filename, err)
+ return
+ }
+ if actual != hash {
+ t.Errorf("file %v has bad MD5: %v, want %v", filename, actual, hash)
+ }
+}
+
+func makeTempDir(t *testing.T) string {
+ dir, err := ioutil.TempDir("", "artifact_test_")
+ if err != nil {
+ t.Errorf("Test failure: cannot create temporary directory: %+v", err)
+ }
+ return dir
+}
+
+func makeTempFiles(t *testing.T, dir string, keys []string, size int) []string {
+ var md5s []string
+ for i, key := range keys {
+ hash := makeTempFile(t, makeFilename(dir, key), size+i)
+ md5s = append(md5s, hash)
+ }
+ return md5s
+}
+
+func makeTempFile(t *testing.T, filename string, size int) string {
+ data := make([]byte, size)
+ for i := 0; i < size; i++ {
+ data[i] = 'z'
+ }
+
+ if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
+ t.Fatalf("cannot create directory for %s: %v", filename, err)
+ }
+ if err := ioutil.WriteFile(filename, data, 0644); err != nil {
+ t.Fatalf("cannot create file %s: %v", filename, err)
+ }
+
+ md5W := md5.New()
+ md5W.Write(data)
+ return base64.StdEncoding.EncodeToString(md5W.Sum(nil))
+}
+
+func makeArtifact(key, hash string) *pb.ArtifactMetadata {
+ return &pb.ArtifactMetadata{
+ Name: key,
+ Md5: hash,
+ Permissions: 0644,
+ }
+}
+
+func makeFilename(dir, key string) string {
+ return filepath.Join(dir, filepath.FromSlash(key))
+}
diff --git a/sdks/go/pkg/beam/artifact/server_test.go b/sdks/go/pkg/beam/artifact/server_test.go
new file mode 100644
index 0000000..85f54a2
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/server_test.go
@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact
+
+import (
+ "fmt"
+ "io"
+ "net"
+ "sync"
+ "testing"
+ "time"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "golang.org/x/net/context"
+ "google.golang.org/grpc"
+)
+
+// startServer starts an in-memory staging and retrieval artifact server
+// and returns a gRPC connection to it.
+func startServer(t *testing.T) *grpc.ClientConn {
+ // If port is zero this will bind an unused port.
+ listener, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ t.Fatalf("Failed to find unused port: %v", err)
+ }
+ endpoint := listener.Addr().String()
+
+ real := &server{m: make(map[string]*manifest)}
+
+ gs := grpc.NewServer()
+ pb.RegisterArtifactStagingServiceServer(gs, real)
+ pb.RegisterArtifactRetrievalServiceServer(gs, real)
+ go gs.Serve(listener)
+
+ t.Logf("server listening on %v", endpoint)
+
+ cc, err := grpcx.Dial(context.Background(), endpoint, time.Minute)
+ if err != nil {
+ t.Fatalf("failed to dial fake server at %v: %v", endpoint, err)
+ }
+ return cc
+}
+
+type data struct {
+ md *pb.ArtifactMetadata
+ chunks [][]byte
+}
+
+type manifest struct {
+ md *pb.Manifest
+ m map[string]*data // key -> data
+ mu sync.Mutex
+}
+
+// server is a in-memory staging and retrieval artifact server for testing.
+type server struct {
+ m map[string]*manifest // token -> manifest
+ mu sync.Mutex
+}
+
+func (s *server) PutArtifact(ps pb.ArtifactStagingService_PutArtifactServer) error {
+ id, err := grpcx.ReadWorkerID(ps.Context())
+ if err != nil {
+ return fmt.Errorf("expected worker id: %v", err)
+ }
+
+ // Read header
+
+ header, err := ps.Recv()
+ if err != nil {
+ return fmt.Errorf("failed to receive header: %v", err)
+ }
+ if header.GetMetadata() == nil {
+ return fmt.Errorf("expected header as first message: %v", header)
+ }
+ key := header.GetMetadata().Name
+
+ // Read chunks
+
+ var chunks [][]byte
+ for {
+ msg, err := ps.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return err
+ }
+
+ if msg.GetData() == nil {
+ return fmt.Errorf("expected data: %v", msg)
+ }
+ if len(msg.GetData().GetData()) == 0 {
+ return fmt.Errorf("expected non-empty data: %v", msg)
+ }
+ chunks = append(chunks, msg.GetData().GetData())
+ }
+
+ // Updated staged artifact. This test implementation will allow updates to artifacts
+ // that are already committed, but real implementations should manage artifacts in a
+ // way that makes that impossible.
+
+ m := s.getManifest(id, true)
+ m.mu.Lock()
+ m.m[key] = &data{chunks: chunks}
+ m.mu.Unlock()
+
+ return ps.SendAndClose(&pb.PutArtifactResponse{})
+}
+
+func (s *server) CommitManifest(ctx context.Context, req *pb.CommitManifestRequest) (*pb.CommitManifestResponse, error) {
+ id, err := grpcx.ReadWorkerID(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("expected worker id: %v", err)
+ }
+
+ m := s.getManifest(id, true)
+ m.mu.Lock()
+ defer m.mu.Unlock()
+
+ // Verify that all artifacts are properly staged. Fail if not.
+
+ artifacts := req.GetManifest().GetArtifact()
+ for _, md := range artifacts {
+ if _, ok := m.m[md.Name]; !ok {
+ return nil, fmt.Errorf("artifact %v not staged", md.Name)
+ }
+ }
+
+ // Update commit. Only one manifest can exist for each staging id.
+
+ for _, md := range artifacts {
+ m.m[md.Name].md = md
+ }
+ m.md = req.GetManifest()
+
+ return &pb.CommitManifestResponse{StagingToken: id}, nil
+}
+
+func (s *server) GetManifest(ctx context.Context, req *pb.GetManifestRequest) (*pb.GetManifestResponse, error) {
+ id, err := grpcx.ReadWorkerID(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("expected worker id: %v", err)
+ }
+
+ m := s.getManifest(id, false)
+ if m == nil || m.md == nil {
+ return nil, fmt.Errorf("manifest for %v not found", id)
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+
+ return &pb.GetManifestResponse{Manifest: m.md}, nil
+}
+
+func (s *server) GetArtifact(req *pb.GetArtifactRequest, stream pb.ArtifactRetrievalService_GetArtifactServer) error {
+ id, err := grpcx.ReadWorkerID(stream.Context())
+ if err != nil {
+ return fmt.Errorf("expected worker id: %v", err)
+ }
+
+ m := s.getManifest(id, false)
+ if m == nil || m.md == nil {
+ return fmt.Errorf("manifest for %v not found", id)
+ }
+
+ // Validate artifact and grab chunks so that we can stream them without
+ // holding the lock.
+
+ m.mu.Lock()
+ elm, ok := m.m[req.GetName()]
+ if !ok || elm.md == nil {
+ m.mu.Unlock()
+ return fmt.Errorf("manifest for %v does not contain artifact %v", id, req.GetName())
+ }
+ chunks := elm.chunks
+ m.mu.Unlock()
+
+ // Send chunks exactly as we received them.
+
+ for _, chunk := range chunks {
+ if err := stream.Send(&pb.ArtifactChunk{Data: chunk}); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *server) getManifest(id string, create bool) *manifest {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ ret, ok := s.m[id]
+ if !ok && create {
+ ret = &manifest{m: make(map[string]*data)}
+ s.m[id] = ret
+ }
+ return ret
+}
diff --git a/sdks/go/pkg/beam/artifact/stage.go b/sdks/go/pkg/beam/artifact/stage.go
new file mode 100644
index 0000000..b87b320
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/stage.go
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "math/rand"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/errorx"
+)
+
+// Commit commits a manifest with the given staged artifacts. It returns the
+// staging token, if successful.
+func Commit(ctx context.Context, client pb.ArtifactStagingServiceClient, artifacts []*pb.ArtifactMetadata) (string, error) {
+ req := &pb.CommitManifestRequest{
+ Manifest: &pb.Manifest{
+ Artifact: artifacts,
+ },
+ }
+ resp, err := client.CommitManifest(ctx, req)
+ if err != nil {
+ return "", err
+ }
+ return resp.GetStagingToken(), nil
+}
+
+// StageDir stages a local directory with relative path keys. Convenience wrapper.
+func StageDir(ctx context.Context, client pb.ArtifactStagingServiceClient, src string) ([]*pb.ArtifactMetadata, error) {
+ list, err := scan(src)
+ if err != nil || len(list) == 0 {
+ return nil, err
+ }
+ return MultiStage(ctx, client, 10, list)
+}
+
+// MultiStage stages a set of local files with the given keys. It returns
+// the full artifact metadate. It retries each artifact a few times.
+// Convenience wrapper.
+func MultiStage(ctx context.Context, client pb.ArtifactStagingServiceClient, cpus int, list []KeyedFile) ([]*pb.ArtifactMetadata, error) {
+ if cpus < 1 {
+ cpus = 1
+ }
+ if len(list) < cpus {
+ cpus = len(list)
+ }
+
+ q := make(chan KeyedFile, len(list))
+ for _, f := range list {
+ q <- f
+ }
+ close(q)
+ var permErr errorx.GuardedError
+
+ ret := make(chan *pb.ArtifactMetadata, len(list))
+
+ var wg sync.WaitGroup
+ for i := 0; i < cpus; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for f := range q {
+ if permErr.Error() != nil {
+ continue
+ }
+
+ const attempts = 3
+
+ var failures []string
+ for {
+ a, err := Stage(ctx, client, f.Key, f.Filename)
+ if err == nil {
+ ret <- a
+ break
+ }
+ if permErr.Error() != nil {
+ break // give up
+ }
+ failures = append(failures, err.Error())
+ if len(failures) > attempts {
+ permErr.TrySetError(fmt.Errorf("failed to stage %v in %v attempts: %v", f.Filename, attempts, strings.Join(failures, "; ")))
+ break // give up
+ }
+ time.Sleep(time.Duration(rand.Intn(5)+1) * time.Second)
+ }
+ }
+ }()
+ }
+ wg.Wait()
+ close(ret)
+
+ return queue2slice(ret), permErr.Error()
+}
+
+// Stage stages a local file as an artifact with the given key. It computes
+// the MD5 and returns the full artifact metadata.
+func Stage(ctx context.Context, client pb.ArtifactStagingServiceClient, key, filename string) (*pb.ArtifactMetadata, error) {
+ stat, err := os.Stat(filename)
+ if err != nil {
+ return nil, err
+ }
+ hash, err := computeMD5(filename)
+ if err != nil {
+ return nil, err
+ }
+ md := &pb.ArtifactMetadata{
+ Name: key,
+ Permissions: uint32(stat.Mode()),
+ Md5: hash,
+ }
+
+ fd, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer fd.Close()
+
+ stream, err := client.PutArtifact(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ header := &pb.PutArtifactRequest{
+ Content: &pb.PutArtifactRequest_Metadata{
+ Metadata: md,
+ },
+ }
+ if err := stream.Send(header); err != nil {
+ stream.CloseAndRecv() // ignore error
+ return nil, fmt.Errorf("failed to send header for %v: %v", filename, err)
+ }
+ stagedHash, err := stageChunks(stream, fd)
+ if err != nil {
+ stream.CloseAndRecv() // ignore error
+ return nil, fmt.Errorf("failed to send chunks for %v: %v", filename, err)
+ }
+ if _, err := stream.CloseAndRecv(); err != nil {
+ return nil, fmt.Errorf("failed to close stream for %v: %v", filename, err)
+ }
+ if hash != stagedHash {
+ return nil, fmt.Errorf("unexpected MD5 for sent chunks for %v: %v, want %v", filename, stagedHash, hash)
+ }
+ return md, nil
+}
+
+func stageChunks(stream pb.ArtifactStagingService_PutArtifactClient, r io.Reader) (string, error) {
+ md5W := md5.New()
+ data := make([]byte, 1<<20)
+ for {
+ n, err := r.Read(data)
+ if n > 0 {
+ if _, err := md5W.Write(data[:n]); err != nil {
+ panic(err) // cannot fail
+ }
+
+ chunk := &pb.PutArtifactRequest{
+ Content: &pb.PutArtifactRequest_Data{
+ Data: &pb.ArtifactChunk{
+ Data: data[:n],
+ },
+ },
+ }
+ if err := stream.Send(chunk); err != nil {
+ return "", fmt.Errorf("chunk send failed: %v", err)
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return "", err
+ }
+ }
+ return base64.StdEncoding.EncodeToString(md5W.Sum(nil)), nil
+}
+
+// KeyedFile is a key and filename pair.
+type KeyedFile struct {
+ Key, Filename string
+}
+
+func scan(dir string) ([]KeyedFile, error) {
+ var ret []KeyedFile
+ if err := walk(dir, "", &ret); err != nil {
+ return nil, fmt.Errorf("failed to scan %v for artifacts to stage: %v", dir, err)
+ }
+ return ret, nil
+}
+
+func walk(dir, key string, accum *[]KeyedFile) error {
+ list, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+
+ for _, elm := range list {
+ k := makeKey(key, elm.Name())
+ f := filepath.Join(dir, elm.Name())
+
+ if elm.IsDir() {
+ walk(f, k, accum)
+ continue
+ }
+ *accum = append(*accum, KeyedFile{k, f})
+ }
+ return nil
+}
+
+func makeKey(prefix, name string) string {
+ if prefix == "" {
+ return name
+ }
+ return path.Join(prefix, name)
+}
diff --git a/sdks/go/pkg/beam/artifact/stage_test.go b/sdks/go/pkg/beam/artifact/stage_test.go
new file mode 100644
index 0000000..f9b5005
--- /dev/null
+++ b/sdks/go/pkg/beam/artifact/stage_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact
+
+import (
+ "context"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/jobmanagement_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "google.golang.org/grpc"
+)
+
+// TestStage verifies that local files can be staged correctly.
+func TestStage(t *testing.T) {
+ cc := startServer(t)
+ defer cc.Close()
+ client := pb.NewArtifactStagingServiceClient(cc)
+
+ ctx := grpcx.WriteWorkerID(context.Background(), "idA")
+ keys := []string{"foo", "bar", "baz/baz/baz"}
+
+ src := makeTempDir(t)
+ defer os.RemoveAll(src)
+ md5s := makeTempFiles(t, src, keys, 300)
+
+ var artifacts []*pb.ArtifactMetadata
+ for _, key := range keys {
+ a, err := Stage(ctx, client, key, makeFilename(src, key))
+ if err != nil {
+ t.Errorf("failed to stage %v: %v", key, err)
+ }
+ artifacts = append(artifacts, a)
+ }
+ if _, err := Commit(ctx, client, artifacts); err != nil {
+ t.Fatalf("failed to commit: %v", err)
+ }
+
+ validate(ctx, cc, t, keys, md5s)
+}
+
+// TestStageDir validates that local files can be staged concurrently.
+func TestStageDir(t *testing.T) {
+ cc := startServer(t)
+ defer cc.Close()
+ client := pb.NewArtifactStagingServiceClient(cc)
+
+ ctx := grpcx.WriteWorkerID(context.Background(), "idB")
+ keys := []string{"1", "2", "3", "4", "a/5", "a/6", "a/7", "a/8", "a/a/9", "a/a/10", "a/b/11", "a/b/12"}
+
+ src := makeTempDir(t)
+ defer os.RemoveAll(src)
+ md5s := makeTempFiles(t, src, keys, 300)
+
+ artifacts, err := StageDir(ctx, client, src)
+ if err != nil {
+ t.Errorf("failed to stage dir %v: %v", src, err)
+ }
+ if _, err := Commit(ctx, client, artifacts); err != nil {
+ t.Fatalf("failed to commit: %v", err)
+ }
+
+ validate(ctx, cc, t, keys, md5s)
+}
+
+func validate(ctx context.Context, cc *grpc.ClientConn, t *testing.T, keys, md5s []string) {
+ rcl := pb.NewArtifactRetrievalServiceClient(cc)
+
+ for i, key := range keys {
+ stream, err := rcl.GetArtifact(ctx, &pb.GetArtifactRequest{Name: key})
+ if err != nil {
+ t.Fatalf("failed to get artifact for %v: %v", key, err)
+ }
+
+ hash, err := retrieveChunks(stream, ioutil.Discard)
+ if err != nil {
+ t.Fatalf("failed to get chunks for %v: %v", key, err)
+ }
+ if hash != md5s[i] {
+ t.Errorf("incorrect MD5: %v, want %v", hash, md5s[i])
+ }
+ }
+}
diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
new file mode 100644
index 0000000..9a31a57
--- /dev/null
+++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
@@ -0,0 +1,2729 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: beam_fn_api.proto
+
+/*
+Package fnexecution_v1 is a generated protocol buffer package.
+
+It is generated from these files:
+ beam_fn_api.proto
+ beam_provision_api.proto
+
+It has these top-level messages:
+ Target
+ RemoteGrpcPort
+ InstructionRequest
+ InstructionResponse
+ RegisterRequest
+ RegisterResponse
+ ProcessBundleDescriptor
+ ProcessBundleRequest
+ ProcessBundleResponse
+ ProcessBundleProgressRequest
+ Metrics
+ ProcessBundleProgressResponse
+ ProcessBundleSplitRequest
+ ElementCountRestriction
+ ElementCountSkipRestriction
+ PrimitiveTransformSplit
+ ProcessBundleSplitResponse
+ Elements
+ StateRequest
+ StateResponse
+ StateKey
+ StateGetRequest
+ StateGetResponse
+ StateAppendRequest
+ StateAppendResponse
+ StateClearRequest
+ StateClearResponse
+ LogEntry
+ LogControl
+ DockerContainer
+ GetProvisionInfoRequest
+ GetProvisionInfoResponse
+ ProvisionInfo
+ Resources
+*/
+package fnexecution_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import org_apache_beam_model_pipeline_v1 "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+import org_apache_beam_model_pipeline_v11 "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+import google_protobuf1 "github.com/golang/protobuf/ptypes/timestamp"
+
+import (
+ context "golang.org/x/net/context"
+ grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type LogEntry_Severity_Enum int32
+
+const (
+ LogEntry_Severity_UNSPECIFIED LogEntry_Severity_Enum = 0
+ // Trace level information, also the default log level unless
+ // another severity is specified.
+ LogEntry_Severity_TRACE LogEntry_Severity_Enum = 1
+ // Debugging information.
+ LogEntry_Severity_DEBUG LogEntry_Severity_Enum = 2
+ // Normal events.
+ LogEntry_Severity_INFO LogEntry_Severity_Enum = 3
+ // Normal but significant events, such as start up, shut down, or
+ // configuration.
+ LogEntry_Severity_NOTICE LogEntry_Severity_Enum = 4
+ // Warning events might cause problems.
+ LogEntry_Severity_WARN LogEntry_Severity_Enum = 5
+ // Error events are likely to cause problems.
+ LogEntry_Severity_ERROR LogEntry_Severity_Enum = 6
+ // Critical events cause severe problems or brief outages and may
+ // indicate that a person must take action.
+ LogEntry_Severity_CRITICAL LogEntry_Severity_Enum = 7
+)
+
+var LogEntry_Severity_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "TRACE",
+ 2: "DEBUG",
+ 3: "INFO",
+ 4: "NOTICE",
+ 5: "WARN",
+ 6: "ERROR",
+ 7: "CRITICAL",
+}
+var LogEntry_Severity_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "TRACE": 1,
+ "DEBUG": 2,
+ "INFO": 3,
+ "NOTICE": 4,
+ "WARN": 5,
+ "ERROR": 6,
+ "CRITICAL": 7,
+}
+
+func (x LogEntry_Severity_Enum) String() string {
+ return proto.EnumName(LogEntry_Severity_Enum_name, int32(x))
+}
+func (LogEntry_Severity_Enum) EnumDescriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{27, 1, 0}
+}
+
+// A representation of an input or output definition on a primitive transform.
+// Stable
+type Target struct {
+ // (Required) The id of the PrimitiveTransform which is the target.
+ PrimitiveTransformReference string `protobuf:"bytes,1,opt,name=primitive_transform_reference,json=primitiveTransformReference" json:"primitive_transform_reference,omitempty"`
+ // (Required) The local name of an input or output defined on the primitive
+ // transform.
+ Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *Target) Reset() { *m = Target{} }
+func (m *Target) String() string { return proto.CompactTextString(m) }
+func (*Target) ProtoMessage() {}
+func (*Target) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *Target) GetPrimitiveTransformReference() string {
+ if m != nil {
+ return m.PrimitiveTransformReference
+ }
+ return ""
+}
+
+func (m *Target) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+// A repeated list of target definitions.
+type Target_List struct {
+ Target []*Target `protobuf:"bytes,1,rep,name=target" json:"target,omitempty"`
+}
+
+func (m *Target_List) Reset() { *m = Target_List{} }
+func (m *Target_List) String() string { return proto.CompactTextString(m) }
+func (*Target_List) ProtoMessage() {}
+func (*Target_List) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 0} }
+
+func (m *Target_List) GetTarget() []*Target {
+ if m != nil {
+ return m.Target
+ }
+ return nil
+}
+
+// A descriptor for connecting to a remote port using the Beam Fn Data API.
+// Allows for communication between two environments (for example between the
+// runner and the SDK).
+// Stable
+type RemoteGrpcPort struct {
+ // (Required) An API descriptor which describes where to
+ // connect to including any authentication that is required.
+ ApiServiceDescriptor *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor `protobuf:"bytes,1,opt,name=api_service_descriptor,json=apiServiceDescriptor" json:"api_service_descriptor,omitempty"`
+}
+
+func (m *RemoteGrpcPort) Reset() { *m = RemoteGrpcPort{} }
+func (m *RemoteGrpcPort) String() string { return proto.CompactTextString(m) }
+func (*RemoteGrpcPort) ProtoMessage() {}
+func (*RemoteGrpcPort) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *RemoteGrpcPort) GetApiServiceDescriptor() *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor {
+ if m != nil {
+ return m.ApiServiceDescriptor
+ }
+ return nil
+}
+
+// A request sent by a runner which the SDK is asked to fulfill.
+// For any unsupported request type, an error should be returned with a
+// matching instruction id.
+// Stable
+type InstructionRequest struct {
+ // (Required) An unique identifier provided by the runner which represents
+ // this requests execution. The InstructionResponse MUST have the matching id.
+ InstructionId string `protobuf:"bytes,1,opt,name=instruction_id,json=instructionId" json:"instruction_id,omitempty"`
+ // (Required) A request that the SDK Harness needs to interpret.
+ //
+ // Types that are valid to be assigned to Request:
+ // *InstructionRequest_Register
+ // *InstructionRequest_ProcessBundle
+ // *InstructionRequest_ProcessBundleProgress
+ // *InstructionRequest_ProcessBundleSplit
+ Request isInstructionRequest_Request `protobuf_oneof:"request"`
+}
+
+func (m *InstructionRequest) Reset() { *m = InstructionRequest{} }
+func (m *InstructionRequest) String() string { return proto.CompactTextString(m) }
+func (*InstructionRequest) ProtoMessage() {}
+func (*InstructionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+type isInstructionRequest_Request interface {
+ isInstructionRequest_Request()
+}
+
+type InstructionRequest_Register struct {
+ Register *RegisterRequest `protobuf:"bytes,1000,opt,name=register,oneof"`
+}
+type InstructionRequest_ProcessBundle struct {
+ ProcessBundle *ProcessBundleRequest `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,oneof"`
+}
+type InstructionRequest_ProcessBundleProgress struct {
+ ProcessBundleProgress *ProcessBundleProgressRequest `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,oneof"`
+}
+type InstructionRequest_ProcessBundleSplit struct {
+ ProcessBundleSplit *ProcessBundleSplitRequest `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,oneof"`
+}
+
+func (*InstructionRequest_Register) isInstructionRequest_Request() {}
+func (*InstructionRequest_ProcessBundle) isInstructionRequest_Request() {}
+func (*InstructionRequest_ProcessBundleProgress) isInstructionRequest_Request() {}
+func (*InstructionRequest_ProcessBundleSplit) isInstructionRequest_Request() {}
+
+func (m *InstructionRequest) GetRequest() isInstructionRequest_Request {
+ if m != nil {
+ return m.Request
+ }
+ return nil
+}
+
+func (m *InstructionRequest) GetInstructionId() string {
+ if m != nil {
+ return m.InstructionId
+ }
+ return ""
+}
+
+func (m *InstructionRequest) GetRegister() *RegisterRequest {
+ if x, ok := m.GetRequest().(*InstructionRequest_Register); ok {
+ return x.Register
+ }
+ return nil
+}
+
+func (m *InstructionRequest) GetProcessBundle() *ProcessBundleRequest {
+ if x, ok := m.GetRequest().(*InstructionRequest_ProcessBundle); ok {
+ return x.ProcessBundle
+ }
+ return nil
+}
+
+func (m *InstructionRequest) GetProcessBundleProgress() *ProcessBundleProgressRequest {
+ if x, ok := m.GetRequest().(*InstructionRequest_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
+ }
+ return nil
+}
+
+func (m *InstructionRequest) GetProcessBundleSplit() *ProcessBundleSplitRequest {
+ if x, ok := m.GetRequest().(*InstructionRequest_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*InstructionRequest) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _InstructionRequest_OneofMarshaler, _InstructionRequest_OneofUnmarshaler, _InstructionRequest_OneofSizer, []interface{}{
+ (*InstructionRequest_Register)(nil),
+ (*InstructionRequest_ProcessBundle)(nil),
+ (*InstructionRequest_ProcessBundleProgress)(nil),
+ (*InstructionRequest_ProcessBundleSplit)(nil),
+ }
+}
+
+func _InstructionRequest_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*InstructionRequest)
+ // request
+ switch x := m.Request.(type) {
+ case *InstructionRequest_Register:
+ b.EncodeVarint(1000<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Register); err != nil {
+ return err
+ }
+ case *InstructionRequest_ProcessBundle:
+ b.EncodeVarint(1001<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundle); err != nil {
+ return err
+ }
+ case *InstructionRequest_ProcessBundleProgress:
+ b.EncodeVarint(1002<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundleProgress); err != nil {
+ return err
+ }
+ case *InstructionRequest_ProcessBundleSplit:
+ b.EncodeVarint(1003<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundleSplit); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("InstructionRequest.Request has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _InstructionRequest_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*InstructionRequest)
+ switch tag {
+ case 1000: // request.register
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(RegisterRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &InstructionRequest_Register{msg}
+ return true, err
+ case 1001: // request.process_bundle
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &InstructionRequest_ProcessBundle{msg}
+ return true, err
+ case 1002: // request.process_bundle_progress
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleProgressRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &InstructionRequest_ProcessBundleProgress{msg}
+ return true, err
+ case 1003: // request.process_bundle_split
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleSplitRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &InstructionRequest_ProcessBundleSplit{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _InstructionRequest_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*InstructionRequest)
+ // request
+ switch x := m.Request.(type) {
+ case *InstructionRequest_Register:
+ s := proto.Size(x.Register)
+ n += proto.SizeVarint(1000<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionRequest_ProcessBundle:
+ s := proto.Size(x.ProcessBundle)
+ n += proto.SizeVarint(1001<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionRequest_ProcessBundleProgress:
+ s := proto.Size(x.ProcessBundleProgress)
+ n += proto.SizeVarint(1002<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionRequest_ProcessBundleSplit:
+ s := proto.Size(x.ProcessBundleSplit)
+ n += proto.SizeVarint(1003<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+// The response for an associated request the SDK had been asked to fulfill.
+// Stable
+type InstructionResponse struct {
+ // (Required) A reference provided by the runner which represents a requests
+ // execution. The InstructionResponse MUST have the matching id when
+ // responding to the runner.
+ InstructionId string `protobuf:"bytes,1,opt,name=instruction_id,json=instructionId" json:"instruction_id,omitempty"`
+ // If this is specified, then this instruction has failed.
+ // A human readable string representing the reason as to why processing has
+ // failed.
+ Error string `protobuf:"bytes,2,opt,name=error" json:"error,omitempty"`
+ // If the instruction did not fail, it is required to return an equivalent
+ // response type depending on the request this matches.
+ //
+ // Types that are valid to be assigned to Response:
+ // *InstructionResponse_Register
+ // *InstructionResponse_ProcessBundle
+ // *InstructionResponse_ProcessBundleProgress
+ // *InstructionResponse_ProcessBundleSplit
+ Response isInstructionResponse_Response `protobuf_oneof:"response"`
+}
+
+func (m *InstructionResponse) Reset() { *m = InstructionResponse{} }
+func (m *InstructionResponse) String() string { return proto.CompactTextString(m) }
+func (*InstructionResponse) ProtoMessage() {}
+func (*InstructionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+type isInstructionResponse_Response interface {
+ isInstructionResponse_Response()
+}
+
+type InstructionResponse_Register struct {
+ Register *RegisterResponse `protobuf:"bytes,1000,opt,name=register,oneof"`
+}
+type InstructionResponse_ProcessBundle struct {
+ ProcessBundle *ProcessBundleResponse `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,oneof"`
+}
+type InstructionResponse_ProcessBundleProgress struct {
+ ProcessBundleProgress *ProcessBundleProgressResponse `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,oneof"`
+}
+type InstructionResponse_ProcessBundleSplit struct {
+ ProcessBundleSplit *ProcessBundleSplitResponse `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,oneof"`
+}
+
+func (*InstructionResponse_Register) isInstructionResponse_Response() {}
+func (*InstructionResponse_ProcessBundle) isInstructionResponse_Response() {}
+func (*InstructionResponse_ProcessBundleProgress) isInstructionResponse_Response() {}
+func (*InstructionResponse_ProcessBundleSplit) isInstructionResponse_Response() {}
+
+func (m *InstructionResponse) GetResponse() isInstructionResponse_Response {
+ if m != nil {
+ return m.Response
+ }
+ return nil
+}
+
+func (m *InstructionResponse) GetInstructionId() string {
+ if m != nil {
+ return m.InstructionId
+ }
+ return ""
+}
+
+func (m *InstructionResponse) GetError() string {
+ if m != nil {
+ return m.Error
+ }
+ return ""
+}
+
+func (m *InstructionResponse) GetRegister() *RegisterResponse {
+ if x, ok := m.GetResponse().(*InstructionResponse_Register); ok {
+ return x.Register
+ }
+ return nil
+}
+
+func (m *InstructionResponse) GetProcessBundle() *ProcessBundleResponse {
+ if x, ok := m.GetResponse().(*InstructionResponse_ProcessBundle); ok {
+ return x.ProcessBundle
+ }
+ return nil
+}
+
+func (m *InstructionResponse) GetProcessBundleProgress() *ProcessBundleProgressResponse {
+ if x, ok := m.GetResponse().(*InstructionResponse_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
+ }
+ return nil
+}
+
+func (m *InstructionResponse) GetProcessBundleSplit() *ProcessBundleSplitResponse {
+ if x, ok := m.GetResponse().(*InstructionResponse_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*InstructionResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _InstructionResponse_OneofMarshaler, _InstructionResponse_OneofUnmarshaler, _InstructionResponse_OneofSizer, []interface{}{
+ (*InstructionResponse_Register)(nil),
+ (*InstructionResponse_ProcessBundle)(nil),
+ (*InstructionResponse_ProcessBundleProgress)(nil),
+ (*InstructionResponse_ProcessBundleSplit)(nil),
+ }
+}
+
+func _InstructionResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*InstructionResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *InstructionResponse_Register:
+ b.EncodeVarint(1000<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Register); err != nil {
+ return err
+ }
+ case *InstructionResponse_ProcessBundle:
+ b.EncodeVarint(1001<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundle); err != nil {
+ return err
+ }
+ case *InstructionResponse_ProcessBundleProgress:
+ b.EncodeVarint(1002<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundleProgress); err != nil {
+ return err
+ }
+ case *InstructionResponse_ProcessBundleSplit:
+ b.EncodeVarint(1003<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessBundleSplit); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("InstructionResponse.Response has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _InstructionResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*InstructionResponse)
+ switch tag {
+ case 1000: // response.register
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(RegisterResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &InstructionResponse_Register{msg}
+ return true, err
+ case 1001: // response.process_bundle
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &InstructionResponse_ProcessBundle{msg}
+ return true, err
+ case 1002: // response.process_bundle_progress
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleProgressResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &InstructionResponse_ProcessBundleProgress{msg}
+ return true, err
+ case 1003: // response.process_bundle_split
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ProcessBundleSplitResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &InstructionResponse_ProcessBundleSplit{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _InstructionResponse_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*InstructionResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *InstructionResponse_Register:
+ s := proto.Size(x.Register)
+ n += proto.SizeVarint(1000<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionResponse_ProcessBundle:
+ s := proto.Size(x.ProcessBundle)
+ n += proto.SizeVarint(1001<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionResponse_ProcessBundleProgress:
+ s := proto.Size(x.ProcessBundleProgress)
+ n += proto.SizeVarint(1002<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *InstructionResponse_ProcessBundleSplit:
+ s := proto.Size(x.ProcessBundleSplit)
+ n += proto.SizeVarint(1003<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+// A list of objects which can be referred to by the runner in
+// future requests.
+// Stable
+type RegisterRequest struct {
+ // (Optional) The set of descriptors used to process bundles.
+ ProcessBundleDescriptor []*ProcessBundleDescriptor `protobuf:"bytes,1,rep,name=process_bundle_descriptor,json=processBundleDescriptor" json:"process_bundle_descriptor,omitempty"`
+}
+
+func (m *RegisterRequest) Reset() { *m = RegisterRequest{} }
+func (m *RegisterRequest) String() string { return proto.CompactTextString(m) }
+func (*RegisterRequest) ProtoMessage() {}
+func (*RegisterRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescriptor {
+ if m != nil {
+ return m.ProcessBundleDescriptor
+ }
+ return nil
+}
+
+// Stable
+type RegisterResponse struct {
+}
+
+func (m *RegisterResponse) Reset() { *m = RegisterResponse{} }
+func (m *RegisterResponse) String() string { return proto.CompactTextString(m) }
+func (*RegisterResponse) ProtoMessage() {}
+func (*RegisterResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+// Definitions that should be used to construct the bundle processing graph.
+type ProcessBundleDescriptor struct {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ // (Required) A map from pipeline-scoped id to PTransform.
+ Transforms map[string]*org_apache_beam_model_pipeline_v1.PTransform `protobuf:"bytes,2,rep,name=transforms" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to PCollection.
+ Pcollections map[string]*org_apache_beam_model_pipeline_v1.PCollection `protobuf:"bytes,3,rep,name=pcollections" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ WindowingStrategies map[string]*org_apache_beam_model_pipeline_v1.WindowingStrategy `protobuf:"bytes,4,rep,name=windowing_strategies,json=windowingStrategies" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to Coder.
+ Coders map[string]*org_apache_beam_model_pipeline_v1.Coder `protobuf:"bytes,5,rep,name=coders" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to Environment.
+ Environments map[string]*org_apache_beam_model_pipeline_v1.Environment `protobuf:"bytes,6,rep,name=environments" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // A descriptor describing the end point to use for State API
+ // calls. Required if the Runner intends to send remote references over the
+ // data plane or if any of the transforms rely on user state or side inputs.
+ StateApiServiceDescriptor *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor `protobuf:"bytes,7,opt,name=state_api_service_descriptor,json=stateApiServiceDescriptor" json:"state_api_service_descriptor,omitempty"`
+}
+
+func (m *ProcessBundleDescriptor) Reset() { *m = ProcessBundleDescriptor{} }
+func (m *ProcessBundleDescriptor) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleDescriptor) ProtoMessage() {}
+func (*ProcessBundleDescriptor) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *ProcessBundleDescriptor) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *ProcessBundleDescriptor) GetTransforms() map[string]*org_apache_beam_model_pipeline_v1.PTransform {
+ if m != nil {
+ return m.Transforms
+ }
+ return nil
+}
+
+func (m *ProcessBundleDescriptor) GetPcollections() map[string]*org_apache_beam_model_pipeline_v1.PCollection {
+ if m != nil {
+ return m.Pcollections
+ }
+ return nil
+}
+
+func (m *ProcessBundleDescriptor) GetWindowingStrategies() map[string]*org_apache_beam_model_pipeline_v1.WindowingStrategy {
+ if m != nil {
+ return m.WindowingStrategies
+ }
+ return nil
+}
+
+func (m *ProcessBundleDescriptor) GetCoders() map[string]*org_apache_beam_model_pipeline_v1.Coder {
+ if m != nil {
+ return m.Coders
+ }
+ return nil
+}
+
+func (m *ProcessBundleDescriptor) GetEnvironments() map[string]*org_apache_beam_model_pipeline_v1.Environment {
+ if m != nil {
+ return m.Environments
+ }
+ return nil
+}
+
+func (m *ProcessBundleDescriptor) GetStateApiServiceDescriptor() *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor {
+ if m != nil {
+ return m.StateApiServiceDescriptor
+ }
+ return nil
+}
+
+// A request to process a given bundle.
+// Stable
+type ProcessBundleRequest struct {
+ // (Required) A reference to the process bundle descriptor that must be
+ // instantiated and executed by the SDK harness.
+ ProcessBundleDescriptorReference string `protobuf:"bytes,1,opt,name=process_bundle_descriptor_reference,json=processBundleDescriptorReference" json:"process_bundle_descriptor_reference,omitempty"`
+ // (Optional) A list of cache tokens that can be used by an SDK to reuse
+ // cached data returned by the State API across multiple bundles.
+ CacheTokens [][]byte `protobuf:"bytes,2,rep,name=cache_tokens,json=cacheTokens,proto3" json:"cache_tokens,omitempty"`
+}
+
+func (m *ProcessBundleRequest) Reset() { *m = ProcessBundleRequest{} }
+func (m *ProcessBundleRequest) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleRequest) ProtoMessage() {}
+func (*ProcessBundleRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *ProcessBundleRequest) GetProcessBundleDescriptorReference() string {
+ if m != nil {
+ return m.ProcessBundleDescriptorReference
+ }
+ return ""
+}
+
+func (m *ProcessBundleRequest) GetCacheTokens() [][]byte {
+ if m != nil {
+ return m.CacheTokens
+ }
+ return nil
+}
+
+// Stable
+type ProcessBundleResponse struct {
+ // (Optional) If metrics reporting is supported by the SDK, this represents
+ // the final metrics to record for this bundle.
+ Metrics *Metrics `protobuf:"bytes,1,opt,name=metrics" json:"metrics,omitempty"`
+}
+
+func (m *ProcessBundleResponse) Reset() { *m = ProcessBundleResponse{} }
+func (m *ProcessBundleResponse) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleResponse) ProtoMessage() {}
+func (*ProcessBundleResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *ProcessBundleResponse) GetMetrics() *Metrics {
+ if m != nil {
+ return m.Metrics
+ }
+ return nil
+}
+
+// A request to report progress information for a given bundle.
+// This is an optional request to be handled and is used to support advanced
+// SDK features such as SplittableDoFn, user level metrics etc.
+type ProcessBundleProgressRequest struct {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ InstructionReference string `protobuf:"bytes,1,opt,name=instruction_reference,json=instructionReference" json:"instruction_reference,omitempty"`
+}
+
+func (m *ProcessBundleProgressRequest) Reset() { *m = ProcessBundleProgressRequest{} }
+func (m *ProcessBundleProgressRequest) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleProgressRequest) ProtoMessage() {}
+func (*ProcessBundleProgressRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *ProcessBundleProgressRequest) GetInstructionReference() string {
+ if m != nil {
+ return m.InstructionReference
+ }
+ return ""
+}
+
+type Metrics struct {
+ Ptransforms map[string]*Metrics_PTransform `protobuf:"bytes,1,rep,name=ptransforms" json:"ptransforms,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ User map[string]*Metrics_User `protobuf:"bytes,2,rep,name=user" json:"user,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *Metrics) Reset() { *m = Metrics{} }
+func (m *Metrics) String() string { return proto.CompactTextString(m) }
+func (*Metrics) ProtoMessage() {}
+func (*Metrics) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+func (m *Metrics) GetPtransforms() map[string]*Metrics_PTransform {
+ if m != nil {
+ return m.Ptransforms
+ }
+ return nil
+}
+
+func (m *Metrics) GetUser() map[string]*Metrics_User {
+ if m != nil {
+ return m.User
+ }
+ return nil
+}
+
+// PTransform level metrics.
+// These metrics are split into processed and active element groups for
+// progress reporting purposes. This allows a Runner to see what is measured,
+// what is estimated and what can be extrapolated to be able to accurately
+// estimate the backlog of remaining work.
+type Metrics_PTransform struct {
+ // (Required): Metrics for processed elements.
+ ProcessedElements *Metrics_PTransform_ProcessedElements `protobuf:"bytes,1,opt,name=processed_elements,json=processedElements" json:"processed_elements,omitempty"`
+ // (Required): Metrics for active elements.
+ ActiveElements *Metrics_PTransform_ActiveElements `protobuf:"bytes,2,opt,name=active_elements,json=activeElements" json:"active_elements,omitempty"`
+ // (Optional): Map from local output name to its watermark.
+ // The watermarks reported are tentative, to get a better sense of progress
+ // while processing a bundle but before it is committed. At bundle commit
+ // time, a Runner needs to also take into account the timers set to compute
+ // the actual watermarks.
+ Watermarks map[string]int64 `protobuf:"bytes,3,rep,name=watermarks" json:"watermarks,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+}
+
+func (m *Metrics_PTransform) Reset() { *m = Metrics_PTransform{} }
+func (m *Metrics_PTransform) String() string { return proto.CompactTextString(m) }
+func (*Metrics_PTransform) ProtoMessage() {}
+func (*Metrics_PTransform) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10, 0} }
+
+func (m *Metrics_PTransform) GetProcessedElements() *Metrics_PTransform_ProcessedElements {
+ if m != nil {
+ return m.ProcessedElements
+ }
+ return nil
+}
+
+func (m *Metrics_PTransform) GetActiveElements() *Metrics_PTransform_ActiveElements {
+ if m != nil {
+ return m.ActiveElements
+ }
+ return nil
+}
+
+func (m *Metrics_PTransform) GetWatermarks() map[string]int64 {
+ if m != nil {
+ return m.Watermarks
+ }
+ return nil
+}
+
+// Metrics that are measured for processed and active element groups.
+type Metrics_PTransform_Measured struct {
+ // (Optional) Map from local input name to number of elements processed
+ // from this input.
+ // If unset, assumed to be the sum of the outputs of all producers to
+ // this transform (for ProcessedElements) and 0 (for ActiveElements).
+ InputElementCounts map[string]int64 `protobuf:"bytes,1,rep,name=input_element_counts,json=inputElementCounts" json:"input_element_counts,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+ // (Required) Map from local output name to number of elements produced
+ // for this output.
+ OutputElementCounts map[string]int64 `protobuf:"bytes,2,rep,name=output_element_counts,json=outputElementCounts" json:"output_element_counts,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+ // (Optional) The total time spent so far in processing the elements in
+ // this group, in seconds.
+ TotalTimeSpent float64 `protobuf:"fixed64,3,opt,name=total_time_spent,json=totalTimeSpent" json:"total_time_spent,omitempty"`
+}
+
+func (m *Metrics_PTransform_Measured) Reset() { *m = Metrics_PTransform_Measured{} }
+func (m *Metrics_PTransform_Measured) String() string { return proto.CompactTextString(m) }
+func (*Metrics_PTransform_Measured) ProtoMessage() {}
+func (*Metrics_PTransform_Measured) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{10, 0, 0}
+}
+
+func (m *Metrics_PTransform_Measured) GetInputElementCounts() map[string]int64 {
+ if m != nil {
+ return m.InputElementCounts
+ }
+ return nil
+}
+
+func (m *Metrics_PTransform_Measured) GetOutputElementCounts() map[string]int64 {
+ if m != nil {
+ return m.OutputElementCounts
+ }
+ return nil
+}
+
+func (m *Metrics_PTransform_Measured) GetTotalTimeSpent() float64 {
+ if m != nil {
+ return m.TotalTimeSpent
+ }
+ return 0
+}
+
+// Metrics for fully processed elements.
+type Metrics_PTransform_ProcessedElements struct {
+ // (Required)
+ Measured *Metrics_PTransform_Measured `protobuf:"bytes,1,opt,name=measured" json:"measured,omitempty"`
+}
+
+func (m *Metrics_PTransform_ProcessedElements) Reset() { *m = Metrics_PTransform_ProcessedElements{} }
+func (m *Metrics_PTransform_ProcessedElements) String() string { return proto.CompactTextString(m) }
+func (*Metrics_PTransform_ProcessedElements) ProtoMessage() {}
+func (*Metrics_PTransform_ProcessedElements) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{10, 0, 1}
+}
+
+func (m *Metrics_PTransform_ProcessedElements) GetMeasured() *Metrics_PTransform_Measured {
+ if m != nil {
+ return m.Measured
+ }
+ return nil
+}
+
+// Metrics for active elements.
+// An element is considered active if the SDK has started but not finished
+// processing it yet.
+type Metrics_PTransform_ActiveElements struct {
+ // (Required)
+ Measured *Metrics_PTransform_Measured `protobuf:"bytes,1,opt,name=measured" json:"measured,omitempty"`
+ // (Optional) Sum of estimated fraction of known work remaining for all
+ // active elements, as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ // TODO: Handle the case when known work is infinite.
+ FractionRemaining float64 `protobuf:"fixed64,2,opt,name=fraction_remaining,json=fractionRemaining" json:"fraction_remaining,omitempty"`
+ // (Optional) Map from local output name to sum of estimated number
+ // of elements remaining for this output from all active elements,
+ // as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ OutputElementsRemaining map[string]int64 `protobuf:"bytes,3,rep,name=output_elements_remaining,json=outputElementsRemaining" json:"output_elements_remaining,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+}
+
+func (m *Metrics_PTransform_ActiveElements) Reset() { *m = Metrics_PTransform_ActiveElements{} }
+func (m *Metrics_PTransform_ActiveElements) String() string { return proto.CompactTextString(m) }
+func (*Metrics_PTransform_ActiveElements) ProtoMessage() {}
+func (*Metrics_PTransform_ActiveElements) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{10, 0, 2}
+}
+
+func (m *Metrics_PTransform_ActiveElements) GetMeasured() *Metrics_PTransform_Measured {
+ if m != nil {
+ return m.Measured
+ }
+ return nil
+}
+
+func (m *Metrics_PTransform_ActiveElements) GetFractionRemaining() float64 {
+ if m != nil {
+ return m.FractionRemaining
+ }
+ return 0
+}
+
+func (m *Metrics_PTransform_ActiveElements) GetOutputElementsRemaining() map[string]int64 {
+ if m != nil {
+ return m.OutputElementsRemaining
+ }
+ return nil
+}
+
+// User defined metrics
+type Metrics_User struct {
+}
+
+func (m *Metrics_User) Reset() { *m = Metrics_User{} }
+func (m *Metrics_User) String() string { return proto.CompactTextString(m) }
+func (*Metrics_User) ProtoMessage() {}
+func (*Metrics_User) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10, 1} }
+
+type ProcessBundleProgressResponse struct {
+ // (Required)
+ Metrics *Metrics `protobuf:"bytes,1,opt,name=metrics" json:"metrics,omitempty"`
+}
+
+func (m *ProcessBundleProgressResponse) Reset() { *m = ProcessBundleProgressResponse{} }
+func (m *ProcessBundleProgressResponse) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleProgressResponse) ProtoMessage() {}
+func (*ProcessBundleProgressResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+
+func (m *ProcessBundleProgressResponse) GetMetrics() *Metrics {
+ if m != nil {
+ return m.Metrics
+ }
+ return nil
+}
+
+type ProcessBundleSplitRequest struct {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ InstructionReference string `protobuf:"bytes,1,opt,name=instruction_reference,json=instructionReference" json:"instruction_reference,omitempty"`
+ // (Required) The fraction of work (when compared to the known amount of work)
+ // the process bundle request should try to split at.
+ Fraction float64 `protobuf:"fixed64,2,opt,name=fraction" json:"fraction,omitempty"`
+}
+
+func (m *ProcessBundleSplitRequest) Reset() { *m = ProcessBundleSplitRequest{} }
+func (m *ProcessBundleSplitRequest) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleSplitRequest) ProtoMessage() {}
+func (*ProcessBundleSplitRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+
+func (m *ProcessBundleSplitRequest) GetInstructionReference() string {
+ if m != nil {
+ return m.InstructionReference
+ }
+ return ""
+}
+
+func (m *ProcessBundleSplitRequest) GetFraction() float64 {
+ if m != nil {
+ return m.Fraction
+ }
+ return 0
+}
+
+// urn:org.apache.beam:restriction:element-count:1.0
+type ElementCountRestriction struct {
+ // A restriction representing the number of elements that should be processed.
+ // Effectively the range [0, count]
+ Count int64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
+}
+
+func (m *ElementCountRestriction) Reset() { *m = ElementCountRestriction{} }
+func (m *ElementCountRestriction) String() string { return proto.CompactTextString(m) }
+func (*ElementCountRestriction) ProtoMessage() {}
+func (*ElementCountRestriction) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+
+func (m *ElementCountRestriction) GetCount() int64 {
+ if m != nil {
+ return m.Count
+ }
+ return 0
+}
+
+// urn:org.apache.beam:restriction:element-count-skip:1.0
+type ElementCountSkipRestriction struct {
+ // A restriction representing the number of elements that should be skipped.
+ // Effectively the range (count, infinity]
+ Count int64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
+}
+
+func (m *ElementCountSkipRestriction) Reset() { *m = ElementCountSkipRestriction{} }
+func (m *ElementCountSkipRestriction) String() string { return proto.CompactTextString(m) }
+func (*ElementCountSkipRestriction) ProtoMessage() {}
+func (*ElementCountSkipRestriction) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+func (m *ElementCountSkipRestriction) GetCount() int64 {
+ if m != nil {
+ return m.Count
+ }
+ return 0
+}
+
+// Each primitive transform that is splittable is defined by a restriction
+// it is currently processing. During splitting, that currently active
+// restriction (R_initial) is split into 2 components:
+// * a restriction (R_done) representing all elements that will be fully
+// processed
+// * a restriction (R_todo) representing all elements that will not be fully
+// processed
+//
+// where:
+// R_initial = R_done ⋃ R_todo
+type PrimitiveTransformSplit struct {
+ // (Required) A reference to a primitive transform with the given id that
+ // is part of the active process bundle request with the given instruction
+ // id.
+ PrimitiveTransformReference string `protobuf:"bytes,1,opt,name=primitive_transform_reference,json=primitiveTransformReference" json:"primitive_transform_reference,omitempty"`
+ // (Required) A function specification describing the restriction
+ // that has been completed by the primitive transform.
+ //
+ // For example, a remote GRPC source will have a specific urn and data
+ // block containing an ElementCountRestriction.
+ CompletedRestriction *org_apache_beam_model_pipeline_v1.FunctionSpec `protobuf:"bytes,2,opt,name=completed_restriction,json=completedRestriction" json:"completed_restriction,omitempty"`
+ // (Required) A function specification describing the restriction
+ // representing the remainder of work for the primitive transform.
+ //
+ // FOr example, a remote GRPC source will have a specific urn and data
+ // block contain an ElemntCountSkipRestriction.
+ RemainingRestriction *org_apache_beam_model_pipeline_v1.FunctionSpec `protobuf:"bytes,3,opt,name=remaining_restriction,json=remainingRestriction" json:"remaining_restriction,omitempty"`
+}
+
+func (m *PrimitiveTransformSplit) Reset() { *m = PrimitiveTransformSplit{} }
+func (m *PrimitiveTransformSplit) String() string { return proto.CompactTextString(m) }
+func (*PrimitiveTransformSplit) ProtoMessage() {}
+func (*PrimitiveTransformSplit) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
+func (m *PrimitiveTransformSplit) GetPrimitiveTransformReference() string {
+ if m != nil {
+ return m.PrimitiveTransformReference
+ }
+ return ""
+}
+
+func (m *PrimitiveTransformSplit) GetCompletedRestriction() *org_apache_beam_model_pipeline_v1.FunctionSpec {
+ if m != nil {
+ return m.CompletedRestriction
+ }
+ return nil
+}
+
+func (m *PrimitiveTransformSplit) GetRemainingRestriction() *org_apache_beam_model_pipeline_v1.FunctionSpec {
+ if m != nil {
+ return m.RemainingRestriction
+ }
+ return nil
+}
+
+type ProcessBundleSplitResponse struct {
+ // If primitive transform B and C are siblings and descendants of A and A, B,
+ // and C report a split. Then B and C's restrictions are relative to A's.
+ // R = A_done
+ // ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ C_todo)
+ // ⋃ A_todo
+ // If there is no descendant of B or C also reporting a split, than
+ // B_boundary = ∅ and C_boundary = ∅
+ //
+ // This restriction is processed and completed by the currently active process
+ // bundle request:
+ // A_done ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ C_done)
+ // and these restrictions will be processed by future process bundle requests:
+ // A_boundary â‹‚ B_todo (passed to SDF B directly)
+ // A_boundary â‹‚ C_todo (passed to SDF C directly)
+ // A_todo (passed to SDF A directly)
+ //
+ // Note that descendants splits should only be reported if it is inexpensive
+ // to compute the boundary restriction intersected with descendants splits.
+ // Also note, that the boundary restriction may represent a set of elements
+ // produced by a parent primitive transform which can not be split at each
+ // element or that there are intermediate unsplittable primitive transforms
+ // between an ancestor splittable function and a descendant splittable
+ // function which may have more than one output per element. Finally note
+ // that the descendant splits should only be reported if the split
+ // information is relatively compact.
+ Splits []*PrimitiveTransformSplit `protobuf:"bytes,1,rep,name=splits" json:"splits,omitempty"`
+}
+
+func (m *ProcessBundleSplitResponse) Reset() { *m = ProcessBundleSplitResponse{} }
+func (m *ProcessBundleSplitResponse) String() string { return proto.CompactTextString(m) }
+func (*ProcessBundleSplitResponse) ProtoMessage() {}
+func (*ProcessBundleSplitResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+
+func (m *ProcessBundleSplitResponse) GetSplits() []*PrimitiveTransformSplit {
+ if m != nil {
+ return m.Splits
+ }
+ return nil
+}
+
+// Messages used to represent logical byte streams.
+// Stable
+type Elements struct {
+ // (Required) A list containing parts of logical byte streams.
+ Data []*Elements_Data `protobuf:"bytes,1,rep,name=data" json:"data,omitempty"`
+}
+
+func (m *Elements) Reset() { *m = Elements{} }
+func (m *Elements) String() string { return proto.CompactTextString(m) }
+func (*Elements) ProtoMessage() {}
+func (*Elements) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+
+func (m *Elements) GetData() []*Elements_Data {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+// Represents multiple encoded elements in nested context for a given named
+// instruction and target.
+type Elements_Data struct {
+ // (Required) A reference to an active instruction request with the given
+ // instruction id.
+ InstructionReference string `protobuf:"bytes,1,opt,name=instruction_reference,json=instructionReference" json:"instruction_reference,omitempty"`
+ // (Required) A definition representing a consumer or producer of this data.
+ // If received by a harness, this represents the consumer within that
+ // harness that should consume these bytes. If sent by a harness, this
+ // represents the producer of these bytes.
+ //
+ // Note that a single element may span multiple Data messages.
+ //
+ // Note that a sending/receiving pair should share the same target
+ // identifier.
+ Target *Target `protobuf:"bytes,2,opt,name=target" json:"target,omitempty"`
+ // (Optional) Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ //
+ // An empty data block represents the end of stream for the given
+ // instruction and target.
+ Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"`
+}
+
+func (m *Elements_Data) Reset() { *m = Elements_Data{} }
+func (m *Elements_Data) String() string { return proto.CompactTextString(m) }
+func (*Elements_Data) ProtoMessage() {}
+func (*Elements_Data) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17, 0} }
+
+func (m *Elements_Data) GetInstructionReference() string {
+ if m != nil {
+ return m.InstructionReference
+ }
+ return ""
+}
+
+func (m *Elements_Data) GetTarget() *Target {
+ if m != nil {
+ return m.Target
+ }
+ return nil
+}
+
+func (m *Elements_Data) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+type StateRequest struct {
+ // (Required) An unique identifier provided by the SDK which represents this
+ // requests execution. The StateResponse corresponding with this request
+ // will have the matching id.
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ // (Required) The associated instruction id of the work that is currently
+ // being processed. This allows for the runner to associate any modifications
+ // to state to be committed with the appropriate work execution.
+ InstructionReference string `protobuf:"bytes,2,opt,name=instruction_reference,json=instructionReference" json:"instruction_reference,omitempty"`
+ // (Required) The state key this request is for.
+ StateKey *StateKey `protobuf:"bytes,3,opt,name=state_key,json=stateKey" json:"state_key,omitempty"`
+ // (Required) The action to take on this request.
+ //
+ // Types that are valid to be assigned to Request:
+ // *StateRequest_Get
+ // *StateRequest_Append
+ // *StateRequest_Clear
+ Request isStateRequest_Request `protobuf_oneof:"request"`
+}
+
+func (m *StateRequest) Reset() { *m = StateRequest{} }
+func (m *StateRequest) String() string { return proto.CompactTextString(m) }
+func (*StateRequest) ProtoMessage() {}
+func (*StateRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+
+type isStateRequest_Request interface {
+ isStateRequest_Request()
+}
+
+type StateRequest_Get struct {
+ Get *StateGetRequest `protobuf:"bytes,1000,opt,name=get,oneof"`
+}
+type StateRequest_Append struct {
+ Append *StateAppendRequest `protobuf:"bytes,1001,opt,name=append,oneof"`
+}
+type StateRequest_Clear struct {
+ Clear *StateClearRequest `protobuf:"bytes,1002,opt,name=clear,oneof"`
+}
+
+func (*StateRequest_Get) isStateRequest_Request() {}
+func (*StateRequest_Append) isStateRequest_Request() {}
+func (*StateRequest_Clear) isStateRequest_Request() {}
+
+func (m *StateRequest) GetRequest() isStateRequest_Request {
+ if m != nil {
+ return m.Request
+ }
+ return nil
+}
+
+func (m *StateRequest) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *StateRequest) GetInstructionReference() string {
+ if m != nil {
+ return m.InstructionReference
+ }
+ return ""
+}
+
+func (m *StateRequest) GetStateKey() *StateKey {
+ if m != nil {
+ return m.StateKey
+ }
+ return nil
+}
+
+func (m *StateRequest) GetGet() *StateGetRequest {
+ if x, ok := m.GetRequest().(*StateRequest_Get); ok {
+ return x.Get
+ }
+ return nil
+}
+
+func (m *StateRequest) GetAppend() *StateAppendRequest {
+ if x, ok := m.GetRequest().(*StateRequest_Append); ok {
+ return x.Append
+ }
+ return nil
+}
+
+func (m *StateRequest) GetClear() *StateClearRequest {
+ if x, ok := m.GetRequest().(*StateRequest_Clear); ok {
+ return x.Clear
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*StateRequest) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _StateRequest_OneofMarshaler, _StateRequest_OneofUnmarshaler, _StateRequest_OneofSizer, []interface{}{
+ (*StateRequest_Get)(nil),
+ (*StateRequest_Append)(nil),
+ (*StateRequest_Clear)(nil),
+ }
+}
+
+func _StateRequest_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*StateRequest)
+ // request
+ switch x := m.Request.(type) {
+ case *StateRequest_Get:
+ b.EncodeVarint(1000<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Get); err != nil {
+ return err
+ }
+ case *StateRequest_Append:
+ b.EncodeVarint(1001<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Append); err != nil {
+ return err
+ }
+ case *StateRequest_Clear:
+ b.EncodeVarint(1002<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Clear); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("StateRequest.Request has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _StateRequest_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*StateRequest)
+ switch tag {
+ case 1000: // request.get
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateGetRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &StateRequest_Get{msg}
+ return true, err
+ case 1001: // request.append
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateAppendRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &StateRequest_Append{msg}
+ return true, err
+ case 1002: // request.clear
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateClearRequest)
+ err := b.DecodeMessage(msg)
+ m.Request = &StateRequest_Clear{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _StateRequest_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*StateRequest)
+ // request
+ switch x := m.Request.(type) {
+ case *StateRequest_Get:
+ s := proto.Size(x.Get)
+ n += proto.SizeVarint(1000<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateRequest_Append:
+ s := proto.Size(x.Append)
+ n += proto.SizeVarint(1001<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateRequest_Clear:
+ s := proto.Size(x.Clear)
+ n += proto.SizeVarint(1002<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type StateResponse struct {
+ // (Required) A reference provided by the SDK which represents a requests
+ // execution. The StateResponse must have the matching id when responding
+ // to the SDK.
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ // (Optional) If this is specified, then the state request has failed.
+ // A human readable string representing the reason as to why the request
+ // failed.
+ Error string `protobuf:"bytes,2,opt,name=error" json:"error,omitempty"`
+ // (Optional) If this is specified, then the result of this state request
+ // can be cached using the supplied token.
+ CacheToken []byte `protobuf:"bytes,3,opt,name=cache_token,json=cacheToken,proto3" json:"cache_token,omitempty"`
+ // A corresponding response matching the request will be populated.
+ //
+ // Types that are valid to be assigned to Response:
+ // *StateResponse_Get
+ // *StateResponse_Append
+ // *StateResponse_Clear
+ Response isStateResponse_Response `protobuf_oneof:"response"`
+}
+
+func (m *StateResponse) Reset() { *m = StateResponse{} }
+func (m *StateResponse) String() string { return proto.CompactTextString(m) }
+func (*StateResponse) ProtoMessage() {}
+func (*StateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+
+type isStateResponse_Response interface {
+ isStateResponse_Response()
+}
+
+type StateResponse_Get struct {
+ Get *StateGetResponse `protobuf:"bytes,1000,opt,name=get,oneof"`
+}
+type StateResponse_Append struct {
+ Append *StateAppendResponse `protobuf:"bytes,1001,opt,name=append,oneof"`
+}
+type StateResponse_Clear struct {
+ Clear *StateClearResponse `protobuf:"bytes,1002,opt,name=clear,oneof"`
+}
+
+func (*StateResponse_Get) isStateResponse_Response() {}
+func (*StateResponse_Append) isStateResponse_Response() {}
+func (*StateResponse_Clear) isStateResponse_Response() {}
+
+func (m *StateResponse) GetResponse() isStateResponse_Response {
+ if m != nil {
+ return m.Response
+ }
+ return nil
+}
+
+func (m *StateResponse) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *StateResponse) GetError() string {
+ if m != nil {
+ return m.Error
+ }
+ return ""
+}
+
+func (m *StateResponse) GetCacheToken() []byte {
+ if m != nil {
+ return m.CacheToken
+ }
+ return nil
+}
+
+func (m *StateResponse) GetGet() *StateGetResponse {
+ if x, ok := m.GetResponse().(*StateResponse_Get); ok {
+ return x.Get
+ }
+ return nil
+}
+
+func (m *StateResponse) GetAppend() *StateAppendResponse {
+ if x, ok := m.GetResponse().(*StateResponse_Append); ok {
+ return x.Append
+ }
+ return nil
+}
+
+func (m *StateResponse) GetClear() *StateClearResponse {
+ if x, ok := m.GetResponse().(*StateResponse_Clear); ok {
+ return x.Clear
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*StateResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _StateResponse_OneofMarshaler, _StateResponse_OneofUnmarshaler, _StateResponse_OneofSizer, []interface{}{
+ (*StateResponse_Get)(nil),
+ (*StateResponse_Append)(nil),
+ (*StateResponse_Clear)(nil),
+ }
+}
+
+func _StateResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*StateResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *StateResponse_Get:
+ b.EncodeVarint(1000<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Get); err != nil {
+ return err
+ }
+ case *StateResponse_Append:
+ b.EncodeVarint(1001<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Append); err != nil {
+ return err
+ }
+ case *StateResponse_Clear:
+ b.EncodeVarint(1002<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Clear); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("StateResponse.Response has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _StateResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*StateResponse)
+ switch tag {
+ case 1000: // response.get
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateGetResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &StateResponse_Get{msg}
+ return true, err
+ case 1001: // response.append
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateAppendResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &StateResponse_Append{msg}
+ return true, err
+ case 1002: // response.clear
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateClearResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &StateResponse_Clear{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _StateResponse_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*StateResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *StateResponse_Get:
+ s := proto.Size(x.Get)
+ n += proto.SizeVarint(1000<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateResponse_Append:
+ s := proto.Size(x.Append)
+ n += proto.SizeVarint(1001<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateResponse_Clear:
+ s := proto.Size(x.Clear)
+ n += proto.SizeVarint(1002<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type StateKey struct {
+ // (Required) One of the following state keys must be set.
+ //
+ // Types that are valid to be assigned to Type:
+ // *StateKey_Runner_
+ // *StateKey_MultimapSideInput_
+ // *StateKey_BagUserState_
+ Type isStateKey_Type `protobuf_oneof:"type"`
+}
+
+func (m *StateKey) Reset() { *m = StateKey{} }
+func (m *StateKey) String() string { return proto.CompactTextString(m) }
+func (*StateKey) ProtoMessage() {}
+func (*StateKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+
+type isStateKey_Type interface {
+ isStateKey_Type()
+}
+
+type StateKey_Runner_ struct {
+ Runner *StateKey_Runner `protobuf:"bytes,1,opt,name=runner,oneof"`
+}
+type StateKey_MultimapSideInput_ struct {
+ MultimapSideInput *StateKey_MultimapSideInput `protobuf:"bytes,2,opt,name=multimap_side_input,json=multimapSideInput,oneof"`
+}
+type StateKey_BagUserState_ struct {
+ BagUserState *StateKey_BagUserState `protobuf:"bytes,3,opt,name=bag_user_state,json=bagUserState,oneof"`
+}
+
+func (*StateKey_Runner_) isStateKey_Type() {}
+func (*StateKey_MultimapSideInput_) isStateKey_Type() {}
+func (*StateKey_BagUserState_) isStateKey_Type() {}
+
+func (m *StateKey) GetType() isStateKey_Type {
+ if m != nil {
+ return m.Type
+ }
+ return nil
+}
+
+func (m *StateKey) GetRunner() *StateKey_Runner {
+ if x, ok := m.GetType().(*StateKey_Runner_); ok {
+ return x.Runner
+ }
+ return nil
+}
+
+func (m *StateKey) GetMultimapSideInput() *StateKey_MultimapSideInput {
+ if x, ok := m.GetType().(*StateKey_MultimapSideInput_); ok {
+ return x.MultimapSideInput
+ }
+ return nil
+}
+
+func (m *StateKey) GetBagUserState() *StateKey_BagUserState {
+ if x, ok := m.GetType().(*StateKey_BagUserState_); ok {
+ return x.BagUserState
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*StateKey) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _StateKey_OneofMarshaler, _StateKey_OneofUnmarshaler, _StateKey_OneofSizer, []interface{}{
+ (*StateKey_Runner_)(nil),
+ (*StateKey_MultimapSideInput_)(nil),
+ (*StateKey_BagUserState_)(nil),
+ }
+}
+
+func _StateKey_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*StateKey)
+ // type
+ switch x := m.Type.(type) {
+ case *StateKey_Runner_:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Runner); err != nil {
+ return err
+ }
+ case *StateKey_MultimapSideInput_:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.MultimapSideInput); err != nil {
+ return err
+ }
+ case *StateKey_BagUserState_:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.BagUserState); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("StateKey.Type has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _StateKey_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*StateKey)
+ switch tag {
+ case 1: // type.runner
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateKey_Runner)
+ err := b.DecodeMessage(msg)
+ m.Type = &StateKey_Runner_{msg}
+ return true, err
+ case 2: // type.multimap_side_input
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateKey_MultimapSideInput)
+ err := b.DecodeMessage(msg)
+ m.Type = &StateKey_MultimapSideInput_{msg}
+ return true, err
+ case 3: // type.bag_user_state
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(StateKey_BagUserState)
+ err := b.DecodeMessage(msg)
+ m.Type = &StateKey_BagUserState_{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _StateKey_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*StateKey)
+ // type
+ switch x := m.Type.(type) {
+ case *StateKey_Runner_:
+ s := proto.Size(x.Runner)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateKey_MultimapSideInput_:
+ s := proto.Size(x.MultimapSideInput)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateKey_BagUserState_:
+ s := proto.Size(x.BagUserState)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type StateKey_Runner struct {
+ // (Required) Opaque information supplied by the runner. Used to support
+ // remote references.
+ Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+}
+
+func (m *StateKey_Runner) Reset() { *m = StateKey_Runner{} }
+func (m *StateKey_Runner) String() string { return proto.CompactTextString(m) }
+func (*StateKey_Runner) ProtoMessage() {}
+func (*StateKey_Runner) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20, 0} }
+
+func (m *StateKey_Runner) GetKey() []byte {
+ if m != nil {
+ return m.Key
+ }
+ return nil
+}
+
+type StateKey_MultimapSideInput struct {
+ // (Required) The id of the PTransform containing a side input.
+ PtransformId string `protobuf:"bytes,1,opt,name=ptransform_id,json=ptransformId" json:"ptransform_id,omitempty"`
+ // (Required) The id of the side input.
+ SideInputId string `protobuf:"bytes,2,opt,name=side_input_id,json=sideInputId" json:"side_input_id,omitempty"`
+ // (Required) The window (after mapping the currently executing elements
+ // window into the side input windows domain) encoded in a nested context.
+ Window []byte `protobuf:"bytes,3,opt,name=window,proto3" json:"window,omitempty"`
+ // (Required) The key encoded in a nested context.
+ Key []byte `protobuf:"bytes,4,opt,name=key,proto3" json:"key,omitempty"`
+}
+
+func (m *StateKey_MultimapSideInput) Reset() { *m = StateKey_MultimapSideInput{} }
+func (m *StateKey_MultimapSideInput) String() string { return proto.CompactTextString(m) }
+func (*StateKey_MultimapSideInput) ProtoMessage() {}
+func (*StateKey_MultimapSideInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20, 1} }
+
+func (m *StateKey_MultimapSideInput) GetPtransformId() string {
+ if m != nil {
+ return m.PtransformId
+ }
+ return ""
+}
+
+func (m *StateKey_MultimapSideInput) GetSideInputId() string {
+ if m != nil {
+ return m.SideInputId
+ }
+ return ""
+}
+
+func (m *StateKey_MultimapSideInput) GetWindow() []byte {
+ if m != nil {
+ return m.Window
+ }
+ return nil
+}
+
+func (m *StateKey_MultimapSideInput) GetKey() []byte {
+ if m != nil {
+ return m.Key
+ }
+ return nil
+}
+
+type StateKey_BagUserState struct {
+ // (Required) The id of the PTransform containing user state.
+ PtransformId string `protobuf:"bytes,1,opt,name=ptransform_id,json=ptransformId" json:"ptransform_id,omitempty"`
+ // (Required) The id of the user state.
+ UserStateId string `protobuf:"bytes,2,opt,name=user_state_id,json=userStateId" json:"user_state_id,omitempty"`
+ // (Required) The window encoded in a nested context.
+ Window []byte `protobuf:"bytes,3,opt,name=window,proto3" json:"window,omitempty"`
+ // (Required) The key of the currently executing element encoded in a
+ // nested context.
+ Key []byte `protobuf:"bytes,4,opt,name=key,proto3" json:"key,omitempty"`
+}
+
+func (m *StateKey_BagUserState) Reset() { *m = StateKey_BagUserState{} }
+func (m *StateKey_BagUserState) String() string { return proto.CompactTextString(m) }
+func (*StateKey_BagUserState) ProtoMessage() {}
+func (*StateKey_BagUserState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20, 2} }
+
+func (m *StateKey_BagUserState) GetPtransformId() string {
+ if m != nil {
+ return m.PtransformId
+ }
+ return ""
+}
+
+func (m *StateKey_BagUserState) GetUserStateId() string {
+ if m != nil {
+ return m.UserStateId
+ }
+ return ""
+}
+
+func (m *StateKey_BagUserState) GetWindow() []byte {
+ if m != nil {
+ return m.Window
+ }
+ return nil
+}
+
+func (m *StateKey_BagUserState) GetKey() []byte {
+ if m != nil {
+ return m.Key
+ }
+ return nil
+}
+
+// A request to get state.
+type StateGetRequest struct {
+ // (Optional) If specified, signals to the runner that the response
+ // should resume from the following continuation token.
+ //
+ // If unspecified, signals to the runner that the response should start
+ // from the beginning of the logical continuable stream.
+ ContinuationToken []byte `protobuf:"bytes,1,opt,name=continuation_token,json=continuationToken,proto3" json:"continuation_token,omitempty"`
+}
+
+func (m *StateGetRequest) Reset() { *m = StateGetRequest{} }
+func (m *StateGetRequest) String() string { return proto.CompactTextString(m) }
+func (*StateGetRequest) ProtoMessage() {}
+func (*StateGetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+
+func (m *StateGetRequest) GetContinuationToken() []byte {
+ if m != nil {
+ return m.ContinuationToken
+ }
+ return nil
+}
+
+// A response to get state representing a logical byte stream which can be
+// continued using the state API.
+type StateGetResponse struct {
+ // (Optional) If specified, represents a token which can be used with the
+ // state API to get the next chunk of this logical byte stream. The end of
+ // the logical byte stream is signalled by this field being unset.
+ ContinuationToken []byte `protobuf:"bytes,1,opt,name=continuation_token,json=continuationToken,proto3" json:"continuation_token,omitempty"`
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+}
+
+func (m *StateGetResponse) Reset() { *m = StateGetResponse{} }
+func (m *StateGetResponse) String() string { return proto.CompactTextString(m) }
+func (*StateGetResponse) ProtoMessage() {}
+func (*StateGetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+
+func (m *StateGetResponse) GetContinuationToken() []byte {
+ if m != nil {
+ return m.ContinuationToken
+ }
+ return nil
+}
+
+func (m *StateGetResponse) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+// A request to append state.
+type StateAppendRequest struct {
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // multiple append requests are concatenated together.
+ Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
+}
+
+func (m *StateAppendRequest) Reset() { *m = StateAppendRequest{} }
+func (m *StateAppendRequest) String() string { return proto.CompactTextString(m) }
+func (*StateAppendRequest) ProtoMessage() {}
+func (*StateAppendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+
+func (m *StateAppendRequest) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+// A response to append state.
+type StateAppendResponse struct {
+}
+
+func (m *StateAppendResponse) Reset() { *m = StateAppendResponse{} }
+func (m *StateAppendResponse) String() string { return proto.CompactTextString(m) }
+func (*StateAppendResponse) ProtoMessage() {}
+func (*StateAppendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+
+// A request to clear state.
+type StateClearRequest struct {
+}
+
+func (m *StateClearRequest) Reset() { *m = StateClearRequest{} }
+func (m *StateClearRequest) String() string { return proto.CompactTextString(m) }
+func (*StateClearRequest) ProtoMessage() {}
+func (*StateClearRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+
+// A response to clear state.
+type StateClearResponse struct {
+}
+
+func (m *StateClearResponse) Reset() { *m = StateClearResponse{} }
+func (m *StateClearResponse) String() string { return proto.CompactTextString(m) }
+func (*StateClearResponse) ProtoMessage() {}
+func (*StateClearResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+
+// A log entry
+type LogEntry struct {
+ // (Required) The severity of the log statement.
+ Severity LogEntry_Severity_Enum `protobuf:"varint,1,opt,name=severity,enum=org.apache.beam.model.fn_execution.v1.LogEntry_Severity_Enum" json:"severity,omitempty"`
+ // (Required) The time at which this log statement occurred.
+ Timestamp *google_protobuf1.Timestamp `protobuf:"bytes,2,opt,name=timestamp" json:"timestamp,omitempty"`
+ // (Required) A human readable message.
+ Message string `protobuf:"bytes,3,opt,name=message" json:"message,omitempty"`
+ // (Optional) An optional trace of the functions involved. For example, in
+ // Java this can include multiple causes and multiple suppressed exceptions.
+ Trace string `protobuf:"bytes,4,opt,name=trace" json:"trace,omitempty"`
+ // (Optional) A reference to the instruction this log statement is associated
+ // with.
+ InstructionReference string `protobuf:"bytes,5,opt,name=instruction_reference,json=instructionReference" json:"instruction_reference,omitempty"`
+ // (Optional) A reference to the primitive transform this log statement is
+ // associated with.
+ PrimitiveTransformReference string `protobuf:"bytes,6,opt,name=primitive_transform_reference,json=primitiveTransformReference" json:"primitive_transform_reference,omitempty"`
+ // (Optional) Human-readable name of the function or method being invoked,
+ // with optional context such as the class or package name. The format can
+ // vary by language. For example:
+ // qual.if.ied.Class.method (Java)
+ // dir/package.func (Go)
+ // module.function (Python)
+ // file.cc:382 (C++)
+ LogLocation string `protobuf:"bytes,7,opt,name=log_location,json=logLocation" json:"log_location,omitempty"`
+ // (Optional) The name of the thread this log statement is associated with.
+ Thread string `protobuf:"bytes,8,opt,name=thread" json:"thread,omitempty"`
+}
+
+func (m *LogEntry) Reset() { *m = LogEntry{} }
+func (m *LogEntry) String() string { return proto.CompactTextString(m) }
+func (*LogEntry) ProtoMessage() {}
+func (*LogEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+
+func (m *LogEntry) GetSeverity() LogEntry_Severity_Enum {
+ if m != nil {
+ return m.Severity
+ }
+ return LogEntry_Severity_UNSPECIFIED
+}
+
+func (m *LogEntry) GetTimestamp() *google_protobuf1.Timestamp {
+ if m != nil {
+ return m.Timestamp
+ }
+ return nil
+}
+
+func (m *LogEntry) GetMessage() string {
+ if m != nil {
+ return m.Message
+ }
+ return ""
+}
+
+func (m *LogEntry) GetTrace() string {
+ if m != nil {
+ return m.Trace
+ }
+ return ""
+}
+
+func (m *LogEntry) GetInstructionReference() string {
+ if m != nil {
+ return m.InstructionReference
+ }
+ return ""
+}
+
+func (m *LogEntry) GetPrimitiveTransformReference() string {
+ if m != nil {
+ return m.PrimitiveTransformReference
+ }
+ return ""
+}
+
+func (m *LogEntry) GetLogLocation() string {
+ if m != nil {
+ return m.LogLocation
+ }
+ return ""
+}
+
+func (m *LogEntry) GetThread() string {
+ if m != nil {
+ return m.Thread
+ }
+ return ""
+}
+
+// A list of log entries, enables buffering and batching of multiple
+// log messages using the logging API.
+type LogEntry_List struct {
+ // (Required) One or or more log messages.
+ LogEntries []*LogEntry `protobuf:"bytes,1,rep,name=log_entries,json=logEntries" json:"log_entries,omitempty"`
+}
+
+func (m *LogEntry_List) Reset() { *m = LogEntry_List{} }
+func (m *LogEntry_List) String() string { return proto.CompactTextString(m) }
+func (*LogEntry_List) ProtoMessage() {}
+func (*LogEntry_List) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27, 0} }
+
+func (m *LogEntry_List) GetLogEntries() []*LogEntry {
+ if m != nil {
+ return m.LogEntries
+ }
+ return nil
+}
+
+// The severity of the event described in a log entry, expressed as one of the
+// severity levels listed below. For your reference, the levels are
+// assigned the listed numeric values. The effect of using numeric values
+// other than those listed is undefined.
+//
+// If you are writing log entries, you should map other severity encodings to
+// one of these standard levels. For example, you might map all of
+// Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`.
+//
+// This list is intentionally not comprehensive; the intent is to provide a
+// common set of "good enough" severity levels so that logging front ends
+// can provide filtering and searching across log types. Users of the API are
+// free not to use all severity levels in their log messages.
+type LogEntry_Severity struct {
+}
+
+func (m *LogEntry_Severity) Reset() { *m = LogEntry_Severity{} }
+func (m *LogEntry_Severity) String() string { return proto.CompactTextString(m) }
+func (*LogEntry_Severity) ProtoMessage() {}
+func (*LogEntry_Severity) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27, 1} }
+
+type LogControl struct {
+}
+
+func (m *LogControl) Reset() { *m = LogControl{} }
+func (m *LogControl) String() string { return proto.CompactTextString(m) }
+func (*LogControl) ProtoMessage() {}
+func (*LogControl) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+
+// A Docker container configuration for launching the SDK harness to execute
+// user specified functions.
+type DockerContainer struct {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ // (Required) The Docker container URI
+ // For example "dataflow.gcr.io/v1beta3/java-batch:1.5.1"
+ Uri string `protobuf:"bytes,2,opt,name=uri" json:"uri,omitempty"`
+ // (Optional) Docker registry specification.
+ // If unspecified, the uri is expected to be able to be fetched without
+ // requiring additional configuration by a runner.
+ RegistryReference string `protobuf:"bytes,3,opt,name=registry_reference,json=registryReference" json:"registry_reference,omitempty"`
+}
+
+func (m *DockerContainer) Reset() { *m = DockerContainer{} }
+func (m *DockerContainer) String() string { return proto.CompactTextString(m) }
+func (*DockerContainer) ProtoMessage() {}
+func (*DockerContainer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+
+func (m *DockerContainer) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *DockerContainer) GetUri() string {
+ if m != nil {
+ return m.Uri
+ }
+ return ""
+}
+
+func (m *DockerContainer) GetRegistryReference() string {
+ if m != nil {
+ return m.RegistryReference
+ }
+ return ""
+}
+
+func init() {
+ proto.RegisterType((*Target)(nil), "org.apache.beam.model.fn_execution.v1.Target")
+ proto.RegisterType((*Target_List)(nil), "org.apache.beam.model.fn_execution.v1.Target.List")
+ proto.RegisterType((*RemoteGrpcPort)(nil), "org.apache.beam.model.fn_execution.v1.RemoteGrpcPort")
+ proto.RegisterType((*InstructionRequest)(nil), "org.apache.beam.model.fn_execution.v1.InstructionRequest")
+ proto.RegisterType((*InstructionResponse)(nil), "org.apache.beam.model.fn_execution.v1.InstructionResponse")
+ proto.RegisterType((*RegisterRequest)(nil), "org.apache.beam.model.fn_execution.v1.RegisterRequest")
+ proto.RegisterType((*RegisterResponse)(nil), "org.apache.beam.model.fn_execution.v1.RegisterResponse")
+ proto.RegisterType((*ProcessBundleDescriptor)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor")
+ proto.RegisterType((*ProcessBundleRequest)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleRequest")
+ proto.RegisterType((*ProcessBundleResponse)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleResponse")
+ proto.RegisterType((*ProcessBundleProgressRequest)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleProgressRequest")
+ proto.RegisterType((*Metrics)(nil), "org.apache.beam.model.fn_execution.v1.Metrics")
+ proto.RegisterType((*Metrics_PTransform)(nil), "org.apache.beam.model.fn_execution.v1.Metrics.PTransform")
+ proto.RegisterType((*Metrics_PTransform_Measured)(nil), "org.apache.beam.model.fn_execution.v1.Metrics.PTransform.Measured")
+ proto.RegisterType((*Metrics_PTransform_ProcessedElements)(nil), "org.apache.beam.model.fn_execution.v1.Metrics.PTransform.ProcessedElements")
+ proto.RegisterType((*Metrics_PTransform_ActiveElements)(nil), "org.apache.beam.model.fn_execution.v1.Metrics.PTransform.ActiveElements")
+ proto.RegisterType((*Metrics_User)(nil), "org.apache.beam.model.fn_execution.v1.Metrics.User")
+ proto.RegisterType((*ProcessBundleProgressResponse)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse")
+ proto.RegisterType((*ProcessBundleSplitRequest)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest")
+ proto.RegisterType((*ElementCountRestriction)(nil), "org.apache.beam.model.fn_execution.v1.ElementCountRestriction")
+ proto.RegisterType((*ElementCountSkipRestriction)(nil), "org.apache.beam.model.fn_execution.v1.ElementCountSkipRestriction")
+ proto.RegisterType((*PrimitiveTransformSplit)(nil), "org.apache.beam.model.fn_execution.v1.PrimitiveTransformSplit")
+ proto.RegisterType((*ProcessBundleSplitResponse)(nil), "org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse")
+ proto.RegisterType((*Elements)(nil), "org.apache.beam.model.fn_execution.v1.Elements")
+ proto.RegisterType((*Elements_Data)(nil), "org.apache.beam.model.fn_execution.v1.Elements.Data")
+ proto.RegisterType((*StateRequest)(nil), "org.apache.beam.model.fn_execution.v1.StateRequest")
+ proto.RegisterType((*StateResponse)(nil), "org.apache.beam.model.fn_execution.v1.StateResponse")
+ proto.RegisterType((*StateKey)(nil), "org.apache.beam.model.fn_execution.v1.StateKey")
+ proto.RegisterType((*StateKey_Runner)(nil), "org.apache.beam.model.fn_execution.v1.StateKey.Runner")
+ proto.RegisterType((*StateKey_MultimapSideInput)(nil), "org.apache.beam.model.fn_execution.v1.StateKey.MultimapSideInput")
+ proto.RegisterType((*StateKey_BagUserState)(nil), "org.apache.beam.model.fn_execution.v1.StateKey.BagUserState")
+ proto.RegisterType((*StateGetRequest)(nil), "org.apache.beam.model.fn_execution.v1.StateGetRequest")
+ proto.RegisterType((*StateGetResponse)(nil), "org.apache.beam.model.fn_execution.v1.StateGetResponse")
+ proto.RegisterType((*StateAppendRequest)(nil), "org.apache.beam.model.fn_execution.v1.StateAppendRequest")
+ proto.RegisterType((*StateAppendResponse)(nil), "org.apache.beam.model.fn_execution.v1.StateAppendResponse")
+ proto.RegisterType((*StateClearRequest)(nil), "org.apache.beam.model.fn_execution.v1.StateClearRequest")
+ proto.RegisterType((*StateClearResponse)(nil), "org.apache.beam.model.fn_execution.v1.StateClearResponse")
+ proto.RegisterType((*LogEntry)(nil), "org.apache.beam.model.fn_execution.v1.LogEntry")
+ proto.RegisterType((*LogEntry_List)(nil), "org.apache.beam.model.fn_execution.v1.LogEntry.List")
+ proto.RegisterType((*LogEntry_Severity)(nil), "org.apache.beam.model.fn_execution.v1.LogEntry.Severity")
+ proto.RegisterType((*LogControl)(nil), "org.apache.beam.model.fn_execution.v1.LogControl")
+ proto.RegisterType((*DockerContainer)(nil), "org.apache.beam.model.fn_execution.v1.DockerContainer")
+ proto.RegisterEnum("org.apache.beam.model.fn_execution.v1.LogEntry_Severity_Enum", LogEntry_Severity_Enum_name, LogEntry_Severity_Enum_value)
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for BeamFnControl service
+
+type BeamFnControlClient interface {
+ // Instructions sent by the runner to the SDK requesting different types
+ // of work.
+ Control(ctx context.Context, opts ...grpc.CallOption) (BeamFnControl_ControlClient, error)
+}
+
+type beamFnControlClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewBeamFnControlClient(cc *grpc.ClientConn) BeamFnControlClient {
+ return &beamFnControlClient{cc}
+}
+
+func (c *beamFnControlClient) Control(ctx context.Context, opts ...grpc.CallOption) (BeamFnControl_ControlClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_BeamFnControl_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.fn_execution.v1.BeamFnControl/Control", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &beamFnControlControlClient{stream}
+ return x, nil
+}
+
+type BeamFnControl_ControlClient interface {
+ Send(*InstructionResponse) error
+ Recv() (*InstructionRequest, error)
+ grpc.ClientStream
+}
+
+type beamFnControlControlClient struct {
+ grpc.ClientStream
+}
+
+func (x *beamFnControlControlClient) Send(m *InstructionResponse) error {
+ return x.ClientStream.SendMsg(m)
+}
+
+func (x *beamFnControlControlClient) Recv() (*InstructionRequest, error) {
+ m := new(InstructionRequest)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for BeamFnControl service
+
+type BeamFnControlServer interface {
+ // Instructions sent by the runner to the SDK requesting different types
+ // of work.
+ Control(BeamFnControl_ControlServer) error
+}
+
+func RegisterBeamFnControlServer(s *grpc.Server, srv BeamFnControlServer) {
+ s.RegisterService(&_BeamFnControl_serviceDesc, srv)
+}
+
+func _BeamFnControl_Control_Handler(srv interface{}, stream grpc.ServerStream) error {
+ return srv.(BeamFnControlServer).Control(&beamFnControlControlServer{stream})
+}
+
+type BeamFnControl_ControlServer interface {
+ Send(*InstructionRequest) error
+ Recv() (*InstructionResponse, error)
+ grpc.ServerStream
+}
+
+type beamFnControlControlServer struct {
+ grpc.ServerStream
+}
+
+func (x *beamFnControlControlServer) Send(m *InstructionRequest) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func (x *beamFnControlControlServer) Recv() (*InstructionResponse, error) {
+ m := new(InstructionResponse)
+ if err := x.ServerStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+var _BeamFnControl_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.fn_execution.v1.BeamFnControl",
+ HandlerType: (*BeamFnControlServer)(nil),
+ Methods: []grpc.MethodDesc{},
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "Control",
+ Handler: _BeamFnControl_Control_Handler,
+ ServerStreams: true,
+ ClientStreams: true,
+ },
+ },
+ Metadata: "beam_fn_api.proto",
+}
+
+// Client API for BeamFnData service
+
+type BeamFnDataClient interface {
+ // Used to send data between harnesses.
+ Data(ctx context.Context, opts ...grpc.CallOption) (BeamFnData_DataClient, error)
+}
+
+type beamFnDataClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewBeamFnDataClient(cc *grpc.ClientConn) BeamFnDataClient {
+ return &beamFnDataClient{cc}
+}
+
+func (c *beamFnDataClient) Data(ctx context.Context, opts ...grpc.CallOption) (BeamFnData_DataClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_BeamFnData_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.fn_execution.v1.BeamFnData/Data", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &beamFnDataDataClient{stream}
+ return x, nil
+}
+
+type BeamFnData_DataClient interface {
+ Send(*Elements) error
+ Recv() (*Elements, error)
+ grpc.ClientStream
+}
+
+type beamFnDataDataClient struct {
+ grpc.ClientStream
+}
+
+func (x *beamFnDataDataClient) Send(m *Elements) error {
+ return x.ClientStream.SendMsg(m)
+}
+
+func (x *beamFnDataDataClient) Recv() (*Elements, error) {
+ m := new(Elements)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for BeamFnData service
+
+type BeamFnDataServer interface {
+ // Used to send data between harnesses.
+ Data(BeamFnData_DataServer) error
+}
+
+func RegisterBeamFnDataServer(s *grpc.Server, srv BeamFnDataServer) {
+ s.RegisterService(&_BeamFnData_serviceDesc, srv)
+}
+
+func _BeamFnData_Data_Handler(srv interface{}, stream grpc.ServerStream) error {
+ return srv.(BeamFnDataServer).Data(&beamFnDataDataServer{stream})
+}
+
+type BeamFnData_DataServer interface {
+ Send(*Elements) error
+ Recv() (*Elements, error)
+ grpc.ServerStream
+}
+
+type beamFnDataDataServer struct {
+ grpc.ServerStream
+}
+
+func (x *beamFnDataDataServer) Send(m *Elements) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func (x *beamFnDataDataServer) Recv() (*Elements, error) {
+ m := new(Elements)
+ if err := x.ServerStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+var _BeamFnData_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.fn_execution.v1.BeamFnData",
+ HandlerType: (*BeamFnDataServer)(nil),
+ Methods: []grpc.MethodDesc{},
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "Data",
+ Handler: _BeamFnData_Data_Handler,
+ ServerStreams: true,
+ ClientStreams: true,
+ },
+ },
+ Metadata: "beam_fn_api.proto",
+}
+
+// Client API for BeamFnState service
+
+type BeamFnStateClient interface {
+ // Used to get/append/clear state stored by the runner on behalf of the SDK.
+ State(ctx context.Context, opts ...grpc.CallOption) (BeamFnState_StateClient, error)
+}
+
+type beamFnStateClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewBeamFnStateClient(cc *grpc.ClientConn) BeamFnStateClient {
+ return &beamFnStateClient{cc}
+}
+
+func (c *beamFnStateClient) State(ctx context.Context, opts ...grpc.CallOption) (BeamFnState_StateClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_BeamFnState_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.fn_execution.v1.BeamFnState/State", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &beamFnStateStateClient{stream}
+ return x, nil
+}
+
+type BeamFnState_StateClient interface {
+ Send(*StateRequest) error
+ Recv() (*StateResponse, error)
+ grpc.ClientStream
+}
+
+type beamFnStateStateClient struct {
+ grpc.ClientStream
+}
+
+func (x *beamFnStateStateClient) Send(m *StateRequest) error {
+ return x.ClientStream.SendMsg(m)
+}
+
+func (x *beamFnStateStateClient) Recv() (*StateResponse, error) {
+ m := new(StateResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for BeamFnState service
+
+type BeamFnStateServer interface {
+ // Used to get/append/clear state stored by the runner on behalf of the SDK.
+ State(BeamFnState_StateServer) error
+}
+
+func RegisterBeamFnStateServer(s *grpc.Server, srv BeamFnStateServer) {
+ s.RegisterService(&_BeamFnState_serviceDesc, srv)
+}
+
+func _BeamFnState_State_Handler(srv interface{}, stream grpc.ServerStream) error {
+ return srv.(BeamFnStateServer).State(&beamFnStateStateServer{stream})
+}
+
+type BeamFnState_StateServer interface {
+ Send(*StateResponse) error
+ Recv() (*StateRequest, error)
+ grpc.ServerStream
+}
+
+type beamFnStateStateServer struct {
+ grpc.ServerStream
+}
+
+func (x *beamFnStateStateServer) Send(m *StateResponse) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func (x *beamFnStateStateServer) Recv() (*StateRequest, error) {
+ m := new(StateRequest)
+ if err := x.ServerStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+var _BeamFnState_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.fn_execution.v1.BeamFnState",
+ HandlerType: (*BeamFnStateServer)(nil),
+ Methods: []grpc.MethodDesc{},
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "State",
+ Handler: _BeamFnState_State_Handler,
+ ServerStreams: true,
+ ClientStreams: true,
+ },
+ },
+ Metadata: "beam_fn_api.proto",
+}
+
+// Client API for BeamFnLogging service
+
+type BeamFnLoggingClient interface {
+ // Allows for the SDK to emit log entries which the runner can
+ // associate with the active job.
+ Logging(ctx context.Context, opts ...grpc.CallOption) (BeamFnLogging_LoggingClient, error)
+}
+
+type beamFnLoggingClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewBeamFnLoggingClient(cc *grpc.ClientConn) BeamFnLoggingClient {
+ return &beamFnLoggingClient{cc}
+}
+
+func (c *beamFnLoggingClient) Logging(ctx context.Context, opts ...grpc.CallOption) (BeamFnLogging_LoggingClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_BeamFnLogging_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.fn_execution.v1.BeamFnLogging/Logging", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &beamFnLoggingLoggingClient{stream}
+ return x, nil
+}
+
+type BeamFnLogging_LoggingClient interface {
+ Send(*LogEntry_List) error
+ Recv() (*LogControl, error)
+ grpc.ClientStream
+}
+
+type beamFnLoggingLoggingClient struct {
+ grpc.ClientStream
+}
+
+func (x *beamFnLoggingLoggingClient) Send(m *LogEntry_List) error {
+ return x.ClientStream.SendMsg(m)
+}
+
+func (x *beamFnLoggingLoggingClient) Recv() (*LogControl, error) {
+ m := new(LogControl)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for BeamFnLogging service
+
+type BeamFnLoggingServer interface {
+ // Allows for the SDK to emit log entries which the runner can
+ // associate with the active job.
+ Logging(BeamFnLogging_LoggingServer) error
+}
+
+func RegisterBeamFnLoggingServer(s *grpc.Server, srv BeamFnLoggingServer) {
+ s.RegisterService(&_BeamFnLogging_serviceDesc, srv)
+}
+
+func _BeamFnLogging_Logging_Handler(srv interface{}, stream grpc.ServerStream) error {
+ return srv.(BeamFnLoggingServer).Logging(&beamFnLoggingLoggingServer{stream})
+}
+
+type BeamFnLogging_LoggingServer interface {
+ Send(*LogControl) error
+ Recv() (*LogEntry_List, error)
+ grpc.ServerStream
+}
+
+type beamFnLoggingLoggingServer struct {
+ grpc.ServerStream
+}
+
+func (x *beamFnLoggingLoggingServer) Send(m *LogControl) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func (x *beamFnLoggingLoggingServer) Recv() (*LogEntry_List, error) {
+ m := new(LogEntry_List)
+ if err := x.ServerStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+var _BeamFnLogging_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.fn_execution.v1.BeamFnLogging",
+ HandlerType: (*BeamFnLoggingServer)(nil),
+ Methods: []grpc.MethodDesc{},
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "Logging",
+ Handler: _BeamFnLogging_Logging_Handler,
+ ServerStreams: true,
+ ClientStreams: true,
+ },
+ },
+ Metadata: "beam_fn_api.proto",
+}
+
+func init() { proto.RegisterFile("beam_fn_api.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+ // 2350 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0x4d, 0x73, 0xdb, 0xc6,
+ 0xf9, 0x37, 0x44, 0x8a, 0xa2, 0x1e, 0x52, 0x14, 0xb5, 0x92, 0x22, 0x1a, 0x71, 0x26, 0x0e, 0xf2,
+ 0xcf, 0x8c, 0x2e, 0xa1, 0xfe, 0x7e, 0x99, 0xd8, 0x4e, 0x93, 0x34, 0x12, 0x45, 0x5b, 0xb4, 0x65,
+ 0x5b, 0x05, 0xa5, 0xb8, 0x4d, 0x67, 0x8a, 0x81, 0x80, 0x15, 0xb3, 0x63, 0x12, 0x40, 0x16, 0x4b,
+ 0xd9, 0xca, 0x64, 0x9a, 0xe9, 0x25, 0x7d, 0x99, 0x76, 0x72, 0xe8, 0x4c, 0xdb, 0x6b, 0xdb, 0x53,
+ 0x4f, 0xfd, 0x0c, 0xfd, 0x08, 0x3d, 0xf7, 0xda, 0x43, 0xda, 0x4e, 0x3f, 0x41, 0x2f, 0x9d, 0x7d,
+ 0x01, 0x08, 0x82, 0xa4, 0x4c, 0x50, 0xea, 0x0d, 0xd8, 0xdd, 0xe7, 0xf7, 0x7b, 0xf6, 0xd9, 0xe7,
+ 0x0d, 0x0b, 0x58, 0x39, 0xc6, 0x76, 0xcf, 0x3a, 0xf1, 0x2c, 0x3b, 0x20, 0xf5, 0x80, 0xfa, 0xcc,
+ 0x47, 0xef, 0xf8, 0xb4, 0x53, 0xb7, 0x03, 0xdb, 0xf9, 0x0c, 0xd7, 0xf9, 0x6c, 0xbd, 0xe7, 0xbb,
+ 0xb8, 0x5b, 0x3f, 0xf1, 0x2c, 0xfc, 0x12, 0x3b, 0x7d, 0x46, 0x7c, 0xaf, 0x7e, 0x7a, 0x43, 0x5f,
+ 0x17, 0x92, 0xb4, 0xef, 0x79, 0x98, 0x0e, 0xa4, 0xf5, 0x65, 0xec, 0xb9, 0x81, 0x4f, 0x3c, 0x16,
+ 0xaa, 0x81, 0x37, 0x3b, 0xbe, 0xdf, 0xe9, 0xe2, 0x2d, 0xf1, 0x76, 0xdc, 0x3f, 0xd9, 0x62, 0xa4,
+ 0x87, 0x43, 0x66, 0xf7, 0x02, 0xb9, 0xc0, 0xf8, 0xb3, 0x06, 0x85, 0x43, 0x9b, 0x76, 0x30, 0x43,
+ 0x3b, 0xf0, 0x46, 0x40, 0x49, 0x8f, 0x30, 0x72, 0x8a, 0x2d, 0x46, 0x6d, 0x2f, 0x3c, 0xf1, 0x69,
+ 0xcf, 0xa2, 0xf8, 0x04, 0x53, 0xec, 0x39, 0xb8, 0xa6, 0x5d, 0xd7, 0x36, 0x17, 0xcd, 0xd7, 0xe3,
+ 0x45, 0x87, 0xd1, 0x1a, 0x33, 0x5a, 0x82, 0x10, 0xe4, 0x3d, 0xbb, 0x87, 0x6b, 0x73, 0x62, 0xa9,
+ 0x78, 0xd6, 0x1f, 0x43, 0x7e, 0x9f, 0x84, 0x0c, 0x35, 0xa1, 0xc0, 0x04, 0x53, 0x4d, 0xbb, 0x9e,
+ 0xdb, 0x2c, 0xdd, 0x7c, 0xb7, 0x3e, 0xd5, 0x5e, 0xeb, 0x52, 0x3d, 0x53, 0x09, 0x1b, 0x5f, 0x41,
+ 0xc5, 0xc4, 0x3d, 0x9f, 0xe1, 0x07, 0x34, 0x70, 0x0e, 0x7c, 0xca, 0x50, 0x0f, 0x5e, 0xb3, 0x03,
+ 0x62, 0x85, 0x98, 0x9e, 0x12, 0x07, 0x5b, 0x2e, 0x0e, 0x1d, 0x4a, 0x02, 0xe6, 0x53, 0xa1, 0x71,
+ 0xe9, 0xe6, 0x9d, 0x09, 0x44, 0x01, 0x09, 0x70, 0x97, 0x78, 0x98, 0x93, 0x6c, 0x07, 0xa4, 0x2d,
+ 0xe5, 0x77, 0x63, 0x71, 0x73, 0xcd, 0x1e, 0x33, 0x6a, 0xfc, 0x27, 0x07, 0xa8, 0xe5, 0x85, 0x8c,
+ 0xf6, 0x1d, 0xae, 0xa2, 0x89, 0x3f, 0xef, 0xe3, 0x90, 0xa1, 0x77, 0xa0, 0x42, 0x06, 0xa3, 0x16,
+ 0x71, 0x95, 0xbd, 0x96, 0x12, 0xa3, 0x2d, 0x17, 0x1d, 0x41, 0x91, 0xe2, 0x0e, 0x09, 0x19, 0xa6,
+ 0xb5, 0x6f, 0x17, 0x84, 0x7e, 0xef, 0x4d, 0x69, 0x08, 0x53, 0xc9, 0x29, 0xc6, 0xbd, 0x2b, 0x66,
+ 0x0c, 0x85, 0x30, 0x54, 0x02, 0xea, 0x3b, 0x38, 0x0c, 0xad, 0xe3, 0xbe, 0xe7, 0x76, 0x71, 0xed,
+ 0x1f, 0x12, 0xfc, 0x3b, 0x53, 0x82, 0x1f, 0x48, 0xe9, 0x1d, 0x21, 0x3c, 0x60, 0x58, 0x0a, 0x92,
+ 0xe3, 0xe8, 0xc7, 0xb0, 0x31, 0x4c, 0x63, 0x05, 0xd4, 0xef, 0x50, 0x1c, 0x86, 0xb5, 0x7f, 0x4a,
+ 0xbe, 0xc6, 0x2c, 0x7c, 0x07, 0x0a, 0x64, 0xc0, 0xbb, 0x1e, 0x8c, 0x9b, 0x47, 0x7d, 0x58, 0x4b,
+ 0xf1, 0x87, 0x41, 0x97, 0xb0, 0xda, 0xbf, 0x24, 0xf9, 0xc7, 0xb3, 0x90, 0xb7, 0x39, 0xc2, 0x80,
+ 0x19, 0x05, 0x23, 0x93, 0x3b, 0x8b, 0xb0, 0x40, 0xe5, 0x02, 0xe3, 0xb7, 0x79, 0x58, 0x1d, 0x3a,
+ 0xfd, 0x30, 0xf0, 0xbd, 0x10, 0x4f, 0x7b, 0xfc, 0x6b, 0x30, 0x8f, 0x29, 0xf5, 0xa9, 0x8a, 0x10,
+ 0xf9, 0x82, 0x3e, 0x19, 0x75, 0x8a, 0x3b, 0x99, 0x9d, 0x42, 0x2a, 0x32, 0xe4, 0x15, 0x27, 0x93,
+ 0xbc, 0xe2, 0x83, 0xd9, 0xbc, 0x22, 0xa6, 0x48, 0xb9, 0xc5, 0x57, 0xaf, 0x74, 0x8b, 0xdd, 0x8b,
+ 0xb9, 0x45, 0x4c, 0x3c, 0xc1, 0x2f, 0x4e, 0xcf, 0xf7, 0x8b, 0xed, 0x0b, 0xf8, 0x45, 0x4c, 0x3d,
+ 0xce, 0x31, 0x80, 0x1f, 0x9c, 0x5c, 0x61, 0xfc, 0x4a, 0x83, 0xe5, 0x54, 0x88, 0xa2, 0x2f, 0xe0,
+ 0x6a, 0x4a, 0xaf, 0xa1, 0xec, 0xc4, 0xd3, 0xe0, 0x47, 0xb3, 0xe8, 0x96, 0x48, 0x52, 0x1b, 0xc1,
+ 0xf8, 0x09, 0x03, 0x41, 0x35, 0xed, 0x1c, 0xc6, 0x1f, 0x01, 0x36, 0x26, 0x00, 0xa1, 0x0a, 0xcc,
+ 0xc5, 0x5e, 0x3b, 0x47, 0x5c, 0xe4, 0x01, 0xc4, 0x55, 0x20, 0xac, 0xcd, 0x09, 0x65, 0x9f, 0x5c,
+ 0x4c, 0xd9, 0x7a, 0x5c, 0x32, 0xc2, 0xa6, 0xc7, 0xe8, 0x99, 0x99, 0x60, 0x40, 0x0c, 0xca, 0x81,
+ 0xe3, 0x77, 0xbb, 0x58, 0xc4, 0x4a, 0x58, 0xcb, 0x09, 0xc6, 0x83, 0x0b, 0x32, 0x1e, 0x24, 0x20,
+ 0x25, 0xe7, 0x10, 0x0b, 0xfa, 0x85, 0x06, 0x6b, 0x2f, 0x88, 0xe7, 0xfa, 0x2f, 0x88, 0xd7, 0xb1,
+ 0x42, 0x46, 0x6d, 0x86, 0x3b, 0x04, 0x87, 0xb5, 0xbc, 0xa0, 0x7f, 0x76, 0x41, 0xfa, 0x67, 0x11,
+ 0x74, 0x3b, 0x46, 0x96, 0x5a, 0xac, 0xbe, 0x18, 0x9d, 0x41, 0xc7, 0x50, 0x70, 0x7c, 0x17, 0xd3,
+ 0xb0, 0x36, 0x2f, 0xd8, 0x1f, 0x5e, 0x90, 0xbd, 0x21, 0xc0, 0x24, 0xa1, 0x42, 0xe6, 0x66, 0xc6,
+ 0xde, 0x29, 0xa1, 0xbe, 0xd7, 0xc3, 0x1e, 0x0b, 0x6b, 0x85, 0x4b, 0x31, 0x73, 0x33, 0x01, 0xa9,
+ 0xcc, 0x9c, 0x64, 0x41, 0x2f, 0xe1, 0x5a, 0xc8, 0x6c, 0x86, 0xad, 0x09, 0x95, 0x7a, 0xe1, 0x62,
+ 0x95, 0xfa, 0xaa, 0x00, 0x1f, 0x37, 0xa5, 0x77, 0x61, 0x39, 0xe5, 0x75, 0xa8, 0x0a, 0xb9, 0xe7,
+ 0xf8, 0x4c, 0xb9, 0x3a, 0x7f, 0x44, 0x0d, 0x98, 0x3f, 0xb5, 0xbb, 0x7d, 0xd9, 0xb8, 0x4c, 0x6e,
+ 0x4d, 0x92, 0x7a, 0x1c, 0x0c, 0xda, 0x1f, 0x29, 0xfb, 0xfe, 0xdc, 0x5d, 0x4d, 0xf7, 0x61, 0x65,
+ 0xc4, 0xe3, 0xc6, 0xf0, 0xed, 0x0e, 0xf3, 0xd5, 0xa7, 0xe1, 0x6b, 0xc4, 0xb0, 0x49, 0xc2, 0x2f,
+ 0xa1, 0x36, 0xc9, 0xc7, 0xc6, 0xf0, 0x3e, 0x1c, 0xe6, 0xbd, 0x3d, 0x05, 0x6f, 0x1a, 0xfd, 0x2c,
+ 0xc9, 0xee, 0x40, 0x29, 0xe1, 0x63, 0x63, 0x08, 0x3f, 0x1a, 0x26, 0xdc, 0x9c, 0x82, 0x50, 0x00,
+ 0xa6, 0x6c, 0x3a, 0xe2, 0x5e, 0x97, 0x63, 0xd3, 0x04, 0x6c, 0x82, 0xd0, 0xf8, 0x99, 0x06, 0x6b,
+ 0xe3, 0xfa, 0x21, 0xf4, 0x18, 0xde, 0x9e, 0x98, 0xce, 0x47, 0x1a, 0xe5, 0xeb, 0x13, 0x12, 0xf3,
+ 0xa0, 0x5b, 0x7e, 0x0b, 0xca, 0x0e, 0x57, 0xcf, 0x62, 0xfe, 0x73, 0xec, 0xc9, 0x1c, 0x5b, 0x36,
+ 0x4b, 0x62, 0xec, 0x50, 0x0c, 0x19, 0x36, 0xac, 0x8f, 0xad, 0xc1, 0x68, 0x0f, 0x16, 0x7a, 0x98,
+ 0x51, 0xe2, 0x84, 0xaa, 0xcb, 0xad, 0x4f, 0x19, 0xc1, 0x8f, 0xa5, 0x94, 0x19, 0x89, 0x1b, 0x6d,
+ 0xb8, 0x76, 0x5e, 0x33, 0x86, 0x6e, 0xc1, 0x7a, 0xb2, 0xb3, 0x49, 0x6f, 0x73, 0x8d, 0x24, 0xbb,
+ 0x21, 0x35, 0x67, 0xfc, 0xa5, 0x02, 0x0b, 0x8a, 0x09, 0xd9, 0x50, 0x0a, 0x12, 0x95, 0x44, 0x96,
+ 0xbd, 0xef, 0x66, 0x53, 0xb7, 0x7e, 0xc0, 0x52, 0xa5, 0x23, 0x89, 0x89, 0xf6, 0x21, 0xdf, 0x0f,
+ 0x31, 0x55, 0x55, 0xea, 0x6e, 0x46, 0xec, 0xa3, 0x10, 0x53, 0x09, 0x2a, 0x50, 0xf4, 0x5f, 0x97,
+ 0x00, 0x06, 0xe1, 0x8d, 0xbe, 0x80, 0xa8, 0xf4, 0x63, 0xd7, 0xc2, 0x5d, 0x2c, 0xf3, 0xa6, 0xb4,
+ 0xfa, 0xa3, 0xac, 0xdb, 0x88, 0x61, 0xa3, 0x54, 0x8a, 0xdd, 0xa6, 0x82, 0x34, 0x57, 0x82, 0xf4,
+ 0x10, 0xfa, 0x1c, 0x96, 0x6d, 0x47, 0x7c, 0x91, 0xc5, 0xc4, 0xd2, 0xbd, 0xf7, 0x66, 0x27, 0xde,
+ 0x16, 0x80, 0x31, 0x6b, 0xc5, 0x1e, 0x7a, 0x47, 0x04, 0xe0, 0x85, 0xcd, 0x30, 0xed, 0xd9, 0xf4,
+ 0x79, 0x54, 0x85, 0x5b, 0xb3, 0xb3, 0x3d, 0x8b, 0xb1, 0x54, 0xc9, 0x1f, 0x80, 0xeb, 0x7f, 0xcf,
+ 0x41, 0xf1, 0x31, 0xb6, 0xc3, 0x3e, 0xc5, 0x2e, 0xfa, 0xa5, 0x06, 0x6b, 0xc4, 0x0b, 0xfa, 0x2c,
+ 0xda, 0xaa, 0xe5, 0xf8, 0x7d, 0x69, 0x69, 0xae, 0xc2, 0xa7, 0xb3, 0xab, 0x10, 0x51, 0xd4, 0x5b,
+ 0x1c, 0x5e, 0x6d, 0xb4, 0x21, 0xc0, 0xa5, 0x4e, 0x88, 0x8c, 0x4c, 0xa0, 0x6f, 0x34, 0x58, 0xf7,
+ 0xfb, 0x6c, 0x8c, 0x3e, 0xd2, 0xc9, 0x7e, 0x78, 0x09, 0xfa, 0x3c, 0x15, 0xf8, 0x63, 0x14, 0x5a,
+ 0xf5, 0x47, 0x67, 0xd0, 0x26, 0x54, 0x99, 0xcf, 0xec, 0xae, 0xc5, 0x3f, 0xe2, 0xad, 0x30, 0xc0,
+ 0x1e, 0xab, 0xe5, 0xae, 0x6b, 0x9b, 0x9a, 0x59, 0x11, 0xe3, 0x87, 0xa4, 0x87, 0xdb, 0x7c, 0x54,
+ 0x6f, 0xc2, 0xc6, 0x84, 0xad, 0x8e, 0xc9, 0x9b, 0x6b, 0xc9, 0xbc, 0x99, 0x4b, 0x26, 0xde, 0xfb,
+ 0x50, 0x9b, 0xa4, 0x61, 0x26, 0x9c, 0x10, 0x56, 0x46, 0x9c, 0x1d, 0xfd, 0x08, 0x8a, 0x3d, 0x65,
+ 0x07, 0x15, 0x4b, 0x3b, 0x17, 0xb7, 0xa8, 0x19, 0x63, 0xea, 0xdf, 0xe4, 0xa0, 0x32, 0xec, 0xe9,
+ 0xff, 0x6b, 0x4a, 0xf4, 0x2e, 0xa0, 0x13, 0x6a, 0x47, 0x69, 0xb2, 0x67, 0x13, 0x8f, 0x78, 0x1d,
+ 0x61, 0x0e, 0xcd, 0x5c, 0x89, 0x66, 0xcc, 0x68, 0x02, 0xfd, 0x5e, 0x83, 0xab, 0xc3, 0x1e, 0x16,
+ 0x26, 0xc4, 0x64, 0xe0, 0xe1, 0xcb, 0x0a, 0xf3, 0x61, 0x5f, 0x0b, 0x63, 0x2d, 0xa4, 0xbf, 0x6d,
+ 0xf8, 0xe3, 0x67, 0xf5, 0x87, 0x70, 0xed, 0x3c, 0xc1, 0x4c, 0x6e, 0xf0, 0x21, 0x2c, 0xa7, 0x92,
+ 0x41, 0x26, 0xf1, 0x02, 0xe4, 0x79, 0xa2, 0xd6, 0xcf, 0xa0, 0x9a, 0x2e, 0x06, 0x63, 0x70, 0x9e,
+ 0x0e, 0x77, 0x03, 0xf7, 0x66, 0xb6, 0x63, 0x52, 0x85, 0x2e, 0x2c, 0xc6, 0xb5, 0x62, 0x0c, 0x67,
+ 0x6b, 0x98, 0xf3, 0xd6, 0x0c, 0x65, 0x28, 0xd9, 0x86, 0x10, 0x78, 0xe3, 0xdc, 0xcf, 0xe1, 0x4b,
+ 0xec, 0x01, 0xba, 0x70, 0x75, 0xe2, 0x9d, 0xc8, 0x4c, 0x0d, 0x00, 0xd2, 0xa1, 0x18, 0x79, 0xbc,
+ 0x8a, 0x80, 0xf8, 0xdd, 0xd8, 0x82, 0x8d, 0x64, 0x46, 0x31, 0x71, 0xc8, 0xb5, 0xe0, 0x53, 0xfc,
+ 0xf8, 0x45, 0x96, 0x15, 0xd8, 0x39, 0x53, 0xbe, 0x18, 0xb7, 0xe0, 0xf5, 0xa4, 0x40, 0xfb, 0x39,
+ 0x09, 0x5e, 0x2d, 0xf4, 0xa7, 0x39, 0xfe, 0xad, 0x9b, 0xbe, 0xab, 0x14, 0x3b, 0xbb, 0x94, 0xbb,
+ 0x4e, 0x17, 0xd6, 0x1d, 0xbf, 0x17, 0x74, 0x31, 0xc3, 0xae, 0x45, 0x07, 0xea, 0xa8, 0xd3, 0xdf,
+ 0x9a, 0xa2, 0xff, 0xbc, 0xdf, 0xf7, 0x84, 0x48, 0x3b, 0xc0, 0x8e, 0xb9, 0x16, 0xa3, 0x25, 0xf7,
+ 0xe6, 0xc2, 0x7a, 0x9c, 0x13, 0x86, 0x58, 0x72, 0x33, 0xb2, 0xc4, 0x68, 0x09, 0x16, 0x83, 0x81,
+ 0x3e, 0xf9, 0xee, 0x03, 0x7d, 0x02, 0x05, 0x71, 0x9d, 0x12, 0x66, 0xbe, 0xb2, 0x18, 0x6b, 0x7d,
+ 0x53, 0xa1, 0x19, 0xff, 0xd6, 0xa0, 0x18, 0x27, 0xe7, 0x3d, 0xc8, 0xbb, 0x36, 0xb3, 0x15, 0xc5,
+ 0xed, 0x29, 0x29, 0xe2, 0xf4, 0xb6, 0x6b, 0x33, 0xdb, 0x14, 0x08, 0xfa, 0x6f, 0x34, 0xc8, 0xf3,
+ 0xd7, 0xd9, 0x1c, 0x77, 0x70, 0x4d, 0x7d, 0xfe, 0xb7, 0xe0, 0xf9, 0xd7, 0xd4, 0x08, 0xa9, 0xed,
+ 0xf0, 0x63, 0x2a, 0x4b, 0xc5, 0x8c, 0x3f, 0xe4, 0xa0, 0xdc, 0xe6, 0x1f, 0xaa, 0x51, 0x64, 0xa5,
+ 0xaf, 0x5c, 0x26, 0x2a, 0x3c, 0x77, 0x8e, 0xc2, 0xfb, 0xb0, 0x28, 0x3f, 0xad, 0x79, 0x36, 0x3a,
+ 0xdf, 0x2b, 0xd2, 0x3a, 0x0b, 0x65, 0x1e, 0xe1, 0x33, 0xb3, 0x18, 0xaa, 0x27, 0xf4, 0x08, 0x72,
+ 0x7c, 0xef, 0x19, 0xaf, 0xa6, 0x05, 0xd0, 0x03, 0x9c, 0xb8, 0x46, 0xe5, 0x28, 0xe8, 0x10, 0x0a,
+ 0x76, 0x10, 0x60, 0xcf, 0x8d, 0xee, 0x1d, 0xef, 0x65, 0xc1, 0xdb, 0x16, 0xa2, 0x03, 0x48, 0x85,
+ 0x85, 0xbe, 0x07, 0xf3, 0x4e, 0x17, 0xdb, 0x34, 0xba, 0x5b, 0xbc, 0x9b, 0x05, 0xb4, 0xc1, 0x25,
+ 0x07, 0x98, 0x12, 0x29, 0x79, 0xc1, 0xfb, 0xb7, 0x39, 0x58, 0x52, 0x87, 0xa4, 0xdc, 0x3f, 0x7d,
+ 0x4a, 0xe3, 0xef, 0x70, 0xdf, 0x84, 0x52, 0xe2, 0x63, 0x4e, 0x9d, 0x3b, 0x0c, 0xbe, 0xe5, 0xd0,
+ 0xfe, 0x90, 0x65, 0xef, 0x64, 0xb6, 0x6c, 0x7c, 0x11, 0x29, 0x4c, 0x7b, 0x94, 0x36, 0xed, 0xfb,
+ 0xb3, 0x98, 0x36, 0xc6, 0x8c, 0x6c, 0x6b, 0xa6, 0x6c, 0x7b, 0x6f, 0x06, 0xdb, 0xc6, 0xa0, 0xca,
+ 0xb8, 0xc9, 0x4b, 0xd2, 0x6f, 0xf3, 0x50, 0x8c, 0xbc, 0x0e, 0x1d, 0x40, 0x41, 0xfe, 0xc2, 0x52,
+ 0xe5, 0xeb, 0xbd, 0x8c, 0x6e, 0x5b, 0x37, 0x85, 0x34, 0x57, 0x5f, 0xe2, 0xa0, 0x10, 0x56, 0x7b,
+ 0xfd, 0x2e, 0x23, 0x3d, 0x3b, 0xb0, 0x42, 0xe2, 0x62, 0x4b, 0x34, 0xf6, 0x2a, 0x92, 0xb7, 0xb3,
+ 0xc2, 0x3f, 0x56, 0x50, 0x6d, 0xe2, 0x62, 0xd1, 0x4f, 0xef, 0x5d, 0x31, 0x57, 0x7a, 0xe9, 0x41,
+ 0xe4, 0x42, 0xe5, 0xd8, 0xee, 0x58, 0xfc, 0xd3, 0xd1, 0x12, 0x71, 0xa4, 0xa2, 0xf0, 0x83, 0xac,
+ 0x7c, 0x3b, 0x76, 0x87, 0xf7, 0x00, 0xe2, 0x7d, 0xef, 0x8a, 0x59, 0x3e, 0x4e, 0xbc, 0xeb, 0x3a,
+ 0x14, 0xe4, 0x76, 0x93, 0x8d, 0x47, 0x59, 0x34, 0x1e, 0xfa, 0xd7, 0x1a, 0xac, 0x8c, 0x28, 0x8b,
+ 0xde, 0x86, 0xa5, 0xc1, 0x37, 0xf2, 0xe0, 0x8f, 0x44, 0x79, 0x30, 0xd8, 0x72, 0x91, 0x01, 0x4b,
+ 0x03, 0x43, 0xf1, 0x45, 0xd2, 0xa9, 0x4b, 0x61, 0x04, 0xd3, 0x72, 0xd1, 0x6b, 0x50, 0x90, 0xb7,
+ 0x95, 0xca, 0xab, 0xd5, 0x5b, 0xa4, 0x48, 0x7e, 0xa0, 0xc8, 0x4f, 0x34, 0x28, 0x27, 0x77, 0x31,
+ 0xb5, 0x0e, 0x03, 0xe3, 0x25, 0x74, 0xe8, 0x47, 0x30, 0x59, 0x74, 0xd8, 0x29, 0x40, 0x9e, 0x9d,
+ 0x05, 0xd8, 0xf8, 0x18, 0x96, 0x53, 0x69, 0x89, 0x37, 0xe8, 0x8e, 0xef, 0x31, 0xe2, 0xf5, 0x6d,
+ 0x91, 0x60, 0x65, 0xa8, 0x4a, 0x43, 0xae, 0x24, 0x67, 0x44, 0xc4, 0x1a, 0x47, 0x50, 0x4d, 0x87,
+ 0x5f, 0x46, 0x88, 0xb8, 0x0c, 0xcc, 0x25, 0xca, 0xc0, 0x26, 0xa0, 0xd1, 0xfc, 0x16, 0xaf, 0xd4,
+ 0x12, 0x2b, 0xd7, 0x61, 0x75, 0x4c, 0xb8, 0x1a, 0xab, 0xb0, 0x32, 0x92, 0xcb, 0x8c, 0x35, 0x85,
+ 0x3a, 0x14, 0x84, 0xc6, 0x5f, 0xf3, 0x50, 0xdc, 0xf7, 0x55, 0xb3, 0xfe, 0x03, 0x28, 0x86, 0xf8,
+ 0x14, 0x53, 0xc2, 0xa4, 0xf7, 0x54, 0x6e, 0x7e, 0x38, 0xa5, 0x8b, 0x46, 0x10, 0xf5, 0xb6, 0x92,
+ 0xaf, 0x37, 0xbd, 0x7e, 0xcf, 0x8c, 0xe1, 0xd0, 0x5d, 0x58, 0x8c, 0x7f, 0x2d, 0xab, 0x70, 0xd3,
+ 0xeb, 0xf2, 0xe7, 0x73, 0x3d, 0xfa, 0xf9, 0x5c, 0x3f, 0x8c, 0x56, 0x98, 0x83, 0xc5, 0xa8, 0xc6,
+ 0x9b, 0xd8, 0x30, 0xb4, 0x3b, 0x32, 0x6c, 0x16, 0xcd, 0xe8, 0x95, 0xe7, 0x59, 0x46, 0x6d, 0x07,
+ 0x8b, 0xc3, 0x5d, 0x34, 0xe5, 0xcb, 0xe4, 0x1a, 0x39, 0x7f, 0x4e, 0x8d, 0x7c, 0x65, 0xbf, 0x57,
+ 0x78, 0x75, 0xbf, 0xf7, 0x16, 0x94, 0xbb, 0x7e, 0xc7, 0xea, 0xfa, 0x8e, 0x38, 0x5f, 0x71, 0x65,
+ 0xbd, 0x68, 0x96, 0xba, 0x7e, 0x67, 0x5f, 0x0d, 0x71, 0x27, 0x65, 0x9f, 0x51, 0x6c, 0xbb, 0xb5,
+ 0xa2, 0x98, 0x54, 0x6f, 0xfa, 0xf7, 0xd5, 0x2f, 0xf0, 0x03, 0xe0, 0xcb, 0x2d, 0xec, 0x31, 0x4a,
+ 0x70, 0xd4, 0x4d, 0x6d, 0x65, 0x3c, 0x03, 0x13, 0xba, 0xf2, 0x89, 0xe0, 0x50, 0xa7, 0x50, 0x8c,
+ 0x8e, 0xc4, 0x38, 0x81, 0x3c, 0x3f, 0x15, 0xb4, 0x0c, 0xa5, 0xa3, 0x27, 0xed, 0x83, 0x66, 0xa3,
+ 0x75, 0xbf, 0xd5, 0xdc, 0xad, 0x5e, 0x41, 0x8b, 0x30, 0x7f, 0x68, 0x6e, 0x37, 0x9a, 0x55, 0x8d,
+ 0x3f, 0xee, 0x36, 0x77, 0x8e, 0x1e, 0x54, 0xe7, 0x50, 0x11, 0xf2, 0xad, 0x27, 0xf7, 0x9f, 0x56,
+ 0x73, 0x08, 0xa0, 0xf0, 0xe4, 0xe9, 0x61, 0xab, 0xd1, 0xac, 0xe6, 0xf9, 0xe8, 0xb3, 0x6d, 0xf3,
+ 0x49, 0x75, 0x9e, 0x2f, 0x6d, 0x9a, 0xe6, 0x53, 0xb3, 0x5a, 0x40, 0x65, 0x28, 0x36, 0xcc, 0xd6,
+ 0x61, 0xab, 0xb1, 0xbd, 0x5f, 0x5d, 0x30, 0xca, 0x00, 0xfb, 0x7e, 0xa7, 0xe1, 0x7b, 0x8c, 0xfa,
+ 0x5d, 0xe3, 0x18, 0x96, 0x77, 0x7d, 0xe7, 0x39, 0xa6, 0x7c, 0xc0, 0x26, 0x3c, 0x41, 0xa5, 0x0b,
+ 0x66, 0x15, 0x72, 0x7d, 0x4a, 0x54, 0x54, 0xf3, 0x47, 0x1e, 0x45, 0xf2, 0x27, 0x25, 0x3d, 0x4b,
+ 0x1c, 0x82, 0x3c, 0xff, 0x95, 0x68, 0x26, 0x36, 0xfd, 0xcd, 0xdf, 0x69, 0xb0, 0xb4, 0x83, 0xed,
+ 0xde, 0x7d, 0x4f, 0xb1, 0xa2, 0xaf, 0x35, 0x58, 0x88, 0x9e, 0xa7, 0xad, 0x7c, 0x63, 0x7e, 0xdb,
+ 0xea, 0xf7, 0x66, 0x91, 0x95, 0x01, 0x77, 0x65, 0x53, 0xfb, 0x7f, 0xed, 0xe6, 0x97, 0x00, 0x52,
+ 0x33, 0xd1, 0x71, 0x7a, 0xaa, 0xf3, 0xdc, 0xca, 0xd8, 0xbe, 0xea, 0x59, 0x05, 0x14, 0xfb, 0x4f,
+ 0x35, 0x28, 0x49, 0x7a, 0x99, 0x6e, 0x5f, 0xc2, 0xbc, 0x7c, 0xb8, 0x95, 0xa5, 0xf6, 0xa8, 0x1d,
+ 0xe9, 0xb7, 0xb3, 0x09, 0xa9, 0x14, 0x23, 0x35, 0xf9, 0x79, 0x7c, 0x44, 0xfb, 0x7e, 0xa7, 0x43,
+ 0xbc, 0x0e, 0x7a, 0x09, 0x0b, 0xd1, 0xe3, 0xed, 0xac, 0x69, 0x86, 0x47, 0x8b, 0x7e, 0x63, 0x7a,
+ 0xa9, 0xc8, 0x19, 0x85, 0x2e, 0x3b, 0xdb, 0xf0, 0x7f, 0x93, 0x24, 0x93, 0x82, 0x3b, 0x8b, 0x52,
+ 0xe1, 0xed, 0x80, 0x7c, 0x5a, 0x49, 0x4c, 0x59, 0xa7, 0x37, 0x8e, 0x0b, 0x22, 0x69, 0xdd, 0xfa,
+ 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0x95, 0xf5, 0x69, 0xa3, 0x23, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go
new file mode 100644
index 0000000..a472885
--- /dev/null
+++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go
@@ -0,0 +1,306 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: beam_provision_api.proto
+
+package fnexecution_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import google_protobuf2 "github.com/golang/protobuf/ptypes/struct"
+
+import (
+ context "golang.org/x/net/context"
+ grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// A request to get the provision info of a SDK harness worker instance.
+type GetProvisionInfoRequest struct {
+}
+
+func (m *GetProvisionInfoRequest) Reset() { *m = GetProvisionInfoRequest{} }
+func (m *GetProvisionInfoRequest) String() string { return proto.CompactTextString(m) }
+func (*GetProvisionInfoRequest) ProtoMessage() {}
+func (*GetProvisionInfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} }
+
+// A response containing the provision info of a SDK harness worker instance.
+type GetProvisionInfoResponse struct {
+ Info *ProvisionInfo `protobuf:"bytes,1,opt,name=info" json:"info,omitempty"`
+}
+
+func (m *GetProvisionInfoResponse) Reset() { *m = GetProvisionInfoResponse{} }
+func (m *GetProvisionInfoResponse) String() string { return proto.CompactTextString(m) }
+func (*GetProvisionInfoResponse) ProtoMessage() {}
+func (*GetProvisionInfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{1} }
+
+func (m *GetProvisionInfoResponse) GetInfo() *ProvisionInfo {
+ if m != nil {
+ return m.Info
+ }
+ return nil
+}
+
+// Runtime provisioning information for a SDK harness worker instance,
+// such as pipeline options, resource constraints and other job metadata
+type ProvisionInfo struct {
+ // (required) The job ID.
+ JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId" json:"job_id,omitempty"`
+ // (required) The job name.
+ JobName string `protobuf:"bytes,2,opt,name=job_name,json=jobName" json:"job_name,omitempty"`
+ // (required) Pipeline options. For non-template jobs, the options are
+ // identical to what is passed to job submission.
+ PipelineOptions *google_protobuf2.Struct `protobuf:"bytes,3,opt,name=pipeline_options,json=pipelineOptions" json:"pipeline_options,omitempty"`
+ // (optional) Resource limits that the SDK harness worker should respect.
+ // Runners may -- but are not required to -- enforce any limits provided.
+ ResourceLimits *Resources `protobuf:"bytes,4,opt,name=resource_limits,json=resourceLimits" json:"resource_limits,omitempty"`
+}
+
+func (m *ProvisionInfo) Reset() { *m = ProvisionInfo{} }
+func (m *ProvisionInfo) String() string { return proto.CompactTextString(m) }
+func (*ProvisionInfo) ProtoMessage() {}
+func (*ProvisionInfo) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{2} }
+
+func (m *ProvisionInfo) GetJobId() string {
+ if m != nil {
+ return m.JobId
+ }
+ return ""
+}
+
+func (m *ProvisionInfo) GetJobName() string {
+ if m != nil {
+ return m.JobName
+ }
+ return ""
+}
+
+func (m *ProvisionInfo) GetPipelineOptions() *google_protobuf2.Struct {
+ if m != nil {
+ return m.PipelineOptions
+ }
+ return nil
+}
+
+func (m *ProvisionInfo) GetResourceLimits() *Resources {
+ if m != nil {
+ return m.ResourceLimits
+ }
+ return nil
+}
+
+// Resources specify limits for local resources, such memory and cpu. It
+// is used to inform SDK harnesses of their allocated footprint.
+type Resources struct {
+ // (optional) Memory usage limits. SDKs can use this value to configure
+ // internal buffer sizes and language specific sizes.
+ Memory *Resources_Memory `protobuf:"bytes,1,opt,name=memory" json:"memory,omitempty"`
+ // (optional) CPU usage limits.
+ Cpu *Resources_Cpu `protobuf:"bytes,2,opt,name=cpu" json:"cpu,omitempty"`
+ // (optional) Disk size limits for the semi-persistent location.
+ SemiPersistentDisk *Resources_Disk `protobuf:"bytes,3,opt,name=semi_persistent_disk,json=semiPersistentDisk" json:"semi_persistent_disk,omitempty"`
+}
+
+func (m *Resources) Reset() { *m = Resources{} }
+func (m *Resources) String() string { return proto.CompactTextString(m) }
+func (*Resources) ProtoMessage() {}
+func (*Resources) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3} }
+
+func (m *Resources) GetMemory() *Resources_Memory {
+ if m != nil {
+ return m.Memory
+ }
+ return nil
+}
+
+func (m *Resources) GetCpu() *Resources_Cpu {
+ if m != nil {
+ return m.Cpu
+ }
+ return nil
+}
+
+func (m *Resources) GetSemiPersistentDisk() *Resources_Disk {
+ if m != nil {
+ return m.SemiPersistentDisk
+ }
+ return nil
+}
+
+// Memory limits.
+type Resources_Memory struct {
+ // (optional) Hard limit in bytes. A zero value means unspecified.
+ Size uint64 `protobuf:"varint,1,opt,name=size" json:"size,omitempty"`
+}
+
+func (m *Resources_Memory) Reset() { *m = Resources_Memory{} }
+func (m *Resources_Memory) String() string { return proto.CompactTextString(m) }
+func (*Resources_Memory) ProtoMessage() {}
+func (*Resources_Memory) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3, 0} }
+
+func (m *Resources_Memory) GetSize() uint64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+// CPU limits.
+type Resources_Cpu struct {
+ // (optional) Shares of a cpu to use. Fractional values, such as "0.2"
+ // or "2.5", are fine. Any value <= 0 means unspecified.
+ Shares float32 `protobuf:"fixed32,1,opt,name=shares" json:"shares,omitempty"`
+}
+
+func (m *Resources_Cpu) Reset() { *m = Resources_Cpu{} }
+func (m *Resources_Cpu) String() string { return proto.CompactTextString(m) }
+func (*Resources_Cpu) ProtoMessage() {}
+func (*Resources_Cpu) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3, 1} }
+
+func (m *Resources_Cpu) GetShares() float32 {
+ if m != nil {
+ return m.Shares
+ }
+ return 0
+}
+
+// Disk limits.
+type Resources_Disk struct {
+ // (optional) Hard limit in bytes. A zero value means unspecified.
+ Size uint64 `protobuf:"varint,1,opt,name=size" json:"size,omitempty"`
+}
+
+func (m *Resources_Disk) Reset() { *m = Resources_Disk{} }
+func (m *Resources_Disk) String() string { return proto.CompactTextString(m) }
+func (*Resources_Disk) ProtoMessage() {}
+func (*Resources_Disk) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3, 2} }
+
+func (m *Resources_Disk) GetSize() uint64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+func init() {
+ proto.RegisterType((*GetProvisionInfoRequest)(nil), "org.apache.beam.model.fn_execution.v1.GetProvisionInfoRequest")
+ proto.RegisterType((*GetProvisionInfoResponse)(nil), "org.apache.beam.model.fn_execution.v1.GetProvisionInfoResponse")
+ proto.RegisterType((*ProvisionInfo)(nil), "org.apache.beam.model.fn_execution.v1.ProvisionInfo")
+ proto.RegisterType((*Resources)(nil), "org.apache.beam.model.fn_execution.v1.Resources")
+ proto.RegisterType((*Resources_Memory)(nil), "org.apache.beam.model.fn_execution.v1.Resources.Memory")
+ proto.RegisterType((*Resources_Cpu)(nil), "org.apache.beam.model.fn_execution.v1.Resources.Cpu")
+ proto.RegisterType((*Resources_Disk)(nil), "org.apache.beam.model.fn_execution.v1.Resources.Disk")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for ProvisionService service
+
+type ProvisionServiceClient interface {
+ // Get provision information for the SDK harness worker instance.
+ GetProvisionInfo(ctx context.Context, in *GetProvisionInfoRequest, opts ...grpc.CallOption) (*GetProvisionInfoResponse, error)
+}
+
+type provisionServiceClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewProvisionServiceClient(cc *grpc.ClientConn) ProvisionServiceClient {
+ return &provisionServiceClient{cc}
+}
+
+func (c *provisionServiceClient) GetProvisionInfo(ctx context.Context, in *GetProvisionInfoRequest, opts ...grpc.CallOption) (*GetProvisionInfoResponse, error) {
+ out := new(GetProvisionInfoResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.fn_execution.v1.ProvisionService/GetProvisionInfo", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// Server API for ProvisionService service
+
+type ProvisionServiceServer interface {
+ // Get provision information for the SDK harness worker instance.
+ GetProvisionInfo(context.Context, *GetProvisionInfoRequest) (*GetProvisionInfoResponse, error)
+}
+
+func RegisterProvisionServiceServer(s *grpc.Server, srv ProvisionServiceServer) {
+ s.RegisterService(&_ProvisionService_serviceDesc, srv)
+}
+
+func _ProvisionService_GetProvisionInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(GetProvisionInfoRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(ProvisionServiceServer).GetProvisionInfo(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.fn_execution.v1.ProvisionService/GetProvisionInfo",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(ProvisionServiceServer).GetProvisionInfo(ctx, req.(*GetProvisionInfoRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+var _ProvisionService_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.fn_execution.v1.ProvisionService",
+ HandlerType: (*ProvisionServiceServer)(nil),
+ Methods: []grpc.MethodDesc{
+ {
+ MethodName: "GetProvisionInfo",
+ Handler: _ProvisionService_GetProvisionInfo_Handler,
+ },
+ },
+ Streams: []grpc.StreamDesc{},
+ Metadata: "beam_provision_api.proto",
+}
+
+func init() { proto.RegisterFile("beam_provision_api.proto", fileDescriptor1) }
+
+var fileDescriptor1 = []byte{
+ // 469 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0xcf, 0x6e, 0xd3, 0x40,
+ 0x10, 0xc6, 0xe5, 0xc6, 0x18, 0x3a, 0x40, 0x1b, 0xad, 0x80, 0xba, 0x56, 0x91, 0x50, 0x04, 0x12,
+ 0xa7, 0x2d, 0x2d, 0x20, 0x6e, 0x20, 0xd2, 0x0a, 0xa8, 0x04, 0xb4, 0xda, 0x9e, 0xe0, 0x62, 0xf9,
+ 0xcf, 0x24, 0xdd, 0x34, 0xde, 0x59, 0x76, 0xd7, 0x11, 0xf0, 0x1a, 0xbc, 0x04, 0x8f, 0xc5, 0x89,
+ 0xe7, 0x40, 0x5e, 0x3b, 0xa1, 0x05, 0x2a, 0xa5, 0xdc, 0xec, 0xf9, 0xf6, 0xfb, 0x79, 0xbf, 0xf1,
+ 0x0c, 0xc4, 0x39, 0x66, 0x55, 0xaa, 0x0d, 0xcd, 0xa4, 0x95, 0xa4, 0xd2, 0x4c, 0x4b, 0xae, 0x0d,
+ 0x39, 0x62, 0x0f, 0xc8, 0x8c, 0x79, 0xa6, 0xb3, 0xe2, 0x04, 0x79, 0x73, 0x88, 0x57, 0x54, 0xe2,
+ 0x94, 0x8f, 0x54, 0x8a, 0x9f, 0xb1, 0xa8, 0x9d, 0x24, 0xc5, 0x67, 0x3b, 0xc9, 0xd6, 0x98, 0x68,
+ 0x3c, 0xc5, 0x6d, 0x6f, 0xca, 0xeb, 0xd1, 0xb6, 0x75, 0xa6, 0x2e, 0x5c, 0x0b, 0x19, 0x6c, 0xc2,
+ 0xc6, 0x6b, 0x74, 0x47, 0x73, 0xfc, 0x81, 0x1a, 0x91, 0xc0, 0x4f, 0x35, 0x5a, 0x37, 0x28, 0x21,
+ 0xfe, 0x5b, 0xb2, 0x9a, 0x94, 0x45, 0xf6, 0x06, 0x42, 0xa9, 0x46, 0x14, 0x07, 0xf7, 0x82, 0x87,
+ 0xd7, 0x77, 0x9f, 0xf0, 0xa5, 0xae, 0xc2, 0xcf, 0xb3, 0x3c, 0x61, 0xf0, 0x23, 0x80, 0x9b, 0xe7,
+ 0xea, 0xec, 0x36, 0x44, 0x13, 0xca, 0x53, 0x59, 0x7a, 0xfa, 0xaa, 0xb8, 0x32, 0xa1, 0xfc, 0xa0,
+ 0x64, 0x9b, 0x70, 0xad, 0x29, 0xab, 0xac, 0xc2, 0x78, 0xc5, 0x0b, 0x57, 0x27, 0x94, 0xbf, 0xcf,
+ 0x2a, 0x64, 0x43, 0xe8, 0x6b, 0xa9, 0x71, 0x2a, 0x15, 0xa6, 0xa4, 0x9b, 0xaf, 0xd9, 0xb8, 0xe7,
+ 0x6f, 0xb6, 0xc1, 0xdb, 0xf4, 0x7c, 0x9e, 0x9e, 0x1f, 0xfb, 0xf4, 0x62, 0x7d, 0x6e, 0x38, 0x6c,
+ 0xcf, 0xb3, 0x0f, 0xb0, 0x6e, 0xd0, 0x52, 0x6d, 0x0a, 0x4c, 0xa7, 0xb2, 0x92, 0xce, 0xc6, 0xa1,
+ 0x47, 0x3c, 0x5a, 0x32, 0x9c, 0xe8, 0xdc, 0x56, 0xac, 0xcd, 0x41, 0x6f, 0x3d, 0x67, 0xf0, 0x73,
+ 0x05, 0x56, 0x17, 0x2a, 0x3b, 0x84, 0xa8, 0xc2, 0x8a, 0xcc, 0x97, 0xae, 0x79, 0xcf, 0x2e, 0xcb,
+ 0xe7, 0xef, 0xbc, 0x5d, 0x74, 0x18, 0xf6, 0x0a, 0x7a, 0x85, 0xae, 0x7d, 0x4f, 0x96, 0xff, 0x15,
+ 0xbf, 0x69, 0x7b, 0xba, 0x16, 0x0d, 0x80, 0x8d, 0xe1, 0x96, 0xc5, 0x4a, 0xa6, 0x1a, 0x8d, 0x95,
+ 0xd6, 0xa1, 0x72, 0x69, 0x29, 0xed, 0x69, 0xd7, 0xc9, 0xa7, 0x97, 0x06, 0xef, 0x4b, 0x7b, 0x2a,
+ 0x58, 0x83, 0x3c, 0x5a, 0x10, 0x9b, 0x5a, 0xb2, 0x05, 0x51, 0x1b, 0x81, 0x31, 0x08, 0xad, 0xfc,
+ 0x8a, 0xbe, 0x13, 0xa1, 0xf0, 0xcf, 0xc9, 0x5d, 0xe8, 0xed, 0xe9, 0x9a, 0xdd, 0x81, 0xc8, 0x9e,
+ 0x64, 0x06, 0xad, 0x17, 0x57, 0x44, 0xf7, 0x96, 0x24, 0x10, 0x36, 0x90, 0x7f, 0x59, 0x77, 0xbf,
+ 0x07, 0xd0, 0x5f, 0xcc, 0xd2, 0x31, 0x9a, 0x99, 0x2c, 0x90, 0x7d, 0x0b, 0xa0, 0xff, 0xe7, 0x1c,
+ 0xb3, 0xe7, 0x4b, 0xa6, 0xb9, 0x60, 0x37, 0x92, 0x17, 0xff, 0xed, 0x6f, 0x17, 0x68, 0xb8, 0x0f,
+ 0xf7, 0x2f, 0x22, 0x9c, 0x05, 0x0c, 0x6f, 0x2c, 0xec, 0x2f, 0xb5, 0xfc, 0xb8, 0x76, 0x46, 0x4d,
+ 0x67, 0x3b, 0x79, 0xe4, 0xc7, 0xfa, 0xf1, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x47, 0x8e, 0xd3,
+ 0xc2, 0x25, 0x04, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/gen.go b/sdks/go/pkg/beam/model/gen.go
new file mode 100644
index 0000000..2c6de37
--- /dev/null
+++ b/sdks/go/pkg/beam/model/gen.go
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+// TODO(herohde) 9/1/2017: for now, install protoc as described on grpc.io before running go generate.
+
+//go:generate protoc -I../../../../../model/pipeline/src/main/proto ../../../../../model/pipeline/src/main/proto/beam_runner_api.proto ../../../../../model/pipeline/src/main/proto/endpoints.proto ../../../../../model/pipeline/src/main/proto/standard_window_fns.proto --go_out=pipeline_v1,plugins=grpc:pipeline_v1
+//go:generate protoc -I../../../../../model/pipeline/src/main/proto -I../../../../../model/job-management/src/main/proto ../../../../../model/job-management/src/main/proto/beam_job_api.proto ../../../../../model/job-management/src/main/proto/beam_artifact_api.proto --go_out=Mbeam_runner_api.proto=github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1,Mendpoints.proto=github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1,jobmanagement_v1,plugins=grpc:jobmanagement_v1
+//go:generate protoc -I../../../../../model/pipeline/src/main/proto -I../../../../../model/fn-execution/src/main/proto ../../../../../model/fn-execution/src/main/proto/beam_fn_api.proto ../../../../../model/fn-execution/src/main/proto/beam_provision_api.proto --go_out=Mbeam_runner_api.proto=github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1,Mendpoints.proto=github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1,fnexecution_v1,plugins=grpc:fnexecution_v1
diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go
new file mode 100644
index 0000000..3a4940e
--- /dev/null
+++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go
@@ -0,0 +1,690 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: beam_artifact_api.proto
+
+package jobmanagement_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+ context "golang.org/x/net/context"
+ grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// An artifact identifier and associated metadata.
+type ArtifactMetadata struct {
+ // (Required) The name of the artifact.
+ Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+ // (Optional) The Unix-like permissions of the artifact
+ Permissions uint32 `protobuf:"varint,2,opt,name=permissions" json:"permissions,omitempty"`
+ // (Optional) The base64-encoded md5 checksum of the artifact. Used, among other things, by
+ // harness boot code to validate the integrity of the artifact.
+ Md5 string `protobuf:"bytes,3,opt,name=md5" json:"md5,omitempty"`
+}
+
+func (m *ArtifactMetadata) Reset() { *m = ArtifactMetadata{} }
+func (m *ArtifactMetadata) String() string { return proto.CompactTextString(m) }
+func (*ArtifactMetadata) ProtoMessage() {}
+func (*ArtifactMetadata) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} }
+
+func (m *ArtifactMetadata) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *ArtifactMetadata) GetPermissions() uint32 {
+ if m != nil {
+ return m.Permissions
+ }
+ return 0
+}
+
+func (m *ArtifactMetadata) GetMd5() string {
+ if m != nil {
+ return m.Md5
+ }
+ return ""
+}
+
+// A collection of artifacts.
+type Manifest struct {
+ Artifact []*ArtifactMetadata `protobuf:"bytes,1,rep,name=artifact" json:"artifact,omitempty"`
+}
+
+func (m *Manifest) Reset() { *m = Manifest{} }
+func (m *Manifest) String() string { return proto.CompactTextString(m) }
+func (*Manifest) ProtoMessage() {}
+func (*Manifest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{1} }
+
+func (m *Manifest) GetArtifact() []*ArtifactMetadata {
+ if m != nil {
+ return m.Artifact
+ }
+ return nil
+}
+
+// A manifest with location information.
+type ProxyManifest struct {
+ Manifest *Manifest `protobuf:"bytes,1,opt,name=manifest" json:"manifest,omitempty"`
+ Location []*ProxyManifest_Location `protobuf:"bytes,2,rep,name=location" json:"location,omitempty"`
+}
+
+func (m *ProxyManifest) Reset() { *m = ProxyManifest{} }
+func (m *ProxyManifest) String() string { return proto.CompactTextString(m) }
+func (*ProxyManifest) ProtoMessage() {}
+func (*ProxyManifest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{2} }
+
+func (m *ProxyManifest) GetManifest() *Manifest {
+ if m != nil {
+ return m.Manifest
+ }
+ return nil
+}
+
+func (m *ProxyManifest) GetLocation() []*ProxyManifest_Location {
+ if m != nil {
+ return m.Location
+ }
+ return nil
+}
+
+type ProxyManifest_Location struct {
+ Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+ Uri string `protobuf:"bytes,2,opt,name=uri" json:"uri,omitempty"`
+}
+
+func (m *ProxyManifest_Location) Reset() { *m = ProxyManifest_Location{} }
+func (m *ProxyManifest_Location) String() string { return proto.CompactTextString(m) }
+func (*ProxyManifest_Location) ProtoMessage() {}
+func (*ProxyManifest_Location) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{2, 0} }
+
+func (m *ProxyManifest_Location) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *ProxyManifest_Location) GetUri() string {
+ if m != nil {
+ return m.Uri
+ }
+ return ""
+}
+
+// A request to get the manifest of a Job.
+type GetManifestRequest struct {
+}
+
+func (m *GetManifestRequest) Reset() { *m = GetManifestRequest{} }
+func (m *GetManifestRequest) String() string { return proto.CompactTextString(m) }
+func (*GetManifestRequest) ProtoMessage() {}
+func (*GetManifestRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3} }
+
+// A response containing a job manifest.
+type GetManifestResponse struct {
+ Manifest *Manifest `protobuf:"bytes,1,opt,name=manifest" json:"manifest,omitempty"`
+}
+
+func (m *GetManifestResponse) Reset() { *m = GetManifestResponse{} }
+func (m *GetManifestResponse) String() string { return proto.CompactTextString(m) }
+func (*GetManifestResponse) ProtoMessage() {}
+func (*GetManifestResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{4} }
+
+func (m *GetManifestResponse) GetManifest() *Manifest {
+ if m != nil {
+ return m.Manifest
+ }
+ return nil
+}
+
+// A request to get an artifact. The artifact must be present in the manifest for the job.
+type GetArtifactRequest struct {
+ // (Required) The name of the artifact to retrieve.
+ Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+}
+
+func (m *GetArtifactRequest) Reset() { *m = GetArtifactRequest{} }
+func (m *GetArtifactRequest) String() string { return proto.CompactTextString(m) }
+func (*GetArtifactRequest) ProtoMessage() {}
+func (*GetArtifactRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{5} }
+
+func (m *GetArtifactRequest) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+// Part of an artifact.
+type ArtifactChunk struct {
+ Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
+}
+
+func (m *ArtifactChunk) Reset() { *m = ArtifactChunk{} }
+func (m *ArtifactChunk) String() string { return proto.CompactTextString(m) }
+func (*ArtifactChunk) ProtoMessage() {}
+func (*ArtifactChunk) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{6} }
+
+func (m *ArtifactChunk) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+// A request to stage an artifact.
+type PutArtifactRequest struct {
+ // (Required)
+ //
+ // Types that are valid to be assigned to Content:
+ // *PutArtifactRequest_Metadata
+ // *PutArtifactRequest_Data
+ Content isPutArtifactRequest_Content `protobuf_oneof:"content"`
+}
+
+func (m *PutArtifactRequest) Reset() { *m = PutArtifactRequest{} }
+func (m *PutArtifactRequest) String() string { return proto.CompactTextString(m) }
+func (*PutArtifactRequest) ProtoMessage() {}
+func (*PutArtifactRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{7} }
+
+type isPutArtifactRequest_Content interface {
+ isPutArtifactRequest_Content()
+}
+
+type PutArtifactRequest_Metadata struct {
+ Metadata *ArtifactMetadata `protobuf:"bytes,1,opt,name=metadata,oneof"`
+}
+type PutArtifactRequest_Data struct {
+ Data *ArtifactChunk `protobuf:"bytes,2,opt,name=data,oneof"`
+}
+
+func (*PutArtifactRequest_Metadata) isPutArtifactRequest_Content() {}
+func (*PutArtifactRequest_Data) isPutArtifactRequest_Content() {}
+
+func (m *PutArtifactRequest) GetContent() isPutArtifactRequest_Content {
+ if m != nil {
+ return m.Content
+ }
+ return nil
+}
+
+func (m *PutArtifactRequest) GetMetadata() *ArtifactMetadata {
+ if x, ok := m.GetContent().(*PutArtifactRequest_Metadata); ok {
+ return x.Metadata
+ }
+ return nil
+}
+
+func (m *PutArtifactRequest) GetData() *ArtifactChunk {
+ if x, ok := m.GetContent().(*PutArtifactRequest_Data); ok {
+ return x.Data
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*PutArtifactRequest) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _PutArtifactRequest_OneofMarshaler, _PutArtifactRequest_OneofUnmarshaler, _PutArtifactRequest_OneofSizer, []interface{}{
+ (*PutArtifactRequest_Metadata)(nil),
+ (*PutArtifactRequest_Data)(nil),
+ }
+}
+
+func _PutArtifactRequest_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*PutArtifactRequest)
+ // content
+ switch x := m.Content.(type) {
+ case *PutArtifactRequest_Metadata:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Metadata); err != nil {
+ return err
+ }
+ case *PutArtifactRequest_Data:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Data); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("PutArtifactRequest.Content has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _PutArtifactRequest_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*PutArtifactRequest)
+ switch tag {
+ case 1: // content.metadata
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ArtifactMetadata)
+ err := b.DecodeMessage(msg)
+ m.Content = &PutArtifactRequest_Metadata{msg}
+ return true, err
+ case 2: // content.data
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ArtifactChunk)
+ err := b.DecodeMessage(msg)
+ m.Content = &PutArtifactRequest_Data{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _PutArtifactRequest_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*PutArtifactRequest)
+ // content
+ switch x := m.Content.(type) {
+ case *PutArtifactRequest_Metadata:
+ s := proto.Size(x.Metadata)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *PutArtifactRequest_Data:
+ s := proto.Size(x.Data)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type PutArtifactResponse struct {
+}
+
+func (m *PutArtifactResponse) Reset() { *m = PutArtifactResponse{} }
+func (m *PutArtifactResponse) String() string { return proto.CompactTextString(m) }
+func (*PutArtifactResponse) ProtoMessage() {}
+func (*PutArtifactResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{8} }
+
+// A request to commit the manifest for a Job. All artifacts must have been successfully uploaded
+// before this call is made.
+type CommitManifestRequest struct {
+ // (Required) The manifest to commit.
+ Manifest *Manifest `protobuf:"bytes,1,opt,name=manifest" json:"manifest,omitempty"`
+}
+
+func (m *CommitManifestRequest) Reset() { *m = CommitManifestRequest{} }
+func (m *CommitManifestRequest) String() string { return proto.CompactTextString(m) }
+func (*CommitManifestRequest) ProtoMessage() {}
+func (*CommitManifestRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{9} }
+
+func (m *CommitManifestRequest) GetManifest() *Manifest {
+ if m != nil {
+ return m.Manifest
+ }
+ return nil
+}
+
+// The result of committing a manifest.
+type CommitManifestResponse struct {
+ // (Required) An opaque token representing the entirety of the staged artifacts.
+ StagingToken string `protobuf:"bytes,1,opt,name=staging_token,json=stagingToken" json:"staging_token,omitempty"`
+}
+
+func (m *CommitManifestResponse) Reset() { *m = CommitManifestResponse{} }
+func (m *CommitManifestResponse) String() string { return proto.CompactTextString(m) }
+func (*CommitManifestResponse) ProtoMessage() {}
+func (*CommitManifestResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{10} }
+
+func (m *CommitManifestResponse) GetStagingToken() string {
+ if m != nil {
+ return m.StagingToken
+ }
+ return ""
+}
+
+func init() {
+ proto.RegisterType((*ArtifactMetadata)(nil), "org.apache.beam.model.job_management.v1.ArtifactMetadata")
+ proto.RegisterType((*Manifest)(nil), "org.apache.beam.model.job_management.v1.Manifest")
+ proto.RegisterType((*ProxyManifest)(nil), "org.apache.beam.model.job_management.v1.ProxyManifest")
+ proto.RegisterType((*ProxyManifest_Location)(nil), "org.apache.beam.model.job_management.v1.ProxyManifest.Location")
+ proto.RegisterType((*GetManifestRequest)(nil), "org.apache.beam.model.job_management.v1.GetManifestRequest")
+ proto.RegisterType((*GetManifestResponse)(nil), "org.apache.beam.model.job_management.v1.GetManifestResponse")
+ proto.RegisterType((*GetArtifactRequest)(nil), "org.apache.beam.model.job_management.v1.GetArtifactRequest")
+ proto.RegisterType((*ArtifactChunk)(nil), "org.apache.beam.model.job_management.v1.ArtifactChunk")
+ proto.RegisterType((*PutArtifactRequest)(nil), "org.apache.beam.model.job_management.v1.PutArtifactRequest")
+ proto.RegisterType((*PutArtifactResponse)(nil), "org.apache.beam.model.job_management.v1.PutArtifactResponse")
+ proto.RegisterType((*CommitManifestRequest)(nil), "org.apache.beam.model.job_management.v1.CommitManifestRequest")
+ proto.RegisterType((*CommitManifestResponse)(nil), "org.apache.beam.model.job_management.v1.CommitManifestResponse")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for ArtifactStagingService service
+
+type ArtifactStagingServiceClient interface {
+ // Stage an artifact to be available during job execution. The first request must contain the
+ // name of the artifact. All future requests must contain sequential chunks of the content of
+ // the artifact.
+ PutArtifact(ctx context.Context, opts ...grpc.CallOption) (ArtifactStagingService_PutArtifactClient, error)
+ // Commit the manifest for a Job. All artifacts must have been successfully uploaded
+ // before this call is made.
+ //
+ // Throws error INVALID_ARGUMENT if not all of the members of the manifest are present
+ CommitManifest(ctx context.Context, in *CommitManifestRequest, opts ...grpc.CallOption) (*CommitManifestResponse, error)
+}
+
+type artifactStagingServiceClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewArtifactStagingServiceClient(cc *grpc.ClientConn) ArtifactStagingServiceClient {
+ return &artifactStagingServiceClient{cc}
+}
+
+func (c *artifactStagingServiceClient) PutArtifact(ctx context.Context, opts ...grpc.CallOption) (ArtifactStagingService_PutArtifactClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_ArtifactStagingService_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.job_management.v1.ArtifactStagingService/PutArtifact", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &artifactStagingServicePutArtifactClient{stream}
+ return x, nil
+}
+
+type ArtifactStagingService_PutArtifactClient interface {
+ Send(*PutArtifactRequest) error
+ CloseAndRecv() (*PutArtifactResponse, error)
+ grpc.ClientStream
+}
+
+type artifactStagingServicePutArtifactClient struct {
+ grpc.ClientStream
+}
+
+func (x *artifactStagingServicePutArtifactClient) Send(m *PutArtifactRequest) error {
+ return x.ClientStream.SendMsg(m)
+}
+
+func (x *artifactStagingServicePutArtifactClient) CloseAndRecv() (*PutArtifactResponse, error) {
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ m := new(PutArtifactResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func (c *artifactStagingServiceClient) CommitManifest(ctx context.Context, in *CommitManifestRequest, opts ...grpc.CallOption) (*CommitManifestResponse, error) {
+ out := new(CommitManifestResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.ArtifactStagingService/CommitManifest", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// Server API for ArtifactStagingService service
+
+type ArtifactStagingServiceServer interface {
+ // Stage an artifact to be available during job execution. The first request must contain the
+ // name of the artifact. All future requests must contain sequential chunks of the content of
+ // the artifact.
+ PutArtifact(ArtifactStagingService_PutArtifactServer) error
+ // Commit the manifest for a Job. All artifacts must have been successfully uploaded
+ // before this call is made.
+ //
+ // Throws error INVALID_ARGUMENT if not all of the members of the manifest are present
+ CommitManifest(context.Context, *CommitManifestRequest) (*CommitManifestResponse, error)
+}
+
+func RegisterArtifactStagingServiceServer(s *grpc.Server, srv ArtifactStagingServiceServer) {
+ s.RegisterService(&_ArtifactStagingService_serviceDesc, srv)
+}
+
+func _ArtifactStagingService_PutArtifact_Handler(srv interface{}, stream grpc.ServerStream) error {
+ return srv.(ArtifactStagingServiceServer).PutArtifact(&artifactStagingServicePutArtifactServer{stream})
+}
+
+type ArtifactStagingService_PutArtifactServer interface {
+ SendAndClose(*PutArtifactResponse) error
+ Recv() (*PutArtifactRequest, error)
+ grpc.ServerStream
+}
+
+type artifactStagingServicePutArtifactServer struct {
+ grpc.ServerStream
+}
+
+func (x *artifactStagingServicePutArtifactServer) SendAndClose(m *PutArtifactResponse) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func (x *artifactStagingServicePutArtifactServer) Recv() (*PutArtifactRequest, error) {
+ m := new(PutArtifactRequest)
+ if err := x.ServerStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func _ArtifactStagingService_CommitManifest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(CommitManifestRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(ArtifactStagingServiceServer).CommitManifest(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.ArtifactStagingService/CommitManifest",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(ArtifactStagingServiceServer).CommitManifest(ctx, req.(*CommitManifestRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+var _ArtifactStagingService_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.job_management.v1.ArtifactStagingService",
+ HandlerType: (*ArtifactStagingServiceServer)(nil),
+ Methods: []grpc.MethodDesc{
+ {
+ MethodName: "CommitManifest",
+ Handler: _ArtifactStagingService_CommitManifest_Handler,
+ },
+ },
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "PutArtifact",
+ Handler: _ArtifactStagingService_PutArtifact_Handler,
+ ClientStreams: true,
+ },
+ },
+ Metadata: "beam_artifact_api.proto",
+}
+
+// Client API for ArtifactRetrievalService service
+
+type ArtifactRetrievalServiceClient interface {
+ // Get the manifest for the job
+ GetManifest(ctx context.Context, in *GetManifestRequest, opts ...grpc.CallOption) (*GetManifestResponse, error)
+ // Get an artifact staged for the job. The requested artifact must be within the manifest
+ GetArtifact(ctx context.Context, in *GetArtifactRequest, opts ...grpc.CallOption) (ArtifactRetrievalService_GetArtifactClient, error)
+}
+
+type artifactRetrievalServiceClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewArtifactRetrievalServiceClient(cc *grpc.ClientConn) ArtifactRetrievalServiceClient {
+ return &artifactRetrievalServiceClient{cc}
+}
+
+func (c *artifactRetrievalServiceClient) GetManifest(ctx context.Context, in *GetManifestRequest, opts ...grpc.CallOption) (*GetManifestResponse, error) {
+ out := new(GetManifestResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.ArtifactRetrievalService/GetManifest", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *artifactRetrievalServiceClient) GetArtifact(ctx context.Context, in *GetArtifactRequest, opts ...grpc.CallOption) (ArtifactRetrievalService_GetArtifactClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_ArtifactRetrievalService_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.job_management.v1.ArtifactRetrievalService/GetArtifact", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &artifactRetrievalServiceGetArtifactClient{stream}
+ if err := x.ClientStream.SendMsg(in); err != nil {
+ return nil, err
+ }
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ return x, nil
+}
+
+type ArtifactRetrievalService_GetArtifactClient interface {
+ Recv() (*ArtifactChunk, error)
+ grpc.ClientStream
+}
+
+type artifactRetrievalServiceGetArtifactClient struct {
+ grpc.ClientStream
+}
+
+func (x *artifactRetrievalServiceGetArtifactClient) Recv() (*ArtifactChunk, error) {
+ m := new(ArtifactChunk)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for ArtifactRetrievalService service
+
+type ArtifactRetrievalServiceServer interface {
+ // Get the manifest for the job
+ GetManifest(context.Context, *GetManifestRequest) (*GetManifestResponse, error)
+ // Get an artifact staged for the job. The requested artifact must be within the manifest
+ GetArtifact(*GetArtifactRequest, ArtifactRetrievalService_GetArtifactServer) error
+}
+
+func RegisterArtifactRetrievalServiceServer(s *grpc.Server, srv ArtifactRetrievalServiceServer) {
+ s.RegisterService(&_ArtifactRetrievalService_serviceDesc, srv)
+}
+
+func _ArtifactRetrievalService_GetManifest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(GetManifestRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(ArtifactRetrievalServiceServer).GetManifest(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.ArtifactRetrievalService/GetManifest",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(ArtifactRetrievalServiceServer).GetManifest(ctx, req.(*GetManifestRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _ArtifactRetrievalService_GetArtifact_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(GetArtifactRequest)
+ if err := stream.RecvMsg(m); err != nil {
+ return err
+ }
+ return srv.(ArtifactRetrievalServiceServer).GetArtifact(m, &artifactRetrievalServiceGetArtifactServer{stream})
+}
+
+type ArtifactRetrievalService_GetArtifactServer interface {
+ Send(*ArtifactChunk) error
+ grpc.ServerStream
+}
+
+type artifactRetrievalServiceGetArtifactServer struct {
+ grpc.ServerStream
+}
+
+func (x *artifactRetrievalServiceGetArtifactServer) Send(m *ArtifactChunk) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+var _ArtifactRetrievalService_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.job_management.v1.ArtifactRetrievalService",
+ HandlerType: (*ArtifactRetrievalServiceServer)(nil),
+ Methods: []grpc.MethodDesc{
+ {
+ MethodName: "GetManifest",
+ Handler: _ArtifactRetrievalService_GetManifest_Handler,
+ },
+ },
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "GetArtifact",
+ Handler: _ArtifactRetrievalService_GetArtifact_Handler,
+ ServerStreams: true,
+ },
+ },
+ Metadata: "beam_artifact_api.proto",
+}
+
+func init() { proto.RegisterFile("beam_artifact_api.proto", fileDescriptor1) }
+
+var fileDescriptor1 = []byte{
+ // 557 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcf, 0x6e, 0xd3, 0x4e,
+ 0x10, 0xee, 0x26, 0x3f, 0xfd, 0x70, 0xc6, 0x0d, 0x8a, 0xb6, 0xb4, 0x58, 0x39, 0x45, 0x5b, 0x09,
+ 0x72, 0xb2, 0x9a, 0x20, 0x90, 0x10, 0x7f, 0xaa, 0xa6, 0x87, 0xf6, 0xd0, 0x48, 0x95, 0x0b, 0x42,
+ 0x2a, 0x87, 0x68, 0x93, 0x6c, 0xd2, 0xa5, 0xd9, 0x5d, 0x63, 0x6f, 0x22, 0xb8, 0x73, 0x40, 0xdc,
+ 0x78, 0x0f, 0x5e, 0x80, 0x17, 0xe0, 0x6d, 0x78, 0x07, 0xe4, 0xf5, 0xda, 0xc4, 0x49, 0x90, 0x92,
+ 0xa8, 0xb7, 0xd1, 0x78, 0xe6, 0x9b, 0x6f, 0xbe, 0x6f, 0x27, 0x81, 0x87, 0x7d, 0x46, 0x45, 0x8f,
+ 0x46, 0x9a, 0x8f, 0xe8, 0x40, 0xf7, 0x68, 0xc8, 0xfd, 0x30, 0x52, 0x5a, 0xe1, 0xc7, 0x2a, 0x1a,
+ 0xfb, 0x34, 0xa4, 0x83, 0x1b, 0xe6, 0x27, 0x35, 0xbe, 0x50, 0x43, 0x36, 0xf1, 0x3f, 0xa8, 0x7e,
+ 0x4f, 0x50, 0x49, 0xc7, 0x4c, 0x30, 0xa9, 0xfd, 0x59, 0x8b, 0x5c, 0x43, 0xed, 0xc4, 0xb6, 0x77,
+ 0x99, 0xa6, 0x43, 0xaa, 0x29, 0xc6, 0xf0, 0x9f, 0xa4, 0x82, 0x79, 0xa8, 0x81, 0x9a, 0x95, 0xc0,
+ 0xc4, 0xb8, 0x01, 0x6e, 0xc8, 0x22, 0xc1, 0xe3, 0x98, 0x2b, 0x19, 0x7b, 0xa5, 0x06, 0x6a, 0x56,
+ 0x83, 0xf9, 0x14, 0xae, 0x41, 0x59, 0x0c, 0x9f, 0x7a, 0x65, 0xd3, 0x94, 0x84, 0x84, 0x82, 0xd3,
+ 0xa5, 0x92, 0x8f, 0x58, 0xac, 0xf1, 0x5b, 0x70, 0x32, 0x9a, 0x1e, 0x6a, 0x94, 0x9b, 0x6e, 0xfb,
+ 0xb9, 0xbf, 0x26, 0x47, 0x7f, 0x91, 0x60, 0x90, 0x43, 0x91, 0xdf, 0x08, 0xaa, 0x97, 0x91, 0xfa,
+ 0xf4, 0x39, 0x1f, 0xd4, 0x05, 0x47, 0xd8, 0xd8, 0x2c, 0xe0, 0xb6, 0x5b, 0x6b, 0x0f, 0xca, 0x40,
+ 0x82, 0x1c, 0x02, 0xbf, 0x07, 0x67, 0xa2, 0x06, 0x54, 0x73, 0x25, 0xbd, 0x92, 0xe1, 0x7d, 0xbc,
+ 0x36, 0x5c, 0x81, 0x98, 0x7f, 0x61, 0x61, 0x82, 0x1c, 0xb0, 0x7e, 0x04, 0x4e, 0x96, 0x5d, 0x29,
+ 0x7a, 0x0d, 0xca, 0xd3, 0x88, 0x1b, 0xb1, 0x2b, 0x41, 0x12, 0x92, 0x07, 0x80, 0xcf, 0x98, 0xce,
+ 0x79, 0xb2, 0x8f, 0x53, 0x16, 0x6b, 0x32, 0x84, 0xbd, 0x42, 0x36, 0x0e, 0x95, 0x8c, 0xd9, 0x1d,
+ 0x4b, 0x41, 0x9a, 0x66, 0x76, 0x66, 0x86, 0x9d, 0xbd, 0x8a, 0x37, 0x39, 0x84, 0x6a, 0x56, 0x76,
+ 0x7a, 0x33, 0x95, 0xb7, 0x49, 0x51, 0x62, 0x9c, 0x29, 0xda, 0x0d, 0x4c, 0x4c, 0x7e, 0x21, 0xc0,
+ 0x97, 0xd3, 0x25, 0xbc, 0x77, 0xe0, 0x08, 0xeb, 0xb3, 0x25, 0xbd, 0xfd, 0x43, 0x39, 0xdf, 0x09,
+ 0x72, 0x30, 0x7c, 0x61, 0x39, 0x94, 0x0c, 0xe8, 0xb3, 0x8d, 0x41, 0xcd, 0x26, 0xe7, 0x3b, 0x29,
+ 0xfb, 0x4e, 0x05, 0xee, 0x0d, 0x94, 0xd4, 0x4c, 0x6a, 0xb2, 0x0f, 0x7b, 0x85, 0x3d, 0x52, 0xf5,
+ 0xc9, 0x08, 0xf6, 0x4f, 0x95, 0x10, 0x7c, 0xd1, 0xad, 0xbb, 0xb6, 0xe5, 0x15, 0x1c, 0x2c, 0xce,
+ 0xb1, 0xfe, 0x1f, 0x42, 0x35, 0xd6, 0x74, 0xcc, 0xe5, 0xb8, 0xa7, 0xd5, 0x2d, 0x93, 0xd6, 0xa3,
+ 0x5d, 0x9b, 0x7c, 0x93, 0xe4, 0xda, 0x3f, 0x4b, 0x70, 0x90, 0x71, 0xbf, 0x4a, 0x3f, 0x5c, 0xb1,
+ 0x68, 0xc6, 0x07, 0x0c, 0x7f, 0x43, 0xe0, 0xce, 0x6d, 0x86, 0x5f, 0xac, 0xff, 0xf2, 0x97, 0x7c,
+ 0xad, 0xbf, 0xdc, 0xae, 0x39, 0x5d, 0xa5, 0x89, 0xf0, 0x77, 0x04, 0xf7, 0x8b, 0x7b, 0xe2, 0xd7,
+ 0x6b, 0x43, 0xae, 0x34, 0xa2, 0x7e, 0xbc, 0x75, 0x7f, 0xca, 0xaa, 0xfd, 0xa3, 0x04, 0xde, 0x5f,
+ 0xaa, 0x3a, 0xe2, 0x6c, 0x46, 0x27, 0x99, 0x7a, 0x5f, 0x11, 0xb8, 0x73, 0x57, 0xb9, 0x81, 0x7a,
+ 0xcb, 0x17, 0xbe, 0x81, 0x7a, 0xab, 0x7e, 0x08, 0xbe, 0xa4, 0x54, 0xb6, 0x30, 0x72, 0xf9, 0xe0,
+ 0xeb, 0x5b, 0x5e, 0xce, 0x11, 0xea, 0x9c, 0xc1, 0xa3, 0x7f, 0xb6, 0x16, 0x3a, 0x3b, 0x6e, 0xd6,
+ 0x7a, 0x12, 0xf2, 0xeb, 0x5a, 0xe1, 0x73, 0x6f, 0xd6, 0xea, 0xff, 0x6f, 0xfe, 0xe4, 0x9e, 0xfc,
+ 0x09, 0x00, 0x00, 0xff, 0xff, 0x80, 0xfd, 0xd5, 0x65, 0xff, 0x06, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
new file mode 100644
index 0000000..575dbd9
--- /dev/null
+++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
@@ -0,0 +1,903 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: beam_job_api.proto
+
+/*
+Package jobmanagement_v1 is a generated protocol buffer package.
+
+It is generated from these files:
+ beam_job_api.proto
+ beam_artifact_api.proto
+
+It has these top-level messages:
+ PrepareJobRequest
+ PrepareJobResponse
+ RunJobRequest
+ RunJobResponse
+ CancelJobRequest
+ CancelJobResponse
+ GetJobStateRequest
+ GetJobStateResponse
+ JobMessagesRequest
+ JobMessage
+ JobMessagesResponse
+ JobState
+ ArtifactMetadata
+ Manifest
+ ProxyManifest
+ GetManifestRequest
+ GetManifestResponse
+ GetArtifactRequest
+ ArtifactChunk
+ PutArtifactRequest
+ PutArtifactResponse
+ CommitManifestRequest
+ CommitManifestResponse
+*/
+package jobmanagement_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import org_apache_beam_model_pipeline_v1 "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+import org_apache_beam_model_pipeline_v11 "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+import google_protobuf1 "github.com/golang/protobuf/ptypes/struct"
+
+import (
+ context "golang.org/x/net/context"
+ grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type JobMessage_MessageImportance int32
+
+const (
+ JobMessage_MESSAGE_IMPORTANCE_UNSPECIFIED JobMessage_MessageImportance = 0
+ JobMessage_JOB_MESSAGE_DEBUG JobMessage_MessageImportance = 1
+ JobMessage_JOB_MESSAGE_DETAILED JobMessage_MessageImportance = 2
+ JobMessage_JOB_MESSAGE_BASIC JobMessage_MessageImportance = 3
+ JobMessage_JOB_MESSAGE_WARNING JobMessage_MessageImportance = 4
+ JobMessage_JOB_MESSAGE_ERROR JobMessage_MessageImportance = 5
+)
+
+var JobMessage_MessageImportance_name = map[int32]string{
+ 0: "MESSAGE_IMPORTANCE_UNSPECIFIED",
+ 1: "JOB_MESSAGE_DEBUG",
+ 2: "JOB_MESSAGE_DETAILED",
+ 3: "JOB_MESSAGE_BASIC",
+ 4: "JOB_MESSAGE_WARNING",
+ 5: "JOB_MESSAGE_ERROR",
+}
+var JobMessage_MessageImportance_value = map[string]int32{
+ "MESSAGE_IMPORTANCE_UNSPECIFIED": 0,
+ "JOB_MESSAGE_DEBUG": 1,
+ "JOB_MESSAGE_DETAILED": 2,
+ "JOB_MESSAGE_BASIC": 3,
+ "JOB_MESSAGE_WARNING": 4,
+ "JOB_MESSAGE_ERROR": 5,
+}
+
+func (x JobMessage_MessageImportance) String() string {
+ return proto.EnumName(JobMessage_MessageImportance_name, int32(x))
+}
+func (JobMessage_MessageImportance) EnumDescriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{9, 0}
+}
+
+type JobState_Enum int32
+
+const (
+ JobState_UNSPECIFIED JobState_Enum = 0
+ JobState_STOPPED JobState_Enum = 1
+ JobState_RUNNING JobState_Enum = 2
+ JobState_DONE JobState_Enum = 3
+ JobState_FAILED JobState_Enum = 4
+ JobState_CANCELLED JobState_Enum = 5
+ JobState_UPDATED JobState_Enum = 6
+ JobState_DRAINING JobState_Enum = 7
+ JobState_DRAINED JobState_Enum = 8
+ JobState_STARTING JobState_Enum = 9
+ JobState_CANCELLING JobState_Enum = 10
+)
+
+var JobState_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "STOPPED",
+ 2: "RUNNING",
+ 3: "DONE",
+ 4: "FAILED",
+ 5: "CANCELLED",
+ 6: "UPDATED",
+ 7: "DRAINING",
+ 8: "DRAINED",
+ 9: "STARTING",
+ 10: "CANCELLING",
+}
+var JobState_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "STOPPED": 1,
+ "RUNNING": 2,
+ "DONE": 3,
+ "FAILED": 4,
+ "CANCELLED": 5,
+ "UPDATED": 6,
+ "DRAINING": 7,
+ "DRAINED": 8,
+ "STARTING": 9,
+ "CANCELLING": 10,
+}
+
+func (x JobState_Enum) String() string {
+ return proto.EnumName(JobState_Enum_name, int32(x))
+}
+func (JobState_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{11, 0} }
+
+// Prepare is a synchronous request that returns a preparationId back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error ALREADY_EXISTS if the jobName is reused. Runners are permitted to deduplicate based on the name of the job.
+// Throws error UNKNOWN for all other issues
+type PrepareJobRequest struct {
+ Pipeline *org_apache_beam_model_pipeline_v1.Pipeline `protobuf:"bytes,1,opt,name=pipeline" json:"pipeline,omitempty"`
+ PipelineOptions *google_protobuf1.Struct `protobuf:"bytes,2,opt,name=pipeline_options,json=pipelineOptions" json:"pipeline_options,omitempty"`
+ JobName string `protobuf:"bytes,3,opt,name=job_name,json=jobName" json:"job_name,omitempty"`
+}
+
+func (m *PrepareJobRequest) Reset() { *m = PrepareJobRequest{} }
+func (m *PrepareJobRequest) String() string { return proto.CompactTextString(m) }
+func (*PrepareJobRequest) ProtoMessage() {}
+func (*PrepareJobRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *PrepareJobRequest) GetPipeline() *org_apache_beam_model_pipeline_v1.Pipeline {
+ if m != nil {
+ return m.Pipeline
+ }
+ return nil
+}
+
+func (m *PrepareJobRequest) GetPipelineOptions() *google_protobuf1.Struct {
+ if m != nil {
+ return m.PipelineOptions
+ }
+ return nil
+}
+
+func (m *PrepareJobRequest) GetJobName() string {
+ if m != nil {
+ return m.JobName
+ }
+ return ""
+}
+
+type PrepareJobResponse struct {
+ // (required) The ID used to associate calls made while preparing the job. preparationId is used
+ // to run the job, as well as in other pre-execution APIs such as Artifact staging.
+ PreparationId string `protobuf:"bytes,1,opt,name=preparation_id,json=preparationId" json:"preparation_id,omitempty"`
+ // An endpoint which exposes the Beam Artifact Staging API. Artifacts used by the job should be
+ // staged to this endpoint, and will be available during job execution.
+ ArtifactStagingEndpoint *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor `protobuf:"bytes,2,opt,name=artifact_staging_endpoint,json=artifactStagingEndpoint" json:"artifact_staging_endpoint,omitempty"`
+}
+
+func (m *PrepareJobResponse) Reset() { *m = PrepareJobResponse{} }
+func (m *PrepareJobResponse) String() string { return proto.CompactTextString(m) }
+func (*PrepareJobResponse) ProtoMessage() {}
+func (*PrepareJobResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *PrepareJobResponse) GetPreparationId() string {
+ if m != nil {
+ return m.PreparationId
+ }
+ return ""
+}
+
+func (m *PrepareJobResponse) GetArtifactStagingEndpoint() *org_apache_beam_model_pipeline_v11.ApiServiceDescriptor {
+ if m != nil {
+ return m.ArtifactStagingEndpoint
+ }
+ return nil
+}
+
+// Run is a synchronous request that returns a jobId back.
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the preparation ID does not exist
+// Throws error UNKNOWN for all other issues
+type RunJobRequest struct {
+ // (required) The ID provided by an earlier call to prepare. Runs the job. All prerequisite tasks
+ // must have been completed.
+ PreparationId string `protobuf:"bytes,1,opt,name=preparation_id,json=preparationId" json:"preparation_id,omitempty"`
+ // (optional) If any artifacts have been staged for this job, contains the staging_token returned
+ // from the CommitManifestResponse.
+ StagingToken string `protobuf:"bytes,2,opt,name=staging_token,json=stagingToken" json:"staging_token,omitempty"`
+}
+
+func (m *RunJobRequest) Reset() { *m = RunJobRequest{} }
+func (m *RunJobRequest) String() string { return proto.CompactTextString(m) }
+func (*RunJobRequest) ProtoMessage() {}
+func (*RunJobRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *RunJobRequest) GetPreparationId() string {
+ if m != nil {
+ return m.PreparationId
+ }
+ return ""
+}
+
+func (m *RunJobRequest) GetStagingToken() string {
+ if m != nil {
+ return m.StagingToken
+ }
+ return ""
+}
+
+type RunJobResponse struct {
+ JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId" json:"job_id,omitempty"`
+}
+
+func (m *RunJobResponse) Reset() { *m = RunJobResponse{} }
+func (m *RunJobResponse) String() string { return proto.CompactTextString(m) }
+func (*RunJobResponse) ProtoMessage() {}
+func (*RunJobResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *RunJobResponse) GetJobId() string {
+ if m != nil {
+ return m.JobId
+ }
+ return ""
+}
+
+// Cancel is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+type CancelJobRequest struct {
+ JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId" json:"job_id,omitempty"`
+}
+
+func (m *CancelJobRequest) Reset() { *m = CancelJobRequest{} }
+func (m *CancelJobRequest) String() string { return proto.CompactTextString(m) }
+func (*CancelJobRequest) ProtoMessage() {}
+func (*CancelJobRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *CancelJobRequest) GetJobId() string {
+ if m != nil {
+ return m.JobId
+ }
+ return ""
+}
+
+// Valid responses include any terminal state or CANCELLING
+type CancelJobResponse struct {
+ State JobState_Enum `protobuf:"varint,1,opt,name=state,enum=org.apache.beam.model.job_management.v1.JobState_Enum" json:"state,omitempty"`
+}
+
+func (m *CancelJobResponse) Reset() { *m = CancelJobResponse{} }
+func (m *CancelJobResponse) String() string { return proto.CompactTextString(m) }
+func (*CancelJobResponse) ProtoMessage() {}
+func (*CancelJobResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *CancelJobResponse) GetState() JobState_Enum {
+ if m != nil {
+ return m.State
+ }
+ return JobState_UNSPECIFIED
+}
+
+// GetState is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+type GetJobStateRequest struct {
+ JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId" json:"job_id,omitempty"`
+}
+
+func (m *GetJobStateRequest) Reset() { *m = GetJobStateRequest{} }
+func (m *GetJobStateRequest) String() string { return proto.CompactTextString(m) }
+func (*GetJobStateRequest) ProtoMessage() {}
+func (*GetJobStateRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *GetJobStateRequest) GetJobId() string {
+ if m != nil {
+ return m.JobId
+ }
+ return ""
+}
+
+type GetJobStateResponse struct {
+ State JobState_Enum `protobuf:"varint,1,opt,name=state,enum=org.apache.beam.model.job_management.v1.JobState_Enum" json:"state,omitempty"`
+}
+
+func (m *GetJobStateResponse) Reset() { *m = GetJobStateResponse{} }
+func (m *GetJobStateResponse) String() string { return proto.CompactTextString(m) }
+func (*GetJobStateResponse) ProtoMessage() {}
+func (*GetJobStateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *GetJobStateResponse) GetState() JobState_Enum {
+ if m != nil {
+ return m.State
+ }
+ return JobState_UNSPECIFIED
+}
+
+// GetJobMessages is a streaming api for streaming job messages from the service
+// One request will connect you to the job and you'll get a stream of job state
+// and job messages back; one is used for logging and the other for detecting
+// the job ended.
+type JobMessagesRequest struct {
+ JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId" json:"job_id,omitempty"`
+}
+
+func (m *JobMessagesRequest) Reset() { *m = JobMessagesRequest{} }
+func (m *JobMessagesRequest) String() string { return proto.CompactTextString(m) }
+func (*JobMessagesRequest) ProtoMessage() {}
+func (*JobMessagesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *JobMessagesRequest) GetJobId() string {
+ if m != nil {
+ return m.JobId
+ }
+ return ""
+}
+
+type JobMessage struct {
+ MessageId string `protobuf:"bytes,1,opt,name=message_id,json=messageId" json:"message_id,omitempty"`
+ Time string `protobuf:"bytes,2,opt,name=time" json:"time,omitempty"`
+ Importance JobMessage_MessageImportance `protobuf:"varint,3,opt,name=importance,enum=org.apache.beam.model.job_management.v1.JobMessage_MessageImportance" json:"importance,omitempty"`
+ MessageText string `protobuf:"bytes,4,opt,name=message_text,json=messageText" json:"message_text,omitempty"`
+}
+
+func (m *JobMessage) Reset() { *m = JobMessage{} }
+func (m *JobMessage) String() string { return proto.CompactTextString(m) }
+func (*JobMessage) ProtoMessage() {}
+func (*JobMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *JobMessage) GetMessageId() string {
+ if m != nil {
+ return m.MessageId
+ }
+ return ""
+}
+
+func (m *JobMessage) GetTime() string {
+ if m != nil {
+ return m.Time
+ }
+ return ""
+}
+
+func (m *JobMessage) GetImportance() JobMessage_MessageImportance {
+ if m != nil {
+ return m.Importance
+ }
+ return JobMessage_MESSAGE_IMPORTANCE_UNSPECIFIED
+}
+
+func (m *JobMessage) GetMessageText() string {
+ if m != nil {
+ return m.MessageText
+ }
+ return ""
+}
+
+type JobMessagesResponse struct {
+ // Types that are valid to be assigned to Response:
+ // *JobMessagesResponse_MessageResponse
+ // *JobMessagesResponse_StateResponse
+ Response isJobMessagesResponse_Response `protobuf_oneof:"response"`
+}
+
+func (m *JobMessagesResponse) Reset() { *m = JobMessagesResponse{} }
+func (m *JobMessagesResponse) String() string { return proto.CompactTextString(m) }
+func (*JobMessagesResponse) ProtoMessage() {}
+func (*JobMessagesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+type isJobMessagesResponse_Response interface {
+ isJobMessagesResponse_Response()
+}
+
+type JobMessagesResponse_MessageResponse struct {
+ MessageResponse *JobMessage `protobuf:"bytes,1,opt,name=message_response,json=messageResponse,oneof"`
+}
+type JobMessagesResponse_StateResponse struct {
+ StateResponse *GetJobStateResponse `protobuf:"bytes,2,opt,name=state_response,json=stateResponse,oneof"`
+}
+
+func (*JobMessagesResponse_MessageResponse) isJobMessagesResponse_Response() {}
+func (*JobMessagesResponse_StateResponse) isJobMessagesResponse_Response() {}
+
+func (m *JobMessagesResponse) GetResponse() isJobMessagesResponse_Response {
+ if m != nil {
+ return m.Response
+ }
+ return nil
+}
+
+func (m *JobMessagesResponse) GetMessageResponse() *JobMessage {
+ if x, ok := m.GetResponse().(*JobMessagesResponse_MessageResponse); ok {
+ return x.MessageResponse
+ }
+ return nil
+}
+
+func (m *JobMessagesResponse) GetStateResponse() *GetJobStateResponse {
+ if x, ok := m.GetResponse().(*JobMessagesResponse_StateResponse); ok {
+ return x.StateResponse
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*JobMessagesResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _JobMessagesResponse_OneofMarshaler, _JobMessagesResponse_OneofUnmarshaler, _JobMessagesResponse_OneofSizer, []interface{}{
+ (*JobMessagesResponse_MessageResponse)(nil),
+ (*JobMessagesResponse_StateResponse)(nil),
+ }
+}
+
+func _JobMessagesResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*JobMessagesResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *JobMessagesResponse_MessageResponse:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.MessageResponse); err != nil {
+ return err
+ }
+ case *JobMessagesResponse_StateResponse:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.StateResponse); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("JobMessagesResponse.Response has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _JobMessagesResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*JobMessagesResponse)
+ switch tag {
+ case 1: // response.message_response
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(JobMessage)
+ err := b.DecodeMessage(msg)
+ m.Response = &JobMessagesResponse_MessageResponse{msg}
+ return true, err
+ case 2: // response.state_response
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(GetJobStateResponse)
+ err := b.DecodeMessage(msg)
+ m.Response = &JobMessagesResponse_StateResponse{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _JobMessagesResponse_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*JobMessagesResponse)
+ // response
+ switch x := m.Response.(type) {
+ case *JobMessagesResponse_MessageResponse:
+ s := proto.Size(x.MessageResponse)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *JobMessagesResponse_StateResponse:
+ s := proto.Size(x.StateResponse)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+// Enumeration of all JobStates
+type JobState struct {
+}
+
+func (m *JobState) Reset() { *m = JobState{} }
+func (m *JobState) String() string { return proto.CompactTextString(m) }
+func (*JobState) ProtoMessage() {}
+func (*JobState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+
+func init() {
+ proto.RegisterType((*PrepareJobRequest)(nil), "org.apache.beam.model.job_management.v1.PrepareJobRequest")
+ proto.RegisterType((*PrepareJobResponse)(nil), "org.apache.beam.model.job_management.v1.PrepareJobResponse")
+ proto.RegisterType((*RunJobRequest)(nil), "org.apache.beam.model.job_management.v1.RunJobRequest")
+ proto.RegisterType((*RunJobResponse)(nil), "org.apache.beam.model.job_management.v1.RunJobResponse")
+ proto.RegisterType((*CancelJobRequest)(nil), "org.apache.beam.model.job_management.v1.CancelJobRequest")
+ proto.RegisterType((*CancelJobResponse)(nil), "org.apache.beam.model.job_management.v1.CancelJobResponse")
+ proto.RegisterType((*GetJobStateRequest)(nil), "org.apache.beam.model.job_management.v1.GetJobStateRequest")
+ proto.RegisterType((*GetJobStateResponse)(nil), "org.apache.beam.model.job_management.v1.GetJobStateResponse")
+ proto.RegisterType((*JobMessagesRequest)(nil), "org.apache.beam.model.job_management.v1.JobMessagesRequest")
+ proto.RegisterType((*JobMessage)(nil), "org.apache.beam.model.job_management.v1.JobMessage")
+ proto.RegisterType((*JobMessagesResponse)(nil), "org.apache.beam.model.job_management.v1.JobMessagesResponse")
+ proto.RegisterType((*JobState)(nil), "org.apache.beam.model.job_management.v1.JobState")
+ proto.RegisterEnum("org.apache.beam.model.job_management.v1.JobMessage_MessageImportance", JobMessage_MessageImportance_name, JobMessage_MessageImportance_value)
+ proto.RegisterEnum("org.apache.beam.model.job_management.v1.JobState_Enum", JobState_Enum_name, JobState_Enum_value)
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for JobService service
+
+type JobServiceClient interface {
+ // Prepare a job for execution. The job will not be executed until a call is made to run with the
+ // returned preparationId.
+ Prepare(ctx context.Context, in *PrepareJobRequest, opts ...grpc.CallOption) (*PrepareJobResponse, error)
+ // Submit the job for execution
+ Run(ctx context.Context, in *RunJobRequest, opts ...grpc.CallOption) (*RunJobResponse, error)
+ // Get the current state of the job
+ GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*GetJobStateResponse, error)
+ // Cancel the job
+ Cancel(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error)
+ // Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
+ GetStateStream(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (JobService_GetStateStreamClient, error)
+ // Subscribe to a stream of state changes and messages from the job
+ GetMessageStream(ctx context.Context, in *JobMessagesRequest, opts ...grpc.CallOption) (JobService_GetMessageStreamClient, error)
+}
+
+type jobServiceClient struct {
+ cc *grpc.ClientConn
+}
+
+func NewJobServiceClient(cc *grpc.ClientConn) JobServiceClient {
+ return &jobServiceClient{cc}
+}
+
+func (c *jobServiceClient) Prepare(ctx context.Context, in *PrepareJobRequest, opts ...grpc.CallOption) (*PrepareJobResponse, error) {
+ out := new(PrepareJobResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.JobService/Prepare", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *jobServiceClient) Run(ctx context.Context, in *RunJobRequest, opts ...grpc.CallOption) (*RunJobResponse, error) {
+ out := new(RunJobResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.JobService/Run", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *jobServiceClient) GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*GetJobStateResponse, error) {
+ out := new(GetJobStateResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.JobService/GetState", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *jobServiceClient) Cancel(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error) {
+ out := new(CancelJobResponse)
+ err := grpc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.JobService/Cancel", in, out, c.cc, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *jobServiceClient) GetStateStream(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (JobService_GetStateStreamClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_JobService_serviceDesc.Streams[0], c.cc, "/org.apache.beam.model.job_management.v1.JobService/GetStateStream", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &jobServiceGetStateStreamClient{stream}
+ if err := x.ClientStream.SendMsg(in); err != nil {
+ return nil, err
+ }
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ return x, nil
+}
+
+type JobService_GetStateStreamClient interface {
+ Recv() (*GetJobStateResponse, error)
+ grpc.ClientStream
+}
+
+type jobServiceGetStateStreamClient struct {
+ grpc.ClientStream
+}
+
+func (x *jobServiceGetStateStreamClient) Recv() (*GetJobStateResponse, error) {
+ m := new(GetJobStateResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func (c *jobServiceClient) GetMessageStream(ctx context.Context, in *JobMessagesRequest, opts ...grpc.CallOption) (JobService_GetMessageStreamClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_JobService_serviceDesc.Streams[1], c.cc, "/org.apache.beam.model.job_management.v1.JobService/GetMessageStream", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &jobServiceGetMessageStreamClient{stream}
+ if err := x.ClientStream.SendMsg(in); err != nil {
+ return nil, err
+ }
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ return x, nil
+}
+
+type JobService_GetMessageStreamClient interface {
+ Recv() (*JobMessagesResponse, error)
+ grpc.ClientStream
+}
+
+type jobServiceGetMessageStreamClient struct {
+ grpc.ClientStream
+}
+
+func (x *jobServiceGetMessageStreamClient) Recv() (*JobMessagesResponse, error) {
+ m := new(JobMessagesResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// Server API for JobService service
+
+type JobServiceServer interface {
+ // Prepare a job for execution. The job will not be executed until a call is made to run with the
+ // returned preparationId.
+ Prepare(context.Context, *PrepareJobRequest) (*PrepareJobResponse, error)
+ // Submit the job for execution
+ Run(context.Context, *RunJobRequest) (*RunJobResponse, error)
+ // Get the current state of the job
+ GetState(context.Context, *GetJobStateRequest) (*GetJobStateResponse, error)
+ // Cancel the job
+ Cancel(context.Context, *CancelJobRequest) (*CancelJobResponse, error)
+ // Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
+ GetStateStream(*GetJobStateRequest, JobService_GetStateStreamServer) error
+ // Subscribe to a stream of state changes and messages from the job
+ GetMessageStream(*JobMessagesRequest, JobService_GetMessageStreamServer) error
+}
+
+func RegisterJobServiceServer(s *grpc.Server, srv JobServiceServer) {
+ s.RegisterService(&_JobService_serviceDesc, srv)
+}
+
+func _JobService_Prepare_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(PrepareJobRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(JobServiceServer).Prepare(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.JobService/Prepare",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(JobServiceServer).Prepare(ctx, req.(*PrepareJobRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _JobService_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(RunJobRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(JobServiceServer).Run(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.JobService/Run",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(JobServiceServer).Run(ctx, req.(*RunJobRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _JobService_GetState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(GetJobStateRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(JobServiceServer).GetState(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.JobService/GetState",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(JobServiceServer).GetState(ctx, req.(*GetJobStateRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _JobService_Cancel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(CancelJobRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(JobServiceServer).Cancel(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/org.apache.beam.model.job_management.v1.JobService/Cancel",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(JobServiceServer).Cancel(ctx, req.(*CancelJobRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _JobService_GetStateStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(GetJobStateRequest)
+ if err := stream.RecvMsg(m); err != nil {
+ return err
+ }
+ return srv.(JobServiceServer).GetStateStream(m, &jobServiceGetStateStreamServer{stream})
+}
+
+type JobService_GetStateStreamServer interface {
+ Send(*GetJobStateResponse) error
+ grpc.ServerStream
+}
+
+type jobServiceGetStateStreamServer struct {
+ grpc.ServerStream
+}
+
+func (x *jobServiceGetStateStreamServer) Send(m *GetJobStateResponse) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+func _JobService_GetMessageStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(JobMessagesRequest)
+ if err := stream.RecvMsg(m); err != nil {
+ return err
+ }
+ return srv.(JobServiceServer).GetMessageStream(m, &jobServiceGetMessageStreamServer{stream})
+}
+
+type JobService_GetMessageStreamServer interface {
+ Send(*JobMessagesResponse) error
+ grpc.ServerStream
+}
+
+type jobServiceGetMessageStreamServer struct {
+ grpc.ServerStream
+}
+
+func (x *jobServiceGetMessageStreamServer) Send(m *JobMessagesResponse) error {
+ return x.ServerStream.SendMsg(m)
+}
+
+var _JobService_serviceDesc = grpc.ServiceDesc{
+ ServiceName: "org.apache.beam.model.job_management.v1.JobService",
+ HandlerType: (*JobServiceServer)(nil),
+ Methods: []grpc.MethodDesc{
+ {
+ MethodName: "Prepare",
+ Handler: _JobService_Prepare_Handler,
+ },
+ {
+ MethodName: "Run",
+ Handler: _JobService_Run_Handler,
+ },
+ {
+ MethodName: "GetState",
+ Handler: _JobService_GetState_Handler,
+ },
+ {
+ MethodName: "Cancel",
+ Handler: _JobService_Cancel_Handler,
+ },
+ },
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "GetStateStream",
+ Handler: _JobService_GetStateStream_Handler,
+ ServerStreams: true,
+ },
+ {
+ StreamName: "GetMessageStream",
+ Handler: _JobService_GetMessageStream_Handler,
+ ServerStreams: true,
+ },
+ },
+ Metadata: "beam_job_api.proto",
+}
+
+func init() { proto.RegisterFile("beam_job_api.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+ // 931 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x56, 0x41, 0x6f, 0xe3, 0x44,
+ 0x14, 0xae, 0xdb, 0x34, 0x4d, 0x5e, 0x9b, 0xd4, 0x9d, 0x52, 0x35, 0x1b, 0x01, 0x5a, 0x8c, 0x60,
+ 0x17, 0xad, 0xe4, 0xdd, 0x76, 0x25, 0x56, 0xec, 0x72, 0x71, 0x62, 0x6f, 0xd6, 0x51, 0x9b, 0x44,
+ 0xe3, 0x54, 0x48, 0x70, 0x30, 0xe3, 0x64, 0x36, 0xb8, 0xd4, 0x1e, 0x63, 0x4f, 0xa2, 0xbd, 0x21,
+ 0x21, 0x71, 0x44, 0xfc, 0x01, 0xfe, 0x00, 0x27, 0x38, 0x70, 0xe3, 0x1f, 0xf1, 0x17, 0xb8, 0xa0,
+ 0x19, 0x8f, 0xdb, 0xa4, 0xed, 0xaa, 0x69, 0x11, 0xe2, 0x94, 0x99, 0xf7, 0xde, 0xf7, 0xcd, 0x37,
+ 0xef, 0x3d, 0xbf, 0x09, 0xa0, 0x80, 0x92, 0xc8, 0x3f, 0x65, 0x81, 0x4f, 0x92, 0xd0, 0x4c, 0x52,
+ 0xc6, 0x19, 0x7a, 0xc0, 0xd2, 0x89, 0x49, 0x12, 0x32, 0xfa, 0x86, 0x9a, 0xc2, 0x6d, 0x46, 0x6c,
+ 0x4c, 0xcf, 0x4c, 0x11, 0x14, 0x91, 0x98, 0x4c, 0x68, 0x44, 0x63, 0x6e, 0xce, 0x0e, 0x9a, 0x7b,
+ 0x12, 0x9c, 0x4e, 0xe3, 0x98, 0xa6, 0x17, 0xf8, 0xe6, 0x36, 0x8d, 0xc7, 0x09, 0x0b, 0x63, 0x9e,
+ 0x29, 0xc3, 0xbb, 0x13, 0xc6, 0x26, 0x67, 0xf4, 0xb1, 0xdc, 0x05, 0xd3, 0xd7, 0x8f, 0x33, 0x9e,
+ 0x4e, 0x47, 0x3c, 0xf7, 0x1a, 0x7f, 0x6a, 0xb0, 0x33, 0x48, 0x69, 0x42, 0x52, 0xda, 0x65, 0x01,
+ 0xa6, 0xdf, 0x4d, 0x69, 0xc6, 0x51, 0x07, 0x2a, 0x49, 0x98, 0xd0, 0xb3, 0x30, 0xa6, 0x0d, 0xed,
+ 0xbe, 0xf6, 0x70, 0xf3, 0xf0, 0x91, 0x79, 0xbd, 0xae, 0x22, 0xcc, 0x9c, 0x1d, 0x98, 0x03, 0xb5,
+ 0xc6, 0xe7, 0x60, 0xd4, 0x02, 0xbd, 0x58, 0xfb, 0x2c, 0xe1, 0x21, 0x8b, 0xb3, 0xc6, 0xaa, 0x24,
+ 0xdc, 0x37, 0x73, 0x5d, 0x66, 0xa1, 0xcb, 0xf4, 0xa4, 0x2e, 0xbc, 0x5d, 0x00, 0xfa, 0x79, 0x3c,
+ 0xba, 0x07, 0x15, 0x71, 0xfb, 0x98, 0x44, 0xb4, 0xb1, 0x76, 0x5f, 0x7b, 0x58, 0xc5, 0x1b, 0xa7,
+ 0x2c, 0xe8, 0x91, 0x88, 0x1a, 0xbf, 0x6b, 0x80, 0xe6, 0xd5, 0x67, 0x09, 0x8b, 0x33, 0x8a, 0x3e,
+ 0x82, 0x7a, 0x22, 0xad, 0x44, 0x30, 0xf8, 0xe1, 0x58, 0x5e, 0xa2, 0x8a, 0x6b, 0x73, 0x56, 0x77,
+ 0x8c, 0x32, 0xb8, 0x47, 0x52, 0x1e, 0xbe, 0x26, 0x23, 0xee, 0x67, 0x9c, 0x4c, 0xc2, 0x78, 0xe2,
+ 0x17, 0xd9, 0x53, 0x2a, 0x9f, 0x2d, 0x71, 0x6d, 0x2b, 0x09, 0x3d, 0x9a, 0xce, 0xc2, 0x11, 0xb5,
+ 0x69, 0x36, 0x4a, 0xc3, 0x84, 0xb3, 0x14, 0xef, 0x17, 0xcc, 0x5e, 0x4e, 0xec, 0x28, 0x5e, 0xe3,
+ 0x2b, 0xa8, 0xe1, 0x69, 0x3c, 0x97, 0xeb, 0x25, 0xc5, 0x7e, 0x08, 0xb5, 0x42, 0x23, 0x67, 0xdf,
+ 0xd2, 0x58, 0x0a, 0xac, 0xe2, 0x2d, 0x65, 0x1c, 0x0a, 0x9b, 0xf1, 0x00, 0xea, 0x05, 0xb9, 0x4a,
+ 0xc5, 0x1e, 0x94, 0x45, 0xf2, 0xce, 0x59, 0xd7, 0x4f, 0x59, 0xe0, 0x8e, 0x8d, 0x4f, 0x40, 0x6f,
+ 0x93, 0x78, 0x44, 0xcf, 0xe6, 0x84, 0xbc, 0x25, 0x94, 0xc0, 0xce, 0x5c, 0xa8, 0xa2, 0x3d, 0x82,
+ 0xf5, 0x8c, 0x13, 0x9e, 0x77, 0x47, 0xfd, 0xf0, 0x53, 0x73, 0xc9, 0xae, 0x35, 0xbb, 0x2c, 0xf0,
+ 0x04, 0xd0, 0x74, 0xe2, 0x69, 0x84, 0x73, 0x12, 0xe3, 0x11, 0xa0, 0x0e, 0xe5, 0x85, 0xeb, 0x06,
+ 0x3d, 0x23, 0xd8, 0x5d, 0x08, 0xfe, 0xaf, 0x14, 0x75, 0x59, 0x70, 0x4c, 0xb3, 0x8c, 0x4c, 0x68,
+ 0x76, 0x83, 0xa2, 0xbf, 0x57, 0x01, 0x2e, 0xa2, 0xd1, 0x7b, 0x00, 0x51, 0xbe, 0xbc, 0x88, 0xac,
+ 0x2a, 0x8b, 0x3b, 0x46, 0x08, 0x4a, 0x3c, 0x8c, 0xa8, 0xaa, 0x9f, 0x5c, 0x23, 0x0a, 0x10, 0x46,
+ 0x09, 0x4b, 0xb9, 0x48, 0xb4, 0x6c, 0xf2, 0xfa, 0xa1, 0x73, 0x9b, 0x1b, 0xa8, 0xb3, 0x4d, 0xf5,
+ 0xeb, 0x9e, 0x93, 0xe1, 0x39, 0x62, 0xf4, 0x01, 0x6c, 0x15, 0xca, 0x38, 0x7d, 0xc3, 0x1b, 0x25,
+ 0x29, 0x61, 0x53, 0xd9, 0x86, 0xf4, 0x0d, 0x37, 0x7e, 0xd3, 0x60, 0xe7, 0x0a, 0x09, 0x32, 0xe0,
+ 0xfd, 0x63, 0xc7, 0xf3, 0xac, 0x8e, 0xe3, 0xbb, 0xc7, 0x83, 0x3e, 0x1e, 0x5a, 0xbd, 0xb6, 0xe3,
+ 0x9f, 0xf4, 0xbc, 0x81, 0xd3, 0x76, 0x5f, 0xba, 0x8e, 0xad, 0xaf, 0xa0, 0x3d, 0xd8, 0xe9, 0xf6,
+ 0x5b, 0x7e, 0x11, 0x67, 0x3b, 0xad, 0x93, 0x8e, 0xae, 0xa1, 0x06, 0xbc, 0xb3, 0x68, 0x1e, 0x5a,
+ 0xee, 0x91, 0x63, 0xeb, 0xab, 0x97, 0x01, 0x2d, 0xcb, 0x73, 0xdb, 0xfa, 0x1a, 0xda, 0x87, 0xdd,
+ 0x79, 0xf3, 0x17, 0x16, 0xee, 0xb9, 0xbd, 0x8e, 0x5e, 0xba, 0x1c, 0xef, 0x60, 0xdc, 0xc7, 0xfa,
+ 0xba, 0xf1, 0x97, 0x06, 0xbb, 0x0b, 0xb5, 0x52, 0x0d, 0xf1, 0x35, 0xe8, 0xc5, 0x65, 0x53, 0x65,
+ 0x53, 0xb3, 0xec, 0xe9, 0x1d, 0x32, 0xfb, 0x6a, 0x05, 0x6f, 0x2b, 0xba, 0xf3, 0x13, 0x28, 0xd4,
+ 0x65, 0xb7, 0x5c, 0xf0, 0xe7, 0x43, 0xe3, 0xf3, 0xa5, 0xf9, 0xaf, 0x69, 0xe4, 0x57, 0x2b, 0xb8,
+ 0x96, 0xcd, 0x1b, 0x5a, 0x00, 0x95, 0xe2, 0x00, 0xe3, 0x57, 0x0d, 0x2a, 0x05, 0xc2, 0xf8, 0x45,
+ 0x83, 0x92, 0x68, 0x5a, 0xb4, 0x0d, 0x9b, 0x8b, 0xb5, 0xd8, 0x84, 0x0d, 0x6f, 0xd8, 0x1f, 0x0c,
+ 0x1c, 0x5b, 0xd7, 0xc4, 0x06, 0x9f, 0xf4, 0x64, 0x12, 0x57, 0x51, 0x05, 0x4a, 0x76, 0xbf, 0xe7,
+ 0xe8, 0x6b, 0x08, 0xa0, 0xfc, 0x32, 0x2f, 0x45, 0x09, 0xd5, 0xa0, 0xda, 0x16, 0x25, 0x3d, 0x12,
+ 0xdb, 0x75, 0x81, 0x38, 0x19, 0xd8, 0xd6, 0xd0, 0xb1, 0xf5, 0x32, 0xda, 0x82, 0x8a, 0x8d, 0x2d,
+ 0x57, 0xe2, 0x37, 0x84, 0x4b, 0xee, 0x1c, 0x5b, 0xaf, 0x08, 0x97, 0x37, 0xb4, 0xf0, 0x50, 0xb8,
+ 0xaa, 0xa8, 0x0e, 0xa0, 0x48, 0xc4, 0x1e, 0x0e, 0xff, 0x28, 0xcb, 0xcf, 0x42, 0xcd, 0x46, 0xf4,
+ 0x83, 0x06, 0x1b, 0x6a, 0x56, 0xa3, 0xe7, 0x4b, 0x67, 0xe8, 0xca, 0xdb, 0xd4, 0x7c, 0x71, 0x27,
+ 0xac, 0x2a, 0xd9, 0x0c, 0xd6, 0xf0, 0x34, 0x46, 0xcb, 0x4f, 0x87, 0x85, 0x59, 0xdd, 0x7c, 0x76,
+ 0x6b, 0x9c, 0x3a, 0xf7, 0x47, 0x0d, 0x2a, 0x1d, 0xca, 0x65, 0xdd, 0xd0, 0x8b, 0xbb, 0xf5, 0x47,
+ 0x2e, 0xe1, 0x5f, 0x35, 0x17, 0xfa, 0x1e, 0xca, 0xf9, 0x30, 0x47, 0x9f, 0x2d, 0xcd, 0x73, 0xf9,
+ 0xa1, 0x68, 0x3e, 0xbf, 0x0b, 0x54, 0x09, 0xf8, 0x49, 0x83, 0x7a, 0x91, 0x08, 0x8f, 0xa7, 0x94,
+ 0x44, 0xff, 0x63, 0x3a, 0x9e, 0x68, 0xe8, 0x67, 0x0d, 0xf4, 0x0e, 0xe5, 0xea, 0x2b, 0xbf, 0xb5,
+ 0xa2, 0xab, 0x8f, 0xc4, 0x2d, 0x14, 0x5d, 0x33, 0xb5, 0x9e, 0x68, 0xad, 0x16, 0x7c, 0xfc, 0x56,
+ 0x82, 0x05, 0x7c, 0xab, 0xdc, 0x65, 0x81, 0x95, 0x84, 0x5f, 0xea, 0x0b, 0x1e, 0x7f, 0x76, 0x10,
+ 0x94, 0xe5, 0x9f, 0xaa, 0xa7, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x20, 0x71, 0x77, 0xfc, 0x61,
+ 0x0a, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go
new file mode 100644
index 0000000..31dc53e
--- /dev/null
+++ b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go
@@ -0,0 +1,3491 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: beam_runner_api.proto
+
+/*
+Package pipeline_v1 is a generated protocol buffer package.
+
+It is generated from these files:
+ beam_runner_api.proto
+ endpoints.proto
+ standard_window_fns.proto
+
+It has these top-level messages:
+ Components
+ MessageWithComponents
+ Pipeline
+ PTransform
+ PCollection
+ ParDoPayload
+ Parameter
+ StateSpec
+ ValueStateSpec
+ BagStateSpec
+ CombiningStateSpec
+ MapStateSpec
+ SetStateSpec
+ TimerSpec
+ IsBounded
+ ReadPayload
+ WindowIntoPayload
+ CombinePayload
+ TestStreamPayload
+ WriteFilesPayload
+ Coder
+ WindowingStrategy
+ MergeStatus
+ AccumulationMode
+ ClosingBehavior
+ OnTimeBehavior
+ OutputTime
+ TimeDomain
+ Trigger
+ TimestampTransform
+ SideInput
+ Environment
+ SdkFunctionSpec
+ FunctionSpec
+ DisplayData
+ ApiServiceDescriptor
+ OAuth2ClientCredentialsGrant
+ FixedWindowsPayload
+ SlidingWindowsPayload
+ SessionsPayload
+*/
+package pipeline_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import google_protobuf "github.com/golang/protobuf/ptypes/any"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type Parameter_Type_Enum int32
+
+const (
+ Parameter_Type_UNSPECIFIED Parameter_Type_Enum = 0
+ Parameter_Type_WINDOW Parameter_Type_Enum = 1
+ Parameter_Type_PIPELINE_OPTIONS Parameter_Type_Enum = 2
+ Parameter_Type_RESTRICTION_TRACKER Parameter_Type_Enum = 3
+)
+
+var Parameter_Type_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "WINDOW",
+ 2: "PIPELINE_OPTIONS",
+ 3: "RESTRICTION_TRACKER",
+}
+var Parameter_Type_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "WINDOW": 1,
+ "PIPELINE_OPTIONS": 2,
+ "RESTRICTION_TRACKER": 3,
+}
+
+func (x Parameter_Type_Enum) String() string {
+ return proto.EnumName(Parameter_Type_Enum_name, int32(x))
+}
+func (Parameter_Type_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 0, 0} }
+
+type IsBounded_Enum int32
+
+const (
+ IsBounded_UNSPECIFIED IsBounded_Enum = 0
+ IsBounded_UNBOUNDED IsBounded_Enum = 1
+ IsBounded_BOUNDED IsBounded_Enum = 2
+)
+
+var IsBounded_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "UNBOUNDED",
+ 2: "BOUNDED",
+}
+var IsBounded_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "UNBOUNDED": 1,
+ "BOUNDED": 2,
+}
+
+func (x IsBounded_Enum) String() string {
+ return proto.EnumName(IsBounded_Enum_name, int32(x))
+}
+func (IsBounded_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{14, 0} }
+
+type MergeStatus_Enum int32
+
+const (
+ MergeStatus_UNSPECIFIED MergeStatus_Enum = 0
+ // The WindowFn does not require merging.
+ // Examples: global window, FixedWindows, SlidingWindows
+ MergeStatus_NON_MERGING MergeStatus_Enum = 1
+ // The WindowFn is merging and the PCollection has not had merging
+ // performed.
+ // Example: Sessions prior to a GroupByKey
+ MergeStatus_NEEDS_MERGE MergeStatus_Enum = 2
+ // The WindowFn is merging and the PCollection has had merging occur
+ // already.
+ // Example: Sessions after a GroupByKey
+ MergeStatus_ALREADY_MERGED MergeStatus_Enum = 3
+)
+
+var MergeStatus_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "NON_MERGING",
+ 2: "NEEDS_MERGE",
+ 3: "ALREADY_MERGED",
+}
+var MergeStatus_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "NON_MERGING": 1,
+ "NEEDS_MERGE": 2,
+ "ALREADY_MERGED": 3,
+}
+
+func (x MergeStatus_Enum) String() string {
+ return proto.EnumName(MergeStatus_Enum_name, int32(x))
+}
+func (MergeStatus_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{22, 0} }
+
+type AccumulationMode_Enum int32
+
+const (
+ AccumulationMode_UNSPECIFIED AccumulationMode_Enum = 0
+ // The aggregation is discarded when it is output
+ AccumulationMode_DISCARDING AccumulationMode_Enum = 1
+ // The aggregation is accumulated across outputs
+ AccumulationMode_ACCUMULATING AccumulationMode_Enum = 2
+)
+
+var AccumulationMode_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "DISCARDING",
+ 2: "ACCUMULATING",
+}
+var AccumulationMode_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "DISCARDING": 1,
+ "ACCUMULATING": 2,
+}
+
+func (x AccumulationMode_Enum) String() string {
+ return proto.EnumName(AccumulationMode_Enum_name, int32(x))
+}
+func (AccumulationMode_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{23, 0} }
+
+type ClosingBehavior_Enum int32
+
+const (
+ ClosingBehavior_UNSPECIFIED ClosingBehavior_Enum = 0
+ // Emit output when a window expires, whether or not there has been
+ // any new data since the last output.
+ ClosingBehavior_EMIT_ALWAYS ClosingBehavior_Enum = 1
+ // Only emit output when new data has arrives since the last output
+ ClosingBehavior_EMIT_IF_NONEMPTY ClosingBehavior_Enum = 2
+)
+
+var ClosingBehavior_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "EMIT_ALWAYS",
+ 2: "EMIT_IF_NONEMPTY",
+}
+var ClosingBehavior_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "EMIT_ALWAYS": 1,
+ "EMIT_IF_NONEMPTY": 2,
+}
+
+func (x ClosingBehavior_Enum) String() string {
+ return proto.EnumName(ClosingBehavior_Enum_name, int32(x))
+}
+func (ClosingBehavior_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{24, 0} }
+
+type OnTimeBehavior_Enum int32
+
+const (
+ OnTimeBehavior_UNSPECIFIED OnTimeBehavior_Enum = 0
+ // Always fire the on-time pane. Even if there is no new data since
+ // the previous firing, an element will be produced.
+ OnTimeBehavior_FIRE_ALWAYS OnTimeBehavior_Enum = 1
+ // Only fire the on-time pane if there is new data since the previous firing.
+ OnTimeBehavior_FIRE_IF_NONEMPTY OnTimeBehavior_Enum = 2
+)
+
+var OnTimeBehavior_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "FIRE_ALWAYS",
+ 2: "FIRE_IF_NONEMPTY",
+}
+var OnTimeBehavior_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "FIRE_ALWAYS": 1,
+ "FIRE_IF_NONEMPTY": 2,
+}
+
+func (x OnTimeBehavior_Enum) String() string {
+ return proto.EnumName(OnTimeBehavior_Enum_name, int32(x))
+}
+func (OnTimeBehavior_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{25, 0} }
+
+type OutputTime_Enum int32
+
+const (
+ OutputTime_UNSPECIFIED OutputTime_Enum = 0
+ // The output has the timestamp of the end of the window.
+ OutputTime_END_OF_WINDOW OutputTime_Enum = 1
+ // The output has the latest timestamp of the input elements since
+ // the last output.
+ OutputTime_LATEST_IN_PANE OutputTime_Enum = 2
+ // The output has the earliest timestamp of the input elements since
+ // the last output.
+ OutputTime_EARLIEST_IN_PANE OutputTime_Enum = 3
+)
+
+var OutputTime_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "END_OF_WINDOW",
+ 2: "LATEST_IN_PANE",
+ 3: "EARLIEST_IN_PANE",
+}
+var OutputTime_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "END_OF_WINDOW": 1,
+ "LATEST_IN_PANE": 2,
+ "EARLIEST_IN_PANE": 3,
+}
+
+func (x OutputTime_Enum) String() string {
+ return proto.EnumName(OutputTime_Enum_name, int32(x))
+}
+func (OutputTime_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{26, 0} }
+
+type TimeDomain_Enum int32
+
+const (
+ TimeDomain_UNSPECIFIED TimeDomain_Enum = 0
+ // Event time is time from the perspective of the data
+ TimeDomain_EVENT_TIME TimeDomain_Enum = 1
+ // Processing time is time from the perspective of the
+ // execution of your pipeline
+ TimeDomain_PROCESSING_TIME TimeDomain_Enum = 2
+ // Synchronized processing time is the minimum of the
+ // processing time of all pending elements.
+ //
+ // The "processing time" of an element refers to
+ // the local processing time at which it was emitted
+ TimeDomain_SYNCHRONIZED_PROCESSING_TIME TimeDomain_Enum = 3
+)
+
+var TimeDomain_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "EVENT_TIME",
+ 2: "PROCESSING_TIME",
+ 3: "SYNCHRONIZED_PROCESSING_TIME",
+}
+var TimeDomain_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "EVENT_TIME": 1,
+ "PROCESSING_TIME": 2,
+ "SYNCHRONIZED_PROCESSING_TIME": 3,
+}
+
+func (x TimeDomain_Enum) String() string {
+ return proto.EnumName(TimeDomain_Enum_name, int32(x))
+}
+func (TimeDomain_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{27, 0} }
+
+type DisplayData_Type_Enum int32
+
+const (
+ DisplayData_Type_UNSPECIFIED DisplayData_Type_Enum = 0
+ DisplayData_Type_STRING DisplayData_Type_Enum = 1
+ DisplayData_Type_INTEGER DisplayData_Type_Enum = 2
+ DisplayData_Type_FLOAT DisplayData_Type_Enum = 3
+ DisplayData_Type_BOOLEAN DisplayData_Type_Enum = 4
+ DisplayData_Type_TIMESTAMP DisplayData_Type_Enum = 5
+ DisplayData_Type_DURATION DisplayData_Type_Enum = 6
+ DisplayData_Type_JAVA_CLASS DisplayData_Type_Enum = 7
+)
+
+var DisplayData_Type_Enum_name = map[int32]string{
+ 0: "UNSPECIFIED",
+ 1: "STRING",
+ 2: "INTEGER",
+ 3: "FLOAT",
+ 4: "BOOLEAN",
+ 5: "TIMESTAMP",
+ 6: "DURATION",
+ 7: "JAVA_CLASS",
+}
+var DisplayData_Type_Enum_value = map[string]int32{
+ "UNSPECIFIED": 0,
+ "STRING": 1,
+ "INTEGER": 2,
+ "FLOAT": 3,
+ "BOOLEAN": 4,
+ "TIMESTAMP": 5,
+ "DURATION": 6,
+ "JAVA_CLASS": 7,
+}
+
+func (x DisplayData_Type_Enum) String() string {
+ return proto.EnumName(DisplayData_Type_Enum_name, int32(x))
+}
+func (DisplayData_Type_Enum) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{34, 2, 0} }
+
+// A set of mappings from id to message. This is included as an optional field
+// on any proto message that may contain references needing resolution.
+type Components struct {
+ // (Required) A map from pipeline-scoped id to PTransform.
+ Transforms map[string]*PTransform `protobuf:"bytes,1,rep,name=transforms" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to PCollection.
+ Pcollections map[string]*PCollection `protobuf:"bytes,2,rep,name=pcollections" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ WindowingStrategies map[string]*WindowingStrategy `protobuf:"bytes,3,rep,name=windowing_strategies,json=windowingStrategies" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to Coder.
+ Coders map[string]*Coder `protobuf:"bytes,4,rep,name=coders" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from pipeline-scoped id to Environment.
+ Environments map[string]*Environment `protobuf:"bytes,5,rep,name=environments" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *Components) Reset() { *m = Components{} }
+func (m *Components) String() string { return proto.CompactTextString(m) }
+func (*Components) ProtoMessage() {}
+func (*Components) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *Components) GetTransforms() map[string]*PTransform {
+ if m != nil {
+ return m.Transforms
+ }
+ return nil
+}
+
+func (m *Components) GetPcollections() map[string]*PCollection {
+ if m != nil {
+ return m.Pcollections
+ }
+ return nil
+}
+
+func (m *Components) GetWindowingStrategies() map[string]*WindowingStrategy {
+ if m != nil {
+ return m.WindowingStrategies
+ }
+ return nil
+}
+
+func (m *Components) GetCoders() map[string]*Coder {
+ if m != nil {
+ return m.Coders
+ }
+ return nil
+}
+
+func (m *Components) GetEnvironments() map[string]*Environment {
+ if m != nil {
+ return m.Environments
+ }
+ return nil
+}
+
+// A disjoint union of all the things that may contain references
+// that require Components to resolve.
+type MessageWithComponents struct {
+ // (Optional) The by-reference components of the root message,
+ // enabling a standalone message.
+ //
+ // If this is absent, it is expected that there are no
+ // references.
+ Components *Components `protobuf:"bytes,1,opt,name=components" json:"components,omitempty"`
+ // (Required) The root message that may contain pointers
+ // that should be resolved by looking inside components.
+ //
+ // Types that are valid to be assigned to Root:
+ // *MessageWithComponents_Coder
+ // *MessageWithComponents_CombinePayload
+ // *MessageWithComponents_SdkFunctionSpec
+ // *MessageWithComponents_ParDoPayload
+ // *MessageWithComponents_Ptransform
+ // *MessageWithComponents_Pcollection
+ // *MessageWithComponents_ReadPayload
+ // *MessageWithComponents_SideInput
+ // *MessageWithComponents_WindowIntoPayload
+ // *MessageWithComponents_WindowingStrategy
+ // *MessageWithComponents_FunctionSpec
+ Root isMessageWithComponents_Root `protobuf_oneof:"root"`
+}
+
+func (m *MessageWithComponents) Reset() { *m = MessageWithComponents{} }
+func (m *MessageWithComponents) String() string { return proto.CompactTextString(m) }
+func (*MessageWithComponents) ProtoMessage() {}
+func (*MessageWithComponents) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+type isMessageWithComponents_Root interface {
+ isMessageWithComponents_Root()
+}
+
+type MessageWithComponents_Coder struct {
+ Coder *Coder `protobuf:"bytes,2,opt,name=coder,oneof"`
+}
+type MessageWithComponents_CombinePayload struct {
+ CombinePayload *CombinePayload `protobuf:"bytes,3,opt,name=combine_payload,json=combinePayload,oneof"`
+}
+type MessageWithComponents_SdkFunctionSpec struct {
+ SdkFunctionSpec *SdkFunctionSpec `protobuf:"bytes,4,opt,name=sdk_function_spec,json=sdkFunctionSpec,oneof"`
+}
+type MessageWithComponents_ParDoPayload struct {
+ ParDoPayload *ParDoPayload `protobuf:"bytes,6,opt,name=par_do_payload,json=parDoPayload,oneof"`
+}
+type MessageWithComponents_Ptransform struct {
+ Ptransform *PTransform `protobuf:"bytes,7,opt,name=ptransform,oneof"`
+}
+type MessageWithComponents_Pcollection struct {
+ Pcollection *PCollection `protobuf:"bytes,8,opt,name=pcollection,oneof"`
+}
+type MessageWithComponents_ReadPayload struct {
+ ReadPayload *ReadPayload `protobuf:"bytes,9,opt,name=read_payload,json=readPayload,oneof"`
+}
+type MessageWithComponents_SideInput struct {
+ SideInput *SideInput `protobuf:"bytes,11,opt,name=side_input,json=sideInput,oneof"`
+}
+type MessageWithComponents_WindowIntoPayload struct {
+ WindowIntoPayload *WindowIntoPayload `protobuf:"bytes,12,opt,name=window_into_payload,json=windowIntoPayload,oneof"`
+}
+type MessageWithComponents_WindowingStrategy struct {
+ WindowingStrategy *WindowingStrategy `protobuf:"bytes,13,opt,name=windowing_strategy,json=windowingStrategy,oneof"`
+}
+type MessageWithComponents_FunctionSpec struct {
+ FunctionSpec *FunctionSpec `protobuf:"bytes,14,opt,name=function_spec,json=functionSpec,oneof"`
+}
+
+func (*MessageWithComponents_Coder) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_CombinePayload) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_SdkFunctionSpec) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_ParDoPayload) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_Ptransform) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_Pcollection) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_ReadPayload) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_SideInput) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_WindowIntoPayload) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_WindowingStrategy) isMessageWithComponents_Root() {}
+func (*MessageWithComponents_FunctionSpec) isMessageWithComponents_Root() {}
+
+func (m *MessageWithComponents) GetRoot() isMessageWithComponents_Root {
+ if m != nil {
+ return m.Root
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetComponents() *Components {
+ if m != nil {
+ return m.Components
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetCoder() *Coder {
+ if x, ok := m.GetRoot().(*MessageWithComponents_Coder); ok {
+ return x.Coder
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetCombinePayload() *CombinePayload {
+ if x, ok := m.GetRoot().(*MessageWithComponents_CombinePayload); ok {
+ return x.CombinePayload
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetSdkFunctionSpec() *SdkFunctionSpec {
+ if x, ok := m.GetRoot().(*MessageWithComponents_SdkFunctionSpec); ok {
+ return x.SdkFunctionSpec
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetParDoPayload() *ParDoPayload {
+ if x, ok := m.GetRoot().(*MessageWithComponents_ParDoPayload); ok {
+ return x.ParDoPayload
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetPtransform() *PTransform {
+ if x, ok := m.GetRoot().(*MessageWithComponents_Ptransform); ok {
+ return x.Ptransform
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetPcollection() *PCollection {
+ if x, ok := m.GetRoot().(*MessageWithComponents_Pcollection); ok {
+ return x.Pcollection
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetReadPayload() *ReadPayload {
+ if x, ok := m.GetRoot().(*MessageWithComponents_ReadPayload); ok {
+ return x.ReadPayload
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetSideInput() *SideInput {
+ if x, ok := m.GetRoot().(*MessageWithComponents_SideInput); ok {
+ return x.SideInput
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetWindowIntoPayload() *WindowIntoPayload {
+ if x, ok := m.GetRoot().(*MessageWithComponents_WindowIntoPayload); ok {
+ return x.WindowIntoPayload
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetWindowingStrategy() *WindowingStrategy {
+ if x, ok := m.GetRoot().(*MessageWithComponents_WindowingStrategy); ok {
+ return x.WindowingStrategy
+ }
+ return nil
+}
+
+func (m *MessageWithComponents) GetFunctionSpec() *FunctionSpec {
+ if x, ok := m.GetRoot().(*MessageWithComponents_FunctionSpec); ok {
+ return x.FunctionSpec
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*MessageWithComponents) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _MessageWithComponents_OneofMarshaler, _MessageWithComponents_OneofUnmarshaler, _MessageWithComponents_OneofSizer, []interface{}{
+ (*MessageWithComponents_Coder)(nil),
+ (*MessageWithComponents_CombinePayload)(nil),
+ (*MessageWithComponents_SdkFunctionSpec)(nil),
+ (*MessageWithComponents_ParDoPayload)(nil),
+ (*MessageWithComponents_Ptransform)(nil),
+ (*MessageWithComponents_Pcollection)(nil),
+ (*MessageWithComponents_ReadPayload)(nil),
+ (*MessageWithComponents_SideInput)(nil),
+ (*MessageWithComponents_WindowIntoPayload)(nil),
+ (*MessageWithComponents_WindowingStrategy)(nil),
+ (*MessageWithComponents_FunctionSpec)(nil),
+ }
+}
+
+func _MessageWithComponents_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*MessageWithComponents)
+ // root
+ switch x := m.Root.(type) {
+ case *MessageWithComponents_Coder:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Coder); err != nil {
+ return err
+ }
+ case *MessageWithComponents_CombinePayload:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.CombinePayload); err != nil {
+ return err
+ }
+ case *MessageWithComponents_SdkFunctionSpec:
+ b.EncodeVarint(4<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.SdkFunctionSpec); err != nil {
+ return err
+ }
+ case *MessageWithComponents_ParDoPayload:
+ b.EncodeVarint(6<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ParDoPayload); err != nil {
+ return err
+ }
+ case *MessageWithComponents_Ptransform:
+ b.EncodeVarint(7<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Ptransform); err != nil {
+ return err
+ }
+ case *MessageWithComponents_Pcollection:
+ b.EncodeVarint(8<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Pcollection); err != nil {
+ return err
+ }
+ case *MessageWithComponents_ReadPayload:
+ b.EncodeVarint(9<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ReadPayload); err != nil {
+ return err
+ }
+ case *MessageWithComponents_SideInput:
+ b.EncodeVarint(11<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.SideInput); err != nil {
+ return err
+ }
+ case *MessageWithComponents_WindowIntoPayload:
+ b.EncodeVarint(12<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.WindowIntoPayload); err != nil {
+ return err
+ }
+ case *MessageWithComponents_WindowingStrategy:
+ b.EncodeVarint(13<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.WindowingStrategy); err != nil {
+ return err
+ }
+ case *MessageWithComponents_FunctionSpec:
+ b.EncodeVarint(14<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.FunctionSpec); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("MessageWithComponents.Root has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _MessageWithComponents_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*MessageWithComponents)
+ switch tag {
+ case 2: // root.coder
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Coder)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_Coder{msg}
+ return true, err
+ case 3: // root.combine_payload
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(CombinePayload)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_CombinePayload{msg}
+ return true, err
+ case 4: // root.sdk_function_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(SdkFunctionSpec)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_SdkFunctionSpec{msg}
+ return true, err
+ case 6: // root.par_do_payload
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ParDoPayload)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_ParDoPayload{msg}
+ return true, err
+ case 7: // root.ptransform
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(PTransform)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_Ptransform{msg}
+ return true, err
+ case 8: // root.pcollection
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(PCollection)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_Pcollection{msg}
+ return true, err
+ case 9: // root.read_payload
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ReadPayload)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_ReadPayload{msg}
+ return true, err
+ case 11: // root.side_input
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(SideInput)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_SideInput{msg}
+ return true, err
+ case 12: // root.window_into_payload
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(WindowIntoPayload)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_WindowIntoPayload{msg}
+ return true, err
+ case 13: // root.windowing_strategy
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(WindowingStrategy)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_WindowingStrategy{msg}
+ return true, err
+ case 14: // root.function_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(FunctionSpec)
+ err := b.DecodeMessage(msg)
+ m.Root = &MessageWithComponents_FunctionSpec{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _MessageWithComponents_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*MessageWithComponents)
+ // root
+ switch x := m.Root.(type) {
+ case *MessageWithComponents_Coder:
+ s := proto.Size(x.Coder)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_CombinePayload:
+ s := proto.Size(x.CombinePayload)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_SdkFunctionSpec:
+ s := proto.Size(x.SdkFunctionSpec)
+ n += proto.SizeVarint(4<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_ParDoPayload:
+ s := proto.Size(x.ParDoPayload)
+ n += proto.SizeVarint(6<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_Ptransform:
+ s := proto.Size(x.Ptransform)
+ n += proto.SizeVarint(7<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_Pcollection:
+ s := proto.Size(x.Pcollection)
+ n += proto.SizeVarint(8<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_ReadPayload:
+ s := proto.Size(x.ReadPayload)
+ n += proto.SizeVarint(9<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_SideInput:
+ s := proto.Size(x.SideInput)
+ n += proto.SizeVarint(11<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_WindowIntoPayload:
+ s := proto.Size(x.WindowIntoPayload)
+ n += proto.SizeVarint(12<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_WindowingStrategy:
+ s := proto.Size(x.WindowingStrategy)
+ n += proto.SizeVarint(13<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *MessageWithComponents_FunctionSpec:
+ s := proto.Size(x.FunctionSpec)
+ n += proto.SizeVarint(14<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+// A Pipeline is a hierarchical graph of PTransforms, linked
+// by PCollections.
+//
+// This is represented by a number of by-reference maps to nodes,
+// PCollections, SDK environments, UDF, etc., for
+// supporting compact reuse and arbitrary graph structure.
+//
+// All of the keys in the maps here are arbitrary strings that are only
+// required to be internally consistent within this proto message.
+type Pipeline struct {
+ // (Required) The coders, UDFs, graph nodes, etc, that make up
+ // this pipeline.
+ Components *Components `protobuf:"bytes,1,opt,name=components" json:"components,omitempty"`
+ // (Required) The ids of all PTransforms that are not contained within another PTransform.
+ // These must be in shallow topological order, so that traversing them recursively
+ // in this order yields a recursively topological traversal.
+ RootTransformIds []string `protobuf:"bytes,2,rep,name=root_transform_ids,json=rootTransformIds" json:"root_transform_ids,omitempty"`
+ // (Optional) Static display data for the pipeline. If there is none,
+ // it may be omitted.
+ DisplayData *DisplayData `protobuf:"bytes,3,opt,name=display_data,json=displayData" json:"display_data,omitempty"`
+}
+
+func (m *Pipeline) Reset() { *m = Pipeline{} }
+func (m *Pipeline) String() string { return proto.CompactTextString(m) }
+func (*Pipeline) ProtoMessage() {}
+func (*Pipeline) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *Pipeline) GetComponents() *Components {
+ if m != nil {
+ return m.Components
+ }
+ return nil
+}
+
+func (m *Pipeline) GetRootTransformIds() []string {
+ if m != nil {
+ return m.RootTransformIds
+ }
+ return nil
+}
+
+func (m *Pipeline) GetDisplayData() *DisplayData {
+ if m != nil {
+ return m.DisplayData
+ }
+ return nil
+}
+
+// An applied PTransform! This does not contain the graph data, but only the
+// fields specific to a graph node that is a Runner API transform
+// between PCollections.
+type PTransform struct {
+ // (Required) A unique name for the application node.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here and be unique, even if it is
+ // autogenerated.
+ UniqueName string `protobuf:"bytes,5,opt,name=unique_name,json=uniqueName" json:"unique_name,omitempty"`
+ // (Optional) A URN and payload that, together, fully defined the semantics
+ // of this transform.
+ //
+ // If absent, this must be an "anonymous" composite transform.
+ //
+ // For primitive transform in the Runner API, this is required, and the
+ // payloads are well-defined messages. When the URN indicates ParDo it
+ // is a ParDoPayload, and so on.
+ //
+ // TODO: document the standardized URNs and payloads
+ // TODO: separate standardized payloads into a separate proto file
+ //
+ // For some special composite transforms, the payload is also officially
+ // defined:
+ //
+ // - when the URN is "urn:beam:transforms:combine" it is a CombinePayload
+ //
+ Spec *FunctionSpec `protobuf:"bytes,1,opt,name=spec" json:"spec,omitempty"`
+ // (Optional) if this node is a composite, a list of the ids of
+ // transforms that it contains.
+ Subtransforms []string `protobuf:"bytes,2,rep,name=subtransforms" json:"subtransforms,omitempty"`
+ // (Required) A map from local names of inputs (unique only with this map, and
+ // likely embedded in the transform payload and serialized user code) to
+ // PCollection ids.
+ //
+ // The payload for this transform may clarify the relationship of these
+ // inputs. For example:
+ //
+ // - for a Flatten transform they are merged
+ // - for a ParDo transform, some may be side inputs
+ //
+ // All inputs are recorded here so that the topological ordering of
+ // the graph is consistent whether or not the payload is understood.
+ //
+ Inputs map[string]string `protobuf:"bytes,3,rep,name=inputs" json:"inputs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Required) A map from local names of outputs (unique only within this map,
+ // and likely embedded in the transform payload and serialized user code)
+ // to PCollection ids.
+ //
+ // The URN or payload for this transform node may clarify the type and
+ // relationship of these outputs. For example:
+ //
+ // - for a ParDo transform, these are tags on PCollections, which will be
+ // embedded in the DoFn.
+ //
+ Outputs map[string]string `protobuf:"bytes,4,rep,name=outputs" json:"outputs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData *DisplayData `protobuf:"bytes,6,opt,name=display_data,json=displayData" json:"display_data,omitempty"`
+}
+
+func (m *PTransform) Reset() { *m = PTransform{} }
+func (m *PTransform) String() string { return proto.CompactTextString(m) }
+func (*PTransform) ProtoMessage() {}
+func (*PTransform) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *PTransform) GetUniqueName() string {
+ if m != nil {
+ return m.UniqueName
+ }
+ return ""
+}
+
+func (m *PTransform) GetSpec() *FunctionSpec {
+ if m != nil {
+ return m.Spec
+ }
+ return nil
+}
+
+func (m *PTransform) GetSubtransforms() []string {
+ if m != nil {
+ return m.Subtransforms
+ }
+ return nil
+}
+
+func (m *PTransform) GetInputs() map[string]string {
+ if m != nil {
+ return m.Inputs
+ }
+ return nil
+}
+
+func (m *PTransform) GetOutputs() map[string]string {
+ if m != nil {
+ return m.Outputs
+ }
+ return nil
+}
+
+func (m *PTransform) GetDisplayData() *DisplayData {
+ if m != nil {
+ return m.DisplayData
+ }
+ return nil
+}
+
+// A PCollection!
+type PCollection struct {
+ // (Required) A unique name for the PCollection.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here, even if it is autogenerated.
+ UniqueName string `protobuf:"bytes,1,opt,name=unique_name,json=uniqueName" json:"unique_name,omitempty"`
+ // (Required) The id of the Coder for this PCollection.
+ CoderId string `protobuf:"bytes,2,opt,name=coder_id,json=coderId" json:"coder_id,omitempty"`
+ // (Required) Whether this PCollection is bounded or unbounded
+ IsBounded IsBounded_Enum `protobuf:"varint,3,opt,name=is_bounded,json=isBounded,enum=org.apache.beam.model.pipeline.v1.IsBounded_Enum" json:"is_bounded,omitempty"`
+ // (Required) The id of the windowing strategy for this PCollection.
+ WindowingStrategyId string `protobuf:"bytes,4,opt,name=windowing_strategy_id,json=windowingStrategyId" json:"windowing_strategy_id,omitempty"`
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData *DisplayData `protobuf:"bytes,5,opt,name=display_data,json=displayData" json:"display_data,omitempty"`
+}
+
+func (m *PCollection) Reset() { *m = PCollection{} }
+func (m *PCollection) String() string { return proto.CompactTextString(m) }
+func (*PCollection) ProtoMessage() {}
+func (*PCollection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *PCollection) GetUniqueName() string {
+ if m != nil {
+ return m.UniqueName
+ }
+ return ""
+}
+
+func (m *PCollection) GetCoderId() string {
+ if m != nil {
+ return m.CoderId
+ }
+ return ""
+}
+
+func (m *PCollection) GetIsBounded() IsBounded_Enum {
+ if m != nil {
+ return m.IsBounded
+ }
+ return IsBounded_UNSPECIFIED
+}
+
+func (m *PCollection) GetWindowingStrategyId() string {
+ if m != nil {
+ return m.WindowingStrategyId
+ }
+ return ""
+}
+
+func (m *PCollection) GetDisplayData() *DisplayData {
+ if m != nil {
+ return m.DisplayData
+ }
+ return nil
+}
+
+// The payload for the primitive ParDo transform.
+type ParDoPayload struct {
+ // (Required) The SdkFunctionSpec of the DoFn.
+ DoFn *SdkFunctionSpec `protobuf:"bytes,1,opt,name=do_fn,json=doFn" json:"do_fn,omitempty"`
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ Parameters []*Parameter `protobuf:"bytes,2,rep,name=parameters" json:"parameters,omitempty"`
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ SideInputs map[string]*SideInput `protobuf:"bytes,3,rep,name=side_inputs,json=sideInputs" json:"side_inputs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Optional) A mapping of local state names to state specifications.
+ StateSpecs map[string]*StateSpec `protobuf:"bytes,4,rep,name=state_specs,json=stateSpecs" json:"state_specs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // (Optional) A mapping of local timer names to timer specifications.
+ TimerSpecs map[string]*TimerSpec `protobuf:"bytes,5,rep,name=timer_specs,json=timerSpecs" json:"timer_specs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ // Whether the DoFn is splittable
+ Splittable bool `protobuf:"varint,6,opt,name=splittable" json:"splittable,omitempty"`
+}
+
+func (m *ParDoPayload) Reset() { *m = ParDoPayload{} }
+func (m *ParDoPayload) String() string { return proto.CompactTextString(m) }
+func (*ParDoPayload) ProtoMessage() {}
+func (*ParDoPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *ParDoPayload) GetDoFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.DoFn
+ }
+ return nil
+}
+
+func (m *ParDoPayload) GetParameters() []*Parameter {
+ if m != nil {
+ return m.Parameters
+ }
+ return nil
+}
+
+func (m *ParDoPayload) GetSideInputs() map[string]*SideInput {
+ if m != nil {
+ return m.SideInputs
+ }
+ return nil
+}
+
+func (m *ParDoPayload) GetStateSpecs() map[string]*StateSpec {
+ if m != nil {
+ return m.StateSpecs
+ }
+ return nil
+}
+
+func (m *ParDoPayload) GetTimerSpecs() map[string]*TimerSpec {
+ if m != nil {
+ return m.TimerSpecs
+ }
+ return nil
+}
+
+func (m *ParDoPayload) GetSplittable() bool {
+ if m != nil {
+ return m.Splittable
+ }
+ return false
+}
+
+// Parameters that a UDF might require.
+//
+// The details of how a runner sends these parameters to the SDK harness
+// are the subject of the Fn API.
+//
+// The details of how an SDK harness delivers them to the UDF is entirely
+// up to the SDK. (for some SDKs there may be parameters that are not
+// represented here if the runner doesn't need to do anything)
+//
+// Here, the parameters are simply indicators to the runner that they
+// need to run the function a particular way.
+//
+// TODO: the evolution of the Fn API will influence what needs explicit
+// representation here
+type Parameter struct {
+ Type Parameter_Type_Enum `protobuf:"varint,1,opt,name=type,enum=org.apache.beam.model.pipeline.v1.Parameter_Type_Enum" json:"type,omitempty"`
+}
+
+func (m *Parameter) Reset() { *m = Parameter{} }
+func (m *Parameter) String() string { return proto.CompactTextString(m) }
+func (*Parameter) ProtoMessage() {}
+func (*Parameter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *Parameter) GetType() Parameter_Type_Enum {
+ if m != nil {
+ return m.Type
+ }
+ return Parameter_Type_UNSPECIFIED
+}
+
+type Parameter_Type struct {
+}
+
+func (m *Parameter_Type) Reset() { *m = Parameter_Type{} }
+func (m *Parameter_Type) String() string { return proto.CompactTextString(m) }
+func (*Parameter_Type) ProtoMessage() {}
+func (*Parameter_Type) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6, 0} }
+
+type StateSpec struct {
+ // Types that are valid to be assigned to Spec:
+ // *StateSpec_ValueSpec
+ // *StateSpec_BagSpec
+ // *StateSpec_CombiningSpec
+ // *StateSpec_MapSpec
+ // *StateSpec_SetSpec
+ Spec isStateSpec_Spec `protobuf_oneof:"spec"`
+}
+
+func (m *StateSpec) Reset() { *m = StateSpec{} }
+func (m *StateSpec) String() string { return proto.CompactTextString(m) }
+func (*StateSpec) ProtoMessage() {}
+func (*StateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+type isStateSpec_Spec interface {
+ isStateSpec_Spec()
+}
+
+type StateSpec_ValueSpec struct {
+ ValueSpec *ValueStateSpec `protobuf:"bytes,1,opt,name=value_spec,json=valueSpec,oneof"`
+}
+type StateSpec_BagSpec struct {
+ BagSpec *BagStateSpec `protobuf:"bytes,2,opt,name=bag_spec,json=bagSpec,oneof"`
+}
+type StateSpec_CombiningSpec struct {
+ CombiningSpec *CombiningStateSpec `protobuf:"bytes,3,opt,name=combining_spec,json=combiningSpec,oneof"`
+}
+type StateSpec_MapSpec struct {
+ MapSpec *MapStateSpec `protobuf:"bytes,4,opt,name=map_spec,json=mapSpec,oneof"`
+}
+type StateSpec_SetSpec struct {
+ SetSpec *SetStateSpec `protobuf:"bytes,5,opt,name=set_spec,json=setSpec,oneof"`
+}
+
+func (*StateSpec_ValueSpec) isStateSpec_Spec() {}
+func (*StateSpec_BagSpec) isStateSpec_Spec() {}
+func (*StateSpec_CombiningSpec) isStateSpec_Spec() {}
+func (*StateSpec_MapSpec) isStateSpec_Spec() {}
+func (*StateSpec_SetSpec) isStateSpec_Spec() {}
+
+func (m *StateSpec) GetSpec() isStateSpec_Spec {
+ if m != nil {
+ return m.Spec
+ }
+ return nil
+}
+
+func (m *StateSpec) GetValueSpec() *ValueStateSpec {
+ if x, ok := m.GetSpec().(*StateSpec_ValueSpec); ok {
+ return x.ValueSpec
+ }
+ return nil
+}
+
+func (m *StateSpec) GetBagSpec() *BagStateSpec {
+ if x, ok := m.GetSpec().(*StateSpec_BagSpec); ok {
+ return x.BagSpec
+ }
+ return nil
+}
+
+func (m *StateSpec) GetCombiningSpec() *CombiningStateSpec {
+ if x, ok := m.GetSpec().(*StateSpec_CombiningSpec); ok {
+ return x.CombiningSpec
+ }
+ return nil
+}
+
+func (m *StateSpec) GetMapSpec() *MapStateSpec {
+ if x, ok := m.GetSpec().(*StateSpec_MapSpec); ok {
+ return x.MapSpec
+ }
+ return nil
+}
+
+func (m *StateSpec) GetSetSpec() *SetStateSpec {
+ if x, ok := m.GetSpec().(*StateSpec_SetSpec); ok {
+ return x.SetSpec
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*StateSpec) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _StateSpec_OneofMarshaler, _StateSpec_OneofUnmarshaler, _StateSpec_OneofSizer, []interface{}{
+ (*StateSpec_ValueSpec)(nil),
+ (*StateSpec_BagSpec)(nil),
+ (*StateSpec_CombiningSpec)(nil),
+ (*StateSpec_MapSpec)(nil),
+ (*StateSpec_SetSpec)(nil),
+ }
+}
+
+func _StateSpec_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*StateSpec)
+ // spec
+ switch x := m.Spec.(type) {
+ case *StateSpec_ValueSpec:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ValueSpec); err != nil {
+ return err
+ }
+ case *StateSpec_BagSpec:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.BagSpec); err != nil {
+ return err
+ }
+ case *StateSpec_CombiningSpec:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.CombiningSpec); err != nil {
+ return err
+ }
+ case *StateSpec_MapSpec:
+ b.EncodeVarint(4<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.MapSpec); err != nil {
+ return err
+ }
+ case *StateSpec_SetSpec:
+ b.EncodeVarint(5<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.SetSpec); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("StateSpec.Spec has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _StateSpec_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*StateSpec)
+ switch tag {
+ case 1: // spec.value_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(ValueStateSpec)
+ err := b.DecodeMessage(msg)
+ m.Spec = &StateSpec_ValueSpec{msg}
+ return true, err
+ case 2: // spec.bag_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(BagStateSpec)
+ err := b.DecodeMessage(msg)
+ m.Spec = &StateSpec_BagSpec{msg}
+ return true, err
+ case 3: // spec.combining_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(CombiningStateSpec)
+ err := b.DecodeMessage(msg)
+ m.Spec = &StateSpec_CombiningSpec{msg}
+ return true, err
+ case 4: // spec.map_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(MapStateSpec)
+ err := b.DecodeMessage(msg)
+ m.Spec = &StateSpec_MapSpec{msg}
+ return true, err
+ case 5: // spec.set_spec
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(SetStateSpec)
+ err := b.DecodeMessage(msg)
+ m.Spec = &StateSpec_SetSpec{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _StateSpec_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*StateSpec)
+ // spec
+ switch x := m.Spec.(type) {
+ case *StateSpec_ValueSpec:
+ s := proto.Size(x.ValueSpec)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateSpec_BagSpec:
+ s := proto.Size(x.BagSpec)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateSpec_CombiningSpec:
+ s := proto.Size(x.CombiningSpec)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateSpec_MapSpec:
+ s := proto.Size(x.MapSpec)
+ n += proto.SizeVarint(4<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *StateSpec_SetSpec:
+ s := proto.Size(x.SetSpec)
+ n += proto.SizeVarint(5<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type ValueStateSpec struct {
+ CoderId string `protobuf:"bytes,1,opt,name=coder_id,json=coderId" json:"coder_id,omitempty"`
+}
+
+func (m *ValueStateSpec) Reset() { *m = ValueStateSpec{} }
+func (m *ValueStateSpec) String() string { return proto.CompactTextString(m) }
+func (*ValueStateSpec) ProtoMessage() {}
+func (*ValueStateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *ValueStateSpec) GetCoderId() string {
+ if m != nil {
+ return m.CoderId
+ }
+ return ""
+}
+
+type BagStateSpec struct {
+ ElementCoderId string `protobuf:"bytes,1,opt,name=element_coder_id,json=elementCoderId" json:"element_coder_id,omitempty"`
+}
+
+func (m *BagStateSpec) Reset() { *m = BagStateSpec{} }
+func (m *BagStateSpec) String() string { return proto.CompactTextString(m) }
+func (*BagStateSpec) ProtoMessage() {}
+func (*BagStateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *BagStateSpec) GetElementCoderId() string {
+ if m != nil {
+ return m.ElementCoderId
+ }
+ return ""
+}
+
+type CombiningStateSpec struct {
+ AccumulatorCoderId string `protobuf:"bytes,1,opt,name=accumulator_coder_id,json=accumulatorCoderId" json:"accumulator_coder_id,omitempty"`
+ CombineFn *SdkFunctionSpec `protobuf:"bytes,2,opt,name=combine_fn,json=combineFn" json:"combine_fn,omitempty"`
+}
+
+func (m *CombiningStateSpec) Reset() { *m = CombiningStateSpec{} }
+func (m *CombiningStateSpec) String() string { return proto.CompactTextString(m) }
+func (*CombiningStateSpec) ProtoMessage() {}
+func (*CombiningStateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+func (m *CombiningStateSpec) GetAccumulatorCoderId() string {
+ if m != nil {
+ return m.AccumulatorCoderId
+ }
+ return ""
+}
+
+func (m *CombiningStateSpec) GetCombineFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.CombineFn
+ }
+ return nil
+}
+
+type MapStateSpec struct {
+ KeyCoderId string `protobuf:"bytes,1,opt,name=key_coder_id,json=keyCoderId" json:"key_coder_id,omitempty"`
+ ValueCoderId string `protobuf:"bytes,2,opt,name=value_coder_id,json=valueCoderId" json:"value_coder_id,omitempty"`
+}
+
+func (m *MapStateSpec) Reset() { *m = MapStateSpec{} }
+func (m *MapStateSpec) String() string { return proto.CompactTextString(m) }
+func (*MapStateSpec) ProtoMessage() {}
+func (*MapStateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+
+func (m *MapStateSpec) GetKeyCoderId() string {
+ if m != nil {
+ return m.KeyCoderId
+ }
+ return ""
+}
+
+func (m *MapStateSpec) GetValueCoderId() string {
+ if m != nil {
+ return m.ValueCoderId
+ }
+ return ""
+}
+
+type SetStateSpec struct {
+ ElementCoderId string `protobuf:"bytes,1,opt,name=element_coder_id,json=elementCoderId" json:"element_coder_id,omitempty"`
+}
+
+func (m *SetStateSpec) Reset() { *m = SetStateSpec{} }
+func (m *SetStateSpec) String() string { return proto.CompactTextString(m) }
+func (*SetStateSpec) ProtoMessage() {}
+func (*SetStateSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+
+func (m *SetStateSpec) GetElementCoderId() string {
+ if m != nil {
+ return m.ElementCoderId
+ }
+ return ""
+}
+
+type TimerSpec struct {
+ TimeDomain TimeDomain_Enum `protobuf:"varint,1,opt,name=time_domain,json=timeDomain,enum=org.apache.beam.model.pipeline.v1.TimeDomain_Enum" json:"time_domain,omitempty"`
+}
+
+func (m *TimerSpec) Reset() { *m = TimerSpec{} }
+func (m *TimerSpec) String() string { return proto.CompactTextString(m) }
+func (*TimerSpec) ProtoMessage() {}
+func (*TimerSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+
+func (m *TimerSpec) GetTimeDomain() TimeDomain_Enum {
+ if m != nil {
+ return m.TimeDomain
+ }
+ return TimeDomain_UNSPECIFIED
+}
+
+type IsBounded struct {
+}
+
+func (m *IsBounded) Reset() { *m = IsBounded{} }
+func (m *IsBounded) String() string { return proto.CompactTextString(m) }
+func (*IsBounded) ProtoMessage() {}
+func (*IsBounded) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+// The payload for the primitive Read transform.
+type ReadPayload struct {
+ // (Required) The SdkFunctionSpec of the source for this Read.
+ Source *SdkFunctionSpec `protobuf:"bytes,1,opt,name=source" json:"source,omitempty"`
+ // (Required) Whether the source is bounded or unbounded
+ IsBounded IsBounded_Enum `protobuf:"varint,2,opt,name=is_bounded,json=isBounded,enum=org.apache.beam.model.pipeline.v1.IsBounded_Enum" json:"is_bounded,omitempty"`
+}
+
+func (m *ReadPayload) Reset() { *m = ReadPayload{} }
+func (m *ReadPayload) String() string { return proto.CompactTextString(m) }
+func (*ReadPayload) ProtoMessage() {}
+func (*ReadPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
+func (m *ReadPayload) GetSource() *SdkFunctionSpec {
+ if m != nil {
+ return m.Source
+ }
+ return nil
+}
+
+func (m *ReadPayload) GetIsBounded() IsBounded_Enum {
+ if m != nil {
+ return m.IsBounded
+ }
+ return IsBounded_UNSPECIFIED
+}
+
+// The payload for the WindowInto transform.
+type WindowIntoPayload struct {
+ // (Required) The SdkFunctionSpec of the WindowFn.
+ WindowFn *SdkFunctionSpec `protobuf:"bytes,1,opt,name=window_fn,json=windowFn" json:"window_fn,omitempty"`
+}
+
+func (m *WindowIntoPayload) Reset() { *m = WindowIntoPayload{} }
+func (m *WindowIntoPayload) String() string { return proto.CompactTextString(m) }
+func (*WindowIntoPayload) ProtoMessage() {}
+func (*WindowIntoPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+
+func (m *WindowIntoPayload) GetWindowFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.WindowFn
+ }
+ return nil
+}
+
+// The payload for the special-but-not-primitive Combine transform.
+type CombinePayload struct {
+ // (Required) The SdkFunctionSpec of the CombineFn.
+ CombineFn *SdkFunctionSpec `protobuf:"bytes,1,opt,name=combine_fn,json=combineFn" json:"combine_fn,omitempty"`
+ // (Required) A reference to the Coder to use for accumulators of the CombineFn
+ AccumulatorCoderId string `protobuf:"bytes,2,opt,name=accumulator_coder_id,json=accumulatorCoderId" json:"accumulator_coder_id,omitempty"`
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ Parameters []*Parameter `protobuf:"bytes,3,rep,name=parameters" json:"parameters,omitempty"`
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ SideInputs map[string]*SideInput `protobuf:"bytes,4,rep,name=side_inputs,json=sideInputs" json:"side_inputs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *CombinePayload) Reset() { *m = CombinePayload{} }
+func (m *CombinePayload) String() string { return proto.CompactTextString(m) }
+func (*CombinePayload) ProtoMessage() {}
+func (*CombinePayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+
+func (m *CombinePayload) GetCombineFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.CombineFn
+ }
+ return nil
+}
+
+func (m *CombinePayload) GetAccumulatorCoderId() string {
+ if m != nil {
+ return m.AccumulatorCoderId
+ }
+ return ""
+}
+
+func (m *CombinePayload) GetParameters() []*Parameter {
+ if m != nil {
+ return m.Parameters
+ }
+ return nil
+}
+
+func (m *CombinePayload) GetSideInputs() map[string]*SideInput {
+ if m != nil {
+ return m.SideInputs
+ }
+ return nil
+}
+
+// The payload for the test-only primitive TestStream
+type TestStreamPayload struct {
+ // (Required) the coder for elements in the TestStream events
+ CoderId string `protobuf:"bytes,1,opt,name=coder_id,json=coderId" json:"coder_id,omitempty"`
+ Events []*TestStreamPayload_Event `protobuf:"bytes,2,rep,name=events" json:"events,omitempty"`
+}
+
+func (m *TestStreamPayload) Reset() { *m = TestStreamPayload{} }
+func (m *TestStreamPayload) String() string { return proto.CompactTextString(m) }
+func (*TestStreamPayload) ProtoMessage() {}
+func (*TestStreamPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+
+func (m *TestStreamPayload) GetCoderId() string {
+ if m != nil {
+ return m.CoderId
+ }
+ return ""
+}
+
+func (m *TestStreamPayload) GetEvents() []*TestStreamPayload_Event {
+ if m != nil {
+ return m.Events
+ }
+ return nil
+}
+
+type TestStreamPayload_Event struct {
+ // Types that are valid to be assigned to Event:
+ // *TestStreamPayload_Event_WatermarkEvent
+ // *TestStreamPayload_Event_ProcessingTimeEvent
+ // *TestStreamPayload_Event_ElementEvent
+ Event isTestStreamPayload_Event_Event `protobuf_oneof:"event"`
+}
+
+func (m *TestStreamPayload_Event) Reset() { *m = TestStreamPayload_Event{} }
+func (m *TestStreamPayload_Event) String() string { return proto.CompactTextString(m) }
+func (*TestStreamPayload_Event) ProtoMessage() {}
+func (*TestStreamPayload_Event) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18, 0} }
+
+type isTestStreamPayload_Event_Event interface {
+ isTestStreamPayload_Event_Event()
+}
+
+type TestStreamPayload_Event_WatermarkEvent struct {
+ WatermarkEvent *TestStreamPayload_Event_AdvanceWatermark `protobuf:"bytes,1,opt,name=watermark_event,json=watermarkEvent,oneof"`
+}
+type TestStreamPayload_Event_ProcessingTimeEvent struct {
+ ProcessingTimeEvent *TestStreamPayload_Event_AdvanceProcessingTime `protobuf:"bytes,2,opt,name=processing_time_event,json=processingTimeEvent,oneof"`
+}
+type TestStreamPayload_Event_ElementEvent struct {
+ ElementEvent *TestStreamPayload_Event_AddElements `protobuf:"bytes,3,opt,name=element_event,json=elementEvent,oneof"`
+}
+
+func (*TestStreamPayload_Event_WatermarkEvent) isTestStreamPayload_Event_Event() {}
+func (*TestStreamPayload_Event_ProcessingTimeEvent) isTestStreamPayload_Event_Event() {}
+func (*TestStreamPayload_Event_ElementEvent) isTestStreamPayload_Event_Event() {}
+
+func (m *TestStreamPayload_Event) GetEvent() isTestStreamPayload_Event_Event {
+ if m != nil {
+ return m.Event
+ }
+ return nil
+}
+
+func (m *TestStreamPayload_Event) GetWatermarkEvent() *TestStreamPayload_Event_AdvanceWatermark {
+ if x, ok := m.GetEvent().(*TestStreamPayload_Event_WatermarkEvent); ok {
+ return x.WatermarkEvent
+ }
+ return nil
+}
+
+func (m *TestStreamPayload_Event) GetProcessingTimeEvent() *TestStreamPayload_Event_AdvanceProcessingTime {
+ if x, ok := m.GetEvent().(*TestStreamPayload_Event_ProcessingTimeEvent); ok {
+ return x.ProcessingTimeEvent
+ }
+ return nil
+}
+
+func (m *TestStreamPayload_Event) GetElementEvent() *TestStreamPayload_Event_AddElements {
+ if x, ok := m.GetEvent().(*TestStreamPayload_Event_ElementEvent); ok {
+ return x.ElementEvent
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*TestStreamPayload_Event) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _TestStreamPayload_Event_OneofMarshaler, _TestStreamPayload_Event_OneofUnmarshaler, _TestStreamPayload_Event_OneofSizer, []interface{}{
+ (*TestStreamPayload_Event_WatermarkEvent)(nil),
+ (*TestStreamPayload_Event_ProcessingTimeEvent)(nil),
+ (*TestStreamPayload_Event_ElementEvent)(nil),
+ }
+}
+
+func _TestStreamPayload_Event_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*TestStreamPayload_Event)
+ // event
+ switch x := m.Event.(type) {
+ case *TestStreamPayload_Event_WatermarkEvent:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.WatermarkEvent); err != nil {
+ return err
+ }
+ case *TestStreamPayload_Event_ProcessingTimeEvent:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ProcessingTimeEvent); err != nil {
+ return err
+ }
+ case *TestStreamPayload_Event_ElementEvent:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ElementEvent); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("TestStreamPayload_Event.Event has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _TestStreamPayload_Event_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*TestStreamPayload_Event)
+ switch tag {
+ case 1: // event.watermark_event
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(TestStreamPayload_Event_AdvanceWatermark)
+ err := b.DecodeMessage(msg)
+ m.Event = &TestStreamPayload_Event_WatermarkEvent{msg}
+ return true, err
+ case 2: // event.processing_time_event
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(TestStreamPayload_Event_AdvanceProcessingTime)
+ err := b.DecodeMessage(msg)
+ m.Event = &TestStreamPayload_Event_ProcessingTimeEvent{msg}
+ return true, err
+ case 3: // event.element_event
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(TestStreamPayload_Event_AddElements)
+ err := b.DecodeMessage(msg)
+ m.Event = &TestStreamPayload_Event_ElementEvent{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _TestStreamPayload_Event_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*TestStreamPayload_Event)
+ // event
+ switch x := m.Event.(type) {
+ case *TestStreamPayload_Event_WatermarkEvent:
+ s := proto.Size(x.WatermarkEvent)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *TestStreamPayload_Event_ProcessingTimeEvent:
+ s := proto.Size(x.ProcessingTimeEvent)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *TestStreamPayload_Event_ElementEvent:
+ s := proto.Size(x.ElementEvent)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type TestStreamPayload_Event_AdvanceWatermark struct {
+ NewWatermark int64 `protobuf:"varint,1,opt,name=new_watermark,json=newWatermark" json:"new_watermark,omitempty"`
+}
+
+func (m *TestStreamPayload_Event_AdvanceWatermark) Reset() {
+ *m = TestStreamPayload_Event_AdvanceWatermark{}
+}
+func (m *TestStreamPayload_Event_AdvanceWatermark) String() string { return proto.CompactTextString(m) }
+func (*TestStreamPayload_Event_AdvanceWatermark) ProtoMessage() {}
+func (*TestStreamPayload_Event_AdvanceWatermark) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{18, 0, 0}
+}
+
+func (m *TestStreamPayload_Event_AdvanceWatermark) GetNewWatermark() int64 {
+ if m != nil {
+ return m.NewWatermark
+ }
+ return 0
+}
+
+type TestStreamPayload_Event_AdvanceProcessingTime struct {
+ AdvanceDuration int64 `protobuf:"varint,1,opt,name=advance_duration,json=advanceDuration" json:"advance_duration,omitempty"`
+}
+
+func (m *TestStreamPayload_Event_AdvanceProcessingTime) Reset() {
+ *m = TestStreamPayload_Event_AdvanceProcessingTime{}
+}
+func (m *TestStreamPayload_Event_AdvanceProcessingTime) String() string {
+ return proto.CompactTextString(m)
+}
+func (*TestStreamPayload_Event_AdvanceProcessingTime) ProtoMessage() {}
+func (*TestStreamPayload_Event_AdvanceProcessingTime) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{18, 0, 1}
+}
+
+func (m *TestStreamPayload_Event_AdvanceProcessingTime) GetAdvanceDuration() int64 {
+ if m != nil {
+ return m.AdvanceDuration
+ }
+ return 0
+}
+
+type TestStreamPayload_Event_AddElements struct {
+ Elements []*TestStreamPayload_TimestampedElement `protobuf:"bytes,1,rep,name=elements" json:"elements,omitempty"`
+}
+
+func (m *TestStreamPayload_Event_AddElements) Reset() { *m = TestStreamPayload_Event_AddElements{} }
+func (m *TestStreamPayload_Event_AddElements) String() string { return proto.CompactTextString(m) }
+func (*TestStreamPayload_Event_AddElements) ProtoMessage() {}
+func (*TestStreamPayload_Event_AddElements) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{18, 0, 2}
+}
+
+func (m *TestStreamPayload_Event_AddElements) GetElements() []*TestStreamPayload_TimestampedElement {
+ if m != nil {
+ return m.Elements
+ }
+ return nil
+}
+
+type TestStreamPayload_TimestampedElement struct {
+ EncodedElement []byte `protobuf:"bytes,1,opt,name=encoded_element,json=encodedElement,proto3" json:"encoded_element,omitempty"`
+ Timestamp int64 `protobuf:"varint,2,opt,name=timestamp" json:"timestamp,omitempty"`
+}
+
+func (m *TestStreamPayload_TimestampedElement) Reset() { *m = TestStreamPayload_TimestampedElement{} }
+func (m *TestStreamPayload_TimestampedElement) String() string { return proto.CompactTextString(m) }
+func (*TestStreamPayload_TimestampedElement) ProtoMessage() {}
+func (*TestStreamPayload_TimestampedElement) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{18, 1}
+}
+
+func (m *TestStreamPayload_TimestampedElement) GetEncodedElement() []byte {
+ if m != nil {
+ return m.EncodedElement
+ }
+ return nil
+}
+
+func (m *TestStreamPayload_TimestampedElement) GetTimestamp() int64 {
+ if m != nil {
+ return m.Timestamp
+ }
+ return 0
+}
+
+// The payload for the special-but-not-primitive WriteFiles transform.
+type WriteFilesPayload struct {
+ // (Required) The SdkFunctionSpec of the FileBasedSink.
+ Sink *SdkFunctionSpec `protobuf:"bytes,1,opt,name=sink" json:"sink,omitempty"`
+ // (Required) The format function.
+ FormatFunction *SdkFunctionSpec `protobuf:"bytes,2,opt,name=format_function,json=formatFunction" json:"format_function,omitempty"`
+ WindowedWrites bool `protobuf:"varint,3,opt,name=windowed_writes,json=windowedWrites" json:"windowed_writes,omitempty"`
+ RunnerDeterminedSharding bool `protobuf:"varint,4,opt,name=runner_determined_sharding,json=runnerDeterminedSharding" json:"runner_determined_sharding,omitempty"`
+ SideInputs map[string]*SideInput `protobuf:"bytes,5,rep,name=side_inputs,json=sideInputs" json:"side_inputs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *WriteFilesPayload) Reset() { *m = WriteFilesPayload{} }
+func (m *WriteFilesPayload) String() string { return proto.CompactTextString(m) }
+func (*WriteFilesPayload) ProtoMessage() {}
+func (*WriteFilesPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+
+func (m *WriteFilesPayload) GetSink() *SdkFunctionSpec {
+ if m != nil {
+ return m.Sink
+ }
+ return nil
+}
+
+func (m *WriteFilesPayload) GetFormatFunction() *SdkFunctionSpec {
+ if m != nil {
+ return m.FormatFunction
+ }
+ return nil
+}
+
+func (m *WriteFilesPayload) GetWindowedWrites() bool {
+ if m != nil {
+ return m.WindowedWrites
+ }
+ return false
+}
+
+func (m *WriteFilesPayload) GetRunnerDeterminedSharding() bool {
+ if m != nil {
+ return m.RunnerDeterminedSharding
+ }
+ return false
+}
+
+func (m *WriteFilesPayload) GetSideInputs() map[string]*SideInput {
+ if m != nil {
+ return m.SideInputs
+ }
+ return nil
+}
+
+// A coder, the binary format for serialization and deserialization of data in
+// a pipeline.
+type Coder struct {
+ // (Required) A specification for the coder, as a URN plus parameters. This
+ // may be a cross-language agreed-upon format, or it may be a "custom coder"
+ // that can only be used by a particular SDK. It does not include component
+ // coders, as it is beneficial for these to be comprehensible to a runner
+ // regardless of whether the binary format is agree-upon.
+ Spec *SdkFunctionSpec `protobuf:"bytes,1,opt,name=spec" json:"spec,omitempty"`
+ // (Optional) If this coder is parametric, such as ListCoder(VarIntCoder),
+ // this is a list of the components. In order for encodings to be identical,
+ // the SdkFunctionSpec and all components must be identical, recursively.
+ ComponentCoderIds []string `protobuf:"bytes,2,rep,name=component_coder_ids,json=componentCoderIds" json:"component_coder_ids,omitempty"`
+}
+
+func (m *Coder) Reset() { *m = Coder{} }
+func (m *Coder) String() string { return proto.CompactTextString(m) }
+func (*Coder) ProtoMessage() {}
+func (*Coder) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+
+func (m *Coder) GetSpec() *SdkFunctionSpec {
+ if m != nil {
+ return m.Spec
+ }
+ return nil
+}
+
+func (m *Coder) GetComponentCoderIds() []string {
+ if m != nil {
+ return m.ComponentCoderIds
+ }
+ return nil
+}
+
+// A windowing strategy describes the window function, triggering, allowed
+// lateness, and accumulation mode for a PCollection.
+//
+// TODO: consider inlining field on PCollection
+type WindowingStrategy struct {
+ // (Required) The SdkFunctionSpec of the UDF that assigns windows,
+ // merges windows, and shifts timestamps before they are
+ // combined according to the OutputTime.
+ WindowFn *SdkFunctionSpec `protobuf:"bytes,1,opt,name=window_fn,json=windowFn" json:"window_fn,omitempty"`
+ // (Required) Whether or not the window fn is merging.
+ //
+ // This knowledge is required for many optimizations.
+ MergeStatus MergeStatus_Enum `protobuf:"varint,2,opt,name=merge_status,json=mergeStatus,enum=org.apache.beam.model.pipeline.v1.MergeStatus_Enum" json:"merge_status,omitempty"`
+ // (Required) The coder for the windows of this PCollection.
+ WindowCoderId string `protobuf:"bytes,3,opt,name=window_coder_id,json=windowCoderId" json:"window_coder_id,omitempty"`
+ // (Required) The trigger to use when grouping this PCollection.
+ Trigger *Trigger `protobuf:"bytes,4,opt,name=trigger" json:"trigger,omitempty"`
+ // (Required) The accumulation mode indicates whether new panes are a full
+ // replacement for prior panes or whether they are deltas to be combined
+ // with other panes (the combine should correspond to whatever the upstream
+ // grouping transform is).
+ AccumulationMode AccumulationMode_Enum `protobuf:"varint,5,opt,name=accumulation_mode,json=accumulationMode,enum=org.apache.beam.model.pipeline.v1.AccumulationMode_Enum" json:"accumulation_mode,omitempty"`
+ // (Required) The OutputTime specifies, for a grouping transform, how to
+ // compute the aggregate timestamp. The window_fn will first possibly shift
+ // it later, then the OutputTime takes the max, min, or ignores it and takes
+ // the end of window.
+ //
+ // This is actually only for input to grouping transforms, but since they
+ // may be introduced in runner-specific ways, it is carried along with the
+ // windowing strategy.
+ OutputTime OutputTime_Enum `protobuf:"varint,6,opt,name=output_time,json=outputTime,enum=org.apache.beam.model.pipeline.v1.OutputTime_Enum" json:"output_time,omitempty"`
+ // (Required) Indicate when output should be omitted upon window expiration.
+ ClosingBehavior ClosingBehavior_Enum `protobuf:"varint,7,opt,name=closing_behavior,json=closingBehavior,enum=org.apache.beam.model.pipeline.v1.ClosingBehavior_Enum" json:"closing_behavior,omitempty"`
+ // (Required) The duration, in milliseconds, beyond the end of a window at
+ // which the window becomes droppable.
+ AllowedLateness int64 `protobuf:"varint,8,opt,name=allowed_lateness,json=allowedLateness" json:"allowed_lateness,omitempty"`
+ // (Required) Indicate whether empty on-time panes should be omitted.
+ OnTimeBehavior OnTimeBehavior_Enum `protobuf:"varint,9,opt,name=OnTimeBehavior,enum=org.apache.beam.model.pipeline.v1.OnTimeBehavior_Enum" json:"OnTimeBehavior,omitempty"`
+ // (Required) Whether or not the window fn assigns inputs to exactly one window
+ //
+ // This knowledge is required for some optimizations
+ AssignsToOneWindow bool `protobuf:"varint,10,opt,name=assigns_to_one_window,json=assignsToOneWindow" json:"assigns_to_one_window,omitempty"`
+}
+
+func (m *WindowingStrategy) Reset() { *m = WindowingStrategy{} }
+func (m *WindowingStrategy) String() string { return proto.CompactTextString(m) }
+func (*WindowingStrategy) ProtoMessage() {}
+func (*WindowingStrategy) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+
+func (m *WindowingStrategy) GetWindowFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.WindowFn
+ }
+ return nil
+}
+
+func (m *WindowingStrategy) GetMergeStatus() MergeStatus_Enum {
+ if m != nil {
+ return m.MergeStatus
+ }
+ return MergeStatus_UNSPECIFIED
+}
+
+func (m *WindowingStrategy) GetWindowCoderId() string {
+ if m != nil {
+ return m.WindowCoderId
+ }
+ return ""
+}
+
+func (m *WindowingStrategy) GetTrigger() *Trigger {
+ if m != nil {
+ return m.Trigger
+ }
+ return nil
+}
+
+func (m *WindowingStrategy) GetAccumulationMode() AccumulationMode_Enum {
+ if m != nil {
+ return m.AccumulationMode
+ }
+ return AccumulationMode_UNSPECIFIED
+}
+
+func (m *WindowingStrategy) GetOutputTime() OutputTime_Enum {
+ if m != nil {
+ return m.OutputTime
+ }
+ return OutputTime_UNSPECIFIED
+}
+
+func (m *WindowingStrategy) GetClosingBehavior() ClosingBehavior_Enum {
+ if m != nil {
+ return m.ClosingBehavior
+ }
+ return ClosingBehavior_UNSPECIFIED
+}
+
+func (m *WindowingStrategy) GetAllowedLateness() int64 {
+ if m != nil {
+ return m.AllowedLateness
+ }
+ return 0
+}
+
+func (m *WindowingStrategy) GetOnTimeBehavior() OnTimeBehavior_Enum {
+ if m != nil {
+ return m.OnTimeBehavior
+ }
+ return OnTimeBehavior_UNSPECIFIED
+}
+
+func (m *WindowingStrategy) GetAssignsToOneWindow() bool {
+ if m != nil {
+ return m.AssignsToOneWindow
+ }
+ return false
+}
+
+// Whether or not a PCollection's WindowFn is non-merging, merging, or
+// merging-but-already-merged, in which case a subsequent GroupByKey is almost
+// always going to do something the user does not want
+type MergeStatus struct {
+}
+
+func (m *MergeStatus) Reset() { *m = MergeStatus{} }
+func (m *MergeStatus) String() string { return proto.CompactTextString(m) }
+func (*MergeStatus) ProtoMessage() {}
+func (*MergeStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+
+// Whether or not subsequent outputs of aggregations should be entire
+// replacement values or just the aggregation of inputs received since
+// the prior output.
+type AccumulationMode struct {
+}
+
+func (m *AccumulationMode) Reset() { *m = AccumulationMode{} }
+func (m *AccumulationMode) String() string { return proto.CompactTextString(m) }
+func (*AccumulationMode) ProtoMessage() {}
+func (*AccumulationMode) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+
+// Controls whether or not an aggregating transform should output data
+// when a window expires.
+type ClosingBehavior struct {
+}
+
+func (m *ClosingBehavior) Reset() { *m = ClosingBehavior{} }
+func (m *ClosingBehavior) String() string { return proto.CompactTextString(m) }
+func (*ClosingBehavior) ProtoMessage() {}
+func (*ClosingBehavior) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+
+// Controls whether or not an aggregating transform should output data
+// when an on-time pane is empty.
+type OnTimeBehavior struct {
+}
+
+func (m *OnTimeBehavior) Reset() { *m = OnTimeBehavior{} }
+func (m *OnTimeBehavior) String() string { return proto.CompactTextString(m) }
+func (*OnTimeBehavior) ProtoMessage() {}
+func (*OnTimeBehavior) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+
+// When a number of windowed, timestamped inputs are aggregated, the timestamp
+// for the resulting output.
+type OutputTime struct {
+}
+
+func (m *OutputTime) Reset() { *m = OutputTime{} }
+func (m *OutputTime) String() string { return proto.CompactTextString(m) }
+func (*OutputTime) ProtoMessage() {}
+func (*OutputTime) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+
+// The different time domains in the Beam model.
+type TimeDomain struct {
+}
+
+func (m *TimeDomain) Reset() { *m = TimeDomain{} }
+func (m *TimeDomain) String() string { return proto.CompactTextString(m) }
+func (*TimeDomain) ProtoMessage() {}
+func (*TimeDomain) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+
+// A small DSL for expressing when to emit new aggregations
+// from a GroupByKey or CombinePerKey
+//
+// A trigger is described in terms of when it is _ready_ to permit output.
+type Trigger struct {
+ // The full disjoint union of possible triggers.
+ //
+ // Types that are valid to be assigned to Trigger:
+ // *Trigger_AfterAll_
+ // *Trigger_AfterAny_
+ // *Trigger_AfterEach_
+ // *Trigger_AfterEndOfWindow_
+ // *Trigger_AfterProcessingTime_
+ // *Trigger_AfterSynchronizedProcessingTime_
+ // *Trigger_Always_
+ // *Trigger_Default_
+ // *Trigger_ElementCount_
+ // *Trigger_Never_
+ // *Trigger_OrFinally_
+ // *Trigger_Repeat_
+ Trigger isTrigger_Trigger `protobuf_oneof:"trigger"`
+}
+
+func (m *Trigger) Reset() { *m = Trigger{} }
+func (m *Trigger) String() string { return proto.CompactTextString(m) }
+func (*Trigger) ProtoMessage() {}
+func (*Trigger) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+
+type isTrigger_Trigger interface {
+ isTrigger_Trigger()
+}
+
+type Trigger_AfterAll_ struct {
+ AfterAll *Trigger_AfterAll `protobuf:"bytes,1,opt,name=after_all,json=afterAll,oneof"`
+}
+type Trigger_AfterAny_ struct {
+ AfterAny *Trigger_AfterAny `protobuf:"bytes,2,opt,name=after_any,json=afterAny,oneof"`
+}
+type Trigger_AfterEach_ struct {
+ AfterEach *Trigger_AfterEach `protobuf:"bytes,3,opt,name=after_each,json=afterEach,oneof"`
+}
+type Trigger_AfterEndOfWindow_ struct {
+ AfterEndOfWindow *Trigger_AfterEndOfWindow `protobuf:"bytes,4,opt,name=after_end_of_window,json=afterEndOfWindow,oneof"`
+}
+type Trigger_AfterProcessingTime_ struct {
+ AfterProcessingTime *Trigger_AfterProcessingTime `protobuf:"bytes,5,opt,name=after_processing_time,json=afterProcessingTime,oneof"`
+}
+type Trigger_AfterSynchronizedProcessingTime_ struct {
+ AfterSynchronizedProcessingTime *Trigger_AfterSynchronizedProcessingTime `protobuf:"bytes,6,opt,name=after_synchronized_processing_time,json=afterSynchronizedProcessingTime,oneof"`
+}
+type Trigger_Always_ struct {
+ Always *Trigger_Always `protobuf:"bytes,12,opt,name=always,oneof"`
+}
+type Trigger_Default_ struct {
+ Default *Trigger_Default `protobuf:"bytes,7,opt,name=default,oneof"`
+}
+type Trigger_ElementCount_ struct {
+ ElementCount *Trigger_ElementCount `protobuf:"bytes,8,opt,name=element_count,json=elementCount,oneof"`
+}
+type Trigger_Never_ struct {
+ Never *Trigger_Never `protobuf:"bytes,9,opt,name=never,oneof"`
+}
+type Trigger_OrFinally_ struct {
+ OrFinally *Trigger_OrFinally `protobuf:"bytes,10,opt,name=or_finally,json=orFinally,oneof"`
+}
+type Trigger_Repeat_ struct {
+ Repeat *Trigger_Repeat `protobuf:"bytes,11,opt,name=repeat,oneof"`
+}
+
+func (*Trigger_AfterAll_) isTrigger_Trigger() {}
+func (*Trigger_AfterAny_) isTrigger_Trigger() {}
+func (*Trigger_AfterEach_) isTrigger_Trigger() {}
+func (*Trigger_AfterEndOfWindow_) isTrigger_Trigger() {}
+func (*Trigger_AfterProcessingTime_) isTrigger_Trigger() {}
+func (*Trigger_AfterSynchronizedProcessingTime_) isTrigger_Trigger() {}
+func (*Trigger_Always_) isTrigger_Trigger() {}
+func (*Trigger_Default_) isTrigger_Trigger() {}
+func (*Trigger_ElementCount_) isTrigger_Trigger() {}
+func (*Trigger_Never_) isTrigger_Trigger() {}
+func (*Trigger_OrFinally_) isTrigger_Trigger() {}
+func (*Trigger_Repeat_) isTrigger_Trigger() {}
+
+func (m *Trigger) GetTrigger() isTrigger_Trigger {
+ if m != nil {
+ return m.Trigger
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterAll() *Trigger_AfterAll {
+ if x, ok := m.GetTrigger().(*Trigger_AfterAll_); ok {
+ return x.AfterAll
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterAny() *Trigger_AfterAny {
+ if x, ok := m.GetTrigger().(*Trigger_AfterAny_); ok {
+ return x.AfterAny
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterEach() *Trigger_AfterEach {
+ if x, ok := m.GetTrigger().(*Trigger_AfterEach_); ok {
+ return x.AfterEach
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterEndOfWindow() *Trigger_AfterEndOfWindow {
+ if x, ok := m.GetTrigger().(*Trigger_AfterEndOfWindow_); ok {
+ return x.AfterEndOfWindow
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterProcessingTime() *Trigger_AfterProcessingTime {
+ if x, ok := m.GetTrigger().(*Trigger_AfterProcessingTime_); ok {
+ return x.AfterProcessingTime
+ }
+ return nil
+}
+
+func (m *Trigger) GetAfterSynchronizedProcessingTime() *Trigger_AfterSynchronizedProcessingTime {
+ if x, ok := m.GetTrigger().(*Trigger_AfterSynchronizedProcessingTime_); ok {
+ return x.AfterSynchronizedProcessingTime
+ }
+ return nil
+}
+
+func (m *Trigger) GetAlways() *Trigger_Always {
+ if x, ok := m.GetTrigger().(*Trigger_Always_); ok {
+ return x.Always
+ }
+ return nil
+}
+
+func (m *Trigger) GetDefault() *Trigger_Default {
+ if x, ok := m.GetTrigger().(*Trigger_Default_); ok {
+ return x.Default
+ }
+ return nil
+}
+
+func (m *Trigger) GetElementCount() *Trigger_ElementCount {
+ if x, ok := m.GetTrigger().(*Trigger_ElementCount_); ok {
+ return x.ElementCount
+ }
+ return nil
+}
+
+func (m *Trigger) GetNever() *Trigger_Never {
+ if x, ok := m.GetTrigger().(*Trigger_Never_); ok {
+ return x.Never
+ }
+ return nil
+}
+
+func (m *Trigger) GetOrFinally() *Trigger_OrFinally {
+ if x, ok := m.GetTrigger().(*Trigger_OrFinally_); ok {
+ return x.OrFinally
+ }
+ return nil
+}
+
+func (m *Trigger) GetRepeat() *Trigger_Repeat {
+ if x, ok := m.GetTrigger().(*Trigger_Repeat_); ok {
+ return x.Repeat
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*Trigger) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _Trigger_OneofMarshaler, _Trigger_OneofUnmarshaler, _Trigger_OneofSizer, []interface{}{
+ (*Trigger_AfterAll_)(nil),
+ (*Trigger_AfterAny_)(nil),
+ (*Trigger_AfterEach_)(nil),
+ (*Trigger_AfterEndOfWindow_)(nil),
+ (*Trigger_AfterProcessingTime_)(nil),
+ (*Trigger_AfterSynchronizedProcessingTime_)(nil),
+ (*Trigger_Always_)(nil),
+ (*Trigger_Default_)(nil),
+ (*Trigger_ElementCount_)(nil),
+ (*Trigger_Never_)(nil),
+ (*Trigger_OrFinally_)(nil),
+ (*Trigger_Repeat_)(nil),
+ }
+}
+
+func _Trigger_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*Trigger)
+ // trigger
+ switch x := m.Trigger.(type) {
+ case *Trigger_AfterAll_:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterAll); err != nil {
+ return err
+ }
+ case *Trigger_AfterAny_:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterAny); err != nil {
+ return err
+ }
+ case *Trigger_AfterEach_:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterEach); err != nil {
+ return err
+ }
+ case *Trigger_AfterEndOfWindow_:
+ b.EncodeVarint(4<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterEndOfWindow); err != nil {
+ return err
+ }
+ case *Trigger_AfterProcessingTime_:
+ b.EncodeVarint(5<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterProcessingTime); err != nil {
+ return err
+ }
+ case *Trigger_AfterSynchronizedProcessingTime_:
+ b.EncodeVarint(6<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AfterSynchronizedProcessingTime); err != nil {
+ return err
+ }
+ case *Trigger_Always_:
+ b.EncodeVarint(12<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Always); err != nil {
+ return err
+ }
+ case *Trigger_Default_:
+ b.EncodeVarint(7<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Default); err != nil {
+ return err
+ }
+ case *Trigger_ElementCount_:
+ b.EncodeVarint(8<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.ElementCount); err != nil {
+ return err
+ }
+ case *Trigger_Never_:
+ b.EncodeVarint(9<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Never); err != nil {
+ return err
+ }
+ case *Trigger_OrFinally_:
+ b.EncodeVarint(10<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.OrFinally); err != nil {
+ return err
+ }
+ case *Trigger_Repeat_:
+ b.EncodeVarint(11<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Repeat); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("Trigger.Trigger has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _Trigger_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*Trigger)
+ switch tag {
+ case 1: // trigger.after_all
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterAll)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterAll_{msg}
+ return true, err
+ case 2: // trigger.after_any
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterAny)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterAny_{msg}
+ return true, err
+ case 3: // trigger.after_each
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterEach)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterEach_{msg}
+ return true, err
+ case 4: // trigger.after_end_of_window
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterEndOfWindow)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterEndOfWindow_{msg}
+ return true, err
+ case 5: // trigger.after_processing_time
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterProcessingTime)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterProcessingTime_{msg}
+ return true, err
+ case 6: // trigger.after_synchronized_processing_time
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_AfterSynchronizedProcessingTime)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_AfterSynchronizedProcessingTime_{msg}
+ return true, err
+ case 12: // trigger.always
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_Always)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_Always_{msg}
+ return true, err
+ case 7: // trigger.default
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_Default)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_Default_{msg}
+ return true, err
+ case 8: // trigger.element_count
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_ElementCount)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_ElementCount_{msg}
+ return true, err
+ case 9: // trigger.never
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_Never)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_Never_{msg}
+ return true, err
+ case 10: // trigger.or_finally
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_OrFinally)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_OrFinally_{msg}
+ return true, err
+ case 11: // trigger.repeat
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(Trigger_Repeat)
+ err := b.DecodeMessage(msg)
+ m.Trigger = &Trigger_Repeat_{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _Trigger_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*Trigger)
+ // trigger
+ switch x := m.Trigger.(type) {
+ case *Trigger_AfterAll_:
+ s := proto.Size(x.AfterAll)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_AfterAny_:
+ s := proto.Size(x.AfterAny)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_AfterEach_:
+ s := proto.Size(x.AfterEach)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_AfterEndOfWindow_:
+ s := proto.Size(x.AfterEndOfWindow)
+ n += proto.SizeVarint(4<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_AfterProcessingTime_:
+ s := proto.Size(x.AfterProcessingTime)
+ n += proto.SizeVarint(5<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_AfterSynchronizedProcessingTime_:
+ s := proto.Size(x.AfterSynchronizedProcessingTime)
+ n += proto.SizeVarint(6<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_Always_:
+ s := proto.Size(x.Always)
+ n += proto.SizeVarint(12<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_Default_:
+ s := proto.Size(x.Default)
+ n += proto.SizeVarint(7<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_ElementCount_:
+ s := proto.Size(x.ElementCount)
+ n += proto.SizeVarint(8<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_Never_:
+ s := proto.Size(x.Never)
+ n += proto.SizeVarint(9<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_OrFinally_:
+ s := proto.Size(x.OrFinally)
+ n += proto.SizeVarint(10<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *Trigger_Repeat_:
+ s := proto.Size(x.Repeat)
+ n += proto.SizeVarint(11<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+// Ready when all subtriggers are ready.
+type Trigger_AfterAll struct {
+ Subtriggers []*Trigger `protobuf:"bytes,1,rep,name=subtriggers" json:"subtriggers,omitempty"`
+}
+
+func (m *Trigger_AfterAll) Reset() { *m = Trigger_AfterAll{} }
+func (m *Trigger_AfterAll) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterAll) ProtoMessage() {}
+func (*Trigger_AfterAll) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 0} }
+
+func (m *Trigger_AfterAll) GetSubtriggers() []*Trigger {
+ if m != nil {
+ return m.Subtriggers
+ }
+ return nil
+}
+
+// Ready when any subtrigger is ready.
+type Trigger_AfterAny struct {
+ Subtriggers []*Trigger `protobuf:"bytes,1,rep,name=subtriggers" json:"subtriggers,omitempty"`
+}
+
+func (m *Trigger_AfterAny) Reset() { *m = Trigger_AfterAny{} }
+func (m *Trigger_AfterAny) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterAny) ProtoMessage() {}
+func (*Trigger_AfterAny) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 1} }
+
+func (m *Trigger_AfterAny) GetSubtriggers() []*Trigger {
+ if m != nil {
+ return m.Subtriggers
+ }
+ return nil
+}
+
+// Starting with the first subtrigger, ready when the _current_ subtrigger
+// is ready. After output, advances the current trigger by one.
+type Trigger_AfterEach struct {
+ Subtriggers []*Trigger `protobuf:"bytes,1,rep,name=subtriggers" json:"subtriggers,omitempty"`
+}
+
+func (m *Trigger_AfterEach) Reset() { *m = Trigger_AfterEach{} }
+func (m *Trigger_AfterEach) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterEach) ProtoMessage() {}
+func (*Trigger_AfterEach) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 2} }
+
+func (m *Trigger_AfterEach) GetSubtriggers() []*Trigger {
+ if m != nil {
+ return m.Subtriggers
+ }
+ return nil
+}
+
+// Ready after the input watermark is past the end of the window.
+//
+// May have implicitly-repeated subtriggers for early and late firings.
+// When the end of the window is reached, the trigger transitions between
+// the subtriggers.
+type Trigger_AfterEndOfWindow struct {
+ // (Optional) A trigger governing output prior to the end of the window.
+ EarlyFirings *Trigger `protobuf:"bytes,1,opt,name=early_firings,json=earlyFirings" json:"early_firings,omitempty"`
+ // (Optional) A trigger governing output after the end of the window.
+ LateFirings *Trigger `protobuf:"bytes,2,opt,name=late_firings,json=lateFirings" json:"late_firings,omitempty"`
+}
+
+func (m *Trigger_AfterEndOfWindow) Reset() { *m = Trigger_AfterEndOfWindow{} }
+func (m *Trigger_AfterEndOfWindow) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterEndOfWindow) ProtoMessage() {}
+func (*Trigger_AfterEndOfWindow) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 3} }
+
+func (m *Trigger_AfterEndOfWindow) GetEarlyFirings() *Trigger {
+ if m != nil {
+ return m.EarlyFirings
+ }
+ return nil
+}
+
+func (m *Trigger_AfterEndOfWindow) GetLateFirings() *Trigger {
+ if m != nil {
+ return m.LateFirings
+ }
+ return nil
+}
+
+// After input arrives, ready when the specified delay has passed.
+type Trigger_AfterProcessingTime struct {
+ // (Required) The transforms to apply to an arriving element's timestamp,
+ // in order
+ TimestampTransforms []*TimestampTransform `protobuf:"bytes,1,rep,name=timestamp_transforms,json=timestampTransforms" json:"timestamp_transforms,omitempty"`
+}
+
+func (m *Trigger_AfterProcessingTime) Reset() { *m = Trigger_AfterProcessingTime{} }
+func (m *Trigger_AfterProcessingTime) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterProcessingTime) ProtoMessage() {}
+func (*Trigger_AfterProcessingTime) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 4} }
+
+func (m *Trigger_AfterProcessingTime) GetTimestampTransforms() []*TimestampTransform {
+ if m != nil {
+ return m.TimestampTransforms
+ }
+ return nil
+}
+
+// Ready whenever upstream processing time has all caught up with
+// the arrival time of an input element
+type Trigger_AfterSynchronizedProcessingTime struct {
+}
+
+func (m *Trigger_AfterSynchronizedProcessingTime) Reset() {
+ *m = Trigger_AfterSynchronizedProcessingTime{}
+}
+func (m *Trigger_AfterSynchronizedProcessingTime) String() string { return proto.CompactTextString(m) }
+func (*Trigger_AfterSynchronizedProcessingTime) ProtoMessage() {}
+func (*Trigger_AfterSynchronizedProcessingTime) Descriptor() ([]byte, []int) {
+ return fileDescriptor0, []int{28, 5}
+}
+
+// The default trigger. Equivalent to Repeat { AfterEndOfWindow } but
+// specially denoted to indicate the user did not alter the triggering.
+type Trigger_Default struct {
+}
+
+func (m *Trigger_Default) Reset() { *m = Trigger_Default{} }
+func (m *Trigger_Default) String() string { return proto.CompactTextString(m) }
+func (*Trigger_Default) ProtoMessage() {}
+func (*Trigger_Default) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 6} }
+
+// Ready whenever the requisite number of input elements have arrived
+type Trigger_ElementCount struct {
+ ElementCount int32 `protobuf:"varint,1,opt,name=element_count,json=elementCount" json:"element_count,omitempty"`
+}
+
+func (m *Trigger_ElementCount) Reset() { *m = Trigger_ElementCount{} }
+func (m *Trigger_ElementCount) String() string { return proto.CompactTextString(m) }
+func (*Trigger_ElementCount) ProtoMessage() {}
+func (*Trigger_ElementCount) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 7} }
+
+func (m *Trigger_ElementCount) GetElementCount() int32 {
+ if m != nil {
+ return m.ElementCount
+ }
+ return 0
+}
+
+// Never ready. There will only be an ON_TIME output and a final
+// output at window expiration.
+type Trigger_Never struct {
+}
+
+func (m *Trigger_Never) Reset() { *m = Trigger_Never{} }
+func (m *Trigger_Never) String() string { return proto.CompactTextString(m) }
+func (*Trigger_Never) ProtoMessage() {}
+func (*Trigger_Never) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 8} }
+
+// Always ready. This can also be expressed as ElementCount(1) but
+// is more explicit.
+type Trigger_Always struct {
+}
+
+func (m *Trigger_Always) Reset() { *m = Trigger_Always{} }
+func (m *Trigger_Always) String() string { return proto.CompactTextString(m) }
+func (*Trigger_Always) ProtoMessage() {}
+func (*Trigger_Always) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 9} }
+
+// Ready whenever either of its subtriggers are ready, but finishes output
+// when the finally subtrigger fires.
+type Trigger_OrFinally struct {
+ // (Required) Trigger governing main output; may fire repeatedly.
+ Main *Trigger `protobuf:"bytes,1,opt,name=main" json:"main,omitempty"`
+ // (Required) Trigger governing termination of output.
+ Finally *Trigger `protobuf:"bytes,2,opt,name=finally" json:"finally,omitempty"`
+}
+
+func (m *Trigger_OrFinally) Reset() { *m = Trigger_OrFinally{} }
+func (m *Trigger_OrFinally) String() string { return proto.CompactTextString(m) }
+func (*Trigger_OrFinally) ProtoMessage() {}
+func (*Trigger_OrFinally) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 10} }
+
+func (m *Trigger_OrFinally) GetMain() *Trigger {
+ if m != nil {
+ return m.Main
+ }
+ return nil
+}
+
+func (m *Trigger_OrFinally) GetFinally() *Trigger {
+ if m != nil {
+ return m.Finally
+ }
+ return nil
+}
+
+// Ready whenever the subtrigger is ready; resets state when the subtrigger
+// completes.
+type Trigger_Repeat struct {
+ // (Require) Trigger that is run repeatedly.
+ Subtrigger *Trigger `protobuf:"bytes,1,opt,name=subtrigger" json:"subtrigger,omitempty"`
+}
+
+func (m *Trigger_Repeat) Reset() { *m = Trigger_Repeat{} }
+func (m *Trigger_Repeat) String() string { return proto.CompactTextString(m) }
+func (*Trigger_Repeat) ProtoMessage() {}
+func (*Trigger_Repeat) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28, 11} }
+
+func (m *Trigger_Repeat) GetSubtrigger() *Trigger {
+ if m != nil {
+ return m.Subtrigger
+ }
+ return nil
+}
+
+// A specification for a transformation on a timestamp.
+//
+// Primarily used by AfterProcessingTime triggers to transform
+// the arrival time of input to a target time for firing.
+type TimestampTransform struct {
+ // Types that are valid to be assigned to TimestampTransform:
+ // *TimestampTransform_Delay_
+ // *TimestampTransform_AlignTo_
+ TimestampTransform isTimestampTransform_TimestampTransform `protobuf_oneof:"timestamp_transform"`
+}
+
+func (m *TimestampTransform) Reset() { *m = TimestampTransform{} }
+func (m *TimestampTransform) String() string { return proto.CompactTextString(m) }
+func (*TimestampTransform) ProtoMessage() {}
+func (*TimestampTransform) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+
+type isTimestampTransform_TimestampTransform interface {
+ isTimestampTransform_TimestampTransform()
+}
+
+type TimestampTransform_Delay_ struct {
+ Delay *TimestampTransform_Delay `protobuf:"bytes,1,opt,name=delay,oneof"`
+}
+type TimestampTransform_AlignTo_ struct {
+ AlignTo *TimestampTransform_AlignTo `protobuf:"bytes,2,opt,name=align_to,json=alignTo,oneof"`
+}
+
+func (*TimestampTransform_Delay_) isTimestampTransform_TimestampTransform() {}
+func (*TimestampTransform_AlignTo_) isTimestampTransform_TimestampTransform() {}
+
+func (m *TimestampTransform) GetTimestampTransform() isTimestampTransform_TimestampTransform {
+ if m != nil {
+ return m.TimestampTransform
+ }
+ return nil
+}
+
+func (m *TimestampTransform) GetDelay() *TimestampTransform_Delay {
+ if x, ok := m.GetTimestampTransform().(*TimestampTransform_Delay_); ok {
+ return x.Delay
+ }
+ return nil
+}
+
+func (m *TimestampTransform) GetAlignTo() *TimestampTransform_AlignTo {
+ if x, ok := m.GetTimestampTransform().(*TimestampTransform_AlignTo_); ok {
+ return x.AlignTo
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*TimestampTransform) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _TimestampTransform_OneofMarshaler, _TimestampTransform_OneofUnmarshaler, _TimestampTransform_OneofSizer, []interface{}{
+ (*TimestampTransform_Delay_)(nil),
+ (*TimestampTransform_AlignTo_)(nil),
+ }
+}
+
+func _TimestampTransform_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*TimestampTransform)
+ // timestamp_transform
+ switch x := m.TimestampTransform.(type) {
+ case *TimestampTransform_Delay_:
+ b.EncodeVarint(1<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Delay); err != nil {
+ return err
+ }
+ case *TimestampTransform_AlignTo_:
+ b.EncodeVarint(2<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.AlignTo); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("TimestampTransform.TimestampTransform has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _TimestampTransform_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*TimestampTransform)
+ switch tag {
+ case 1: // timestamp_transform.delay
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(TimestampTransform_Delay)
+ err := b.DecodeMessage(msg)
+ m.TimestampTransform = &TimestampTransform_Delay_{msg}
+ return true, err
+ case 2: // timestamp_transform.align_to
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(TimestampTransform_AlignTo)
+ err := b.DecodeMessage(msg)
+ m.TimestampTransform = &TimestampTransform_AlignTo_{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _TimestampTransform_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*TimestampTransform)
+ // timestamp_transform
+ switch x := m.TimestampTransform.(type) {
+ case *TimestampTransform_Delay_:
+ s := proto.Size(x.Delay)
+ n += proto.SizeVarint(1<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case *TimestampTransform_AlignTo_:
+ s := proto.Size(x.AlignTo)
+ n += proto.SizeVarint(2<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type TimestampTransform_Delay struct {
+ // (Required) The delay, in milliseconds.
+ DelayMillis int64 `protobuf:"varint,1,opt,name=delay_millis,json=delayMillis" json:"delay_millis,omitempty"`
+}
+
+func (m *TimestampTransform_Delay) Reset() { *m = TimestampTransform_Delay{} }
+func (m *TimestampTransform_Delay) String() string { return proto.CompactTextString(m) }
+func (*TimestampTransform_Delay) ProtoMessage() {}
+func (*TimestampTransform_Delay) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29, 0} }
+
+func (m *TimestampTransform_Delay) GetDelayMillis() int64 {
+ if m != nil {
+ return m.DelayMillis
+ }
+ return 0
+}
+
+type TimestampTransform_AlignTo struct {
+ // (Required) A duration to which delays should be quantized
+ // in milliseconds.
+ Period int64 `protobuf:"varint,3,opt,name=period" json:"period,omitempty"`
+ // (Required) An offset from 0 for the quantization specified by
+ // alignment_size, in milliseconds
+ Offset int64 `protobuf:"varint,4,opt,name=offset" json:"offset,omitempty"`
+}
+
+func (m *TimestampTransform_AlignTo) Reset() { *m = TimestampTransform_AlignTo{} }
+func (m *TimestampTransform_AlignTo) String() string { return proto.CompactTextString(m) }
+func (*TimestampTransform_AlignTo) ProtoMessage() {}
+func (*TimestampTransform_AlignTo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29, 1} }
+
+func (m *TimestampTransform_AlignTo) GetPeriod() int64 {
+ if m != nil {
+ return m.Period
+ }
+ return 0
+}
+
+func (m *TimestampTransform_AlignTo) GetOffset() int64 {
+ if m != nil {
+ return m.Offset
+ }
+ return 0
+}
+
+// A specification for how to "side input" a PCollection.
+type SideInput struct {
+ // (Required) URN of the access pattern required by the `view_fn` to present
+ // the desired SDK-specific interface to a UDF.
+ //
+ // This access pattern defines the SDK harness <-> Runner Harness RPC
+ // interface for accessing a side input.
+ //
+ // The only access pattern intended for Beam, because of its superior
+ // performance possibilities, is "urn:beam:sideinput:multimap" (or some such
+ // URN)
+ AccessPattern *FunctionSpec `protobuf:"bytes,1,opt,name=access_pattern,json=accessPattern" json:"access_pattern,omitempty"`
+ // (Required) The SdkFunctionSpec of the UDF that adapts a particular
+ // access_pattern to a user-facing view type.
+ //
+ // For example, View.asSingleton() may include a `view_fn` that adapts a
+ // specially-designed multimap to a single value per window.
+ ViewFn *SdkFunctionSpec `protobuf:"bytes,2,opt,name=view_fn,json=viewFn" json:"view_fn,omitempty"`
+ // (Required) The SdkFunctionSpec of the UDF that maps a main input window
+ // to a side input window.
+ //
+ // For example, when the main input is in fixed windows of one hour, this
+ // can specify that the side input should be accessed according to the day
+ // in which that hour falls.
+ WindowMappingFn *SdkFunctionSpec `protobuf:"bytes,3,opt,name=window_mapping_fn,json=windowMappingFn" json:"window_mapping_fn,omitempty"`
+}
+
+func (m *SideInput) Reset() { *m = SideInput{} }
+func (m *SideInput) String() string { return proto.CompactTextString(m) }
+func (*SideInput) ProtoMessage() {}
+func (*SideInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
+
+func (m *SideInput) GetAccessPattern() *FunctionSpec {
+ if m != nil {
+ return m.AccessPattern
+ }
+ return nil
+}
+
+func (m *SideInput) GetViewFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.ViewFn
+ }
+ return nil
+}
+
+func (m *SideInput) GetWindowMappingFn() *SdkFunctionSpec {
+ if m != nil {
+ return m.WindowMappingFn
+ }
+ return nil
+}
+
+// An environment for executing UDFs. Generally an SDK container URL, but
+// there can be many for a single SDK, for example to provide dependency
+// isolation.
+type Environment struct {
+ // (Required) The URL of a container
+ //
+ // TODO: reconcile with Fn API's DockerContainer structure by
+ // adding adequate metadata to know how to interpret the container
+ Url string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"`
+}
+
+func (m *Environment) Reset() { *m = Environment{} }
+func (m *Environment) String() string { return proto.CompactTextString(m) }
+func (*Environment) ProtoMessage() {}
+func (*Environment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
+
+func (m *Environment) GetUrl() string {
+ if m != nil {
+ return m.Url
+ }
+ return ""
+}
+
+// A specification of a user defined function.
+//
+type SdkFunctionSpec struct {
+ // (Required) A full specification of this function.
+ Spec *FunctionSpec `protobuf:"bytes,1,opt,name=spec" json:"spec,omitempty"`
+ // (Required) Reference to an execution environment capable of
+ // invoking this function.
+ EnvironmentId string `protobuf:"bytes,2,opt,name=environment_id,json=environmentId" json:"environment_id,omitempty"`
+}
+
+func (m *SdkFunctionSpec) Reset() { *m = SdkFunctionSpec{} }
+func (m *SdkFunctionSpec) String() string { return proto.CompactTextString(m) }
+func (*SdkFunctionSpec) ProtoMessage() {}
+func (*SdkFunctionSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
+
+func (m *SdkFunctionSpec) GetSpec() *FunctionSpec {
+ if m != nil {
+ return m.Spec
+ }
+ return nil
+}
+
+func (m *SdkFunctionSpec) GetEnvironmentId() string {
+ if m != nil {
+ return m.EnvironmentId
+ }
+ return ""
+}
+
+// A URN along with a parameter object whose schema is determined by the
+// URN.
+//
+// This structure is reused in two distinct, but compatible, ways:
+//
+// 1. This can be a specification of the function over PCollections
+// that a PTransform computes.
+// 2. This can be a specification of a user-defined function, possibly
+// SDK-specific. (external to this message must be adequate context
+// to indicate the environment in which the UDF can be understood).
+//
+// Though not explicit in this proto, there are two possibilities
+// for the relationship of a runner to this specification that
+// one should bear in mind:
+//
+// 1. The runner understands the URN. For example, it might be
+// a well-known URN like "urn:beam:transform:Top" or
+// "urn:beam:windowfn:FixedWindows" with
+// an agreed-upon payload (e.g. a number or duration,
+// respectively).
+// 2. The runner does not understand the URN. It might be an
+// SDK specific URN such as "urn:beam:dofn:javasdk:1.0"
+// that indicates to the SDK what the payload is,
+// such as a serialized Java DoFn from a particular
+// version of the Beam Java SDK. The payload will often
+// then be an opaque message such as bytes in a
+// language-specific serialization format.
+type FunctionSpec struct {
+ // (Required) A URN that describes the accompanying payload.
+ // For any URN that is not recognized (by whomever is inspecting
+ // it) the parameter payload should be treated as opaque and
+ // passed as-is.
+ Urn string `protobuf:"bytes,1,opt,name=urn" json:"urn,omitempty"`
+ // (Optional) The data specifying any parameters to the URN. If
+ // the URN does not require any arguments, this may be omitted.
+ Payload []byte `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
+}
+
+func (m *FunctionSpec) Reset() { *m = FunctionSpec{} }
+func (m *FunctionSpec) String() string { return proto.CompactTextString(m) }
+func (*FunctionSpec) ProtoMessage() {}
+func (*FunctionSpec) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
+
+func (m *FunctionSpec) GetUrn() string {
+ if m != nil {
+ return m.Urn
+ }
+ return ""
+}
+
+func (m *FunctionSpec) GetPayload() []byte {
+ if m != nil {
+ return m.Payload
+ }
+ return nil
+}
+
+// TODO: transfer javadoc here
+type DisplayData struct {
+ // (Required) The list of display data.
+ Items []*DisplayData_Item `protobuf:"bytes,1,rep,name=items" json:"items,omitempty"`
+}
+
+func (m *DisplayData) Reset() { *m = DisplayData{} }
+func (m *DisplayData) String() string { return proto.CompactTextString(m) }
+func (*DisplayData) ProtoMessage() {}
+func (*DisplayData) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
+
+func (m *DisplayData) GetItems() []*DisplayData_Item {
+ if m != nil {
+ return m.Items
+ }
+ return nil
+}
+
+// A complete identifier for a DisplayData.Item
+type DisplayData_Identifier struct {
+ // (Required) The transform originating this display data.
+ TransformId string `protobuf:"bytes,1,opt,name=transform_id,json=transformId" json:"transform_id,omitempty"`
+ // (Optional) The URN indicating the type of the originating transform,
+ // if there is one.
+ TransformUrn string `protobuf:"bytes,2,opt,name=transform_urn,json=transformUrn" json:"transform_urn,omitempty"`
+ Key string `protobuf:"bytes,3,opt,name=key" json:"key,omitempty"`
+}
+
+func (m *DisplayData_Identifier) Reset() { *m = DisplayData_Identifier{} }
+func (m *DisplayData_Identifier) String() string { return proto.CompactTextString(m) }
+func (*DisplayData_Identifier) ProtoMessage() {}
+func (*DisplayData_Identifier) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34, 0} }
+
+func (m *DisplayData_Identifier) GetTransformId() string {
+ if m != nil {
+ return m.TransformId
+ }
+ return ""
+}
+
+func (m *DisplayData_Identifier) GetTransformUrn() string {
+ if m != nil {
+ return m.TransformUrn
+ }
+ return ""
+}
+
+func (m *DisplayData_Identifier) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+// A single item of display data.
+type DisplayData_Item struct {
+ // (Required)
+ Id *DisplayData_Identifier `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ // (Required)
+ Type DisplayData_Type_Enum `protobuf:"varint,2,opt,name=type,enum=org.apache.beam.model.pipeline.v1.DisplayData_Type_Enum" json:"type,omitempty"`
+ // (Required)
+ Value *google_protobuf.Any `protobuf:"bytes,3,opt,name=value" json:"value,omitempty"`
+ // (Optional)
+ ShortValue *google_protobuf.Any `protobuf:"bytes,4,opt,name=short_value,json=shortValue" json:"short_value,omitempty"`
+ // (Optional)
+ Label string `protobuf:"bytes,5,opt,name=label" json:"label,omitempty"`
+ // (Optional)
+ LinkUrl string `protobuf:"bytes,6,opt,name=link_url,json=linkUrl" json:"link_url,omitempty"`
+}
+
+func (m *DisplayData_Item) Reset() { *m = DisplayData_Item{} }
+func (m *DisplayData_Item) String() string { return proto.CompactTextString(m) }
+func (*DisplayData_Item) ProtoMessage() {}
+func (*DisplayData_Item) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34, 1} }
+
+func (m *DisplayData_Item) GetId() *DisplayData_Identifier {
+ if m != nil {
+ return m.Id
+ }
+ return nil
+}
+
+func (m *DisplayData_Item) GetType() DisplayData_Type_Enum {
+ if m != nil {
+ return m.Type
+ }
+ return DisplayData_Type_UNSPECIFIED
+}
+
+func (m *DisplayData_Item) GetValue() *google_protobuf.Any {
+ if m != nil {
+ return m.Value
+ }
+ return nil
+}
+
+func (m *DisplayData_Item) GetShortValue() *google_protobuf.Any {
+ if m != nil {
+ return m.ShortValue
+ }
+ return nil
+}
+
+func (m *DisplayData_Item) GetLabel() string {
+ if m != nil {
+ return m.Label
+ }
+ return ""
+}
+
+func (m *DisplayData_Item) GetLinkUrl() string {
+ if m != nil {
+ return m.LinkUrl
+ }
+ return ""
+}
+
+type DisplayData_Type struct {
+}
+
+func (m *DisplayData_Type) Reset() { *m = DisplayData_Type{} }
+func (m *DisplayData_Type) String() string { return proto.CompactTextString(m) }
+func (*DisplayData_Type) ProtoMessage() {}
+func (*DisplayData_Type) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34, 2} }
+
+func init() {
+ proto.RegisterType((*Components)(nil), "org.apache.beam.model.pipeline.v1.Components")
+ proto.RegisterType((*MessageWithComponents)(nil), "org.apache.beam.model.pipeline.v1.MessageWithComponents")
+ proto.RegisterType((*Pipeline)(nil), "org.apache.beam.model.pipeline.v1.Pipeline")
+ proto.RegisterType((*PTransform)(nil), "org.apache.beam.model.pipeline.v1.PTransform")
+ proto.RegisterType((*PCollection)(nil), "org.apache.beam.model.pipeline.v1.PCollection")
+ proto.RegisterType((*ParDoPayload)(nil), "org.apache.beam.model.pipeline.v1.ParDoPayload")
+ proto.RegisterType((*Parameter)(nil), "org.apache.beam.model.pipeline.v1.Parameter")
+ proto.RegisterType((*Parameter_Type)(nil), "org.apache.beam.model.pipeline.v1.Parameter.Type")
+ proto.RegisterType((*StateSpec)(nil), "org.apache.beam.model.pipeline.v1.StateSpec")
+ proto.RegisterType((*ValueStateSpec)(nil), "org.apache.beam.model.pipeline.v1.ValueStateSpec")
+ proto.RegisterType((*BagStateSpec)(nil), "org.apache.beam.model.pipeline.v1.BagStateSpec")
+ proto.RegisterType((*CombiningStateSpec)(nil), "org.apache.beam.model.pipeline.v1.CombiningStateSpec")
+ proto.RegisterType((*MapStateSpec)(nil), "org.apache.beam.model.pipeline.v1.MapStateSpec")
+ proto.RegisterType((*SetStateSpec)(nil), "org.apache.beam.model.pipeline.v1.SetStateSpec")
+ proto.RegisterType((*TimerSpec)(nil), "org.apache.beam.model.pipeline.v1.TimerSpec")
+ proto.RegisterType((*IsBounded)(nil), "org.apache.beam.model.pipeline.v1.IsBounded")
+ proto.RegisterType((*ReadPayload)(nil), "org.apache.beam.model.pipeline.v1.ReadPayload")
+ proto.RegisterType((*WindowIntoPayload)(nil), "org.apache.beam.model.pipeline.v1.WindowIntoPayload")
+ proto.RegisterType((*CombinePayload)(nil), "org.apache.beam.model.pipeline.v1.CombinePayload")
+ proto.RegisterType((*TestStreamPayload)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload")
+ proto.RegisterType((*TestStreamPayload_Event)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload.Event")
+ proto.RegisterType((*TestStreamPayload_Event_AdvanceWatermark)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceWatermark")
+ proto.RegisterType((*TestStreamPayload_Event_AdvanceProcessingTime)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceProcessingTime")
+ proto.RegisterType((*TestStreamPayload_Event_AddElements)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements")
+ proto.RegisterType((*TestStreamPayload_TimestampedElement)(nil), "org.apache.beam.model.pipeline.v1.TestStreamPayload.TimestampedElement")
+ proto.RegisterType((*WriteFilesPayload)(nil), "org.apache.beam.model.pipeline.v1.WriteFilesPayload")
+ proto.RegisterType((*Coder)(nil), "org.apache.beam.model.pipeline.v1.Coder")
+ proto.RegisterType((*WindowingStrategy)(nil), "org.apache.beam.model.pipeline.v1.WindowingStrategy")
+ proto.RegisterType((*MergeStatus)(nil), "org.apache.beam.model.pipeline.v1.MergeStatus")
+ proto.RegisterType((*AccumulationMode)(nil), "org.apache.beam.model.pipeline.v1.AccumulationMode")
+ proto.RegisterType((*ClosingBehavior)(nil), "org.apache.beam.model.pipeline.v1.ClosingBehavior")
+ proto.RegisterType((*OnTimeBehavior)(nil), "org.apache.beam.model.pipeline.v1.OnTimeBehavior")
+ proto.RegisterType((*OutputTime)(nil), "org.apache.beam.model.pipeline.v1.OutputTime")
+ proto.RegisterType((*TimeDomain)(nil), "org.apache.beam.model.pipeline.v1.TimeDomain")
+ proto.RegisterType((*Trigger)(nil), "org.apache.beam.model.pipeline.v1.Trigger")
+ proto.RegisterType((*Trigger_AfterAll)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterAll")
+ proto.RegisterType((*Trigger_AfterAny)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterAny")
+ proto.RegisterType((*Trigger_AfterEach)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterEach")
+ proto.RegisterType((*Trigger_AfterEndOfWindow)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow")
+ proto.RegisterType((*Trigger_AfterProcessingTime)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime")
+ proto.RegisterType((*Trigger_AfterSynchronizedProcessingTime)(nil), "org.apache.beam.model.pipeline.v1.Trigger.AfterSynchronizedProcessingTime")
+ proto.RegisterType((*Trigger_Default)(nil), "org.apache.beam.model.pipeline.v1.Trigger.Default")
+ proto.RegisterType((*Trigger_ElementCount)(nil), "org.apache.beam.model.pipeline.v1.Trigger.ElementCount")
+ proto.RegisterType((*Trigger_Never)(nil), "org.apache.beam.model.pipeline.v1.Trigger.Never")
+ proto.RegisterType((*Trigger_Always)(nil), "org.apache.beam.model.pipeline.v1.Trigger.Always")
+ proto.RegisterType((*Trigger_OrFinally)(nil), "org.apache.beam.model.pipeline.v1.Trigger.OrFinally")
+ proto.RegisterType((*Trigger_Repeat)(nil), "org.apache.beam.model.pipeline.v1.Trigger.Repeat")
+ proto.RegisterType((*TimestampTransform)(nil), "org.apache.beam.model.pipeline.v1.TimestampTransform")
+ proto.RegisterType((*TimestampTransform_Delay)(nil), "org.apache.beam.model.pipeline.v1.TimestampTransform.Delay")
+ proto.RegisterType((*TimestampTransform_AlignTo)(nil), "org.apache.beam.model.pipeline.v1.TimestampTransform.AlignTo")
+ proto.RegisterType((*SideInput)(nil), "org.apache.beam.model.pipeline.v1.SideInput")
+ proto.RegisterType((*Environment)(nil), "org.apache.beam.model.pipeline.v1.Environment")
+ proto.RegisterType((*SdkFunctionSpec)(nil), "org.apache.beam.model.pipeline.v1.SdkFunctionSpec")
+ proto.RegisterType((*FunctionSpec)(nil), "org.apache.beam.model.pipeline.v1.FunctionSpec")
+ proto.RegisterType((*DisplayData)(nil), "org.apache.beam.model.pipeline.v1.DisplayData")
+ proto.RegisterType((*DisplayData_Identifier)(nil), "org.apache.beam.model.pipeline.v1.DisplayData.Identifier")
+ proto.RegisterType((*DisplayData_Item)(nil), "org.apache.beam.model.pipeline.v1.DisplayData.Item")
+ proto.RegisterType((*DisplayData_Type)(nil), "org.apache.beam.model.pipeline.v1.DisplayData.Type")
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.Parameter_Type_Enum", Parameter_Type_Enum_name, Parameter_Type_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.IsBounded_Enum", IsBounded_Enum_name, IsBounded_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.MergeStatus_Enum", MergeStatus_Enum_name, MergeStatus_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.AccumulationMode_Enum", AccumulationMode_Enum_name, AccumulationMode_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.ClosingBehavior_Enum", ClosingBehavior_Enum_name, ClosingBehavior_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.OnTimeBehavior_Enum", OnTimeBehavior_Enum_name, OnTimeBehavior_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.OutputTime_Enum", OutputTime_Enum_name, OutputTime_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.TimeDomain_Enum", TimeDomain_Enum_name, TimeDomain_Enum_value)
+ proto.RegisterEnum("org.apache.beam.model.pipeline.v1.DisplayData_Type_Enum", DisplayData_Type_Enum_name, DisplayData_Type_Enum_value)
+}
+
+func init() { proto.RegisterFile("beam_runner_api.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+ // 3390 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5b, 0xcd, 0x73, 0x23, 0x49,
+ 0x56, 0xd7, 0xf7, 0xc7, 0x93, 0x2c, 0xcb, 0xe9, 0x6e, 0xd0, 0x28, 0x36, 0x98, 0x9e, 0x62, 0x81,
+ 0x66, 0x18, 0x34, 0xdb, 0xee, 0x1d, 0x66, 0x7a, 0x86, 0x9d, 0x5d, 0x59, 0x2a, 0xb5, 0xd4, 0x63,
+ 0x4b, 0x9a, 0x92, 0xdc, 0xa6, 0x67, 0x97, 0xae, 0x49, 0xab, 0x52, 0x72, 0x85, 0x4b, 0x59, 0xa2,
+ 0xaa, 0x64, 0x87, 0x08, 0x36, 0xf6, 0x46, 0x10, 0xc1, 0x05, 0x8e, 0x7b, 0x85, 0x23, 0x27, 0x76,
+ 0x08, 0x22, 0x38, 0xf3, 0x27, 0x70, 0xe3, 0xbf, 0x20, 0x08, 0xee, 0x44, 0x7e, 0x54, 0xa9, 0x24,
+ 0xd9, 0x3d, 0x25, 0xd9, 0x41, 0x70, 0x73, 0x3d, 0xd5, 0xfb, 0xbd, 0x57, 0x2f, 0xdf, 0x67, 0x66,
+ 0x1a, 0x1e, 0x5f, 0x10, 0x3c, 0xd5, 0x9d, 0x39, 0xa5, 0xc4, 0xd1, 0xf1, 0xcc, 0xac, 0xcd, 0x1c,
+ 0xdb, 0xb3, 0xd1, 0x07, 0xb6, 0x33, 0xa9, 0xe1, 0x19, 0x1e, 0x5d, 0x92, 0x1a, 0x7b, 0xa3, 0x36,
+ 0xb5, 0x0d, 0x62, 0xd5, 0x66, 0xe6, 0x8c, 0x58, 0x26, 0x25, 0xb5, 0xeb, 0x67, 0xd5, 0xf7, 0x26,
+ 0xb6, 0x3d, 0xb1, 0xc8, 0xc7, 0x9c, 0xe1, 0x62, 0x3e, 0xfe, 0x18, 0xd3, 0x85, 0xe0, 0x56, 0xfe,
+ 0x35, 0x07, 0xd0, 0xb0, 0xa7, 0x33, 0x9b, 0x12, 0xea, 0xb9, 0xe8, 0xcf, 0x01, 0x3c, 0x07, 0x53,
+ 0x77, 0x6c, 0x3b, 0x53, 0xb7, 0x12, 0x7f, 0x92, 0x7c, 0x5a, 0x38, 0xfa, 0x49, 0xed, 0x7b, 0x25,
+ 0xd4, 0x96, 0x10, 0xb5, 0x61, 0xc0, 0xaf, 0x52, 0xcf, 0x59, 0x68, 0x21, 0x40, 0x34, 0x82, 0xe2,
+ 0x6c, 0x64, 0x5b, 0x16, 0x19, 0x79, 0xa6, 0x4d, 0xdd, 0x4a, 0x82, 0x0b, 0xf8, 0xe9, 0x76, 0x02,
+ 0xfa, 0x21, 0x04, 0x21, 0x62, 0x05, 0x14, 0x2d, 0xe0, 0xd1, 0x8d, 0x49, 0x0d, 0xfb, 0xc6, 0xa4,
+ 0x13, 0xdd, 0xf5, 0x1c, 0xec, 0x91, 0x89, 0x49, 0xdc, 0x4a, 0x92, 0x0b, 0x6b, 0x6d, 0x27, 0xec,
+ 0xdc, 0x47, 0x1a, 0x04, 0x40, 0x42, 0xe6, 0xe1, 0xcd, 0xe6, 0x2f, 0xe8, 0x6b, 0xc8, 0x8c, 0x6c,
+ 0x83, 0x38, 0x6e, 0x25, 0xc5, 0x85, 0xbd, 0xd8, 0x4e, 0x58, 0x83, 0xf3, 0x0a, 0x7c, 0x09, 0xc4,
+ 0x4c, 0x46, 0xe8, 0xb5, 0xe9, 0xd8, 0x74, 0xca, 0xde, 0xa9, 0xa4, 0x77, 0x31, 0x99, 0x1a, 0x42,
+ 0x90, 0x26, 0x0b, 0x83, 0x56, 0x2d, 0xd8, 0x5f, 0x5b, 0x36, 0x54, 0x86, 0xe4, 0x15, 0x59, 0x54,
+ 0xe2, 0x4f, 0xe2, 0x4f, 0xf3, 0x1a, 0xfb, 0x13, 0x35, 0x20, 0x7d, 0x8d, 0xad, 0x39, 0xa9, 0x24,
+ 0x9e, 0xc4, 0x9f, 0x16, 0x8e, 0xfe, 0x38, 0x82, 0x0a, 0xfd, 0x00, 0x55, 0x13, 0xbc, 0x9f, 0x27,
+ 0x3e, 0x8b, 0x57, 0x6d, 0x38, 0xd8, 0x58, 0xc3, 0x5b, 0xe4, 0x35, 0x57, 0xe5, 0xd5, 0xa2, 0xc8,
+ 0x6b, 0x04, 0xb0, 0x61, 0x81, 0x7f, 0x05, 0x95, 0xbb, 0xd6, 0xf1, 0x16, 0xb9, 0xaf, 0x56, 0xe5,
+ 0xfe, 0x38, 0x82, 0xdc, 0x75, 0xf4, 0x45, 0x58, 0xfa, 0x08, 0x0a, 0xa1, 0x85, 0xbd, 0x45, 0xe0,
+ 0x97, 0xab, 0x02, 0x9f, 0x46, 0x5a, 0x5b, 0x83, 0x38, 0x6b, 0x36, 0xdd, 0x58, 0xe4, 0x87, 0xb1,
+ 0x69, 0x08, 0x36, 0x24, 0x50, 0xf9, 0xb7, 0x1c, 0x3c, 0x3e, 0x25, 0xae, 0x8b, 0x27, 0xe4, 0xdc,
+ 0xf4, 0x2e, 0x43, 0x39, 0xe4, 0x14, 0x60, 0x14, 0x3c, 0x71, 0xe1, 0xd1, 0x9c, 0x65, 0x09, 0xa1,
+ 0x85, 0x00, 0xd0, 0xcf, 0x20, 0xcd, 0x43, 0x61, 0x5b, 0xeb, 0xb4, 0x63, 0x9a, 0x60, 0x44, 0xbf,
+ 0x80, 0xfd, 0x91, 0x3d, 0xbd, 0x30, 0x29, 0xd1, 0x67, 0x78, 0x61, 0xd9, 0xd8, 0xa8, 0x24, 0x39,
+ 0xd6, 0xb3, 0x68, 0x5a, 0x31, 0xce, 0xbe, 0x60, 0x6c, 0xc7, 0xb4, 0xd2, 0x68, 0x85, 0x82, 0xbe,
+ 0x85, 0x03, 0xd7, 0xb8, 0xd2, 0xc7, 0x73, 0xca, 0xfd, 0x4e, 0x77, 0x67, 0x64, 0x54, 0x49, 0x71,
+ 0xfc, 0xa3, 0x08, 0xf8, 0x03, 0xe3, 0xaa, 0x25, 0x59, 0x07, 0x33, 0x32, 0x6a, 0xc7, 0xb4, 0x7d,
+ 0x77, 0x95, 0x84, 0xce, 0xa1, 0x34, 0xc3, 0x8e, 0x6e, 0xd8, 0x81, 0xfa, 0x19, 0x0e, 0xff, 0x71,
+ 0x94, 0x88, 0xc0, 0x4e, 0xd3, 0x5e, 0x2a, 0x5f, 0x9c, 0x85, 0x9e, 0x51, 0x0f, 0x60, 0x16, 0x64,
+ 0xe7, 0x4a, 0x76, 0x87, 0xb0, 0x6e, 0xc7, 0xb4, 0x10, 0x04, 0xd2, 0xa0, 0x10, 0x4a, 0xc5, 0x95,
+ 0xdc, 0x2e, 0x81, 0xdb, 0x8e, 0x69, 0x61, 0x10, 0x34, 0x80, 0xa2, 0x43, 0xb0, 0x11, 0x7c, 0x7b,
+ 0x3e, 0x32, 0xa8, 0x46, 0xb0, 0xb1, 0xfc, 0xf4, 0x82, 0xb3, 0x7c, 0x64, 0x3e, 0xea, 0x9a, 0x06,
+ 0xd1, 0x4d, 0x3a, 0x9b, 0x7b, 0x95, 0x02, 0x87, 0xfc, 0x28, 0xca, 0x6a, 0x99, 0x06, 0xe9, 0x30,
+ 0x9e, 0x76, 0x4c, 0xcb, 0xbb, 0xfe, 0x03, 0x1a, 0x83, 0x2c, 0x07, 0xba, 0x49, 0xbd, 0xe5, 0x32,
+ 0x15, 0xb7, 0x4c, 0x20, 0x1d, 0xea, 0x85, 0xd6, 0xea, 0xe0, 0x66, 0x9d, 0x88, 0x08, 0xa0, 0x8d,
+ 0xd2, 0xb6, 0xa8, 0xec, 0xed, 0x9e, 0xa7, 0x96, 0x62, 0x42, 0x44, 0xf4, 0x1a, 0xf6, 0x56, 0xdd,
+ 0xb9, 0x14, 0xd9, 0xdf, 0xd6, 0x7c, 0xb9, 0x38, 0x0e, 0x3d, 0x1f, 0x67, 0x20, 0xe5, 0xd8, 0xb6,
+ 0xa7, 0xfc, 0x67, 0x1c, 0x72, 0x7d, 0xc9, 0xf4, 0xd0, 0xe9, 0xe2, 0x23, 0x40, 0x4c, 0x86, 0x1e,
+ 0x38, 0xa5, 0x6e, 0x1a, 0xa2, 0xd1, 0xc8, 0x6b, 0x65, 0xf6, 0x4b, 0xe0, 0xbb, 0x1d, 0x83, 0x15,
+ 0xec, 0xa2, 0x61, 0xba, 0x33, 0x0b, 0x2f, 0x74, 0x03, 0x7b, 0x58, 0xe6, 0x85, 0x28, 0xce, 0xd5,
+ 0x14, 0x6c, 0x4d, 0xec, 0x61, 0xad, 0x60, 0x2c, 0x1f, 0x94, 0xbf, 0x4d, 0x01, 0x2c, 0x03, 0x04,
+ 0xbd, 0x0f, 0x85, 0x39, 0x35, 0xff, 0x62, 0x4e, 0x74, 0x8a, 0xa7, 0xa4, 0x92, 0xe6, 0xb9, 0x18,
+ 0x04, 0xa9, 0x8b, 0xa7, 0x04, 0x35, 0x20, 0xc5, 0x6d, 0x1c, 0xdf, 0xc9, 0xc6, 0x1a, 0x67, 0x46,
+ 0x3f, 0x84, 0x3d, 0x77, 0x7e, 0x11, 0x6a, 0xdd, 0xc4, 0x07, 0xaf, 0x12, 0x59, 0x7b, 0xc2, 0x1d,
+ 0xde, 0xef, 0x85, 0x5e, 0x6c, 0x15, 0xeb, 0x35, 0xee, 0xeb, 0x7e, 0x7b, 0x22, 0x80, 0xd0, 0x10,
+ 0xb2, 0xf6, 0xdc, 0xe3, 0x98, 0xa2, 0xe5, 0xf9, 0x7c, 0x3b, 0xcc, 0x9e, 0x60, 0x16, 0xa0, 0x3e,
+ 0xd4, 0xc6, 0xb2, 0x64, 0xee, 0xbd, 0x2c, 0xd5, 0x17, 0x50, 0x08, 0xe9, 0x7f, 0x4b, 0x69, 0x7c,
+ 0x14, 0x2e, 0x8d, 0xf9, 0x70, 0x6d, 0xfd, 0x1c, 0x8a, 0x61, 0x35, 0xb7, 0xe1, 0x55, 0xfe, 0x21,
+ 0x01, 0x85, 0x50, 0x72, 0x5b, 0x77, 0x87, 0xf8, 0x86, 0x3b, 0xbc, 0x07, 0x39, 0x5e, 0xb5, 0x74,
+ 0xd3, 0x90, 0x68, 0x59, 0xfe, 0xdc, 0x31, 0x50, 0x1f, 0xc0, 0x74, 0xf5, 0x0b, 0x7b, 0x4e, 0x0d,
+ 0x22, 0x4a, 0x58, 0x29, 0x52, 0x09, 0xeb, 0xb8, 0xc7, 0x82, 0xa7, 0xa6, 0xd2, 0xf9, 0x54, 0xcb,
+ 0x9b, 0xfe, 0x33, 0x3a, 0x82, 0xc7, 0x9b, 0xf9, 0x84, 0x49, 0x4e, 0x71, 0xc9, 0x1b, 0x3d, 0xee,
+ 0xa2, 0x63, 0x6c, 0xac, 0x4d, 0xfa, 0xfe, 0x21, 0xf3, 0x9b, 0x0c, 0x14, 0xc3, 0x85, 0x0a, 0xbd,
+ 0x84, 0xb4, 0x61, 0xeb, 0x63, 0x2a, 0x83, 0x62, 0x87, 0x3a, 0xaa, 0xa5, 0x0c, 0xbb, 0x45, 0xd1,
+ 0x09, 0xc0, 0x0c, 0x3b, 0x78, 0x4a, 0x3c, 0xd6, 0x94, 0x8b, 0x71, 0xe3, 0xa3, 0x68, 0x65, 0x53,
+ 0x30, 0x69, 0x21, 0x7e, 0xf4, 0x2d, 0x14, 0x96, 0x55, 0xc3, 0x0f, 0xa2, 0x9f, 0x6e, 0x59, 0x85,
+ 0x97, 0x35, 0xc4, 0x1f, 0x90, 0x82, 0x3a, 0x22, 0x24, 0x78, 0xd8, 0x23, 0x3c, 0xed, 0xfa, 0x21,
+ 0xb5, 0xbd, 0x04, 0x06, 0xc1, 0xac, 0x10, 0x48, 0x08, 0x08, 0x4c, 0x82, 0x67, 0x4e, 0x89, 0x23,
+ 0x25, 0xa4, 0x77, 0x93, 0x30, 0x64, 0x10, 0x61, 0x09, 0x5e, 0x40, 0x40, 0xbf, 0x03, 0xe0, 0xce,
+ 0x2c, 0xd3, 0xf3, 0xf0, 0x85, 0x45, 0x78, 0xe8, 0xe6, 0xb4, 0x10, 0xa5, 0x7a, 0x05, 0xfb, 0x6b,
+ 0x26, 0xb8, 0x25, 0xa2, 0x8e, 0x57, 0x1b, 0xd5, 0xad, 0x6a, 0x73, 0x38, 0x76, 0x99, 0xb0, 0x55,
+ 0x6b, 0x3c, 0x90, 0x30, 0x1f, 0x74, 0x4d, 0xd8, 0x9a, 0x61, 0x1e, 0x46, 0x58, 0x00, 0x1a, 0xce,
+ 0x2c, 0xdf, 0xc5, 0x21, 0x1f, 0xb8, 0x29, 0x7a, 0x05, 0x29, 0x6f, 0x31, 0x13, 0x09, 0xa5, 0x74,
+ 0xf4, 0x27, 0xdb, 0xb8, 0x78, 0x6d, 0xb8, 0x98, 0x11, 0x91, 0x1a, 0x38, 0x46, 0xf5, 0x1b, 0x48,
+ 0x31, 0x92, 0xa2, 0x41, 0x8a, 0x51, 0xd1, 0x3e, 0x14, 0xce, 0xba, 0x83, 0xbe, 0xda, 0xe8, 0xb4,
+ 0x3a, 0x6a, 0xb3, 0x1c, 0x43, 0x00, 0x99, 0xf3, 0x4e, 0xb7, 0xd9, 0x3b, 0x2f, 0xc7, 0xd1, 0x23,
+ 0x28, 0xf7, 0x3b, 0x7d, 0xf5, 0xa4, 0xd3, 0x55, 0xf5, 0x5e, 0x7f, 0xd8, 0xe9, 0x75, 0x07, 0xe5,
+ 0x04, 0xfa, 0x6d, 0x38, 0xd4, 0xd4, 0xc1, 0x50, 0xeb, 0x34, 0x18, 0x45, 0x1f, 0x6a, 0xf5, 0xc6,
+ 0x57, 0xaa, 0x56, 0x4e, 0x2a, 0xff, 0x9c, 0x84, 0x7c, 0x60, 0x3b, 0xa4, 0x01, 0xf0, 0x0f, 0xd2,
+ 0x43, 0x15, 0x30, 0x4a, 0x46, 0x7b, 0xcd, 0x98, 0x02, 0x18, 0xd6, 0x8b, 0x71, 0x18, 0x8e, 0x79,
+ 0x02, 0xb9, 0x0b, 0x3c, 0x11, 0x88, 0x89, 0xc8, 0x35, 0xf5, 0x18, 0x4f, 0xc2, 0x78, 0xd9, 0x0b,
+ 0x3c, 0xe1, 0x68, 0x6f, 0x41, 0xf6, 0xfb, 0x3c, 0x43, 0x32, 0x4c, 0xd1, 0x22, 0x7c, 0x12, 0x79,
+ 0x74, 0xe0, 0xd9, 0x73, 0x89, 0xbc, 0x17, 0xc0, 0xf9, 0xda, 0x4e, 0xf1, 0x2c, 0x3c, 0x34, 0x44,
+ 0xd1, 0xf6, 0x14, 0xcf, 0x56, 0xb4, 0x9d, 0xe2, 0x99, 0x8f, 0xe6, 0x12, 0x4f, 0xa0, 0xa5, 0x23,
+ 0xa3, 0x0d, 0x88, 0xb7, 0x82, 0xe6, 0x12, 0xcf, 0x6f, 0xd7, 0x18, 0x92, 0xf2, 0x47, 0x50, 0x5a,
+ 0x35, 0xf8, 0x4a, 0x91, 0x8a, 0xaf, 0x14, 0x29, 0xe5, 0x33, 0x28, 0x86, 0x6d, 0x89, 0x9e, 0x42,
+ 0x99, 0x58, 0x84, 0x4d, 0x8f, 0xfa, 0x1a, 0x4b, 0x49, 0xd2, 0x1b, 0x92, 0xf3, 0xd7, 0x71, 0x40,
+ 0x9b, 0x26, 0x43, 0x3f, 0x82, 0x47, 0x78, 0x34, 0x9a, 0x4f, 0xe7, 0x16, 0xf6, 0x6c, 0x67, 0x1d,
+ 0x04, 0x85, 0x7e, 0x93, 0x40, 0xe8, 0x6b, 0xde, 0x51, 0xf2, 0x79, 0x6f, 0x4c, 0xa5, 0x0f, 0xec,
+ 0x52, 0x42, 0xf2, 0x12, 0xa5, 0x45, 0x95, 0xd7, 0x50, 0x0c, 0xdb, 0x1c, 0x3d, 0x81, 0xe2, 0x15,
+ 0x59, 0xac, 0x2b, 0x03, 0x57, 0x64, 0xe1, 0x2b, 0xf1, 0x43, 0x28, 0x09, 0xd7, 0x5e, 0xab, 0xe6,
+ 0x45, 0x4e, 0x6d, 0x2c, 0xad, 0x15, 0xb6, 0xfe, 0x16, 0xd6, 0xfa, 0x16, 0xf2, 0x41, 0x5a, 0x40,
+ 0x03, 0x91, 0xd4, 0x75, 0xc3, 0x9e, 0x62, 0x93, 0xca, 0x24, 0x70, 0x14, 0x31, 0xb3, 0x34, 0x39,
+ 0x93, 0x48, 0x00, 0x3c, 0x8f, 0x0b, 0x82, 0xf2, 0x33, 0xc8, 0x07, 0x9d, 0x83, 0xf2, 0xfc, 0xae,
+ 0x5c, 0xb0, 0x07, 0xf9, 0xb3, 0xee, 0x71, 0xef, 0xac, 0xdb, 0x54, 0x9b, 0xe5, 0x38, 0x2a, 0x40,
+ 0xd6, 0x7f, 0x48, 0x28, 0xff, 0x14, 0x87, 0x42, 0x68, 0x08, 0x43, 0xaf, 0x20, 0xe3, 0xda, 0x73,
+ 0x67, 0x44, 0xee, 0x51, 0xd7, 0x25, 0xc2, 0x5a, 0x33, 0x94, 0xb8, 0x7f, 0x33, 0xa4, 0x18, 0x70,
+ 0xb0, 0x31, 0x86, 0xa1, 0x1e, 0xe4, 0xe5, 0x64, 0x77, 0xaf, 0x6e, 0x24, 0x27, 0x40, 0x5a, 0x54,
+ 0xf9, 0x97, 0x24, 0x94, 0x56, 0xf7, 0x14, 0xd6, 0xfc, 0x35, 0xfe, 0x00, 0xfe, 0x7a, 0x67, 0xd0,
+ 0x24, 0xee, 0x0c, 0x9a, 0xd5, 0x4e, 0x29, 0x79, 0xcf, 0x4e, 0xe9, 0x62, 0xb5, 0x53, 0x12, 0x7d,
+ 0x4c, 0x7d, 0xeb, 0xed, 0x96, 0x77, 0xf5, 0x4a, 0xff, 0xa7, 0x7d, 0x84, 0xf2, 0x1f, 0x19, 0x38,
+ 0x18, 0x12, 0xd7, 0x1b, 0x78, 0x0e, 0xc1, 0x53, 0x7f, 0xe5, 0xee, 0xce, 0x83, 0x48, 0x83, 0x0c,
+ 0xb9, 0xe6, 0x23, 0x6d, 0x22, 0xf2, 0x5c, 0xb4, 0x21, 0xa0, 0xa6, 0x32, 0x08, 0x4d, 0x22, 0x55,
+ 0xff, 0x2b, 0x05, 0x69, 0x4e, 0x41, 0xd7, 0xb0, 0x7f, 0x83, 0x3d, 0xe2, 0x4c, 0xb1, 0x73, 0xa5,
+ 0xf3, 0x5f, 0xa5, 0xdf, 0x7c, 0xb5, 0xbb, 0x98, 0x5a, 0xdd, 0xb8, 0xc6, 0x74, 0x44, 0xce, 0x7d,
+ 0xe0, 0x76, 0x4c, 0x2b, 0x05, 0x52, 0x84, 0xdc, 0xbf, 0x8e, 0xc3, 0xe3, 0x99, 0x63, 0x8f, 0x88,
+ 0xeb, 0xb2, 0x82, 0xc8, 0x93, 0x8e, 0x10, 0x2f, 0xec, 0xdb, 0xbf, 0xbf, 0xf8, 0x7e, 0x00, 0xcf,
+ 0x92, 0x53, 0x3b, 0xa6, 0x1d, 0xce, 0x56, 0x28, 0x42, 0x91, 0x29, 0xec, 0xf9, 0x89, 0x52, 0xc8,
+ 0x17, 0x65, 0xb9, 0x75, 0x2f, 0xf9, 0x86, 0x2a, 0x20, 0xdd, 0x76, 0x4c, 0x2b, 0x4a, 0x78, 0xfe,
+ 0x5b, 0xf5, 0x53, 0x28, 0xaf, 0x5b, 0x07, 0xfd, 0x2e, 0xec, 0x51, 0x72, 0xa3, 0x07, 0x16, 0xe2,
+ 0x2b, 0x90, 0xd4, 0x8a, 0x94, 0xdc, 0x04, 0x2f, 0x55, 0x8f, 0xe1, 0xf1, 0xad, 0xdf, 0x85, 0xfe,
+ 0x10, 0xca, 0x58, 0xfc, 0xa0, 0x1b, 0x73, 0x07, 0xf3, 0xfd, 0x32, 0x01, 0xb0, 0x2f, 0xe9, 0x4d,
+ 0x49, 0xae, 0x3a, 0x50, 0x08, 0xe9, 0x86, 0x46, 0x90, 0x93, 0xba, 0xf9, 0x27, 0x34, 0x2f, 0x77,
+ 0xfa, 0x6a, 0xa6, 0x86, 0xeb, 0xe1, 0xe9, 0x8c, 0xf8, 0xd8, 0x5a, 0x00, 0x7c, 0x9c, 0x85, 0x34,
+ 0xb7, 0x6b, 0xf5, 0xe7, 0x80, 0x36, 0x5f, 0x44, 0x7f, 0x00, 0xfb, 0x84, 0x32, 0x57, 0x37, 0x74,
+ 0xc9, 0xc2, 0x95, 0x2f, 0x6a, 0x25, 0x49, 0xf6, 0x5f, 0xfc, 0x01, 0xe4, 0x3d, 0x9f, 0x9d, 0xfb,
+ 0x48, 0x52, 0x5b, 0x12, 0x94, 0xff, 0x4e, 0xc2, 0xc1, 0xb9, 0x63, 0x7a, 0xa4, 0x65, 0x5a, 0xc4,
+ 0xf5, 0xa3, 0xaa, 0x05, 0x29, 0xd7, 0xa4, 0x57, 0xf7, 0x19, 0xfe, 0x18, 0x3f, 0xfa, 0x39, 0xec,
+ 0x8f, 0x6d, 0x67, 0x8a, 0xbd, 0x60, 0x73, 0xf6, 0x1e, 0xcd, 0x40, 0x49, 0x40, 0xf9, 0x34, 0x66,
+ 0x01, 0x91, 0xd3, 0x89, 0xa1, 0xdf, 0xb0, 0x4f, 0x70, 0xb9, 0x0b, 0xe6, 0xb4, 0x92, 0x4f, 0xe6,
+ 0x1f, 0xe6, 0xa2, 0x3f, 0x85, 0xaa, 0x3c, 0xb3, 0x33, 0x58, 0x6e, 0x9c, 0x9a, 0x94, 0x18, 0xba,
+ 0x7b, 0x89, 0x1d, 0xc3, 0xa4, 0x13, 0xde, 0xf3, 0xe5, 0xb4, 0x8a, 0x78, 0xa3, 0x19, 0xbc, 0x30,
+ 0x90, 0xbf, 0x23, 0xb2, 0x9a, 0x48, 0xc5, 0xb8, 0xd6, 0x8c, 0xb2, 0xd5, 0xb7, 0x6e, 0xd6, 0xff,
+ 0x3f, 0xb9, 0xf4, 0x57, 0x90, 0xe6, 0x55, 0x87, 0x2f, 0xf4, 0xb2, 0xf1, 0xdf, 0x6d, 0xa1, 0x59,
+ 0xfb, 0x53, 0x83, 0xc3, 0x60, 0x07, 0x30, 0xa8, 0x75, 0xfe, 0x1e, 0xd8, 0x41, 0xf0, 0x93, 0x2c,
+ 0x75, 0xae, 0xf2, 0x37, 0x19, 0xbf, 0xd4, 0x87, 0x77, 0x3d, 0x1f, 0xba, 0xd4, 0xa3, 0xd7, 0x50,
+ 0x9c, 0x12, 0x67, 0x42, 0x74, 0x36, 0x7e, 0xcf, 0x5d, 0xd9, 0xa4, 0x3c, 0x8f, 0xd2, 0xdf, 0x33,
+ 0xb6, 0x01, 0xe7, 0x12, 0x6d, 0x4a, 0x61, 0xba, 0xa4, 0xa0, 0xdf, 0xf7, 0x5d, 0x6f, 0x59, 0xd7,
+ 0x93, 0x7c, 0x95, 0xf6, 0x04, 0xd9, 0x2f, 0xe9, 0x4d, 0xc8, 0x7a, 0x8e, 0x39, 0x99, 0x10, 0x47,
+ 0x8e, 0x16, 0x1f, 0x46, 0xc9, 0x13, 0x82, 0x43, 0xf3, 0x59, 0x11, 0x81, 0x83, 0xa0, 0x5d, 0x30,
+ 0x6d, 0xaa, 0x33, 0x16, 0x3e, 0x5c, 0x94, 0x8e, 0x3e, 0x8b, 0x80, 0x57, 0x0f, 0xf1, 0x9e, 0xda,
+ 0x86, 0x1c, 0x34, 0xcb, 0x78, 0x8d, 0xcc, 0x5a, 0x58, 0xb1, 0xfb, 0xc7, 0x8b, 0x0a, 0xdf, 0x36,
+ 0x88, 0xd6, 0xc2, 0x8a, 0xad, 0x39, 0x96, 0xa3, 0x64, 0x0b, 0x6b, 0x07, 0x04, 0x74, 0x01, 0xe5,
+ 0x91, 0x65, 0xf3, 0x52, 0x75, 0x41, 0x2e, 0xf1, 0xb5, 0x69, 0x3b, 0xfc, 0x98, 0xa3, 0x74, 0xf4,
+ 0x69, 0x94, 0x5e, 0x44, 0xb0, 0x1e, 0x4b, 0x4e, 0x01, 0xbf, 0x3f, 0x5a, 0xa5, 0xf2, 0x44, 0x6e,
+ 0x59, 0x3c, 0x0f, 0x58, 0xd8, 0x23, 0x94, 0xb8, 0x2e, 0x3f, 0xf8, 0x60, 0x89, 0x5c, 0xd0, 0x4f,
+ 0x24, 0x99, 0x0d, 0x93, 0x3d, 0xca, 0x14, 0xf3, 0x99, 0xf9, 0x61, 0x46, 0xb4, 0x71, 0x7d, 0x95,
+ 0x51, 0xe8, 0xb2, 0x86, 0x86, 0x9e, 0xc1, 0x63, 0xec, 0xba, 0xe6, 0x84, 0xba, 0xba, 0x67, 0xeb,
+ 0x36, 0x25, 0xba, 0x70, 0x88, 0x0a, 0xf0, 0x2c, 0x83, 0xe4, 0x8f, 0x43, 0xbb, 0x47, 0x89, 0xf0,
+ 0x7f, 0xe5, 0x17, 0x50, 0x08, 0x39, 0x9b, 0x72, 0x7a, 0x57, 0x9b, 0xbf, 0x0f, 0x85, 0x6e, 0xaf,
+ 0xab, 0x9f, 0xaa, 0xda, 0xcb, 0x4e, 0xf7, 0x65, 0x39, 0xce, 0x09, 0xaa, 0xda, 0x1c, 0x70, 0x92,
+ 0x5a, 0x4e, 0x20, 0x04, 0xa5, 0xfa, 0x89, 0xa6, 0xd6, 0x9b, 0x6f, 0x04, 0xa9, 0x59, 0x4e, 0x2a,
+ 0xa7, 0x50, 0x5e, 0x5f, 0x7f, 0xe5, 0xc5, 0x5d, 0x22, 0x4a, 0x00, 0xcd, 0xce, 0xa0, 0x51, 0xd7,
+ 0x9a, 0x42, 0x42, 0x19, 0x8a, 0xf5, 0x46, 0xe3, 0xec, 0xf4, 0xec, 0xa4, 0x3e, 0x64, 0x94, 0x84,
+ 0xf2, 0x35, 0xec, 0xaf, 0xad, 0x89, 0xf2, 0xe5, 0x3b, 0x14, 0x56, 0x4f, 0x3b, 0x43, 0xbd, 0x7e,
+ 0x72, 0x5e, 0x7f, 0x33, 0x10, 0x1b, 0x15, 0x9c, 0xd0, 0x69, 0xe9, 0xdd, 0x5e, 0x57, 0x3d, 0xed,
+ 0x0f, 0xdf, 0x94, 0x13, 0x4a, 0x7f, 0x7d, 0x49, 0xde, 0x89, 0xd8, 0xea, 0x68, 0xea, 0x0a, 0x22,
+ 0x27, 0xac, 0x22, 0x5e, 0x00, 0x2c, 0x5d, 0x52, 0x19, 0xde, 0x85, 0x76, 0x00, 0x7b, 0x6a, 0xb7,
+ 0xa9, 0xf7, 0x5a, 0x7a, 0xb0, 0x95, 0x82, 0xa0, 0x74, 0x52, 0x1f, 0xaa, 0x83, 0xa1, 0xde, 0xe9,
+ 0xea, 0xfd, 0x7a, 0x97, 0x59, 0x95, 0x69, 0x5d, 0xd7, 0x4e, 0x3a, 0x61, 0x6a, 0x52, 0xb1, 0x00,
+ 0x96, 0x93, 0x9b, 0xf2, 0xf6, 0x1d, 0x16, 0x55, 0x5f, 0xab, 0xdd, 0xa1, 0x3e, 0xec, 0x9c, 0xaa,
+ 0xe5, 0x38, 0x3a, 0x84, 0xfd, 0xbe, 0xd6, 0x6b, 0xa8, 0x83, 0x41, 0xa7, 0xfb, 0x52, 0x10, 0x13,
+ 0xe8, 0x09, 0xfc, 0x60, 0xf0, 0xa6, 0xdb, 0x68, 0x6b, 0xbd, 0x6e, 0xe7, 0x1b, 0xb5, 0xa9, 0xaf,
+ 0xbf, 0x91, 0x54, 0xfe, 0xb1, 0x0c, 0x59, 0x99, 0x16, 0x90, 0x06, 0x79, 0x3c, 0xf6, 0x88, 0xa3,
+ 0x63, 0xcb, 0x92, 0x49, 0xf2, 0x79, 0xf4, 0xac, 0x52, 0xab, 0x33, 0xde, 0xba, 0x65, 0xb5, 0x63,
+ 0x5a, 0x0e, 0xcb, 0xbf, 0x43, 0x98, 0x74, 0x21, 0x6b, 0xcb, 0xf6, 0x98, 0x74, 0xb1, 0xc4, 0xa4,
+ 0x0b, 0x74, 0x06, 0x20, 0x30, 0x09, 0x1e, 0x5d, 0xca, 0xe6, 0xf0, 0xc7, 0xdb, 0x82, 0xaa, 0x78,
+ 0x74, 0xd9, 0x8e, 0x69, 0x42, 0x3b, 0xf6, 0x80, 0x2c, 0x38, 0x94, 0xb0, 0xd4, 0xd0, 0xed, 0xb1,
+ 0x1f, 0x5f, 0x22, 0xbd, 0x7e, 0xb1, 0x35, 0x3e, 0x35, 0x7a, 0x63, 0x11, 0x88, 0xed, 0x98, 0x56,
+ 0xc6, 0x6b, 0x34, 0xe4, 0xc1, 0x63, 0x21, 0x6d, 0xad, 0xe5, 0x96, 0x7b, 0x3b, 0x5f, 0x6e, 0x2b,
+ 0x6f, 0xb3, 0xb5, 0xc6, 0x9b, 0x64, 0xf4, 0xeb, 0x38, 0x28, 0x42, 0xac, 0xbb, 0xa0, 0xa3, 0x4b,
+ 0xc7, 0xa6, 0xe6, 0x5f, 0x12, 0x63, 0x43, 0x07, 0x71, 0x26, 0xf3, 0x6a, 0x5b, 0x1d, 0x06, 0x21,
+ 0xcc, 0x0d, 0x7d, 0xde, 0xc7, 0xef, 0x7e, 0x05, 0x7d, 0x05, 0x19, 0x6c, 0xdd, 0xe0, 0x85, 0x2b,
+ 0xcf, 0x56, 0x9f, 0x6d, 0x23, 0x9e, 0x33, 0xb6, 0x63, 0x9a, 0x84, 0x40, 0x5d, 0xc8, 0x1a, 0x64,
+ 0x8c, 0xe7, 0x96, 0x27, 0xcf, 0xbe, 0x8f, 0xb6, 0x40, 0x6b, 0x0a, 0xce, 0x76, 0x4c, 0xf3, 0x41,
+ 0xd0, 0xdb, 0xe5, 0x4c, 0x32, 0xb2, 0xe7, 0xd4, 0x93, 0xe7, 0xdf, 0x9f, 0x6e, 0x81, 0xaa, 0xfa,
+ 0x9b, 0x3c, 0x73, 0xea, 0x85, 0x86, 0x10, 0xfe, 0x8c, 0xda, 0x90, 0xa6, 0xe4, 0x9a, 0x38, 0xf2,
+ 0x08, 0xfc, 0x47, 0x5b, 0xe0, 0x76, 0x19, 0x5f, 0x3b, 0xa6, 0x09, 0x00, 0x16, 0x1d, 0xb6, 0xa3,
+ 0x8f, 0x4d, 0x8a, 0x2d, 0x6b, 0xc1, 0xab, 0xc3, 0x76, 0xd1, 0xd1, 0x73, 0x5a, 0x82, 0x97, 0x45,
+ 0x87, 0xed, 0x3f, 0xb0, 0xd5, 0x71, 0xc8, 0x8c, 0x60, 0xff, 0x44, 0x7d, 0x9b, 0xd5, 0xd1, 0x38,
+ 0x23, 0x5b, 0x1d, 0x01, 0x51, 0xfd, 0x33, 0xc8, 0xf9, 0xd9, 0x02, 0x9d, 0x40, 0x81, 0x9f, 0x64,
+ 0xf2, 0x57, 0xfd, 0xa9, 0x67, 0x9b, 0x6e, 0x26, 0xcc, 0xbe, 0x44, 0xa6, 0x8b, 0x07, 0x46, 0x7e,
+ 0x03, 0xf9, 0x20, 0x71, 0x3c, 0x30, 0xf4, 0x6f, 0xe2, 0x50, 0x5e, 0x4f, 0x1a, 0xa8, 0x07, 0x7b,
+ 0x04, 0x3b, 0xd6, 0x42, 0x1f, 0x9b, 0x8e, 0x49, 0x27, 0xfe, 0xf1, 0xf9, 0x36, 0x42, 0x8a, 0x1c,
+ 0xa0, 0x25, 0xf8, 0xd1, 0x29, 0x14, 0x59, 0x13, 0x13, 0xe0, 0x25, 0xb6, 0xc6, 0x2b, 0x30, 0x7e,
+ 0x09, 0x57, 0xfd, 0x15, 0x1c, 0xde, 0x92, 0x78, 0xd0, 0x25, 0x3c, 0x0a, 0x66, 0x40, 0x7d, 0xe3,
+ 0xbe, 0xe1, 0x27, 0x11, 0xf7, 0x2d, 0x39, 0xfb, 0xf2, 0x82, 0xd9, 0xa1, 0xb7, 0x41, 0x73, 0xab,
+ 0x1f, 0xc0, 0xfb, 0xdf, 0x93, 0x75, 0xaa, 0x79, 0xc8, 0xca, 0x58, 0xae, 0x3e, 0x87, 0x62, 0x38,
+ 0x00, 0xd9, 0x84, 0xbf, 0x1a, 0xd0, 0xcc, 0xbc, 0xe9, 0xd5, 0xa8, 0xac, 0x66, 0x21, 0xcd, 0xa3,
+ 0xab, 0x9a, 0x83, 0x8c, 0x48, 0x31, 0xd5, 0xbf, 0x8f, 0x43, 0x3e, 0x08, 0x11, 0xf4, 0x25, 0xa4,
+ 0x82, 0x5d, 0xd9, 0xed, 0x6c, 0xc9, 0xf9, 0x58, 0x1b, 0xef, 0x47, 0xea, 0xf6, 0xcb, 0xe1, 0xb3,
+ 0x56, 0x87, 0x90, 0x11, 0x21, 0x86, 0x5e, 0x01, 0x2c, 0x1d, 0x6b, 0x07, 0xad, 0x42, 0xdc, 0xc7,
+ 0xf9, 0x60, 0xc4, 0x50, 0xfe, 0x3d, 0x11, 0xda, 0x29, 0x58, 0xde, 0x7f, 0x18, 0x40, 0xda, 0x20,
+ 0x16, 0x5e, 0x48, 0x41, 0x5f, 0xec, 0xb4, 0xb8, 0xb5, 0x26, 0x83, 0x60, 0xf9, 0x8b, 0x63, 0xa1,
+ 0x6f, 0x20, 0x87, 0x2d, 0x73, 0x42, 0x75, 0xcf, 0x96, 0x36, 0xf9, 0xc9, 0x6e, 0xb8, 0x75, 0x86,
+ 0x32, 0xb4, 0x59, 0x16, 0xc7, 0xe2, 0xcf, 0xea, 0x87, 0x90, 0xe6, 0xd2, 0xd0, 0x07, 0x50, 0xe4,
+ 0xd2, 0xf4, 0xa9, 0x69, 0x59, 0xa6, 0x2b, 0x77, 0x67, 0x0a, 0x9c, 0x76, 0xca, 0x49, 0xd5, 0x17,
+ 0x90, 0x95, 0x08, 0xe8, 0xb7, 0x20, 0x33, 0x23, 0x8e, 0x69, 0x8b, 0x59, 0x2c, 0xa9, 0xc9, 0x27,
+ 0x46, 0xb7, 0xc7, 0x63, 0x97, 0x78, 0xbc, 0x49, 0x48, 0x6a, 0xf2, 0xe9, 0xf8, 0x31, 0x1c, 0xde,
+ 0x12, 0x03, 0xca, 0xdf, 0x25, 0x20, 0x1f, 0x0c, 0xcd, 0xe8, 0x35, 0x94, 0xf0, 0x88, 0x39, 0xab,
+ 0x3e, 0xc3, 0x9e, 0x47, 0x1c, 0xba, 0xeb, 0x2d, 0x91, 0x3d, 0x01, 0xd3, 0x17, 0x28, 0xe8, 0x2b,
+ 0xc8, 0x5e, 0x9b, 0xe4, 0xe6, 0x7e, 0xc7, 0x23, 0x19, 0x06, 0xd1, 0xa2, 0xe8, 0x2d, 0xc8, 0x2b,
+ 0x44, 0xfa, 0x14, 0xcf, 0x66, 0xac, 0x3f, 0x18, 0x53, 0xd9, 0x71, 0xed, 0x02, 0x2b, 0x67, 0xdb,
+ 0x53, 0x81, 0xd5, 0xa2, 0xca, 0xfb, 0x50, 0x08, 0xdd, 0x41, 0x44, 0x65, 0x48, 0xce, 0x1d, 0xcb,
+ 0xdf, 0x97, 0x98, 0x3b, 0x96, 0xf2, 0x4b, 0xd8, 0x5f, 0x03, 0x79, 0x98, 0x4b, 0x35, 0xbf, 0x07,
+ 0xa5, 0xd0, 0x2d, 0xd9, 0xe5, 0xf6, 0xf9, 0x5e, 0x88, 0xda, 0x31, 0x94, 0xcf, 0xa1, 0xb8, 0x22,
+ 0x9b, 0x2b, 0x48, 0x97, 0x0a, 0x52, 0x54, 0x81, 0x6c, 0xf8, 0xe2, 0x61, 0x51, 0xf3, 0x1f, 0x95,
+ 0xff, 0x49, 0x41, 0x21, 0x74, 0x2d, 0x02, 0x75, 0x20, 0x6d, 0x7a, 0x24, 0x48, 0x85, 0xcf, 0xb7,
+ 0xbb, 0x55, 0x51, 0xeb, 0x78, 0x64, 0xaa, 0x09, 0x84, 0xea, 0x18, 0xa0, 0x63, 0x10, 0xea, 0x99,
+ 0x63, 0x93, 0x38, 0xcc, 0x99, 0xc3, 0x37, 0xa2, 0xa4, 0x76, 0x05, 0x6f, 0x79, 0x19, 0x8a, 0x65,
+ 0xbb, 0xe5, 0x2b, 0xec, 0x0b, 0xe4, 0x81, 0x55, 0x40, 0x3c, 0x73, 0xa8, 0xbf, 0x2b, 0x94, 0x0c,
+ 0x76, 0x85, 0xaa, 0xdf, 0x25, 0x20, 0xc5, 0xe4, 0xa2, 0x0e, 0x24, 0x24, 0x70, 0xb4, 0x9b, 0x45,
+ 0x2b, 0x8a, 0x07, 0x9a, 0x6a, 0x09, 0xd3, 0x40, 0x27, 0xf2, 0x34, 0x3b, 0x11, 0x79, 0x9b, 0x21,
+ 0x0c, 0xb6, 0x76, 0x9e, 0x8d, 0x3e, 0xf4, 0xf7, 0xad, 0x84, 0x53, 0x3e, 0xaa, 0x89, 0xeb, 0xf0,
+ 0x35, 0xff, 0x3a, 0x7c, 0xad, 0x4e, 0xfd, 0x0b, 0xbb, 0xe8, 0x13, 0x28, 0xb8, 0x97, 0xb6, 0xe3,
+ 0xe9, 0x82, 0x23, 0xf5, 0x0e, 0x0e, 0xe0, 0x2f, 0xf2, 0x93, 0x51, 0xf4, 0x08, 0xd2, 0x16, 0xbe,
+ 0x20, 0x96, 0xbc, 0xdf, 0x25, 0x1e, 0xd0, 0x7b, 0x90, 0xb3, 0x4c, 0x7a, 0xa5, 0x33, 0x7f, 0xcd,
+ 0x88, 0xe3, 0x01, 0xf6, 0x7c, 0xe6, 0x58, 0xd5, 0x5f, 0xca, 0x33, 0xf6, 0xf9, 0x3b, 0xce, 0xd8,
+ 0x07, 0x43, 0x4d, 0x4c, 0xc2, 0x05, 0xc8, 0x76, 0xba, 0x43, 0xf5, 0xa5, 0xaa, 0x95, 0x13, 0x28,
+ 0x0f, 0xe9, 0xd6, 0x49, 0xaf, 0x3e, 0x2c, 0x27, 0xc5, 0x61, 0x5b, 0xef, 0x44, 0xad, 0x77, 0xcb,
+ 0x29, 0xb4, 0x07, 0x79, 0x36, 0xaf, 0x0d, 0x86, 0xf5, 0xd3, 0x7e, 0x39, 0x8d, 0x8a, 0x90, 0x6b,
+ 0x9e, 0x69, 0xf5, 0x61, 0xa7, 0xd7, 0x2d, 0x67, 0xd8, 0x24, 0xf8, 0xaa, 0xfe, 0xba, 0xae, 0x37,
+ 0x4e, 0xea, 0x83, 0x41, 0x39, 0x7b, 0xfc, 0x05, 0x7c, 0xff, 0xbf, 0x0d, 0x1c, 0xe7, 0x35, 0xbe,
+ 0x2b, 0x59, 0x9f, 0x99, 0xdf, 0x14, 0x7c, 0xba, 0x7e, 0xfd, 0xec, 0x22, 0xc3, 0xcd, 0xf0, 0xfc,
+ 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x4c, 0x56, 0x9e, 0x5a, 0x91, 0x30, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go
new file mode 100644
index 0000000..72a230d
--- /dev/null
+++ b/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go
@@ -0,0 +1,160 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: endpoints.proto
+
+package pipeline_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type ApiServiceDescriptor struct {
+ // (Required) The URL to connect to.
+ Url string `protobuf:"bytes,2,opt,name=url" json:"url,omitempty"`
+ // (Optional) The method for authentication. If unspecified, access to the
+ // url is already being performed in a trusted context (e.g. localhost,
+ // private network).
+ //
+ // Types that are valid to be assigned to Authentication:
+ // *ApiServiceDescriptor_Oauth2ClientCredentialsGrant
+ Authentication isApiServiceDescriptor_Authentication `protobuf_oneof:"authentication"`
+}
+
+func (m *ApiServiceDescriptor) Reset() { *m = ApiServiceDescriptor{} }
+func (m *ApiServiceDescriptor) String() string { return proto.CompactTextString(m) }
+func (*ApiServiceDescriptor) ProtoMessage() {}
+func (*ApiServiceDescriptor) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} }
+
+type isApiServiceDescriptor_Authentication interface {
+ isApiServiceDescriptor_Authentication()
+}
+
+type ApiServiceDescriptor_Oauth2ClientCredentialsGrant struct {
+ Oauth2ClientCredentialsGrant *OAuth2ClientCredentialsGrant `protobuf:"bytes,3,opt,name=oauth2_client_credentials_grant,json=oauth2ClientCredentialsGrant,oneof"`
+}
+
+func (*ApiServiceDescriptor_Oauth2ClientCredentialsGrant) isApiServiceDescriptor_Authentication() {}
+
+func (m *ApiServiceDescriptor) GetAuthentication() isApiServiceDescriptor_Authentication {
+ if m != nil {
+ return m.Authentication
+ }
+ return nil
+}
+
+func (m *ApiServiceDescriptor) GetUrl() string {
+ if m != nil {
+ return m.Url
+ }
+ return ""
+}
+
+func (m *ApiServiceDescriptor) GetOauth2ClientCredentialsGrant() *OAuth2ClientCredentialsGrant {
+ if x, ok := m.GetAuthentication().(*ApiServiceDescriptor_Oauth2ClientCredentialsGrant); ok {
+ return x.Oauth2ClientCredentialsGrant
+ }
+ return nil
+}
+
+// XXX_OneofFuncs is for the internal use of the proto package.
+func (*ApiServiceDescriptor) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
+ return _ApiServiceDescriptor_OneofMarshaler, _ApiServiceDescriptor_OneofUnmarshaler, _ApiServiceDescriptor_OneofSizer, []interface{}{
+ (*ApiServiceDescriptor_Oauth2ClientCredentialsGrant)(nil),
+ }
+}
+
+func _ApiServiceDescriptor_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
+ m := msg.(*ApiServiceDescriptor)
+ // authentication
+ switch x := m.Authentication.(type) {
+ case *ApiServiceDescriptor_Oauth2ClientCredentialsGrant:
+ b.EncodeVarint(3<<3 | proto.WireBytes)
+ if err := b.EncodeMessage(x.Oauth2ClientCredentialsGrant); err != nil {
+ return err
+ }
+ case nil:
+ default:
+ return fmt.Errorf("ApiServiceDescriptor.Authentication has unexpected type %T", x)
+ }
+ return nil
+}
+
+func _ApiServiceDescriptor_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
+ m := msg.(*ApiServiceDescriptor)
+ switch tag {
+ case 3: // authentication.oauth2_client_credentials_grant
+ if wire != proto.WireBytes {
+ return true, proto.ErrInternalBadWireType
+ }
+ msg := new(OAuth2ClientCredentialsGrant)
+ err := b.DecodeMessage(msg)
+ m.Authentication = &ApiServiceDescriptor_Oauth2ClientCredentialsGrant{msg}
+ return true, err
+ default:
+ return false, nil
+ }
+}
+
+func _ApiServiceDescriptor_OneofSizer(msg proto.Message) (n int) {
+ m := msg.(*ApiServiceDescriptor)
+ // authentication
+ switch x := m.Authentication.(type) {
+ case *ApiServiceDescriptor_Oauth2ClientCredentialsGrant:
+ s := proto.Size(x.Oauth2ClientCredentialsGrant)
+ n += proto.SizeVarint(3<<3 | proto.WireBytes)
+ n += proto.SizeVarint(uint64(s))
+ n += s
+ case nil:
+ default:
+ panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
+ }
+ return n
+}
+
+type OAuth2ClientCredentialsGrant struct {
+ // (Required) The URL to submit a "client_credentials" grant type request for
+ // an OAuth access token which will be used as a bearer token for requests.
+ Url string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"`
+}
+
+func (m *OAuth2ClientCredentialsGrant) Reset() { *m = OAuth2ClientCredentialsGrant{} }
+func (m *OAuth2ClientCredentialsGrant) String() string { return proto.CompactTextString(m) }
+func (*OAuth2ClientCredentialsGrant) ProtoMessage() {}
+func (*OAuth2ClientCredentialsGrant) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{1} }
+
+func (m *OAuth2ClientCredentialsGrant) GetUrl() string {
+ if m != nil {
+ return m.Url
+ }
+ return ""
+}
+
+func init() {
+ proto.RegisterType((*ApiServiceDescriptor)(nil), "org.apache.beam.model.pipeline.v1.ApiServiceDescriptor")
+ proto.RegisterType((*OAuth2ClientCredentialsGrant)(nil), "org.apache.beam.model.pipeline.v1.OAuth2ClientCredentialsGrant")
+}
+
+func init() { proto.RegisterFile("endpoints.proto", fileDescriptor1) }
+
+var fileDescriptor1 = []byte{
+ // 235 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x90, 0xb1, 0x4a, 0x03, 0x41,
+ 0x10, 0x86, 0x5d, 0x03, 0x42, 0x36, 0xa0, 0xe1, 0xb0, 0x48, 0x11, 0x30, 0xa6, 0x4a, 0xb5, 0x98,
+ 0x58, 0x5a, 0x48, 0x2e, 0x8a, 0x76, 0x42, 0xec, 0x6c, 0x96, 0xcd, 0xde, 0x90, 0x0c, 0x6c, 0x76,
+ 0x96, 0xb9, 0xc9, 0x3d, 0x83, 0x2f, 0xe6, 0x7b, 0xc9, 0x46, 0x4e, 0x1b, 0xc9, 0x75, 0xc3, 0xfc,
+ 0xf0, 0xfd, 0xfc, 0x9f, 0xbe, 0x82, 0x58, 0x25, 0xc2, 0x28, 0xb5, 0x49, 0x4c, 0x42, 0xc5, 0x2d,
+ 0xf1, 0xd6, 0xb8, 0xe4, 0xfc, 0x0e, 0xcc, 0x06, 0xdc, 0xde, 0xec, 0xa9, 0x82, 0x60, 0x12, 0x26,
+ 0x08, 0x18, 0xc1, 0x34, 0xf3, 0xe9, 0x97, 0xd2, 0xd7, 0xcb, 0x84, 0xef, 0xc0, 0x0d, 0x7a, 0x78,
+ 0x82, 0xda, 0x33, 0x26, 0x21, 0x2e, 0x86, 0xba, 0x77, 0xe0, 0x30, 0x3a, 0x9f, 0xa8, 0x59, 0x7f,
+ 0x9d, 0xcf, 0xe2, 0x53, 0xe9, 0x1b, 0x72, 0x07, 0xd9, 0x2d, 0xac, 0x0f, 0x08, 0x51, 0xac, 0x67,
+ 0xa8, 0x20, 0x0a, 0xba, 0x50, 0xdb, 0x2d, 0xbb, 0x28, 0xa3, 0xde, 0x44, 0xcd, 0x06, 0x8b, 0x47,
+ 0xd3, 0x59, 0x6c, 0xde, 0x96, 0x99, 0xb4, 0x3a, 0x82, 0x56, 0x7f, 0x9c, 0x97, 0x8c, 0x79, 0x3d,
+ 0x5b, 0x8f, 0x7f, 0x9a, 0xfe, 0xcf, 0xcb, 0xa1, 0xbe, 0xcc, 0x71, 0xfe, 0x79, 0x27, 0x48, 0x71,
+ 0x7a, 0xa7, 0xc7, 0xa7, 0x88, 0xed, 0x1c, 0xf5, 0x3b, 0xa7, 0x7c, 0xd0, 0xdd, 0x7a, 0xca, 0xfe,
+ 0x73, 0xab, 0xf4, 0x63, 0xd0, 0xfe, 0x6d, 0x33, 0xdf, 0x5c, 0x1c, 0x05, 0xdf, 0x7f, 0x07, 0x00,
+ 0x00, 0xff, 0xff, 0x98, 0xdf, 0xf8, 0x2f, 0x73, 0x01, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go
new file mode 100644
index 0000000..082724d
--- /dev/null
+++ b/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go
@@ -0,0 +1,120 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: standard_window_fns.proto
+
+package pipeline_v1
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import google_protobuf1 "github.com/golang/protobuf/ptypes/duration"
+import google_protobuf2 "github.com/golang/protobuf/ptypes/timestamp"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// beam:windowfn:fixed_windows:v0.1
+type FixedWindowsPayload struct {
+ Size *google_protobuf1.Duration `protobuf:"bytes,1,opt,name=size" json:"size,omitempty"`
+ Offset *google_protobuf2.Timestamp `protobuf:"bytes,2,opt,name=offset" json:"offset,omitempty"`
+}
+
+func (m *FixedWindowsPayload) Reset() { *m = FixedWindowsPayload{} }
+func (m *FixedWindowsPayload) String() string { return proto.CompactTextString(m) }
+func (*FixedWindowsPayload) ProtoMessage() {}
+func (*FixedWindowsPayload) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} }
+
+func (m *FixedWindowsPayload) GetSize() *google_protobuf1.Duration {
+ if m != nil {
+ return m.Size
+ }
+ return nil
+}
+
+func (m *FixedWindowsPayload) GetOffset() *google_protobuf2.Timestamp {
+ if m != nil {
+ return m.Offset
+ }
+ return nil
+}
+
+// beam:windowfn:sliding_windows:v0.1
+type SlidingWindowsPayload struct {
+ Size *google_protobuf1.Duration `protobuf:"bytes,1,opt,name=size" json:"size,omitempty"`
+ Offset *google_protobuf2.Timestamp `protobuf:"bytes,2,opt,name=offset" json:"offset,omitempty"`
+ Period *google_protobuf1.Duration `protobuf:"bytes,3,opt,name=period" json:"period,omitempty"`
+}
+
+func (m *SlidingWindowsPayload) Reset() { *m = SlidingWindowsPayload{} }
+func (m *SlidingWindowsPayload) String() string { return proto.CompactTextString(m) }
+func (*SlidingWindowsPayload) ProtoMessage() {}
+func (*SlidingWindowsPayload) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{1} }
+
+func (m *SlidingWindowsPayload) GetSize() *google_protobuf1.Duration {
+ if m != nil {
+ return m.Size
+ }
+ return nil
+}
+
+func (m *SlidingWindowsPayload) GetOffset() *google_protobuf2.Timestamp {
+ if m != nil {
+ return m.Offset
+ }
+ return nil
+}
+
+func (m *SlidingWindowsPayload) GetPeriod() *google_protobuf1.Duration {
+ if m != nil {
+ return m.Period
+ }
+ return nil
+}
+
+// beam:windowfn:session_windows:v0.1
+type SessionsPayload struct {
+ GapSize *google_protobuf1.Duration `protobuf:"bytes,1,opt,name=gap_size,json=gapSize" json:"gap_size,omitempty"`
+}
+
+func (m *SessionsPayload) Reset() { *m = SessionsPayload{} }
+func (m *SessionsPayload) String() string { return proto.CompactTextString(m) }
+func (*SessionsPayload) ProtoMessage() {}
+func (*SessionsPayload) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{2} }
+
+func (m *SessionsPayload) GetGapSize() *google_protobuf1.Duration {
+ if m != nil {
+ return m.GapSize
+ }
+ return nil
+}
+
+func init() {
+ proto.RegisterType((*FixedWindowsPayload)(nil), "org.apache.beam.model.pipeline.v1.FixedWindowsPayload")
+ proto.RegisterType((*SlidingWindowsPayload)(nil), "org.apache.beam.model.pipeline.v1.SlidingWindowsPayload")
+ proto.RegisterType((*SessionsPayload)(nil), "org.apache.beam.model.pipeline.v1.SessionsPayload")
+}
+
+func init() { proto.RegisterFile("standard_window_fns.proto", fileDescriptor2) }
+
+var fileDescriptor2 = []byte{
+ // 277 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x92, 0x4d, 0x4b, 0xc3, 0x40,
+ 0x10, 0x86, 0xa9, 0x4a, 0x95, 0xed, 0x41, 0x8c, 0x08, 0x69, 0x0e, 0x7e, 0xe4, 0xe4, 0xc5, 0x2d,
+ 0xa9, 0xfe, 0x82, 0x2a, 0xf5, 0x2a, 0x8d, 0x20, 0x78, 0x09, 0x13, 0x77, 0xb2, 0x0e, 0x24, 0x3b,
+ 0x4b, 0x76, 0xfb, 0x61, 0xff, 0x93, 0xff, 0x51, 0xc8, 0x87, 0x07, 0x3d, 0xd4, 0x93, 0xe7, 0x79,
+ 0xde, 0xf7, 0x19, 0x76, 0x47, 0x8c, 0x9d, 0x07, 0xa3, 0xa0, 0x56, 0xd9, 0x9a, 0x8c, 0xe2, 0x75,
+ 0x56, 0x18, 0x27, 0x6d, 0xcd, 0x9e, 0x83, 0x2b, 0xae, 0xb5, 0x04, 0x0b, 0x6f, 0xef, 0x28, 0x73,
+ 0x84, 0x4a, 0x56, 0xac, 0xb0, 0x94, 0x96, 0x2c, 0x96, 0x64, 0x50, 0xae, 0x92, 0xe8, 0x5c, 0x33,
+ 0xeb, 0x12, 0x27, 0x4d, 0x20, 0x5f, 0x16, 0x13, 0xb5, 0xac, 0xc1, 0x13, 0x9b, 0xb6, 0x22, 0xba,
+ 0xf8, 0x39, 0xf7, 0x54, 0xa1, 0xf3, 0x50, 0xd9, 0x16, 0x88, 0x37, 0xe2, 0x74, 0x4e, 0x1b, 0x54,
+ 0x2f, 0x8d, 0xdc, 0x3d, 0xc1, 0x47, 0xc9, 0xa0, 0x82, 0x1b, 0x71, 0xe0, 0x68, 0x8b, 0xe1, 0xe0,
+ 0x72, 0x70, 0x3d, 0x9a, 0x8e, 0x65, 0x5b, 0x23, 0xfb, 0x1a, 0xf9, 0xd0, 0x69, 0x16, 0x0d, 0x16,
+ 0x4c, 0xc5, 0x90, 0x8b, 0xc2, 0xa1, 0x0f, 0xf7, 0x9a, 0x40, 0xf4, 0x2b, 0xf0, 0xdc, 0x7b, 0x17,
+ 0x1d, 0x19, 0x7f, 0x0e, 0xc4, 0x59, 0x5a, 0x92, 0x22, 0xa3, 0xff, 0x5d, 0x1e, 0x24, 0x62, 0x68,
+ 0xb1, 0x26, 0x56, 0xe1, 0xfe, 0x2e, 0x49, 0x07, 0xc6, 0x8f, 0xe2, 0x38, 0x45, 0xe7, 0x88, 0xcd,
+ 0xf7, 0xa2, 0x77, 0xe2, 0x48, 0x83, 0xcd, 0xfe, 0xb6, 0xec, 0xa1, 0x06, 0x9b, 0xd2, 0x16, 0x67,
+ 0xf7, 0x62, 0xf7, 0xc7, 0xce, 0x4e, 0xd2, 0xee, 0x2c, 0xda, 0xb7, 0x99, 0x1b, 0xf7, 0x3a, 0xea,
+ 0xe7, 0xd9, 0x2a, 0xc9, 0x87, 0x8d, 0xe0, 0xf6, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x82, 0x58, 0x88,
+ 0x91, 0x3f, 0x02, 0x00, 0x00,
+}
diff --git a/sdks/go/pkg/beam/provision/provision_test.go b/sdks/go/pkg/beam/provision/provision_test.go
new file mode 100644
index 0000000..f29bc9b
--- /dev/null
+++ b/sdks/go/pkg/beam/provision/provision_test.go
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package provision
+
+import (
+ "reflect"
+ "testing"
+)
+
+type s struct {
+ A int `json:"a,omitempty"`
+ B string `json:"b,omitempty"`
+ C bool `json:"c,omitempty"`
+ D *s `json:"d,omitempty"`
+}
+
+// TestConversions verifies that we can process proto structs via JSON.
+func TestConversions(t *testing.T) {
+ tests := []s{
+ s{},
+ s{A: 2},
+ s{B: "foo"},
+ s{C: true},
+ s{D: &s{A: 3}},
+ s{A: 1, B: "bar", C: true, D: &s{A: 3, B: "baz"}},
+ }
+
+ for _, test := range tests {
+ enc, err := OptionsToProto(test)
+ if err != nil {
+ t.Errorf("Failed to marshal %v: %v", test, err)
+ }
+ var ret s
+ if err := ProtoToOptions(enc, &ret); err != nil {
+ t.Errorf("Failed to unmarshal %v from %v: %v", test, enc, err)
+ }
+ if !reflect.DeepEqual(test, ret) {
+ t.Errorf("Unmarshal(Marshal(%v)) = %v, want %v", test, ret, test)
+ }
+ }
+}
diff --git a/sdks/go/pkg/beam/provision/provison.go b/sdks/go/pkg/beam/provision/provison.go
new file mode 100644
index 0000000..4fbfd23
--- /dev/null
+++ b/sdks/go/pkg/beam/provision/provison.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package provision contains utilities for obtaining runtime provision,
+// information -- such as pipeline options.
+package provision
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "time"
+
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "github.com/golang/protobuf/jsonpb"
+ google_protobuf "github.com/golang/protobuf/ptypes/struct"
+)
+
+// Info returns the runtime provisioning info for the worker.
+func Info(ctx context.Context, endpoint string) (*pb.ProvisionInfo, error) {
+ cc, err := grpcx.Dial(ctx, endpoint, 2*time.Minute)
+ if err != nil {
+ return nil, err
+ }
+ defer cc.Close()
+
+ client := pb.NewProvisionServiceClient(cc)
+
+ resp, err := client.GetProvisionInfo(ctx, &pb.GetProvisionInfoRequest{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to get manifest: %v", err)
+ }
+ return resp.GetInfo(), nil
+}
+
+// OptionsToProto converts pipeline options to a proto struct via JSON.
+func OptionsToProto(v interface{}) (*google_protobuf.Struct, error) {
+ data, err := json.Marshal(v)
+ if err != nil {
+ return nil, err
+ }
+ return JSONToProto(string(data))
+}
+
+// JSONToProto converts JSON-encoded pipeline options to a proto struct.
+func JSONToProto(data string) (*google_protobuf.Struct, error) {
+ var out google_protobuf.Struct
+ if err := jsonpb.UnmarshalString(string(data), &out); err != nil {
+ return nil, err
+ }
+ return &out, nil
+}
+
+// ProtoToOptions converts pipeline options from a proto struct via JSON.
+func ProtoToOptions(opt *google_protobuf.Struct, v interface{}) error {
+ data, err := ProtoToJSON(opt)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal([]byte(data), v)
+}
+
+// ProtoToJSON converts pipeline options from a proto struct to JSON.
+func ProtoToJSON(opt *google_protobuf.Struct) (string, error) {
+ return (&jsonpb.Marshaler{}).MarshalToString(opt)
+}
diff --git a/sdks/go/pkg/beam/util/errorx/guarded.go b/sdks/go/pkg/beam/util/errorx/guarded.go
new file mode 100644
index 0000000..cc0b07b
--- /dev/null
+++ b/sdks/go/pkg/beam/util/errorx/guarded.go
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errorx contains utilities for handling errors.
+package errorx
+
+import "sync"
+
+// GuardedError is a concurrency-safe error wrapper. It is sticky
+// in that the first error won't be overwritten.
+type GuardedError struct {
+ err error
+ mu sync.Mutex
+}
+
+// Error returns the guarded error.
+func (g *GuardedError) Error() error {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ return g.err
+}
+
+// TrySetError sets the error, if not already set. Returns true iff the
+// error was set.
+func (g *GuardedError) TrySetError(err error) bool {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ upd := g.err == nil
+ if upd {
+ g.err = err
+ }
+ return upd
+}
diff --git a/sdks/go/pkg/beam/util/execx/exec.go b/sdks/go/pkg/beam/util/execx/exec.go
new file mode 100644
index 0000000..b4978ef
--- /dev/null
+++ b/sdks/go/pkg/beam/util/execx/exec.go
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package execx contains wrappers and utilities for the exec package.
+package execx
+
+import (
+ "os"
+ "os/exec"
+)
+
+// Execute runs the program with the given arguments. It attaches stdio to the
+// child process.
+func Execute(prog string, args ...string) error {
+ cmd := exec.Command(prog, args...)
+ cmd.Stdin = os.Stdin
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+
+ return cmd.Run()
+}
diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go b/sdks/go/pkg/beam/util/gcsx/gcs.go
new file mode 100644
index 0000000..2e04be0
--- /dev/null
+++ b/sdks/go/pkg/beam/util/gcsx/gcs.go
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gcsx contains utilities for working with Google Cloud Storage (GCS).
+package gcsx
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/url"
+
+ "golang.org/x/oauth2/google"
+ "google.golang.org/api/storage/v1"
+)
+
+// NewClient creates a new GCS client with default application credentials.
+func NewClient(ctx context.Context, scope string) (*storage.Service, error) {
+ cl, err := google.DefaultClient(ctx, scope)
+ if err != nil {
+ return nil, err
+ }
+ return storage.New(cl)
+}
+
+// WriteObject writes the given content to the specified object. If the object
+// already exist, it is overwritten.
+func WriteObject(client *storage.Service, bucket, object string, r io.Reader) error {
+ obj := &storage.Object{
+ Name: object,
+ Bucket: bucket,
+ }
+ _, err := client.Objects.Insert(bucket, obj).Media(r).Do()
+ return err
+}
+
+// ReadObject reads the content of the given object in full.
+func ReadObject(client *storage.Service, bucket, object string) ([]byte, error) {
+ resp, err := client.Objects.Get(bucket, object).Download()
+ if err != nil {
+ return nil, err
+ }
+ return ioutil.ReadAll(resp.Body)
+}
+
+// MakeObject creates a object location from bucket and path. For example,
+// MakeObject("foo", "bar/baz") returns "gs://foo/bar/baz". The bucket
+// must be non-empty.
+func MakeObject(bucket, path string) string {
+ if bucket == "" {
+ panic("bucket must be non-empty")
+ }
+ return fmt.Sprintf("gs://%v/%v", bucket, path)
+}
+
+// ParseObject deconstructs a GCS object name into (bucket, name).
+func ParseObject(object string) (bucket, path string, err error) {
+ parsed, err := url.Parse(object)
+ if err != nil {
+ return "", "", err
+ }
+
+ if parsed.Scheme != "gs" {
+ return "", "", fmt.Errorf("object %s must have 'gs' scheme", object)
+ }
+ if parsed.Host == "" {
+ return "", "", fmt.Errorf("object %s must have bucket", object)
+ }
+ if parsed.Path == "" {
+ return parsed.Host, "", nil
+ }
+
+ // remove leading "/" in URL path
+ return parsed.Host, parsed.Path[1:], nil
+}
diff --git a/sdks/go/pkg/beam/util/grpcx/dial.go b/sdks/go/pkg/beam/util/grpcx/dial.go
new file mode 100644
index 0000000..8467ace
--- /dev/null
+++ b/sdks/go/pkg/beam/util/grpcx/dial.go
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcx
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "google.golang.org/grpc"
+)
+
+// Dial is a convenience wrapper over grpc.Dial that specifies an insecure,
+// blocking connection with a timeout.
+func Dial(ctx context.Context, endpoint string, timeout time.Duration) (*grpc.ClientConn, error) {
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+
+ cc, err := grpc.DialContext(ctx, endpoint, grpc.WithInsecure(), grpc.WithBlock())
+ if err != nil {
+ return nil, fmt.Errorf("failed to dial server at %v: %v", endpoint, err)
+ }
+ return cc, nil
+}
diff --git a/sdks/go/pkg/beam/util/grpcx/metadata.go b/sdks/go/pkg/beam/util/grpcx/metadata.go
new file mode 100644
index 0000000..eed51ed
--- /dev/null
+++ b/sdks/go/pkg/beam/util/grpcx/metadata.go
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package grpcx contains utilities for working with gRPC.
+package grpcx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "google.golang.org/grpc/metadata"
+)
+
+const idKey = "id"
+
+// ReadWorkerID reads the worker ID from an incoming gRPC request context.
+func ReadWorkerID(ctx context.Context) (string, error) {
+ md, ok := metadata.FromIncomingContext(ctx)
+ if !ok {
+ return "", errors.New("failed to read metadata from context")
+ }
+ id, ok := md[idKey]
+ if !ok || len(id) < 1 {
+ return "", fmt.Errorf("failed to find worker id in metadata %v", md)
+ }
+ if len(id) > 1 {
+ return "", fmt.Errorf("multiple worker ids in metadata: %v", id)
+ }
+ return id[0], nil
+}
+
+// WriteWorkerID write the worker ID to an outgoing gRPC request context. It
+// merges the information with any existing gRPC metadata.
+func WriteWorkerID(ctx context.Context, id string) context.Context {
+ md := metadata.New(map[string]string{
+ idKey: id,
+ })
+ if old, ok := metadata.FromOutgoingContext(ctx); ok {
+ md = metadata.Join(md, old)
+ }
+ return metadata.NewOutgoingContext(ctx, md)
+}
diff --git a/sdks/go/pkg/beam/util/syscallx/syscall.go b/sdks/go/pkg/beam/util/syscallx/syscall.go
new file mode 100644
index 0000000..ca352ec
--- /dev/null
+++ b/sdks/go/pkg/beam/util/syscallx/syscall.go
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package syscallx provides system call utilities that attempt to hide platform
+// differences. Operations return UnsupportedErr if not implemented on the
+// given platform. Consumers of this package should generally treat that
+// error specially.
+package syscallx
+
+import (
+ "errors"
+)
+
+// UnsupportedErr is the error returned for unsupported operations.
+var UnsupportedErr = errors.New("not supported on platform")
diff --git a/sdks/go/pkg/beam/util/syscallx/syscall_default.go b/sdks/go/pkg/beam/util/syscallx/syscall_default.go
new file mode 100644
index 0000000..ccc9324
--- /dev/null
+++ b/sdks/go/pkg/beam/util/syscallx/syscall_default.go
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !linux
+
+package syscallx
+
+// PhysicalMemorySize returns the total physical memory size.
+func PhysicalMemorySize() (uint64, error) {
+ return 0, UnsupportedErr
+}
+
+// FreeDiskSpace returns the free disk space for a given path.
+func FreeDiskSpace(path string) (uint64, error) {
+ return 0, UnsupportedErr
+}
diff --git a/sdks/go/pkg/beam/util/syscallx/syscall_linux.go b/sdks/go/pkg/beam/util/syscallx/syscall_linux.go
new file mode 100644
index 0000000..c639f876
--- /dev/null
+++ b/sdks/go/pkg/beam/util/syscallx/syscall_linux.go
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package syscallx
+
+import "syscall"
+
+// PhysicalMemorySize returns the total physical memory size.
+func PhysicalMemorySize() (uint64, error) {
+ var info syscall.Sysinfo_t
+ if err := syscall.Sysinfo(&info); err != nil {
+ return 0, err
+ }
+ return info.Totalram, nil
+}
+
+// FreeDiskSpace returns the free disk space for a given path.
+func FreeDiskSpace(path string) (uint64, error) {
+ var stat syscall.Statfs_t
+ if err := syscall.Statfs(path, &stat); err != nil {
+ return 0, err
+ }
+ return stat.Bavail * uint64(stat.Bsize), nil
+}
diff --git a/sdks/go/pom.xml b/sdks/go/pom.xml
new file mode 100644
index 0000000..016677a
--- /dev/null
+++ b/sdks/go/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-go</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: SDKs :: Go</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+ <go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-pkg-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.dir}/pkg</outputDirectory>
+ <resources>
+ <resource>
+ <directory>pkg</directory>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.dir}/cmd</outputDirectory>
+ <resources>
+ <resource>
+ <directory>cmd</directory>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- export pkg/ sources as zip for inclusion elsewhere -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>export-go-pkg-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <descriptors>
+ <descriptor>descriptor.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>google.golang.org/api/storage/v1</package>
+ <package>github.com/spf13/cobra</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/sdks/go/cmd/beamctl</package>
+ </packages>
+ <resultName>beamctl</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/sdks/go/cmd/beamctl</package>
+ </packages>
+ <resultName>linux_amd64/beamctl</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <phase>test</phase>
+ <configuration>
+ <packages>
+ <folder>./...</folder>
+ </packages>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/java/build-tools/pom.xml b/sdks/java/build-tools/pom.xml
index 5a2c498..d9b16c1 100644
--- a/sdks/java/build-tools/pom.xml
+++ b/sdks/java/build-tools/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
index ebbaa7d..b2a74a7 100644
--- a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
@@ -81,6 +81,14 @@
<!--
+ ANNOTATIONS CHECKS
+
+ -->
+
+ <module name="MissingDeprecated" />
+
+ <!--
+
IMPORT CHECKS
-->
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index 8ff0cb0..04d4baf 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -26,13 +26,18 @@
<!-- The uncallable method error fails on @ProcessElement style methods -->
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS"/>
+ <!-- Suppress checking of AutoValue internals -->
+ <Match>
+ <Class name="~.*AutoValue_.*"/>
+ </Match>
+
<!--
Suppressed findbugs issues. All new issues should include a comment why they're
suppressed.
Suppressions should go in this file rather than inline using @SuppressFBWarnings to avoid
unapproved artifact license.
- -->
+ -->
<Match>
<Class name="org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver"/>
<Method name="onCompleted"/>
@@ -311,52 +316,10 @@
<!--[BEAM-398] Possible double check of field-->
</Match>
<Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="done"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="lastRecordStart"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="offsetOfLastSplitPoint"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="splitPointsSeen"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="startOffset"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
- <Field name="stopOffset"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-407] Inconsistent synchronization-->
- </Match>
- <Match>
<Class name="org.apache.beam.sdk.testing.WindowSupplier"/>
<Field name="windows"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
- <!--[BEAM-409] Inconsistent synchronization -->
- </Match>
- <Match>
- <Class name="org.apache.beam.sdk.transforms.ApproximateQuantiles$ApproximateQuantilesCombineFn"/>
- <Method name="create"/>
- <Bug pattern="ICAST_INT_CAST_TO_DOUBLE_PASSED_TO_CEIL"/>
- <!--[BEAM-409] Integral value cast to double and then passed to Math.ceil-->
+ <!--[BEAM-407] Inconsistent synchronization -->
</Match>
<Match>
<Class name="org.apache.beam.sdk.transforms.ApproximateQuantiles$QuantileBuffer"/>
@@ -405,4 +368,27 @@
<Bug pattern="NM_CLASS_NOT_EXCEPTION"/>
<!-- It is clear from the name that this class holds either StorageObject or IOException. -->
</Match>
+
+ <Match>
+ <Class name="org.apache.beam.runners.direct.ParDoMultiOverrideFactory$StatefulParDo"/>
+ <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
+ <!-- PTransforms do not actually support serialization. -->
+ </Match>
+
+ <Match>
+ <Class name="org.apache.beam.sdk.options.ProxyInvocationHandler"/>
+ <Field name="~.*"/>
+ <Bug pattern="SE_BAD_FIELD"/>
+ <!--
+ ProxyInvocationHandler implements Serializable only for the sake of throwing an informative
+ exception in writeObject()
+ -->
+ </Match>
+
+ <Match>
+ <!--
+ Classes in this package is auto-generated, let's disable the findbugs for it.
+ -->
+ <Package name="org.apache.beam.sdk.extensions.sql.impl.parser.impl"/>
+ </Match>
</FindBugsFilter>
diff --git a/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/Dockerfile b/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/Dockerfile
new file mode 100644
index 0000000..35d164a
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/Dockerfile
@@ -0,0 +1,49 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM maven:3-jdk-7
+
+# Download OS dependencies
+RUN apt-get update && \
+ apt-get install -y \
+ libsnappy1 \
+ python-pip \
+ python-virtualenv \
+ python-dev \
+ rsync \
+ && rm -rf /var/lib/apt/lists/*
+
+# Add the entrypoint script that downloads the beam sources on run
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# Create beam user to validate the build on user space
+ENV USER=user \
+ UID=9999 \
+ HOME=/home/user
+RUN groupadd --system --gid=$UID $USER; \
+ useradd --system --uid=$UID --gid $USER $USER;
+RUN mkdir -p $HOME; \
+ chown -R $USER:$USER $HOME;
+USER $USER
+WORKDIR $HOME
+
+ENV URL=https://github.com/apache/beam/archive/master.zip
+ENV SRC_FILE=master.zip
+ENV SRC_DIR=beam-master
diff --git a/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/docker-entrypoint.sh b/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/docker-entrypoint.sh
new file mode 100755
index 0000000..589e6ba
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/file/openjdk7/docker-entrypoint.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wget --no-verbose -O $SRC_FILE $URL
+unzip -q $SRC_FILE
+rm $SRC_FILE
+ln -s $HOME/$SRC_DIR $HOME/beam
+cd $HOME/beam
+exec "$@"
diff --git a/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/Dockerfile b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/Dockerfile
new file mode 100644
index 0000000..23032fa
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/Dockerfile
@@ -0,0 +1,49 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM maven:3-jdk-8
+
+# Download OS dependencies
+RUN apt-get update && \
+ apt-get install -y \
+ libsnappy1v5 \
+ python-pip \
+ python-virtualenv \
+ python-dev \
+ rsync \
+ && rm -rf /var/lib/apt/lists/*
+
+# Add the entrypoint script that downloads the beam sources on run
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# Create beam user to validate the build on user space
+ENV USER=user \
+ UID=9999 \
+ HOME=/home/user
+RUN groupadd --system --gid=$UID $USER; \
+ useradd --system --uid=$UID --gid $USER $USER;
+RUN mkdir -p $HOME; \
+ chown -R $USER:$USER $HOME;
+USER $USER
+WORKDIR $HOME
+
+ENV URL=https://github.com/apache/beam/archive/master.zip
+ENV SRC_FILE=master.zip
+ENV SRC_DIR=beam-master
diff --git a/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
new file mode 100755
index 0000000..589e6ba
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wget --no-verbose -O $SRC_FILE $URL
+unzip -q $SRC_FILE
+rm $SRC_FILE
+ln -s $HOME/$SRC_DIR $HOME/beam
+cd $HOME/beam
+exec "$@"
diff --git a/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/Dockerfile b/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/Dockerfile
new file mode 100644
index 0000000..26b5955
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/Dockerfile
@@ -0,0 +1,53 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM maven:3-jdk-8
+
+# Download OS dependencies
+RUN apt-get update && \
+ apt-get install -y \
+ libsnappy1v5 \
+ python-pip \
+ python-virtualenv \
+ python-dev \
+ rsync \
+ && rm -rf /var/lib/apt/lists/*
+
+# Add the entrypoint script that downloads the beam sources on run
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# Create beam user to validate the build on user space
+ENV USER=user \
+ UID=9999 \
+ HOME=/home/user
+RUN groupadd --system --gid=$UID $USER; \
+ useradd --system --uid=$UID --gid $USER $USER;
+RUN mkdir -p $HOME; \
+ chown -R $USER:$USER $HOME;
+USER $USER
+WORKDIR $HOME
+
+ARG URL=https://github.com/apache/beam
+ENV BRANCH=master
+
+RUN git clone $URL beam; \
+ cd beam; \
+ git config --local --add remote.origin.fetch '+refs/pull/*/head:refs/remotes/origin/pr/*'; \
+ git fetch --quiet --all;
diff --git a/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/docker-entrypoint.sh b/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/docker-entrypoint.sh
new file mode 100755
index 0000000..c25842d
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/git/openjdk8/docker-entrypoint.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cd $HOME/beam
+git fetch --quiet --all
+git checkout $BRANCH
+exec "$@"
diff --git a/sdks/java/build-tools/src/main/resources/docker/release/python2/Dockerfile b/sdks/java/build-tools/src/main/resources/docker/release/python2/Dockerfile
new file mode 100644
index 0000000..551fe9a
--- /dev/null
+++ b/sdks/java/build-tools/src/main/resources/docker/release/python2/Dockerfile
@@ -0,0 +1,21 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM python:2
+
+RUN pip install --user apache-beam[gcp]
diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile
new file mode 100644
index 0000000..7fb325d
--- /dev/null
+++ b/sdks/java/container/Dockerfile
@@ -0,0 +1,28 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM openjdk:8
+MAINTAINER "Apache Beam <dev@beam.apache.org>"
+
+ADD target/slf4j-api.jar /opt/apache/beam/jars/
+ADD target/slf4j-jdk14.jar /opt/apache/beam/jars/
+ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
+
+ADD target/linux_amd64/boot /opt/apache/beam/
+
+ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/java/container/boot.go b/sdks/java/container/boot.go
new file mode 100644
index 0000000..1c80e0b
--- /dev/null
+++ b/sdks/java/container/boot.go
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// boot is the boot code for the Java SDK harness container. It is responsible
+// for retrieving staged files and invoking the JVM correctly.
+package main
+
+import (
+ "context"
+ "flag"
+ "log"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/artifact"
+ fnpb "github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1"
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/provision"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/execx"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/syscallx"
+ "github.com/golang/protobuf/proto"
+)
+
+var (
+ // Contract: https://s.apache.org/beam-fn-api-container-contract.
+
+ id = flag.String("id", "", "Local identifier (required).")
+ loggingEndpoint = flag.String("logging_endpoint", "", "Logging endpoint (required).")
+ artifactEndpoint = flag.String("artifact_endpoint", "", "Artifact endpoint (required).")
+ provisionEndpoint = flag.String("provision_endpoint", "", "Provision endpoint (required).")
+ controlEndpoint = flag.String("control_endpoint", "", "Control endpoint (required).")
+ semiPersistDir = flag.String("semi_persist_dir", "/tmp", "Local semi-persistent directory (optional).")
+)
+
+func main() {
+ flag.Parse()
+ if *id == "" {
+ log.Fatal("No id provided.")
+ }
+ if *loggingEndpoint == "" {
+ log.Fatal("No logging endpoint provided.")
+ }
+ if *artifactEndpoint == "" {
+ log.Fatal("No artifact endpoint provided.")
+ }
+ if *provisionEndpoint == "" {
+ log.Fatal("No provision endpoint provided.")
+ }
+ if *controlEndpoint == "" {
+ log.Fatal("No control endpoint provided.")
+ }
+
+ log.Printf("Initializing java harness: %v", strings.Join(os.Args, " "))
+
+ ctx := grpcx.WriteWorkerID(context.Background(), *id)
+
+ // (1) Obtain the pipeline options
+
+ info, err := provision.Info(ctx, *provisionEndpoint)
+ if err != nil {
+ log.Fatalf("Failed to obtain provisioning information: %v", err)
+ }
+ options, err := provision.ProtoToJSON(info.GetPipelineOptions())
+ if err != nil {
+ log.Fatalf("Failed to convert pipeline options: %v", err)
+ }
+
+ // (2) Retrieve the staged user jars. We ignore any disk limit,
+ // because the staged jars are mandatory.
+
+ dir := filepath.Join(*semiPersistDir, "staged")
+
+ artifacts, err := artifact.Materialize(ctx, *artifactEndpoint, dir)
+ if err != nil {
+ log.Fatalf("Failed to retrieve staged files: %v", err)
+ }
+
+ // (3) Invoke the Java harness, preserving artifact ordering in classpath.
+
+ os.Setenv("PIPELINE_OPTIONS", options)
+ os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pb.ApiServiceDescriptor{Url: *loggingEndpoint}))
+ os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pb.ApiServiceDescriptor{Url: *controlEndpoint}))
+
+ const jarsDir = "/opt/apache/beam/jars"
+ cp := []string{
+ filepath.Join(jarsDir, "slf4j-api.jar"),
+ filepath.Join(jarsDir, "slf4j-jdk14.jar"),
+ filepath.Join(jarsDir, "beam-sdks-java-harness.jar"),
+ }
+ for _, md := range artifacts {
+ cp = append(cp, filepath.Join(dir, filepath.FromSlash(md.Name)))
+ }
+
+ args := []string{
+ "-Xmx" + strconv.FormatUint(heapSizeLimit(info), 10),
+ "-XX:-OmitStackTraceInFastThrow",
+ "-cp", strings.Join(cp, ":"),
+ "org.apache.beam.fn.harness.FnHarness",
+ }
+
+ log.Printf("Executing: java %v", strings.Join(args, " "))
+
+ log.Fatalf("Java exited: %v", execx.Execute("java", args...))
+}
+
+// heapSizeLimit returns 80% of the runner limit, if provided. If not provided,
+// it returns 70% of the physical memory on the machine. If it cannot determine
+// that value, it returns 1GB. This is an imperfect heuristic. It aims to
+// ensure there is memory for non-heap use and other overhead, while also not
+// underutilizing the machine.
+func heapSizeLimit(info *fnpb.ProvisionInfo) uint64 {
+ if provided := info.GetResourceLimits().GetMemory().GetSize(); provided > 0 {
+ return (provided * 80) / 100
+ }
+ if size, err := syscallx.PhysicalMemorySize(); err == nil {
+ return (size * 70) / 100
+ }
+ return 1 << 30
+}
diff --git a/sdks/java/container/pom.xml b/sdks/java/container/pom.xml
new file mode 100644
index 0000000..0d0ca7a
--- /dev/null
+++ b/sdks/java/container/pom.xml
@@ -0,0 +1,184 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-container</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: SDKs :: Java :: Container</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+ <go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.base}/github.com/apache/beam/cmd/boot</outputDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>*.go</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- CAVEAT: for latest shared files, run mvn install in sdks/go -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ <classifier>pkg-sources</classifier>
+ <overWrite>true</overWrite>
+ <outputDirectory>${go.source.dir}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+
+ <execution>
+ <id>copy-dependent-jars</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/target</outputDirectory>
+ <stripVersion>true</stripVersion>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <overWrite>true</overWrite>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <version>${slf4j.version}</version>
+ <overWrite>true</overWrite>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-harness</artifactId>
+ <overWrite>true</overWrite>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>google.golang.org/api/storage/v1</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>boot</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>linux_amd64/boot</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <configuration>
+ <repository>${docker-repository-root}/java</repository>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 882657b..5cead58 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -57,6 +57,85 @@
<testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>net.bytebuddy:byte-buddy</include>
+ <include>org.apache.commons:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <excludes>
+ <!-- com.google.common is too generic, need to exclude guava-testlib -->
+ <exclude>com.google.common.**.testing.*</exclude>
+ </excludes>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.sdk.repackaged.com.google.common
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.sdk.repackaged.com.google.thirdparty
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.sdk.repackaged.com.google.protobuf
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>net.bytebuddy</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.sdk.repackaged.net.bytebuddy
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.sdk.repackaged.org.apache.commons
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
@@ -120,92 +199,6 @@
</execution>
</executions>
</plugin>
-
- <!-- Ensure that the Maven jar plugin runs before the Maven
- shade plugin by listing the plugin higher within the file. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-and-repackage</id>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadeTestJar>true</shadeTestJar>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>net.bytebuddy:byte-buddy</include>
- <include>org.apache.commons:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <excludes>
- <!-- com.google.common is too generic, need to exclude guava-testlib -->
- <exclude>com.google.common.**.testing.*</exclude>
- </excludes>
- <!--suppress MavenModelInspection -->
- <shadedPattern>
- org.apache.beam.sdk.repackaged.com.google.common
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.thirdparty</pattern>
- <!--suppress MavenModelInspection -->
- <shadedPattern>
- org.apache.beam.sdk.repackaged.com.google.thirdparty
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.protobuf</pattern>
- <!--suppress MavenModelInspection -->
- <shadedPattern>
- org.apache.beam.sdk.repackaged.com.google.protobuf
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>net.bytebuddy</pattern>
- <!--suppress MavenModelInspection -->
- <shadedPattern>
- org.apache.beam.sdk.repackaged.net.bytebuddy
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <!--suppress MavenModelInspection -->
- <shadedPattern>
- org.apache.beam.sdk.repackaged.org.apache.commons
- </shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
@@ -234,6 +227,11 @@
</dependency>
<dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
@@ -266,7 +264,6 @@
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
- <version>1.1.4-M3</version>
</dependency>
<dependency>
@@ -310,9 +307,20 @@
<!-- test dependencies -->
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
+ <exclusions>
+ <!-- We only rely on the standard_coders.yaml -->
+ <exclusion>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-construction</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 83496a5..5358f7d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk;
+import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
@@ -28,6 +29,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.io.Read;
@@ -179,6 +181,12 @@
return begin().apply(name, root);
}
+ @Internal
+ public static Pipeline forTransformHierarchy(
+ TransformHierarchy transforms, PipelineOptions options) {
+ return new Pipeline(transforms, options);
+ }
+
/**
* <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
*
@@ -205,7 +213,7 @@
public CompositeBehavior enterCompositeTransform(Node node) {
if (!node.isRootNode()) {
for (PTransformOverride override : overrides) {
- if (override.getMatcher().matches(node.toAppliedPTransform())) {
+ if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
matched.put(node, override);
}
}
@@ -227,7 +235,7 @@
@Override
public void visitPrimitiveTransform(Node node) {
for (PTransformOverride override : overrides) {
- if (override.getMatcher().matches(node.toAppliedPTransform())) {
+ if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
matched.put(node, override);
}
}
@@ -238,7 +246,7 @@
private void replace(final PTransformOverride override) {
final Set<Node> matches = new HashSet<>();
final Set<Node> freedNodes = new HashSet<>();
- transforms.visit(
+ traverseTopologically(
new PipelineVisitor.Defaults() {
@Override
public CompositeBehavior enterCompositeTransform(Node node) {
@@ -247,7 +255,8 @@
freedNodes.add(node);
return CompositeBehavior.ENTER_TRANSFORM;
}
- if (!node.isRootNode() && override.getMatcher().matches(node.toAppliedPTransform())) {
+ if (!node.isRootNode()
+ && override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
matches.add(node);
// This node will be freed. When we visit any of its children, they will also be freed
freedNodes.add(node);
@@ -259,7 +268,7 @@
public void visitPrimitiveTransform(Node node) {
if (freedNodes.contains(node.getEnclosingNode())) {
freedNodes.add(node);
- } else if (override.getMatcher().matches(node.toAppliedPTransform())) {
+ } else if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
matches.add(node);
freedNodes.add(node);
}
@@ -334,8 +343,14 @@
@Internal
public interface PipelineVisitor {
/**
- * Called for each composite transform after all topological predecessors have been visited
- * but before any of its component transforms.
+ * Called before visiting anything values or transforms, as many uses of a visitor require
+ * access to the {@link Pipeline} object itself.
+ */
+ void enterPipeline(Pipeline p);
+
+ /**
+ * Called for each composite transform after all topological predecessors have been visited but
+ * before any of its component transforms.
*
* <p>The return value controls whether or not child transforms are visited.
*/
@@ -360,6 +375,11 @@
void visitValue(PValue value, TransformHierarchy.Node producer);
/**
+ * Called when all values and transforms in a {@link Pipeline} have been visited.
+ */
+ void leavePipeline(Pipeline pipeline);
+
+ /**
* Control enum for indicating whether or not a traversal should process the contents of
* a composite transform or not.
*/
@@ -373,6 +393,22 @@
* User implementations can override just those methods they are interested in.
*/
class Defaults implements PipelineVisitor {
+
+ @Nullable private Pipeline pipeline;
+
+ protected Pipeline getPipeline() {
+ if (pipeline == null) {
+ throw new IllegalStateException(
+ "Illegal access to pipeline after visitor traversal was completed");
+ }
+ return pipeline;
+ }
+
+ @Override
+ public void enterPipeline(Pipeline pipeline) {
+ this.pipeline = checkNotNull(pipeline);
+ }
+
@Override
public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
return CompositeBehavior.ENTER_TRANSFORM;
@@ -386,6 +422,11 @@
@Override
public void visitValue(PValue value, TransformHierarchy.Node producer) { }
+
+ @Override
+ public void leavePipeline(Pipeline pipeline) {
+ this.pipeline = null;
+ }
}
}
@@ -406,7 +447,9 @@
*/
@Internal
public void traverseTopologically(PipelineVisitor visitor) {
+ visitor.enterPipeline(this);
transforms.visit(visitor);
+ visitor.leavePipeline(this);
}
/**
@@ -444,16 +487,24 @@
/////////////////////////////////////////////////////////////////////////////
// Below here are internal operations, never called by users.
- private final TransformHierarchy transforms = new TransformHierarchy(this);
+ private final TransformHierarchy transforms;
private Set<String> usedFullNames = new HashSet<>();
- private CoderRegistry coderRegistry;
+
+ /** Lazily initialized; access via {@link #getCoderRegistry()}. */
+ @Nullable private CoderRegistry coderRegistry;
+
private final List<String> unstableNames = new ArrayList<>();
private final PipelineOptions defaultOptions;
- protected Pipeline(PipelineOptions options) {
+ private Pipeline(TransformHierarchy transforms, PipelineOptions options) {
+ this.transforms = transforms;
this.defaultOptions = options;
}
+ protected Pipeline(PipelineOptions options) {
+ this(new TransformHierarchy(), options);
+ }
+
@Override
public String toString() {
return "Pipeline#" + hashCode();
@@ -495,7 +546,7 @@
PTransformOverrideFactory<InputT, OutputT, TransformT> replacementFactory) {
PTransformReplacement<InputT, OutputT> replacement =
replacementFactory.getReplacementTransform(
- (AppliedPTransform<InputT, OutputT, TransformT>) original.toAppliedPTransform());
+ (AppliedPTransform<InputT, OutputT, TransformT>) original.toAppliedPTransform(this));
if (replacement.getTransform() == original.getTransform()) {
return;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
index b60de63..4a9c30a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
@@ -46,14 +46,14 @@
State cancel() throws IOException;
/**
- * Waits until the pipeline finishes and returns the final status.
- * It times out after the given duration.
+ * Waits until the pipeline finishes and returns the final status. It times out after the given
+ * duration.
*
- * @param duration The time to wait for the pipeline to finish.
- * Provide a value less than 1 ms for an infinite wait.
- *
+ * @param duration The time to wait for the pipeline to finish. Provide a value less than 1 ms for
+ * an infinite wait.
* @return The final state of the pipeline or null on timeout.
- * @throws UnsupportedOperationException if the runner does not support cancellation.
+ * @throws UnsupportedOperationException if the runner does not support waiting to finish with a
+ * timeout.
*/
State waitUntilFinish(Duration duration);
@@ -61,7 +61,7 @@
* Waits until the pipeline finishes and returns the final status.
*
* @return The final state of the pipeline.
- * @throws UnsupportedOperationException if the runner does not support cancellation.
+ * @throws UnsupportedOperationException if the runner does not support waiting to finish.
*/
State waitUntilFinish();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
index 8224ebb..fecc407 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
@@ -72,8 +72,9 @@
OUTPUT_TIME,
/**
- * <a href="https://s.apache.org/splittable-do-fn">Splittable DoFn</a>.
- * Do not use: API is unstable and runner support is incomplete.
+ * <a href="https://s.apache.org/splittable-do-fn">Splittable DoFn</a>. See <a
+ * href="https://beam.apache.org/documentation/runners/capability-matrix/">capability matrix</a>
+ * for runner support.
*/
SPLITTABLE_DO_FN,
@@ -93,6 +94,12 @@
CORE_RUNNERS_ONLY,
/** Experimental feature related to making the encoded element type available from a Coder. */
- CODER_TYPE_ENCODING
+ CODER_TYPE_ENCODING,
+
+ /**
+ * Experimental APIs related to <a href="https://s.apache.org/context-fn">contextful
+ * closures</a>.
+ */
+ CONTEXTFUL,
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/package-info.java
index df42eda..9f8cfd0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/package-info.java
@@ -18,4 +18,8 @@
/**
* Defines annotations used across the SDK.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.annotations;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
new file mode 100644
index 0000000..70fbf58
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.coders;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.BeamRecordType;
+
+/**
+ * A {@link Coder} for {@link BeamRecord}. It wraps the {@link Coder} for each element directly.
+ */
+@Experimental
+public class BeamRecordCoder extends CustomCoder<BeamRecord> {
+ private static final BitSetCoder nullListCoder = BitSetCoder.of();
+
+ private BeamRecordType recordType;
+ private List<Coder> coders;
+
+ private BeamRecordCoder(BeamRecordType recordType, List<Coder> coders) {
+ this.recordType = recordType;
+ this.coders = coders;
+ }
+
+ public static BeamRecordCoder of(BeamRecordType recordType, List<Coder> coderArray){
+ if (recordType.getFieldCount() != coderArray.size()) {
+ throw new IllegalArgumentException("Coder size doesn't match with field size");
+ }
+ return new BeamRecordCoder(recordType, coderArray);
+ }
+
+ public BeamRecordType getRecordType() {
+ return recordType;
+ }
+
+ @Override
+ public void encode(BeamRecord value, OutputStream outStream)
+ throws CoderException, IOException {
+ nullListCoder.encode(scanNullFields(value), outStream);
+ for (int idx = 0; idx < value.getFieldCount(); ++idx) {
+ if (value.getFieldValue(idx) == null) {
+ continue;
+ }
+
+ coders.get(idx).encode(value.getFieldValue(idx), outStream);
+ }
+ }
+
+ @Override
+ public BeamRecord decode(InputStream inStream) throws CoderException, IOException {
+ BitSet nullFields = nullListCoder.decode(inStream);
+
+ List<Object> fieldValues = new ArrayList<>(recordType.getFieldCount());
+ for (int idx = 0; idx < recordType.getFieldCount(); ++idx) {
+ if (nullFields.get(idx)) {
+ fieldValues.add(null);
+ } else {
+ fieldValues.add(coders.get(idx).decode(inStream));
+ }
+ }
+ BeamRecord record = new BeamRecord(recordType, fieldValues);
+
+ return record;
+ }
+
+ /**
+ * Scan {@link BeamRecord} to find fields with a NULL value.
+ */
+ private BitSet scanNullFields(BeamRecord record){
+ BitSet nullFields = new BitSet(record.getFieldCount());
+ for (int idx = 0; idx < record.getFieldCount(); ++idx) {
+ if (record.getFieldValue(idx) == null) {
+ nullFields.set(idx);
+ }
+ }
+ return nullFields;
+ }
+
+ @Override
+ public void verifyDeterministic()
+ throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
+ for (Coder c : coders) {
+ c.verifyDeterministic();
+ }
+ }
+
+ public List<Coder> getCoders() {
+ return Collections.unmodifiableList(coders);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BooleanCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BooleanCoder.java
new file mode 100644
index 0000000..e7f7543
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BooleanCoder.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.coders;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/** A {@link Coder} for {@link Boolean}. */
+public class BooleanCoder extends AtomicCoder<Boolean> {
+ private static final ByteCoder BYTE_CODER = ByteCoder.of();
+
+ private static final BooleanCoder INSTANCE = new BooleanCoder();
+
+ /** Returns the singleton instance of {@link BooleanCoder}. */
+ public static BooleanCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(Boolean value, OutputStream os) throws IOException {
+ BYTE_CODER.encode(value ? (byte) 1 : 0, os);
+ }
+
+ @Override
+ public Boolean decode(InputStream is) throws IOException {
+ return BYTE_CODER.decode(is) == 1;
+ }
+
+ @Override
+ public boolean consistentWithEquals() {
+ return true;
+ }
+
+ @Override
+ public boolean isRegisterByteSizeObserverCheap(Boolean value) {
+ return true;
+ }
+
+ @Override
+ protected long getEncodedElementByteSize(Boolean value) throws Exception {
+ return 1;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index edcc3a8..78a4a02 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -56,7 +56,13 @@
* @param <T> the type of values being encoded and decoded
*/
public abstract class Coder<T> implements Serializable {
- /** The context in which encoding or decoding is being done. */
+ /**
+ * The context in which encoding or decoding is being done.
+ *
+ * @deprecated to implement a coder, do not use any `Context`. Just implement only those abstract
+ * methods which do not accept a `Context` and leave the default implementations for methods
+ * accepting a `Context`.
+ */
@Deprecated
@Experimental(Kind.CODER_CONTEXT)
public static class Context {
@@ -127,6 +133,8 @@
* @throws IOException if writing to the {@code OutputStream} fails
* for some reason
* @throws CoderException if the value could not be encoded for some reason
+ *
+ * @deprecated only implement and call {@link #encode(Object value, OutputStream)}
*/
@Deprecated
@Experimental(Kind.CODER_CONTEXT)
@@ -152,6 +160,8 @@
* @throws IOException if reading from the {@code InputStream} fails
* for some reason
* @throws CoderException if the value could not be decoded for some reason
+ *
+ * @deprecated only implement and call {@link #decode(InputStream)}
*/
@Deprecated
@Experimental(Kind.CODER_CONTEXT)
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
index 2ba548a..012d6de 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
@@ -43,6 +43,12 @@
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.CannotProvideCoderException.ReasonCode;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.ReadableFileCoder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.io.fs.MetadataCoder;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.fs.ResourceIdCoder;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.util.CoderUtils;
@@ -89,6 +95,8 @@
private CommonTypes() {
ImmutableMap.Builder<Class<?>, CoderProvider> builder = ImmutableMap.builder();
+ builder.put(Boolean.class,
+ CoderProviders.fromStaticMethods(Boolean.class, BooleanCoder.class));
builder.put(Byte.class,
CoderProviders.fromStaticMethods(Byte.class, ByteCoder.class));
builder.put(BitSet.class,
@@ -109,6 +117,14 @@
CoderProviders.fromStaticMethods(Long.class, VarLongCoder.class));
builder.put(Map.class,
CoderProviders.fromStaticMethods(Map.class, MapCoder.class));
+ builder.put(Metadata.class,
+ CoderProviders.fromStaticMethods(Metadata.class, MetadataCoder.class));
+ builder.put(ResourceId.class,
+ CoderProviders.fromStaticMethods(ResourceId.class, ResourceIdCoder.class));
+ builder.put(
+ FileIO.ReadableFile.class,
+ CoderProviders.fromStaticMethods(
+ FileIO.ReadableFile.class, ReadableFileCoder.class));
builder.put(Set.class,
CoderProviders.fromStaticMethods(Set.class, SetCoder.class));
builder.put(String.class,
@@ -147,9 +163,13 @@
Set<CoderProviderRegistrar> registrars = Sets.newTreeSet(ObjectsClassComparator.INSTANCE);
registrars.addAll(Lists.newArrayList(
ServiceLoader.load(CoderProviderRegistrar.class, ReflectHelpers.findClassLoader())));
+
+ // DefaultCoder should have the highest precedence and SerializableCoder the lowest
+ codersToRegister.addAll(new DefaultCoder.DefaultCoderProviderRegistrar().getCoderProviders());
for (CoderProviderRegistrar registrar : registrars) {
codersToRegister.addAll(registrar.getCoderProviders());
}
+ codersToRegister.add(SerializableCoder.getCoderProvider());
REGISTERED_CODER_FACTORIES = ImmutableList.copyOf(codersToRegister);
}
@@ -234,6 +254,9 @@
* type uses the given {@link Coder}.
*
* @throws CannotProvideCoderException if a {@link Coder} cannot be provided
+ *
+ * @deprecated This method is to change in an unknown backwards incompatible way once support for
+ * this functionality is refined.
*/
@Deprecated
@Internal
@@ -254,6 +277,9 @@
* used for its input elements.
*
* @throws CannotProvideCoderException if a {@link Coder} cannot be provided
+ *
+ * @deprecated This method is to change in an unknown backwards incompatible way once support for
+ * this functionality is refined.
*/
@Deprecated
@Internal
@@ -276,6 +302,9 @@
* subclass, given {@link Coder Coders} to use for all other type parameters (if any).
*
* @throws CannotProvideCoderException if a {@link Coder} cannot be provided
+ *
+ * @deprecated This method is to change in an unknown backwards incompatible way once support for
+ * this functionality is refined.
*/
@Deprecated
@Internal
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
index 6eff9e9..57ab122 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.coders;
-import com.google.auto.service.AutoService;
import com.google.common.collect.ImmutableList;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
@@ -27,6 +26,7 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.List;
+import javax.annotation.CheckForNull;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.slf4j.Logger;
@@ -50,6 +50,7 @@
@Target(ElementType.TYPE)
@SuppressWarnings("rawtypes")
public @interface DefaultCoder {
+ @CheckForNull
Class<? extends Coder> value();
/**
@@ -57,7 +58,6 @@
* the {@code @DefaultCoder} annotation to provide {@link CoderProvider coder providers} that
* creates {@link Coder}s.
*/
- @AutoService(CoderProviderRegistrar.class)
class DefaultCoderProviderRegistrar implements CoderProviderRegistrar {
@Override
@@ -88,22 +88,23 @@
clazz.getName()));
}
- if (defaultAnnotation.value() == null) {
+ Class<? extends Coder> defaultAnnotationValue = defaultAnnotation.value();
+ if (defaultAnnotationValue == null) {
throw new CannotProvideCoderException(
- String.format("Class %s has a @DefaultCoder annotation with a null value.",
- clazz.getName()));
+ String.format(
+ "Class %s has a @DefaultCoder annotation with a null value.", clazz.getName()));
}
LOG.debug("DefaultCoder annotation found for {} with value {}",
- clazz, defaultAnnotation.value());
+ clazz, defaultAnnotationValue);
Method coderProviderMethod;
try {
- coderProviderMethod = defaultAnnotation.value().getMethod("getCoderProvider");
+ coderProviderMethod = defaultAnnotationValue.getMethod("getCoderProvider");
} catch (NoSuchMethodException e) {
throw new CannotProvideCoderException(String.format(
"Unable to find 'public static CoderProvider getCoderProvider()' on %s",
- defaultAnnotation.value()),
+ defaultAnnotationValue),
e);
}
@@ -117,7 +118,7 @@
| ExceptionInInitializerError e) {
throw new CannotProvideCoderException(String.format(
"Unable to invoke 'public static CoderProvider getCoderProvider()' on %s",
- defaultAnnotation.value()),
+ defaultAnnotationValue),
e);
}
return coderProvider.coderFor(typeDescriptor, componentCoders);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
index 648493e..e4fadef 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
@@ -17,11 +17,13 @@
*/
package org.apache.beam.sdk.coders;
-import com.google.common.base.Converter;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
+import java.io.UTFDataFormatException;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.joda.time.Instant;
@@ -39,53 +41,46 @@
private static final InstantCoder INSTANCE = new InstantCoder();
private static final TypeDescriptor<Instant> TYPE_DESCRIPTOR = new TypeDescriptor<Instant>() {};
- private static final BigEndianLongCoder LONG_CODER = BigEndianLongCoder.of();
-
private InstantCoder() {}
- private static final Converter<Instant, Long> ORDER_PRESERVING_CONVERTER =
- new LexicographicLongConverter();
-
- /**
- * Converts {@link Instant} to a {@code Long} representing its millis-since-epoch,
- * but shifted so that the byte representation of negative values are lexicographically
- * ordered before the byte representation of positive values.
- *
- * <p>This deliberately utilizes the well-defined overflow for {@code Long} values.
- * See http://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.18.2
- */
- private static class LexicographicLongConverter extends Converter<Instant, Long> {
-
- @Override
- protected Long doForward(Instant instant) {
- return instant.getMillis() - Long.MIN_VALUE;
- }
-
- @Override
- protected Instant doBackward(Long shiftedMillis) {
- return new Instant(shiftedMillis + Long.MIN_VALUE);
- }
- }
-
@Override
- public void encode(Instant value, OutputStream outStream)
- throws CoderException, IOException {
+ public void encode(Instant value, OutputStream outStream) throws CoderException, IOException {
if (value == null) {
throw new CoderException("cannot encode a null Instant");
}
- LONG_CODER.encode(ORDER_PRESERVING_CONVERTER.convert(value), outStream);
+
+ // Converts {@link Instant} to a {@code long} representing its millis-since-epoch,
+ // but shifted so that the byte representation of negative values are lexicographically
+ // ordered before the byte representation of positive values.
+ //
+ // This deliberately utilizes the well-defined underflow for {@code long} values.
+ // See http://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.18.2
+ long shiftedMillis = value.getMillis() - Long.MIN_VALUE;
+ new DataOutputStream(outStream).writeLong(shiftedMillis);
}
@Override
- public Instant decode(InputStream inStream)
- throws CoderException, IOException {
- return ORDER_PRESERVING_CONVERTER.reverse().convert(LONG_CODER.decode(inStream));
+ public Instant decode(InputStream inStream) throws CoderException, IOException {
+ long shiftedMillis;
+ try {
+ shiftedMillis = new DataInputStream(inStream).readLong();
+ } catch (EOFException | UTFDataFormatException exn) {
+ // These exceptions correspond to decoding problems, so change
+ // what kind of exception they're branded as.
+ throw new CoderException(exn);
+ }
+
+ // Produces an {@link Instant} from a {@code long} representing its millis-since-epoch,
+ // but shifted so that the byte representation of negative values are lexicographically
+ // ordered before the byte representation of positive values.
+ //
+ // This deliberately utilizes the well-defined overflow for {@code long} values.
+ // See http://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.18.2
+ return new Instant(shiftedMillis + Long.MIN_VALUE);
}
@Override
- public void verifyDeterministic() {
- LONG_CODER.verifyDeterministic();
- }
+ public void verifyDeterministic() {}
/**
* {@inheritDoc}
@@ -104,15 +99,15 @@
*/
@Override
public boolean isRegisterByteSizeObserverCheap(Instant value) {
- return LONG_CODER.isRegisterByteSizeObserverCheap(
- ORDER_PRESERVING_CONVERTER.convert(value));
+ return true;
}
@Override
- public void registerByteSizeObserver(
- Instant value, ElementByteSizeObserver observer) throws Exception {
- LONG_CODER.registerByteSizeObserver(
- ORDER_PRESERVING_CONVERTER.convert(value), observer);
+ protected long getEncodedElementByteSize(Instant value) throws Exception {
+ if (value == null) {
+ throw new CoderException("cannot encode a null Instant");
+ }
+ return 8;
}
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
index b24f66d..9466ad1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
@@ -26,7 +26,6 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
-import javax.annotation.Nullable;
import org.apache.beam.sdk.util.VarInt;
/**
@@ -126,7 +125,7 @@
* {@inheritDoc}
*/
@Override
- public boolean isRegisterByteSizeObserverCheap(@Nullable T value) {
+ public boolean isRegisterByteSizeObserverCheap(T value) {
return valueCoder.isRegisterByteSizeObserverCheap(value);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
index 6691876..1330125 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.coders;
-import com.google.auto.service.AutoService;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.io.InputStream;
@@ -26,6 +25,7 @@
import java.io.OutputStream;
import java.io.Serializable;
import java.util.List;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.values.TypeDescriptor;
/**
@@ -80,7 +80,6 @@
* A {@link CoderProviderRegistrar} which registers a {@link CoderProvider} which can handle
* serializable types.
*/
- @AutoService(CoderProviderRegistrar.class)
public static class SerializableCoderProviderRegistrar implements CoderProviderRegistrar {
@Override
@@ -107,7 +106,9 @@
}
private final Class<T> type;
- private transient TypeDescriptor<T> typeDescriptor;
+
+ /** Access via {@link #getEncodedTypeDescriptor()}. */
+ @Nullable private transient TypeDescriptor<T> typeDescriptor;
protected SerializableCoder(Class<T> type, TypeDescriptor<T> typeDescriptor) {
this.type = type;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java
new file mode 100644
index 0000000..a86b198
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.coders;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.values.ShardedKey;
+
+/** A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}. */
+@VisibleForTesting
+public class ShardedKeyCoder<KeyT> extends StructuredCoder<ShardedKey<KeyT>> {
+ public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
+ return new ShardedKeyCoder<>(keyCoder);
+ }
+
+ private final Coder<KeyT> keyCoder;
+ private final VarIntCoder shardNumberCoder;
+
+ protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
+ this.keyCoder = keyCoder;
+ this.shardNumberCoder = VarIntCoder.of();
+ }
+
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return Arrays.asList(keyCoder);
+ }
+
+ @Override
+ public void encode(ShardedKey<KeyT> key, OutputStream outStream)
+ throws IOException {
+ keyCoder.encode(key.getKey(), outStream);
+ shardNumberCoder.encode(key.getShardNumber(), outStream);
+ }
+
+ @Override
+ public ShardedKey<KeyT> decode(InputStream inStream)
+ throws IOException {
+ return ShardedKey.of(keyCoder.decode(inStream), shardNumberCoder.decode(inStream));
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ keyCoder.verifyDeterministic();
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index 2eb662b..fe17e8b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -17,10 +17,7 @@
*/
package org.apache.beam.sdk.coders;
-import java.io.ByteArrayOutputStream;
-import java.util.Collections;
import java.util.List;
-import org.apache.beam.sdk.values.TypeDescriptor;
/**
* An abstract base class to implement a {@link Coder} that defines equality, hashing, and printing
@@ -48,12 +45,7 @@
* <p>The default components will be equal to the value returned by {@link #getCoderArguments()}.
*/
public List<? extends Coder<?>> getComponents() {
- List<? extends Coder<?>> coderArguments = getCoderArguments();
- if (coderArguments == null) {
- return Collections.emptyList();
- } else {
- return coderArguments;
- }
+ return getCoderArguments();
}
/**
@@ -99,36 +91,4 @@
return builder.toString();
}
- /**
- * {@inheritDoc}
- *
- * @return {@code false} for {@link StructuredCoder} unless overridden.
- */
- @Override
- public boolean consistentWithEquals() {
- return false;
- }
-
- @Override
- public Object structuralValue(T value) {
- if (value != null && consistentWithEquals()) {
- return value;
- } else {
- try {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- encode(value, os, Context.OUTER);
- return new StructuralByteArray(os.toByteArray());
- } catch (Exception exn) {
- throw new IllegalArgumentException(
- "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public TypeDescriptor<T> getEncodedTypeDescriptor() {
- return (TypeDescriptor<T>)
- TypeDescriptor.of(getClass()).resolveType(new TypeDescriptor<T>() {}.getType());
- }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
index 3e1ff7f..82b63f0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
@@ -19,6 +19,7 @@
import java.io.InputStream;
import java.io.OutputStream;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.values.TypeDescriptor;
/**
@@ -34,6 +35,7 @@
private static final VoidCoder INSTANCE = new VoidCoder();
private static final TypeDescriptor<Void> TYPE_DESCRIPTOR = new TypeDescriptor<Void>() {};
+ private static final Object STRUCTURAL_VOID_VALUE = new Object();
private VoidCoder() {}
@@ -43,6 +45,7 @@
}
@Override
+ @Nullable
public Void decode(InputStream inStream) {
// Nothing to read!
return null;
@@ -51,14 +54,9 @@
@Override
public void verifyDeterministic() {}
- /**
- * {@inheritDoc}
- *
- * @return {@code true}. {@link VoidCoder} is (vacuously) injective.
- */
@Override
- public boolean consistentWithEquals() {
- return true;
+ public Object structuralValue(Void value) {
+ return STRUCTURAL_VOID_VALUE;
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
index 5693077..9b612233 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
@@ -42,4 +42,8 @@
* types.
*
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.coders;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index 6af0e79..2cc0f52 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -18,12 +18,14 @@
package org.apache.beam.sdk.io;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment;
import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import com.google.common.io.BaseEncoding;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.avro.Schema;
@@ -33,76 +35,160 @@
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
-import org.apache.beam.sdk.io.Read.Bounded;
+import org.apache.beam.sdk.io.FileIO.MatchConfiguration;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.Duration;
/**
* {@link PTransform}s for reading and writing Avro files.
*
- * <p>To read a {@link PCollection} from one or more Avro files, use {@code AvroIO.read()},
- * using {@link AvroIO.Read#from} to specify the filename or filepattern to read from.
- * See {@link FileSystems} for information on supported file systems and filepatterns.
+ * <h2>Reading Avro files</h2>
*
- * <p>To read specific records, such as Avro-generated classes, use {@link #read(Class)}.
- * To read {@link GenericRecord GenericRecords}, use {@link #readGenericRecords(Schema)} which takes
- * a {@link Schema} object, or {@link #readGenericRecords(String)} which takes an Avro schema in a
+ * <p>To read a {@link PCollection} from one or more Avro files with the same schema known at
+ * pipeline construction time, use {@link #read}, using {@link AvroIO.Read#from} to specify the
+ * filename or filepattern to read from. If the filepatterns to be read are themselves in a {@link
+ * PCollection}, apply {@link #readAll}. If the schema is unknown at pipeline construction time, use
+ * {@link #parseGenericRecords} or {@link #parseAllGenericRecords}.
+ *
+ * <p>Many configuration options below apply to several or all of these transforms.
+ *
+ * <p>See {@link FileSystems} for information on supported file systems and filepatterns.
+ *
+ * <h3>Filepattern expansion and watching</h3>
+ *
+ * <p>By default, {@link #read} prohibits filepatterns that match no files, and {@link #readAll}
+ * allows them in case the filepattern contains a glob wildcard character. Use {@link
+ * Read#withEmptyMatchTreatment} to configure this behavior.
+ *
+ * <p>By default, the filepatterns are expanded only once. {@link Read#watchForNewFiles}
+ * allows streaming of new files matching the filepattern(s).
+ *
+ * <h3>Reading records of a known schema</h3>
+ *
+ * <p>To read specific records, such as Avro-generated classes, use {@link #read(Class)}. To read
+ * {@link GenericRecord GenericRecords}, use {@link #readGenericRecords(Schema)} which takes a
+ * {@link Schema} object, or {@link #readGenericRecords(String)} which takes an Avro schema in a
* JSON-encoded string form. An exception will be thrown if a record doesn't match the specified
- * schema.
+ * schema. Likewise, to read a {@link PCollection} of filepatterns, apply {@link
+ * #readAllGenericRecords}.
*
* <p>For example:
- * <pre> {@code
+ *
+ * <pre>{@code
* Pipeline p = ...;
*
- * // A simple Read of a local file (only runs locally):
+ * // Read Avro-generated classes from files on GCS
* PCollection<AvroAutoGenClass> records =
- * p.apply(AvroIO.read(AvroAutoGenClass.class).from("/path/to/file.avro"));
+ * p.apply(AvroIO.read(AvroAutoGenClass.class).from("gs://my_bucket/path/to/records-*.avro"));
*
- * // A Read from a GCS file (runs locally and using remote execution):
+ * // Read GenericRecord's of the given schema from files on GCS
* Schema schema = new Schema.Parser().parse(new File("schema.avsc"));
* PCollection<GenericRecord> records =
* p.apply(AvroIO.readGenericRecords(schema)
* .from("gs://my_bucket/path/to/records-*.avro"));
- * } </pre>
+ * }</pre>
*
- * <p>To write a {@link PCollection} to one or more Avro files, use {@link AvroIO.Write}, using
- * {@code AvroIO.write().to(String)} to specify the output filename prefix. The default
- * {@link DefaultFilenamePolicy} will use this prefix, in conjunction with a
- * {@link ShardNameTemplate} (set via {@link Write#withShardNameTemplate(String)}) and optional
- * filename suffix (set via {@link Write#withSuffix(String)}, to generate output filenames in a
- * sharded way. You can override this default write filename policy using
- * {@link Write#withFilenamePolicy(FileBasedSink.FilenamePolicy)} to specify a custom file naming
- * policy.
+ * <h3>Reading records of an unknown schema</h3>
*
- * <p>By default, all input is put into the global window before writing. If per-window writes are
- * desired - for example, when using a streaming runner -
- * {@link AvroIO.Write#withWindowedWrites()} will cause windowing and triggering to be
- * preserved. When producing windowed writes, the number of output shards must be set explicitly
- * using {@link AvroIO.Write#withNumShards(int)}; some runners may set this for you to a
- * runner-chosen value, so you may need not set it yourself. A
- * {@link FileBasedSink.FilenamePolicy} must be set, and unique windows and triggers must produce
- * unique filenames.
- *
- * <p>To write specific records, such as Avro-generated classes, use {@link #write(Class)}.
- * To write {@link GenericRecord GenericRecords}, use either {@link #writeGenericRecords(Schema)}
- * which takes a {@link Schema} object, or {@link #writeGenericRecords(String)} which takes a schema
- * in a JSON-encoded string form. An exception will be thrown if a record doesn't match the
- * specified schema.
+ * <p>To read records from files whose schema is unknown at pipeline construction time or differs
+ * between files, use {@link #parseGenericRecords} - in this case, you will need to specify a
+ * parsing function for converting each {@link GenericRecord} into a value of your custom type.
+ * Likewise, to read a {@link PCollection} of filepatterns with unknown schema, use {@link
+ * #parseAllGenericRecords}.
*
* <p>For example:
- * <pre> {@code
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<Foo> records =
+ * p.apply(AvroIO.parseGenericRecords(new SerializableFunction<GenericRecord, Foo>() {
+ * public Foo apply(GenericRecord record) {
+ * // If needed, access the schema of the record using record.getSchema()
+ * return ...;
+ * }
+ * }));
+ * }</pre>
+ *
+ * <h3>Reading from a {@link PCollection} of filepatterns</h3>
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<String> filepatterns = p.apply(...);
+ * PCollection<AvroAutoGenClass> records =
+ * filepatterns.apply(AvroIO.read(AvroAutoGenClass.class));
+ * PCollection<GenericRecord> genericRecords =
+ * filepatterns.apply(AvroIO.readGenericRecords(schema));
+ * PCollection<Foo> records =
+ * filepatterns.apply(AvroIO.parseAllGenericRecords(new SerializableFunction...);
+ * }</pre>
+ *
+ * <h3>Streaming new files matching a filepattern</h3>
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<AvroAutoGenClass> lines = p.apply(AvroIO
+ * .read(AvroAutoGenClass.class)
+ * .from("gs://my_bucket/path/to/records-*.avro")
+ * .watchForNewFiles(
+ * // Check for new files every minute
+ * Duration.standardMinutes(1),
+ * // Stop watching the filepattern if no new files appear within an hour
+ * afterTimeSinceNewOutput(Duration.standardHours(1))));
+ * }</pre>
+ *
+ * <h3>Reading a very large number of files</h3>
+ *
+ * <p>If it is known that the filepattern will match a very large number of files (e.g. tens of
+ * thousands or more), use {@link Read#withHintMatchesManyFiles} for better performance and
+ * scalability. Note that it may decrease performance if the filepattern matches only a small number
+ * of files.
+ *
+ * <h2>Writing Avro files</h2>
+ *
+ * <p>To write a {@link PCollection} to one or more Avro files, use {@link AvroIO.Write}, using
+ * {@code AvroIO.write().to(String)} to specify the output filename prefix. The default {@link
+ * DefaultFilenamePolicy} will use this prefix, in conjunction with a {@link ShardNameTemplate} (set
+ * via {@link Write#withShardNameTemplate(String)}) and optional filename suffix (set via {@link
+ * Write#withSuffix(String)}, to generate output filenames in a sharded way. You can override this
+ * default write filename policy using {@link Write#to(FileBasedSink.FilenamePolicy)} to specify a
+ * custom file naming policy.
+ *
+ * <p>By default, {@link AvroIO.Write} produces output files that are compressed using the {@link
+ * org.apache.avro.file.Codec CodecFactory.deflateCodec(6)}. This default can be changed or
+ * overridden using {@link AvroIO.Write#withCodec}.
+ *
+ * <h3>Writing specific or generic records</h3>
+ *
+ * <p>To write specific records, such as Avro-generated classes, use {@link #write(Class)}. To write
+ * {@link GenericRecord GenericRecords}, use either {@link #writeGenericRecords(Schema)} which takes
+ * a {@link Schema} object, or {@link #writeGenericRecords(String)} which takes a schema in a
+ * JSON-encoded string form. An exception will be thrown if a record doesn't match the specified
+ * schema.
+ *
+ * <p>For example:
+ *
+ * <pre>{@code
* // A simple Write to a local file (only runs locally):
* PCollection<AvroAutoGenClass> records = ...;
* records.apply(AvroIO.write(AvroAutoGenClass.class).to("/path/to/file.avro"));
@@ -113,11 +199,64 @@
* records.apply("WriteToAvro", AvroIO.writeGenericRecords(schema)
* .to("gs://my_bucket/path/to/numbers")
* .withSuffix(".avro"));
- * } </pre>
+ * }</pre>
*
- * <p>By default, {@link AvroIO.Write} produces output files that are compressed using the
- * {@link org.apache.avro.file.Codec CodecFactory.deflateCodec(6)}. This default can
- * be changed or overridden using {@link AvroIO.Write#withCodec}.
+ * <h3>Writing windowed or unbounded data</h3>
+ *
+ * <p>By default, all input is put into the global window before writing. If per-window writes are
+ * desired - for example, when using a streaming runner - {@link AvroIO.Write#withWindowedWrites()}
+ * will cause windowing and triggering to be preserved. When producing windowed writes with a
+ * streaming runner that supports triggers, the number of output shards must be set explicitly using
+ * {@link AvroIO.Write#withNumShards(int)}; some runners may set this for you to a runner-chosen
+ * value, so you may need not set it yourself. A {@link FileBasedSink.FilenamePolicy} must be set,
+ * and unique windows and triggers must produce unique filenames.
+ *
+ * <h3>Writing data to multiple destinations</h3>
+ *
+ * <p>The following shows a more-complex example of AvroIO.Write usage, generating dynamic file
+ * destinations as well as a dynamic Avro schema per file. In this example, a PCollection of user
+ * events (e.g. actions on a website) is written out to Avro files. Each event contains the user id
+ * as an integer field. We want events for each user to go into a specific directory for that user,
+ * and each user's data should be written with a specific schema for that user; a side input is
+ * used, so the schema can be calculated in a different stage.
+ *
+ * <pre>{@code
+ * // This is the user class that controls dynamic destinations for this avro write. The input to
+ * // AvroIO.Write will be UserEvent, and we will be writing GenericRecords to the file (in order
+ * // to have dynamic schemas). Everything is per userid, so we define a dynamic destination type
+ * // of Integer.
+ * class UserDynamicAvroDestinations
+ * extends DynamicAvroDestinations<UserEvent, Integer, GenericRecord> {
+ * private final PCollectionView<Map<Integer, String>> userToSchemaMap;
+ * public UserDynamicAvroDestinations( PCollectionView<Map<Integer, String>> userToSchemaMap) {
+ * this.userToSchemaMap = userToSchemaMap;
+ * }
+ * public GenericRecord formatRecord(UserEvent record) {
+ * return formatUserRecord(record, getSchema(record.getUserId()));
+ * }
+ * public Schema getSchema(Integer userId) {
+ * return new Schema.Parser().parse(sideInput(userToSchemaMap).get(userId));
+ * }
+ * public Integer getDestination(UserEvent record) {
+ * return record.getUserId();
+ * }
+ * public Integer getDefaultDestination() {
+ * return 0;
+ * }
+ * public FilenamePolicy getFilenamePolicy(Integer userId) {
+ * return DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(baseDir + "/user-"
+ * + userId + "/events"));
+ * }
+ * public List<PCollectionView<?>> getSideInputs() {
+ * return ImmutableList.<PCollectionView<?>>of(userToSchemaMap);
+ * }
+ * }
+ * PCollection<UserEvents> events = ...;
+ * PCollectionView<Map<Integer, String>> userToSchemaMap = events.apply(
+ * "ComputePerUserSchemas", new ComputePerUserSchemas());
+ * events.apply("WriteAvros", AvroIO.<Integer>writeCustomTypeToGenericRecords()
+ * .to(new UserDynamicAvroDestinations(userToSchemaMap)));
+ * }</pre>
*/
public class AvroIO {
/**
@@ -127,16 +266,46 @@
*/
public static <T> Read<T> read(Class<T> recordClass) {
return new AutoValue_AvroIO_Read.Builder<T>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
.setRecordClass(recordClass)
.setSchema(ReflectData.get().getSchema(recordClass))
+ .setHintMatchesManyFiles(false)
+ .build();
+ }
+
+ /** Like {@link #read}, but reads each filepattern in the input {@link PCollection}. */
+ public static <T> ReadAll<T> readAll(Class<T> recordClass) {
+ return new AutoValue_AvroIO_ReadAll.Builder<T>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
+ .setRecordClass(recordClass)
+ .setSchema(ReflectData.get().getSchema(recordClass))
+ // 64MB is a reasonable value that allows to amortize the cost of opening files,
+ // but is not so large as to exhaust a typical runner's maximum amount of output per
+ // ProcessElement call.
+ .setDesiredBundleSizeBytes(64 * 1024 * 1024L)
.build();
}
/** Reads Avro file(s) containing records of the specified schema. */
public static Read<GenericRecord> readGenericRecords(Schema schema) {
return new AutoValue_AvroIO_Read.Builder<GenericRecord>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
.setRecordClass(GenericRecord.class)
.setSchema(schema)
+ .setHintMatchesManyFiles(false)
+ .build();
+ }
+
+ /**
+ * Like {@link #readGenericRecords(Schema)}, but reads each filepattern in the input {@link
+ * PCollection}.
+ */
+ public static ReadAll<GenericRecord> readAllGenericRecords(Schema schema) {
+ return new AutoValue_AvroIO_ReadAll.Builder<GenericRecord>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
+ .setRecordClass(GenericRecord.class)
+ .setSchema(schema)
+ .setDesiredBundleSizeBytes(64 * 1024 * 1024L)
.build();
}
@@ -149,22 +318,90 @@
}
/**
+ * Like {@link #readGenericRecords(String)}, but reads each filepattern in the input {@link
+ * PCollection}.
+ */
+ public static ReadAll<GenericRecord> readAllGenericRecords(String schema) {
+ return readAllGenericRecords(new Schema.Parser().parse(schema));
+ }
+
+ /**
+ * Reads Avro file(s) containing records of an unspecified schema and converting each record to a
+ * custom type.
+ */
+ public static <T> Parse<T> parseGenericRecords(SerializableFunction<GenericRecord, T> parseFn) {
+ return new AutoValue_AvroIO_Parse.Builder<T>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
+ .setParseFn(parseFn)
+ .setHintMatchesManyFiles(false)
+ .build();
+ }
+
+ /**
+ * Like {@link #parseGenericRecords(SerializableFunction)}, but reads each filepattern in the
+ * input {@link PCollection}.
+ */
+ public static <T> ParseAll<T> parseAllGenericRecords(
+ SerializableFunction<GenericRecord, T> parseFn) {
+ return new AutoValue_AvroIO_ParseAll.Builder<T>()
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
+ .setParseFn(parseFn)
+ .setDesiredBundleSizeBytes(64 * 1024 * 1024L)
+ .build();
+ }
+
+ /**
* Writes a {@link PCollection} to an Avro file (or multiple Avro files matching a sharding
* pattern).
*/
public static <T> Write<T> write(Class<T> recordClass) {
- return AvroIO.<T>defaultWriteBuilder()
- .setRecordClass(recordClass)
- .setSchema(ReflectData.get().getSchema(recordClass))
- .build();
+ return new Write<>(
+ AvroIO.<T, T>defaultWriteBuilder()
+ .setGenericRecords(false)
+ .setSchema(ReflectData.get().getSchema(recordClass))
+ .build());
}
/** Writes Avro records of the specified schema. */
public static Write<GenericRecord> writeGenericRecords(Schema schema) {
- return AvroIO.<GenericRecord>defaultWriteBuilder()
- .setRecordClass(GenericRecord.class)
- .setSchema(schema)
- .build();
+ return new Write<>(
+ AvroIO.<GenericRecord, GenericRecord>defaultWriteBuilder()
+ .setGenericRecords(true)
+ .setSchema(schema)
+ .build());
+ }
+
+ /**
+ * A {@link PTransform} that writes a {@link PCollection} to an avro file (or multiple avro files
+ * matching a sharding pattern), with each element of the input collection encoded into its own
+ * record of type OutputT.
+ *
+ * <p>This version allows you to apply {@link AvroIO} writes to a PCollection of a custom type
+ * {@link UserT}. A format mechanism that converts the input type {@link UserT} to the output type
+ * that will be written to the file must be specified. If using a custom {@link
+ * DynamicAvroDestinations} object this is done using {@link
+ * DynamicAvroDestinations#formatRecord}, otherwise the {@link
+ * AvroIO.TypedWrite#withFormatFunction} can be used to specify a format function.
+ *
+ * <p>The advantage of using a custom type is that is it allows a user-provided {@link
+ * DynamicAvroDestinations} object, set via {@link AvroIO.Write#to(DynamicAvroDestinations)} to
+ * examine the custom type when choosing a destination.
+ *
+ * <p>If the output type is {@link GenericRecord} use {@link #writeCustomTypeToGenericRecords()}
+ * instead.
+ */
+ public static <UserT, OutputT> TypedWrite<UserT, Void, OutputT> writeCustomType() {
+ return AvroIO.<UserT, OutputT>defaultWriteBuilder().setGenericRecords(false).build();
+ }
+
+ /**
+ * Similar to {@link #writeCustomType()}, but specialized for the case where the output type is
+ * {@link GenericRecord}. A schema must be specified either in {@link
+ * DynamicAvroDestinations#getSchema} or if not using dynamic destinations, by using {@link
+ * TypedWrite#withSchema(Schema)}.
+ */
+ public static <UserT> TypedWrite<UserT, Void, GenericRecord> writeCustomTypeToGenericRecords() {
+ return AvroIO.<UserT, GenericRecord>defaultWriteBuilder().setGenericRecords(true).build();
}
/**
@@ -174,95 +411,459 @@
return writeGenericRecords(new Schema.Parser().parse(schema));
}
- private static <T> Write.Builder<T> defaultWriteBuilder() {
- return new AutoValue_AvroIO_Write.Builder<T>()
+ private static <UserT, OutputT> TypedWrite.Builder<UserT, Void, OutputT> defaultWriteBuilder() {
+ return new AutoValue_AvroIO_TypedWrite.Builder<UserT, Void, OutputT>()
.setFilenameSuffix(null)
.setShardTemplate(null)
.setNumShards(0)
- .setCodec(Write.DEFAULT_CODEC)
+ .setCodec(TypedWrite.DEFAULT_SERIALIZABLE_CODEC)
.setMetadata(ImmutableMap.<String, Object>of())
.setWindowedWrites(false);
}
- /** Implementation of {@link #read}. */
+ /** Implementation of {@link #read} and {@link #readGenericRecords}. */
@AutoValue
public abstract static class Read<T> extends PTransform<PBegin, PCollection<T>> {
- @Nullable abstract String getFilepattern();
+ @Nullable abstract ValueProvider<String> getFilepattern();
+ abstract MatchConfiguration getMatchConfiguration();
@Nullable abstract Class<T> getRecordClass();
@Nullable abstract Schema getSchema();
+ abstract boolean getHintMatchesManyFiles();
abstract Builder<T> toBuilder();
@AutoValue.Builder
abstract static class Builder<T> {
- abstract Builder<T> setFilepattern(String filepattern);
+ abstract Builder<T> setFilepattern(ValueProvider<String> filepattern);
+ abstract Builder<T> setMatchConfiguration(MatchConfiguration matchConfiguration);
abstract Builder<T> setRecordClass(Class<T> recordClass);
abstract Builder<T> setSchema(Schema schema);
+ abstract Builder<T> setHintMatchesManyFiles(boolean hintManyFiles);
abstract Read<T> build();
}
- /** Reads from the given filename or filepattern. */
- public Read<T> from(String filepattern) {
+ /**
+ * Reads from the given filename or filepattern.
+ *
+ * <p>If it is known that the filepattern will match a very large number of files (at least tens
+ * of thousands), use {@link #withHintMatchesManyFiles} for better performance and scalability.
+ */
+ public Read<T> from(ValueProvider<String> filepattern) {
return toBuilder().setFilepattern(filepattern).build();
}
+ /** Like {@link #from(ValueProvider)}. */
+ public Read<T> from(String filepattern) {
+ return from(StaticValueProvider.of(filepattern));
+ }
+
+
+ /** Sets the {@link MatchConfiguration}. */
+ public Read<T> withMatchConfiguration(MatchConfiguration matchConfiguration) {
+ return toBuilder().setMatchConfiguration(matchConfiguration).build();
+ }
+
+ /** Configures whether or not a filepattern matching no files is allowed. */
+ public Read<T> withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /**
+ * Continuously watches for new files matching the filepattern, polling it at the given
+ * interval, until the given termination condition is reached. The returned {@link PCollection}
+ * is unbounded.
+ *
+ * <p>This works only in runners supporting {@link Kind#SPLITTABLE_DO_FN}.
+ */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public Read<T> watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ /**
+ * Hints that the filepattern specified in {@link #from(String)} matches a very large number of
+ * files.
+ *
+ * <p>This hint may cause a runner to execute the transform differently, in a way that improves
+ * performance for this case, but it may worsen performance if the filepattern matches only a
+ * small number of files (e.g., in a runner that supports dynamic work rebalancing, it will
+ * happen less efficiently within individual files).
+ */
+ public Read<T> withHintMatchesManyFiles() {
+ return toBuilder().setHintMatchesManyFiles(true).build();
+ }
+
@Override
public PCollection<T> expand(PBegin input) {
- if (getFilepattern() == null) {
- throw new IllegalStateException(
- "need to set the filepattern of an AvroIO.Read transform");
- }
- if (getSchema() == null) {
- throw new IllegalStateException("need to set the schema of an AvroIO.Read transform");
- }
+ checkNotNull(getFilepattern(), "filepattern");
+ checkNotNull(getSchema(), "schema");
- @SuppressWarnings("unchecked")
- Bounded<T> read =
- getRecordClass() == GenericRecord.class
- ? (Bounded<T>) org.apache.beam.sdk.io.Read.from(
- AvroSource.from(getFilepattern()).withSchema(getSchema()))
- : org.apache.beam.sdk.io.Read.from(
- AvroSource.from(getFilepattern()).withSchema(getRecordClass()));
+ if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) {
+ return input.apply(
+ "Read",
+ org.apache.beam.sdk.io.Read.from(
+ createSource(
+ getFilepattern(),
+ getMatchConfiguration().getEmptyMatchTreatment(),
+ getRecordClass(),
+ getSchema())));
+ }
+ // All other cases go through ReadAll.
- PCollection<T> pcol = input.getPipeline().apply("Read", read);
- // Honor the default output coder that would have been used by this PTransform.
- pcol.setCoder(getDefaultOutputCoder());
- return pcol;
+ ReadAll<T> readAll =
+ (getRecordClass() == GenericRecord.class)
+ ? (ReadAll<T>) readAllGenericRecords(getSchema())
+ : readAll(getRecordClass());
+ readAll = readAll.withMatchConfiguration(getMatchConfiguration());
+ return input
+ .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of()))
+ .apply("Via ReadAll", readAll);
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
- .addIfNotNull(DisplayData.item("filePattern", getFilepattern())
- .withLabel("Input File Pattern"));
+ .addIfNotNull(
+ DisplayData.item("filePattern", getFilepattern()).withLabel("Input File Pattern"))
+ .include("matchConfiguration", getMatchConfiguration());
}
- @Override
- protected Coder<T> getDefaultOutputCoder() {
- return AvroCoder.of(getRecordClass(), getSchema());
+ @SuppressWarnings("unchecked")
+ private static <T> AvroSource<T> createSource(
+ ValueProvider<String> filepattern,
+ EmptyMatchTreatment emptyMatchTreatment,
+ Class<T> recordClass,
+ Schema schema) {
+ AvroSource<?> source =
+ AvroSource.from(filepattern).withEmptyMatchTreatment(emptyMatchTreatment);
+ return recordClass == GenericRecord.class
+ ? (AvroSource<T>) source.withSchema(schema)
+ : source.withSchema(recordClass);
}
}
/////////////////////////////////////////////////////////////////////////////
+ /** Implementation of {@link #readAll}. */
+ @AutoValue
+ public abstract static class ReadAll<T> extends PTransform<PCollection<String>, PCollection<T>> {
+ abstract MatchConfiguration getMatchConfiguration();
+ @Nullable abstract Class<T> getRecordClass();
+ @Nullable abstract Schema getSchema();
+ abstract long getDesiredBundleSizeBytes();
+
+ abstract Builder<T> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setMatchConfiguration(MatchConfiguration matchConfiguration);
+ abstract Builder<T> setRecordClass(Class<T> recordClass);
+ abstract Builder<T> setSchema(Schema schema);
+ abstract Builder<T> setDesiredBundleSizeBytes(long desiredBundleSizeBytes);
+
+ abstract ReadAll<T> build();
+ }
+
+
+ /** Sets the {@link MatchConfiguration}. */
+ public ReadAll<T> withMatchConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setMatchConfiguration(configuration).build();
+ }
+
+ /** Like {@link Read#withEmptyMatchTreatment}. */
+ public ReadAll<T> withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /** Like {@link Read#watchForNewFiles}. */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public ReadAll<T> watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ @VisibleForTesting
+ ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
+ }
+
+ @Override
+ public PCollection<T> expand(PCollection<String> input) {
+ checkNotNull(getSchema(), "schema");
+ return input
+ .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration()))
+ .apply(FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT))
+ .apply(
+ "Read all via FileBasedSource",
+ new ReadAllViaFileBasedSource<>(
+ getDesiredBundleSizeBytes(),
+ new CreateSourceFn<>(getRecordClass(), getSchema().toString()),
+ AvroCoder.of(getRecordClass(), getSchema())));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.include("matchConfiguration", getMatchConfiguration());
+ }
+ }
+
+ private static class CreateSourceFn<T>
+ implements SerializableFunction<String, FileBasedSource<T>> {
+ private final Class<T> recordClass;
+ private final Supplier<Schema> schemaSupplier;
+
+ public CreateSourceFn(Class<T> recordClass, String jsonSchema) {
+ this.recordClass = recordClass;
+ this.schemaSupplier = AvroUtils.serializableSchemaSupplier(jsonSchema);
+ }
+
+ @Override
+ public FileBasedSource<T> apply(String input) {
+ return Read.createSource(
+ StaticValueProvider.of(input),
+ EmptyMatchTreatment.DISALLOW,
+ recordClass,
+ schemaSupplier.get());
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+
+ /** Implementation of {@link #parseGenericRecords}. */
+ @AutoValue
+ public abstract static class Parse<T> extends PTransform<PBegin, PCollection<T>> {
+ @Nullable abstract ValueProvider<String> getFilepattern();
+ abstract MatchConfiguration getMatchConfiguration();
+ abstract SerializableFunction<GenericRecord, T> getParseFn();
+ @Nullable abstract Coder<T> getCoder();
+ abstract boolean getHintMatchesManyFiles();
+
+ abstract Builder<T> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setFilepattern(ValueProvider<String> filepattern);
+ abstract Builder<T> setMatchConfiguration(MatchConfiguration matchConfiguration);
+ abstract Builder<T> setParseFn(SerializableFunction<GenericRecord, T> parseFn);
+ abstract Builder<T> setCoder(Coder<T> coder);
+ abstract Builder<T> setHintMatchesManyFiles(boolean hintMatchesManyFiles);
+
+ abstract Parse<T> build();
+ }
+
+ /** Reads from the given filename or filepattern. */
+ public Parse<T> from(String filepattern) {
+ return from(StaticValueProvider.of(filepattern));
+ }
+
+ /** Like {@link #from(String)}. */
+ public Parse<T> from(ValueProvider<String> filepattern) {
+ return toBuilder().setFilepattern(filepattern).build();
+ }
+
+ /** Sets the {@link MatchConfiguration}. */
+ public Parse<T> withMatchConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setMatchConfiguration(configuration).build();
+ }
+
+ /** Like {@link Read#withEmptyMatchTreatment}. */
+ public Parse<T> withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /** Like {@link Read#watchForNewFiles}. */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public Parse<T> watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ /** Sets a coder for the result of the parse function. */
+ public Parse<T> withCoder(Coder<T> coder) {
+ return toBuilder().setCoder(coder).build();
+ }
+
+ /** Like {@link Read#withHintMatchesManyFiles()}. */
+ public Parse<T> withHintMatchesManyFiles() {
+ return toBuilder().setHintMatchesManyFiles(true).build();
+ }
+
+ @Override
+ public PCollection<T> expand(PBegin input) {
+ checkNotNull(getFilepattern(), "filepattern");
+ Coder<T> coder = inferCoder(getCoder(), getParseFn(), input.getPipeline().getCoderRegistry());
+
+ if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) {
+ return input.apply(
+ org.apache.beam.sdk.io.Read.from(
+ AvroSource.from(getFilepattern()).withParseFn(getParseFn(), coder)));
+ }
+ // All other cases go through ParseAllGenericRecords.
+ return input
+ .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of()))
+ .apply(
+ "Via ParseAll",
+ parseAllGenericRecords(getParseFn())
+ .withCoder(coder)
+ .withMatchConfiguration(getMatchConfiguration()));
+ }
+
+ private static <T> Coder<T> inferCoder(
+ @Nullable Coder<T> explicitCoder,
+ SerializableFunction<GenericRecord, T> parseFn,
+ CoderRegistry coderRegistry) {
+ if (explicitCoder != null) {
+ return explicitCoder;
+ }
+ // If a coder was not specified explicitly, infer it from parse fn.
+ try {
+ return coderRegistry.getCoder(TypeDescriptors.outputOf(parseFn));
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalArgumentException(
+ "Unable to infer coder for output of parseFn. Specify it explicitly using withCoder().",
+ e);
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder
+ .addIfNotNull(
+ DisplayData.item("filePattern", getFilepattern()).withLabel("Input File Pattern"))
+ .add(DisplayData.item("parseFn", getParseFn().getClass()).withLabel("Parse function"))
+ .include("matchConfiguration", getMatchConfiguration());
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+
+ /** Implementation of {@link #parseAllGenericRecords}. */
+ @AutoValue
+ public abstract static class ParseAll<T> extends PTransform<PCollection<String>, PCollection<T>> {
+ abstract MatchConfiguration getMatchConfiguration();
+ abstract SerializableFunction<GenericRecord, T> getParseFn();
+ @Nullable abstract Coder<T> getCoder();
+ abstract long getDesiredBundleSizeBytes();
+
+ abstract Builder<T> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setMatchConfiguration(MatchConfiguration matchConfiguration);
+ abstract Builder<T> setParseFn(SerializableFunction<GenericRecord, T> parseFn);
+ abstract Builder<T> setCoder(Coder<T> coder);
+ abstract Builder<T> setDesiredBundleSizeBytes(long desiredBundleSizeBytes);
+
+ abstract ParseAll<T> build();
+ }
+
+ /** Sets the {@link MatchConfiguration}. */
+ public ParseAll<T> withMatchConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setMatchConfiguration(configuration).build();
+ }
+
+ /** Like {@link Read#withEmptyMatchTreatment}. */
+ public ParseAll<T> withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /** Like {@link Read#watchForNewFiles}. */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public ParseAll<T> watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ /** Specifies the coder for the result of the {@code parseFn}. */
+ public ParseAll<T> withCoder(Coder<T> coder) {
+ return toBuilder().setCoder(coder).build();
+ }
+
+ @VisibleForTesting
+ ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
+ }
+
+ @Override
+ public PCollection<T> expand(PCollection<String> input) {
+ final Coder<T> coder =
+ Parse.inferCoder(getCoder(), getParseFn(), input.getPipeline().getCoderRegistry());
+ final SerializableFunction<GenericRecord, T> parseFn = getParseFn();
+ final SerializableFunction<String, FileBasedSource<T>> createSource =
+ new CreateParseSourceFn<>(parseFn, coder);
+ return input
+ .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration()))
+ .apply(FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT))
+ .apply(
+ "Parse all via FileBasedSource",
+ new ReadAllViaFileBasedSource<>(getDesiredBundleSizeBytes(), createSource, coder));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder
+ .add(DisplayData.item("parseFn", getParseFn().getClass()).withLabel("Parse function"))
+ .include("matchConfiguration", getMatchConfiguration());
+ }
+
+ private static class CreateParseSourceFn<T>
+ implements SerializableFunction<String, FileBasedSource<T>> {
+ private final SerializableFunction<GenericRecord, T> parseFn;
+ private final Coder<T> coder;
+
+ public CreateParseSourceFn(SerializableFunction<GenericRecord, T> parseFn, Coder<T> coder) {
+ this.parseFn = parseFn;
+ this.coder = coder;
+ }
+
+ @Override
+ public FileBasedSource<T> apply(String input) {
+ return AvroSource.from(input).withParseFn(parseFn, coder);
+ }
+ }
+ }
+
+ // ///////////////////////////////////////////////////////////////////////////
+
/** Implementation of {@link #write}. */
@AutoValue
- public abstract static class Write<T> extends PTransform<PCollection<T>, PDone> {
- private static final SerializableAvroCodecFactory DEFAULT_CODEC =
- new SerializableAvroCodecFactory(CodecFactory.deflateCodec(6));
- // This should be a multiple of 4 to not get a partial encoded byte.
- private static final int METADATA_BYTES_MAX_LENGTH = 40;
+ public abstract static class TypedWrite<UserT, DestinationT, OutputT>
+ extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>> {
+ static final CodecFactory DEFAULT_CODEC = CodecFactory.deflateCodec(6);
+ static final SerializableAvroCodecFactory DEFAULT_SERIALIZABLE_CODEC =
+ new SerializableAvroCodecFactory(DEFAULT_CODEC);
+
+ @Nullable
+ abstract SerializableFunction<UserT, OutputT> getFormatFunction();
@Nullable abstract ValueProvider<ResourceId> getFilenamePrefix();
@Nullable abstract String getShardTemplate();
@Nullable abstract String getFilenameSuffix();
+
+ @Nullable
+ abstract ValueProvider<ResourceId> getTempDirectory();
+
abstract int getNumShards();
- @Nullable abstract Class<T> getRecordClass();
+
+ abstract boolean getGenericRecords();
+
@Nullable abstract Schema getSchema();
abstract boolean getWindowedWrites();
@Nullable abstract FilenamePolicy getFilenamePolicy();
+
+ @Nullable
+ abstract DynamicAvroDestinations<UserT, DestinationT, OutputT> getDynamicDestinations();
+
/**
* The codec used to encode the blocks in the Avro file. String value drawn from those in
* https://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/file/CodecFactory.html
@@ -271,22 +872,43 @@
/** Avro file metadata. */
abstract ImmutableMap<String, Object> getMetadata();
- abstract Builder<T> toBuilder();
+ abstract Builder<UserT, DestinationT, OutputT> toBuilder();
@AutoValue.Builder
- abstract static class Builder<T> {
- abstract Builder<T> setFilenamePrefix(ValueProvider<ResourceId> filenamePrefix);
- abstract Builder<T> setFilenameSuffix(String filenameSuffix);
- abstract Builder<T> setNumShards(int numShards);
- abstract Builder<T> setShardTemplate(String shardTemplate);
- abstract Builder<T> setRecordClass(Class<T> recordClass);
- abstract Builder<T> setSchema(Schema schema);
- abstract Builder<T> setWindowedWrites(boolean windowedWrites);
- abstract Builder<T> setFilenamePolicy(FilenamePolicy filenamePolicy);
- abstract Builder<T> setCodec(SerializableAvroCodecFactory codec);
- abstract Builder<T> setMetadata(ImmutableMap<String, Object> metadata);
+ abstract static class Builder<UserT, DestinationT, OutputT> {
+ abstract Builder<UserT, DestinationT, OutputT> setFormatFunction(
+ SerializableFunction<UserT, OutputT> formatFunction);
- abstract Write<T> build();
+ abstract Builder<UserT, DestinationT, OutputT> setFilenamePrefix(
+ ValueProvider<ResourceId> filenamePrefix);
+
+ abstract Builder<UserT, DestinationT, OutputT> setFilenameSuffix(String filenameSuffix);
+
+ abstract Builder<UserT, DestinationT, OutputT> setTempDirectory(
+ ValueProvider<ResourceId> tempDirectory);
+
+ abstract Builder<UserT, DestinationT, OutputT> setNumShards(int numShards);
+
+ abstract Builder<UserT, DestinationT, OutputT> setShardTemplate(String shardTemplate);
+
+ abstract Builder<UserT, DestinationT, OutputT> setGenericRecords(boolean genericRecords);
+
+ abstract Builder<UserT, DestinationT, OutputT> setSchema(Schema schema);
+
+ abstract Builder<UserT, DestinationT, OutputT> setWindowedWrites(boolean windowedWrites);
+
+ abstract Builder<UserT, DestinationT, OutputT> setFilenamePolicy(
+ FilenamePolicy filenamePolicy);
+
+ abstract Builder<UserT, DestinationT, OutputT> setCodec(SerializableAvroCodecFactory codec);
+
+ abstract Builder<UserT, DestinationT, OutputT> setMetadata(
+ ImmutableMap<String, Object> metadata);
+
+ abstract Builder<UserT, DestinationT, OutputT> setDynamicDestinations(
+ DynamicAvroDestinations<UserT, DestinationT, OutputT> dynamicDestinations);
+
+ abstract TypedWrite<UserT, DestinationT, OutputT> build();
}
/**
@@ -296,77 +918,134 @@
* <p>The name of the output files will be determined by the {@link FilenamePolicy} used.
*
* <p>By default, a {@link DefaultFilenamePolicy} will build output filenames using the
- * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and
- * a common suffix (if supplied using {@link #withSuffix(String)}). This default can be
- * overridden using {@link #withFilenamePolicy(FilenamePolicy)}.
+ * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and a
+ * common suffix (if supplied using {@link #withSuffix(String)}). This default can be overridden
+ * using {@link #to(FilenamePolicy)}.
*/
- public Write<T> to(String outputPrefix) {
+ public TypedWrite<UserT, DestinationT, OutputT> to(String outputPrefix) {
return to(FileBasedSink.convertToFileResourceIfPossible(outputPrefix));
}
/**
* Writes to file(s) with the given output prefix. See {@link FileSystems} for information on
- * supported file systems.
- *
- * <p>The name of the output files will be determined by the {@link FilenamePolicy} used.
+ * supported file systems. This prefix is used by the {@link DefaultFilenamePolicy} to generate
+ * filenames.
*
* <p>By default, a {@link DefaultFilenamePolicy} will build output filenames using the
- * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and
- * a common suffix (if supplied using {@link #withSuffix(String)}). This default can be
- * overridden using {@link #withFilenamePolicy(FilenamePolicy)}.
+ * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and a
+ * common suffix (if supplied using {@link #withSuffix(String)}). This default can be overridden
+ * using {@link #to(FilenamePolicy)}.
+ *
+ * <p>This default policy can be overridden using {@link #to(FilenamePolicy)}, in which case
+ * {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should not be set.
+ * Custom filename policies do not automatically see this prefix - you should explicitly pass
+ * the prefix into your {@link FilenamePolicy} object if you need this.
+ *
+ * <p>If {@link #withTempDirectory} has not been called, this filename prefix will be used to
+ * infer a directory for temporary files.
*/
@Experimental(Kind.FILESYSTEM)
- public Write<T> to(ResourceId outputPrefix) {
+ public TypedWrite<UserT, DestinationT, OutputT> to(ResourceId outputPrefix) {
return toResource(StaticValueProvider.of(outputPrefix));
}
- /**
- * Like {@link #to(String)}.
- */
- public Write<T> to(ValueProvider<String> outputPrefix) {
- return toResource(NestedValueProvider.of(outputPrefix,
- new SerializableFunction<String, ResourceId>() {
- @Override
- public ResourceId apply(String input) {
- return FileBasedSink.convertToFileResourceIfPossible(input);
- }
- }));
+ private static class OutputPrefixToResourceId
+ implements SerializableFunction<String, ResourceId> {
+ @Override
+ public ResourceId apply(String input) {
+ return FileBasedSink.convertToFileResourceIfPossible(input);
+ }
}
- /**
- * Like {@link #to(ResourceId)}.
- */
+ /** Like {@link #to(String)}. */
+ public TypedWrite<UserT, DestinationT, OutputT> to(ValueProvider<String> outputPrefix) {
+ return toResource(
+ NestedValueProvider.of(
+ outputPrefix,
+ // The function cannot be created as an anonymous class here since the enclosed class
+ // may contain unserializable members.
+ new OutputPrefixToResourceId()));
+ }
+
+ /** Like {@link #to(ResourceId)}. */
@Experimental(Kind.FILESYSTEM)
- public Write<T> toResource(ValueProvider<ResourceId> outputPrefix) {
+ public TypedWrite<UserT, DestinationT, OutputT> toResource(
+ ValueProvider<ResourceId> outputPrefix) {
return toBuilder().setFilenamePrefix(outputPrefix).build();
}
/**
- * Configures the {@link FileBasedSink.FilenamePolicy} that will be used to name written files.
+ * Writes to files named according to the given {@link FileBasedSink.FilenamePolicy}. A
+ * directory for temporary files must be specified using {@link #withTempDirectory}.
*/
- public Write<T> withFilenamePolicy(FilenamePolicy filenamePolicy) {
+ @Experimental(Kind.FILESYSTEM)
+ public TypedWrite<UserT, DestinationT, OutputT> to(FilenamePolicy filenamePolicy) {
return toBuilder().setFilenamePolicy(filenamePolicy).build();
}
/**
+ * Use a {@link DynamicAvroDestinations} object to vend {@link FilenamePolicy} objects. These
+ * objects can examine the input record when creating a {@link FilenamePolicy}. A directory for
+ * temporary files must be specified using {@link #withTempDirectory}.
+ */
+ @Experimental(Kind.FILESYSTEM)
+ public <NewDestinationT> TypedWrite<UserT, NewDestinationT, OutputT> to(
+ DynamicAvroDestinations<UserT, NewDestinationT, OutputT> dynamicDestinations) {
+ return toBuilder()
+ .setDynamicDestinations((DynamicAvroDestinations) dynamicDestinations)
+ .build();
+ }
+
+ /**
+ * Sets the the output schema. Can only be used when the output type is {@link GenericRecord}
+ * and when not using {@link #to(DynamicAvroDestinations)}.
+ */
+ public TypedWrite<UserT, DestinationT, OutputT> withSchema(Schema schema) {
+ return toBuilder().setSchema(schema).build();
+ }
+
+ /**
+ * Specifies a format function to convert {@link UserT} to the output type. If {@link
+ * #to(DynamicAvroDestinations)} is used, {@link DynamicAvroDestinations#formatRecord} must be
+ * used instead.
+ */
+ public TypedWrite<UserT, DestinationT, OutputT> withFormatFunction(
+ SerializableFunction<UserT, OutputT> formatFunction) {
+ return toBuilder().setFormatFunction(formatFunction).build();
+ }
+
+ /** Set the base directory used to generate temporary files. */
+ @Experimental(Kind.FILESYSTEM)
+ public TypedWrite<UserT, DestinationT, OutputT> withTempDirectory(
+ ValueProvider<ResourceId> tempDirectory) {
+ return toBuilder().setTempDirectory(tempDirectory).build();
+ }
+
+ /** Set the base directory used to generate temporary files. */
+ @Experimental(Kind.FILESYSTEM)
+ public TypedWrite<UserT, DestinationT, OutputT> withTempDirectory(ResourceId tempDirectory) {
+ return withTempDirectory(StaticValueProvider.of(tempDirectory));
+ }
+
+ /**
* Uses the given {@link ShardNameTemplate} for naming output files. This option may only be
- * used when {@link #withFilenamePolicy(FilenamePolicy)} has not been configured.
+ * used when using one of the default filename-prefix to() overrides.
*
* <p>See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are
* used.
*/
- public Write<T> withShardNameTemplate(String shardTemplate) {
+ public TypedWrite<UserT, DestinationT, OutputT> withShardNameTemplate(String shardTemplate) {
return toBuilder().setShardTemplate(shardTemplate).build();
}
/**
- * Configures the filename suffix for written files. This option may only be used when
- * {@link #withFilenamePolicy(FilenamePolicy)} has not been configured.
+ * Configures the filename suffix for written files. This option may only be used when using one
+ * of the default filename-prefix to() overrides.
*
* <p>See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are
* used.
*/
- public Write<T> withSuffix(String filenameSuffix) {
+ public TypedWrite<UserT, DestinationT, OutputT> withSuffix(String filenameSuffix) {
return toBuilder().setFilenameSuffix(filenameSuffix).build();
}
@@ -380,7 +1059,7 @@
*
* @param numShards the number of shards to use, or 0 to let the system decide.
*/
- public Write<T> withNumShards(int numShards) {
+ public TypedWrite<UserT, DestinationT, OutputT> withNumShards(int numShards) {
checkArgument(numShards >= 0);
return toBuilder().setNumShards(numShards).build();
}
@@ -395,23 +1074,22 @@
*
* <p>This is equivalent to {@code .withNumShards(1).withShardNameTemplate("")}
*/
- public Write<T> withoutSharding() {
+ public TypedWrite<UserT, DestinationT, OutputT> withoutSharding() {
return withNumShards(1).withShardNameTemplate("");
}
/**
* Preserves windowing of input elements and writes them to files based on the element's window.
*
- * <p>Requires use of {@link #withFilenamePolicy(FileBasedSink.FilenamePolicy)}. Filenames will
- * be generated using {@link FilenamePolicy#windowedFilename}. See also
- * {@link WriteFiles#withWindowedWrites()}.
+ * <p>If using {@link #to(FileBasedSink.FilenamePolicy)}. Filenames will be generated using
+ * {@link FilenamePolicy#windowedFilename}. See also {@link WriteFiles#withWindowedWrites()}.
*/
- public Write<T> withWindowedWrites() {
+ public TypedWrite<UserT, DestinationT, OutputT> withWindowedWrites() {
return toBuilder().setWindowedWrites(true).build();
}
/** Writes to Avro file(s) compressed using specified codec. */
- public Write<T> withCodec(CodecFactory codec) {
+ public TypedWrite<UserT, DestinationT, OutputT> withCodec(CodecFactory codec) {
return toBuilder().setCodec(new SerializableAvroCodecFactory(codec)).build();
}
@@ -420,7 +1098,7 @@
*
* <p>Supported value types are String, Long, and byte[].
*/
- public Write<T> withMetadata(Map<String, Object> metadata) {
+ public TypedWrite<UserT, DestinationT, OutputT> withMetadata(Map<String, Object> metadata) {
Map<String, String> badKeys = Maps.newLinkedHashMap();
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
Object v = entry.getValue();
@@ -435,32 +1113,61 @@
return toBuilder().setMetadata(ImmutableMap.copyOf(metadata)).build();
}
- @Override
- public PDone expand(PCollection<T> input) {
- checkState(getFilenamePrefix() != null,
- "Need to set the filename prefix of an AvroIO.Write transform.");
- checkState(
- (getFilenamePolicy() == null)
- || (getShardTemplate() == null && getFilenameSuffix() == null),
- "Cannot set a filename policy and also a filename template or suffix.");
- checkState(getSchema() != null,
- "Need to set the schema of an AvroIO.Write transform.");
- checkState(!getWindowedWrites() || (getFilenamePolicy() != null),
- "When using windowed writes, a filename policy must be set via withFilenamePolicy().");
+ DynamicAvroDestinations<UserT, DestinationT, OutputT> resolveDynamicDestinations() {
+ DynamicAvroDestinations<UserT, DestinationT, OutputT> dynamicDestinations =
+ getDynamicDestinations();
+ if (dynamicDestinations == null) {
+ // In this case DestinationT is Void.
+ FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
+ if (usedFilenamePolicy == null) {
+ usedFilenamePolicy =
+ DefaultFilenamePolicy.fromStandardParameters(
+ getFilenamePrefix(),
+ getShardTemplate(),
+ getFilenameSuffix(),
+ getWindowedWrites());
+ }
+ dynamicDestinations =
+ (DynamicAvroDestinations<UserT, DestinationT, OutputT>)
+ constantDestinations(
+ usedFilenamePolicy,
+ getSchema(),
+ getMetadata(),
+ getCodec().getCodec(),
+ getFormatFunction());
+ }
+ return dynamicDestinations;
+ }
- FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
- if (usedFilenamePolicy == null) {
- usedFilenamePolicy = DefaultFilenamePolicy.constructUsingStandardParameters(
- getFilenamePrefix(), getShardTemplate(), getFilenameSuffix());
+ @Override
+ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
+ checkArgument(
+ getFilenamePrefix() != null || getTempDirectory() != null,
+ "Need to set either the filename prefix or the tempDirectory of a AvroIO.Write "
+ + "transform.");
+ if (getFilenamePolicy() != null) {
+ checkArgument(
+ getShardTemplate() == null && getFilenameSuffix() == null,
+ "shardTemplate and filenameSuffix should only be used with the default "
+ + "filename policy");
+ }
+ if (getDynamicDestinations() != null) {
+ checkArgument(
+ getFormatFunction() == null,
+ "A format function should not be specified "
+ + "with DynamicDestinations. Use DynamicDestinations.formatRecord instead");
+ } else {
+ checkArgument(
+ getSchema() != null, "Unless using DynamicDestinations, .withSchema() is required.");
}
- WriteFiles<T> write = WriteFiles.to(
- new AvroSink<>(
- getFilenamePrefix(),
- usedFilenamePolicy,
- AvroCoder.of(getRecordClass(), getSchema()),
- getCodec(),
- getMetadata()));
+ ValueProvider<ResourceId> tempDirectory = getTempDirectory();
+ if (tempDirectory == null) {
+ tempDirectory = getFilenamePrefix();
+ }
+ WriteFiles<UserT, DestinationT, OutputT> write =
+ WriteFiles.to(
+ new AvroSink<>(tempDirectory, resolveDynamicDestinations(), getGenericRecords()));
if (getNumShards() > 0) {
write = write.withNumShards(getNumShards());
}
@@ -473,57 +1180,156 @@
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- checkState(
- getFilenamePrefix() != null,
- "Unable to populate DisplayData for invalid AvroIO.Write (unset output prefix).");
- String outputPrefixString = null;
- if (getFilenamePrefix().isAccessible()) {
- ResourceId dir = getFilenamePrefix().get();
- outputPrefixString = dir.toString();
- } else {
- outputPrefixString = getFilenamePrefix().toString();
- }
+ resolveDynamicDestinations().populateDisplayData(builder);
builder
- .add(DisplayData.item("schema", getRecordClass())
- .withLabel("Record Schema"))
- .addIfNotNull(DisplayData.item("filePrefix", outputPrefixString)
- .withLabel("Output File Prefix"))
- .addIfNotNull(DisplayData.item("shardNameTemplate", getShardTemplate())
- .withLabel("Output Shard Name Template"))
- .addIfNotNull(DisplayData.item("fileSuffix", getFilenameSuffix())
- .withLabel("Output File Suffix"))
- .addIfNotDefault(DisplayData.item("numShards", getNumShards())
- .withLabel("Maximum Output Shards"),
- 0)
- .addIfNotDefault(DisplayData.item("codec", getCodec().toString())
- .withLabel("Avro Compression Codec"),
- DEFAULT_CODEC.toString());
- builder.include("Metadata", new Metadata());
- }
-
- private class Metadata implements HasDisplayData {
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- for (Map.Entry<String, Object> entry : getMetadata().entrySet()) {
- DisplayData.Type type = DisplayData.inferType(entry.getValue());
- if (type != null) {
- builder.add(DisplayData.item(entry.getKey(), type, entry.getValue()));
- } else {
- String base64 = BaseEncoding.base64().encode((byte[]) entry.getValue());
- String repr = base64.length() <= METADATA_BYTES_MAX_LENGTH
- ? base64 : base64.substring(0, METADATA_BYTES_MAX_LENGTH) + "...";
- builder.add(DisplayData.item(entry.getKey(), repr));
- }
- }
- }
- }
-
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
+ .addIfNotDefault(
+ DisplayData.item("numShards", getNumShards()).withLabel("Maximum Output Shards"), 0)
+ .addIfNotNull(
+ DisplayData.item("tempDirectory", getTempDirectory())
+ .withLabel("Directory for temporary files"));
}
}
+ /**
+ * This class is used as the default return value of {@link AvroIO#write}
+ *
+ * <p>All methods in this class delegate to the appropriate method of {@link AvroIO.TypedWrite}.
+ * This class exists for backwards compatibility, and will be removed in Beam 3.0.
+ */
+ public static class Write<T> extends PTransform<PCollection<T>, PDone> {
+ @VisibleForTesting TypedWrite<T, ?, T> inner;
+
+ Write(TypedWrite<T, ?, T> inner) {
+ this.inner = inner;
+ }
+
+ /** See {@link TypedWrite#to(String)}. */
+ public Write<T> to(String outputPrefix) {
+ return new Write<>(
+ inner
+ .to(FileBasedSink.convertToFileResourceIfPossible(outputPrefix))
+ .withFormatFunction(SerializableFunctions.<T>identity()));
+ }
+
+ /** See {@link TypedWrite#to(ResourceId)} . */
+ @Experimental(Kind.FILESYSTEM)
+ public Write<T> to(ResourceId outputPrefix) {
+ return new Write<T>(
+ inner.to(outputPrefix).withFormatFunction(SerializableFunctions.<T>identity()));
+ }
+
+ /** See {@link TypedWrite#to(ValueProvider)}. */
+ public Write<T> to(ValueProvider<String> outputPrefix) {
+ return new Write<>(
+ inner.to(outputPrefix).withFormatFunction(SerializableFunctions.<T>identity()));
+ }
+
+ /** See {@link TypedWrite#to(ResourceId)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write<T> toResource(ValueProvider<ResourceId> outputPrefix) {
+ return new Write<>(
+ inner.toResource(outputPrefix).withFormatFunction(SerializableFunctions.<T>identity()));
+ }
+
+ /** See {@link TypedWrite#to(FilenamePolicy)}. */
+ public Write<T> to(FilenamePolicy filenamePolicy) {
+ return new Write<>(
+ inner.to(filenamePolicy).withFormatFunction(SerializableFunctions.<T>identity()));
+ }
+
+ /** See {@link TypedWrite#to(DynamicAvroDestinations)}. */
+ public Write<T> to(DynamicAvroDestinations<T, ?, T> dynamicDestinations) {
+ return new Write<>(inner.to(dynamicDestinations).withFormatFunction(null));
+ }
+
+ /** See {@link TypedWrite#withSchema}. */
+ public Write<T> withSchema(Schema schema) {
+ return new Write<>(inner.withSchema(schema));
+ }
+ /** See {@link TypedWrite#withTempDirectory(ValueProvider)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write<T> withTempDirectory(ValueProvider<ResourceId> tempDirectory) {
+ return new Write<>(inner.withTempDirectory(tempDirectory));
+ }
+
+ /** See {@link TypedWrite#withTempDirectory(ResourceId)}. */
+ public Write<T> withTempDirectory(ResourceId tempDirectory) {
+ return new Write<>(inner.withTempDirectory(tempDirectory));
+ }
+
+ /** See {@link TypedWrite#withShardNameTemplate}. */
+ public Write<T> withShardNameTemplate(String shardTemplate) {
+ return new Write<>(inner.withShardNameTemplate(shardTemplate));
+ }
+
+ /** See {@link TypedWrite#withSuffix}. */
+ public Write<T> withSuffix(String filenameSuffix) {
+ return new Write<>(inner.withSuffix(filenameSuffix));
+ }
+
+ /** See {@link TypedWrite#withNumShards}. */
+ public Write<T> withNumShards(int numShards) {
+ return new Write<>(inner.withNumShards(numShards));
+ }
+
+ /** See {@link TypedWrite#withoutSharding}. */
+ public Write<T> withoutSharding() {
+ return new Write<>(inner.withoutSharding());
+ }
+
+ /** See {@link TypedWrite#withWindowedWrites}. */
+ public Write<T> withWindowedWrites() {
+ return new Write<>(inner.withWindowedWrites());
+ }
+
+ /** See {@link TypedWrite#withCodec}. */
+ public Write<T> withCodec(CodecFactory codec) {
+ return new Write<>(inner.withCodec(codec));
+ }
+
+ /** Specify that output filenames are wanted.
+ *
+ * <p>The nested {@link TypedWrite}transform always has access to output filenames, however
+ * due to backwards-compatibility concerns, {@link Write} cannot return them. This method
+ * simply returns the inner {@link TypedWrite} transform which has {@link WriteFilesResult} as
+ * its output type, allowing access to output files.
+ *
+ * <p>The supplied {@code DestinationT} type must be: the same as that supplied in {@link
+ * #to(DynamicAvroDestinations)} if that method was used, or {@code Void} otherwise.
+ */
+ public <DestinationT> TypedWrite<T, DestinationT, T> withOutputFilenames() {
+ return (TypedWrite) inner;
+ }
+
+ /** See {@link TypedWrite#withMetadata} . */
+ public Write<T> withMetadata(Map<String, Object> metadata) {
+ return new Write<>(inner.withMetadata(metadata));
+ }
+
+ @Override
+ public PDone expand(PCollection<T> input) {
+ inner.expand(input);
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ inner.populateDisplayData(builder);
+ }
+ }
+
+ /**
+ * Returns a {@link DynamicAvroDestinations} that always returns the same {@link FilenamePolicy},
+ * schema, metadata, and codec.
+ */
+ public static <UserT, OutputT> DynamicAvroDestinations<UserT, Void, OutputT> constantDestinations(
+ FilenamePolicy filenamePolicy,
+ Schema schema,
+ Map<String, Object> metadata,
+ CodecFactory codec,
+ SerializableFunction<UserT, OutputT> formatFunction) {
+ return new ConstantAvroDestination<>(filenamePolicy, schema, metadata, codec, formatFunction);
+ }
/////////////////////////////////////////////////////////////////////////////
/** Disallow construction of utility class. */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
index 6c36266..888db85 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
@@ -17,91 +17,90 @@
*/
package org.apache.beam.sdk.io;
-import com.google.common.collect.ImmutableMap;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.util.MimeTypes;
/** A {@link FileBasedSink} for Avro files. */
-class AvroSink<T> extends FileBasedSink<T> {
- private final AvroCoder<T> coder;
- private final SerializableAvroCodecFactory codec;
- private final ImmutableMap<String, Object> metadata;
+class AvroSink<UserT, DestinationT, OutputT> extends FileBasedSink<UserT, DestinationT, OutputT> {
+ private final DynamicAvroDestinations<UserT, DestinationT, OutputT> dynamicDestinations;
+ private final boolean genericRecords;
AvroSink(
ValueProvider<ResourceId> outputPrefix,
- FilenamePolicy filenamePolicy,
- AvroCoder<T> coder,
- SerializableAvroCodecFactory codec,
- ImmutableMap<String, Object> metadata) {
+ DynamicAvroDestinations<UserT, DestinationT, OutputT> dynamicDestinations,
+ boolean genericRecords) {
// Avro handle compression internally using the codec.
- super(outputPrefix, filenamePolicy, CompressionType.UNCOMPRESSED);
- this.coder = coder;
- this.codec = codec;
- this.metadata = metadata;
+ super(outputPrefix, dynamicDestinations, Compression.UNCOMPRESSED);
+ this.dynamicDestinations = dynamicDestinations;
+ this.genericRecords = genericRecords;
}
@Override
- public WriteOperation<T> createWriteOperation() {
- return new AvroWriteOperation<>(this, coder, codec, metadata);
+ public DynamicAvroDestinations<UserT, DestinationT, OutputT> getDynamicDestinations() {
+ return (DynamicAvroDestinations<UserT, DestinationT, OutputT>) super.getDynamicDestinations();
+ }
+
+ @Override
+ public WriteOperation<DestinationT, OutputT> createWriteOperation() {
+ return new AvroWriteOperation<>(this, genericRecords);
}
/** A {@link WriteOperation WriteOperation} for Avro files. */
- private static class AvroWriteOperation<T> extends WriteOperation<T> {
- private final AvroCoder<T> coder;
- private final SerializableAvroCodecFactory codec;
- private final ImmutableMap<String, Object> metadata;
+ private static class AvroWriteOperation<DestinationT, OutputT>
+ extends WriteOperation<DestinationT, OutputT> {
+ private final DynamicAvroDestinations<?, DestinationT, ?> dynamicDestinations;
+ private final boolean genericRecords;
- private AvroWriteOperation(AvroSink<T> sink,
- AvroCoder<T> coder,
- SerializableAvroCodecFactory codec,
- ImmutableMap<String, Object> metadata) {
+ private AvroWriteOperation(AvroSink<?, DestinationT, OutputT> sink, boolean genericRecords) {
super(sink);
- this.coder = coder;
- this.codec = codec;
- this.metadata = metadata;
+ this.dynamicDestinations = sink.getDynamicDestinations();
+ this.genericRecords = genericRecords;
}
@Override
- public Writer<T> createWriter() throws Exception {
- return new AvroWriter<>(this, coder, codec, metadata);
+ public Writer<DestinationT, OutputT> createWriter() throws Exception {
+ return new AvroWriter<>(this, dynamicDestinations, genericRecords);
}
}
/** A {@link Writer Writer} for Avro files. */
- private static class AvroWriter<T> extends Writer<T> {
- private final AvroCoder<T> coder;
- private DataFileWriter<T> dataFileWriter;
- private SerializableAvroCodecFactory codec;
- private final ImmutableMap<String, Object> metadata;
+ private static class AvroWriter<DestinationT, OutputT> extends Writer<DestinationT, OutputT> {
+ private DataFileWriter<OutputT> dataFileWriter;
+ private final DynamicAvroDestinations<?, DestinationT, ?> dynamicDestinations;
+ private final boolean genericRecords;
- public AvroWriter(WriteOperation<T> writeOperation,
- AvroCoder<T> coder,
- SerializableAvroCodecFactory codec,
- ImmutableMap<String, Object> metadata) {
+ public AvroWriter(
+ WriteOperation<DestinationT, OutputT> writeOperation,
+ DynamicAvroDestinations<?, DestinationT, ?> dynamicDestinations,
+ boolean genericRecords) {
super(writeOperation, MimeTypes.BINARY);
- this.coder = coder;
- this.codec = codec;
- this.metadata = metadata;
+ this.dynamicDestinations = dynamicDestinations;
+ this.genericRecords = genericRecords;
}
@SuppressWarnings("deprecation") // uses internal test functionality.
@Override
protected void prepareWrite(WritableByteChannel channel) throws Exception {
- DatumWriter<T> datumWriter = coder.getType().equals(GenericRecord.class)
- ? new GenericDatumWriter<T>(coder.getSchema())
- : new ReflectDatumWriter<T>(coder.getSchema());
+ DestinationT destination = getDestination();
+ CodecFactory codec = dynamicDestinations.getCodec(destination);
+ Schema schema = dynamicDestinations.getSchema(destination);
+ Map<String, Object> metadata = dynamicDestinations.getMetadata(destination);
- dataFileWriter = new DataFileWriter<>(datumWriter).setCodec(codec.getCodec());
+ DatumWriter<OutputT> datumWriter =
+ genericRecords
+ ? new GenericDatumWriter<OutputT>(schema)
+ : new ReflectDatumWriter<OutputT>(schema);
+ dataFileWriter = new DataFileWriter<>(datumWriter).setCodec(codec);
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
Object v = entry.getValue();
if (v instanceof String) {
@@ -116,11 +115,11 @@
+ v.getClass().getSimpleName());
}
}
- dataFileWriter.create(coder.getSchema(), Channels.newOutputStream(channel));
+ dataFileWriter.create(schema, Channels.newOutputStream(channel));
}
@Override
- public void write(T value) throws Exception {
+ public void write(OutputT value) throws Exception {
dataFileWriter.append(value);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
index 37bbe46..a2610df 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
@@ -17,17 +17,21 @@
*/
package org.apache.beam.sdk.io;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
import java.io.ObjectStreamException;
import java.io.PushbackInputStream;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
@@ -37,6 +41,7 @@
import java.util.WeakHashMap;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;
+import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
@@ -51,9 +56,13 @@
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.PCollection;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
@@ -127,117 +136,184 @@
// The default sync interval is 64k.
private static final long DEFAULT_MIN_BUNDLE_SIZE = 2 * DataFileConstants.DEFAULT_SYNC_INTERVAL;
- // The JSON schema used to encode records.
- private final String readSchemaString;
+ // Use cases of AvroSource are:
+ // 1) AvroSource<GenericRecord> Reading GenericRecord records with a specified schema.
+ // 2) AvroSource<Foo> Reading records of a generated Avro class Foo.
+ // 3) AvroSource<T> Reading GenericRecord records with an unspecified schema
+ // and converting them to type T.
+ // | Case 1 | Case 2 | Case 3 |
+ // type | GenericRecord | Foo | GenericRecord |
+ // readerSchemaString | non-null | non-null | null |
+ // parseFn | null | null | non-null |
+ // outputCoder | null | null | non-null |
+ private static class Mode<T> implements Serializable {
+ private final Class<?> type;
- // The JSON schema that was used to write the source Avro file (may differ from the schema we will
- // use to read from it).
- private final String fileSchemaString;
+ // The JSON schema used to decode records.
+ @Nullable
+ private String readerSchemaString;
- // The type of the records contained in the file.
- private final Class<T> type;
+ @Nullable
+ private final SerializableFunction<GenericRecord, T> parseFn;
- // The following metadata fields are not user-configurable. They are extracted from the object
- // container file header upon subsource creation.
+ @Nullable
+ private final Coder<T> outputCoder;
- // The codec used to encode the blocks in the Avro file. String value drawn from those in
- // https://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/file/CodecFactory.html
- private final String codec;
+ private Mode(
+ Class<?> type,
+ @Nullable String readerSchemaString,
+ @Nullable SerializableFunction<GenericRecord, T> parseFn,
+ @Nullable Coder<T> outputCoder) {
+ this.type = type;
+ this.readerSchemaString = internSchemaString(readerSchemaString);
+ this.parseFn = parseFn;
+ this.outputCoder = outputCoder;
+ }
- // The object container file's 16-byte sync marker.
- private final byte[] syncMarker;
+ private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
+ is.defaultReadObject();
+ readerSchemaString = internSchemaString(readerSchemaString);
+ }
- // Default output coder, lazily initialized.
- private transient AvroCoder<T> coder = null;
+ private Coder<T> getOutputCoder() {
+ if (parseFn == null) {
+ return AvroCoder.of((Class<T>) type, internOrParseSchemaString(readerSchemaString));
+ } else {
+ return outputCoder;
+ }
+ }
- // Schema of the file, lazily initialized.
- private transient Schema fileSchema;
+ private void validate() {
+ if (parseFn == null) {
+ checkArgument(
+ readerSchemaString != null,
+ "schema must be specified using withSchema() when not using a parse fn");
+ }
+ }
+ }
- // Schema used to encode records, lazily initialized.
- private transient Schema readSchema;
+ private static Mode<GenericRecord> readGenericRecordsWithSchema(String schema) {
+ return new Mode<>(GenericRecord.class, schema, null, null);
+ }
+ private static <T> Mode<T> readGeneratedClasses(Class<T> clazz) {
+ return new Mode<>(clazz, ReflectData.get().getSchema(clazz).toString(), null, null);
+ }
+ private static <T> Mode<T> parseGenericRecords(
+ SerializableFunction<GenericRecord, T> parseFn, Coder<T> outputCoder) {
+ return new Mode<>(GenericRecord.class, null, parseFn, outputCoder);
+ }
+
+ private final Mode<T> mode;
/**
- * Creates an {@link AvroSource} that reads from the given file name or pattern ("glob"). The
- * returned source can be further configured by calling {@link #withSchema} to return a type other
- * than {@link GenericRecord}.
+ * Reads from the given file name or pattern ("glob"). The returned source needs to be further
+ * configured by calling {@link #withSchema} to return a type other than {@link GenericRecord}.
*/
+ public static AvroSource<GenericRecord> from(ValueProvider<String> fileNameOrPattern) {
+ return new AvroSource<>(
+ fileNameOrPattern,
+ EmptyMatchTreatment.DISALLOW,
+ DEFAULT_MIN_BUNDLE_SIZE,
+ readGenericRecordsWithSchema(null /* will need to be specified in withSchema */));
+ }
+
+ /** Like {@link #from(ValueProvider)}. */
public static AvroSource<GenericRecord> from(String fileNameOrPattern) {
- return new AvroSource<>(
- fileNameOrPattern, DEFAULT_MIN_BUNDLE_SIZE, null, GenericRecord.class, null, null);
+ return from(ValueProvider.StaticValueProvider.of(fileNameOrPattern));
}
- /**
- * Returns an {@link AvroSource} that's like this one but reads files containing records that
- * conform to the given schema.
- *
- * <p>Does not modify this object.
- */
+ public AvroSource<T> withEmptyMatchTreatment(EmptyMatchTreatment emptyMatchTreatment) {
+ return new AvroSource<T>(
+ getFileOrPatternSpecProvider(),
+ emptyMatchTreatment,
+ getMinBundleSize(),
+ mode);
+ }
+
+ /** Reads files containing records that conform to the given schema. */
public AvroSource<GenericRecord> withSchema(String schema) {
+ checkArgument(schema != null, "schema can not be null");
return new AvroSource<>(
- getFileOrPatternSpec(), getMinBundleSize(), schema, GenericRecord.class, codec, syncMarker);
+ getFileOrPatternSpecProvider(),
+ getEmptyMatchTreatment(),
+ getMinBundleSize(),
+ readGenericRecordsWithSchema(schema));
}
- /**
- * Returns an {@link AvroSource} that's like this one but reads files containing records that
- * conform to the given schema.
- *
- * <p>Does not modify this object.
- */
+ /** Like {@link #withSchema(String)}. */
public AvroSource<GenericRecord> withSchema(Schema schema) {
- return new AvroSource<>(getFileOrPatternSpec(), getMinBundleSize(), schema.toString(),
- GenericRecord.class, codec, syncMarker);
+ checkArgument(schema != null, "schema can not be null");
+ return withSchema(schema.toString());
}
- /**
- * Returns an {@link AvroSource} that's like this one but reads files containing records of the
- * type of the given class.
- *
- * <p>Does not modify this object.
- */
+ /** Reads files containing records of the given class. */
public <X> AvroSource<X> withSchema(Class<X> clazz) {
- return new AvroSource<X>(getFileOrPatternSpec(), getMinBundleSize(),
- ReflectData.get().getSchema(clazz).toString(), clazz, codec, syncMarker);
+ checkArgument(clazz != null, "clazz can not be null");
+ return new AvroSource<>(
+ getFileOrPatternSpecProvider(),
+ getEmptyMatchTreatment(),
+ getMinBundleSize(),
+ readGeneratedClasses(clazz));
}
/**
- * Returns an {@link AvroSource} that's like this one but uses the supplied minimum bundle size.
- * Refer to {@link OffsetBasedSource} for a description of {@code minBundleSize} and its use.
- *
- * <p>Does not modify this object.
+ * Reads {@link GenericRecord} of unspecified schema and maps them to instances of a custom type
+ * using the given {@code parseFn} and encoded using the given coder.
+ */
+ public <X> AvroSource<X> withParseFn(
+ SerializableFunction<GenericRecord, X> parseFn, Coder<X> coder) {
+ checkArgument(parseFn != null, "parseFn can not be null");
+ checkArgument(coder != null, "coder can not be null");
+ return new AvroSource<>(
+ getFileOrPatternSpecProvider(),
+ getEmptyMatchTreatment(),
+ getMinBundleSize(),
+ parseGenericRecords(parseFn, coder));
+ }
+
+ /**
+ * Sets the minimum bundle size. Refer to {@link OffsetBasedSource} for a description of {@code
+ * minBundleSize} and its use.
*/
public AvroSource<T> withMinBundleSize(long minBundleSize) {
return new AvroSource<>(
- getFileOrPatternSpec(), minBundleSize, readSchemaString, type, codec, syncMarker);
+ getFileOrPatternSpecProvider(), getEmptyMatchTreatment(), minBundleSize, mode);
}
- private AvroSource(String fileNameOrPattern, long minBundleSize, String schema, Class<T> type,
- String codec, byte[] syncMarker) {
- super(fileNameOrPattern, minBundleSize);
- this.readSchemaString = internSchemaString(schema);
- this.codec = codec;
- this.syncMarker = syncMarker;
- this.type = type;
- this.fileSchemaString = null;
+ /** Constructor for FILEPATTERN mode. */
+ private AvroSource(
+ ValueProvider<String> fileNameOrPattern,
+ EmptyMatchTreatment emptyMatchTreatment,
+ long minBundleSize,
+ Mode<T> mode) {
+ super(fileNameOrPattern, emptyMatchTreatment, minBundleSize);
+ this.mode = mode;
}
- private AvroSource(Metadata metadata, long minBundleSize, long startOffset, long endOffset,
- String schema, Class<T> type, String codec, byte[] syncMarker, String fileSchema) {
+ /** Constructor for SINGLE_FILE_OR_SUBRANGE mode. */
+ private AvroSource(
+ Metadata metadata,
+ long minBundleSize,
+ long startOffset,
+ long endOffset,
+ Mode<T> mode) {
super(metadata, minBundleSize, startOffset, endOffset);
- this.readSchemaString = internSchemaString(schema);
- this.codec = codec;
- this.syncMarker = syncMarker;
- this.type = type;
- this.fileSchemaString = internSchemaString(fileSchema);
+ this.mode = mode;
}
@Override
public void validate() {
- // AvroSource objects do not need to be configured with more than a file pattern. Overridden to
- // make this explicit.
super.validate();
+ mode.validate();
}
- @Deprecated // Added to let DataflowRunner migrate off of this; to be deleted.
+ /**
+ * Used by the Dataflow worker. Do not introduce new usages. Do not delete without confirming that
+ * Dataflow ValidatesRunner tests pass.
+ *
+ * @deprecated Used by Dataflow worker
+ */
+ @Deprecated
public BlockBasedSource<T> createForSubrangeOfFile(String fileName, long start, long end)
throws IOException {
return createForSubrangeOfFile(FileSystems.matchSingleFileSpec(fileName), start, end);
@@ -245,37 +321,7 @@
@Override
public BlockBasedSource<T> createForSubrangeOfFile(Metadata fileMetadata, long start, long end) {
- byte[] syncMarker = this.syncMarker;
- String codec = this.codec;
- String readSchemaString = this.readSchemaString;
- String fileSchemaString = this.fileSchemaString;
- // codec and syncMarker are initially null when the source is created, as they differ
- // across input files and must be read from the file. Here, when we are creating a source
- // for a subrange of a file, we can initialize these values. When the resulting AvroSource
- // is further split, they do not need to be read again.
- if (codec == null || syncMarker == null || fileSchemaString == null) {
- AvroMetadata metadata;
- try {
- metadata = readMetadataFromFile(fileMetadata.resourceId());
- } catch (IOException e) {
- throw new RuntimeException("Error reading metadata from file " + fileMetadata, e);
- }
- codec = metadata.getCodec();
- syncMarker = metadata.getSyncMarker();
- fileSchemaString = metadata.getSchemaString();
- // If the source was created with a null schema, use the schema that we read from the file's
- // metadata.
- if (readSchemaString == null) {
- readSchemaString = metadata.getSchemaString();
- }
- }
- // Note that if the fileSchemaString is equivalent to the readSchemaString, "intern"ing
- // the string will occur within the constructor and return the same reference as the
- // readSchemaString. This allows for Java to have an efficient serialization since it
- // will only encode the schema once while just storing pointers to the encoded version
- // within this source.
- return new AvroSource<>(fileMetadata, getMinBundleSize(), start, end, readSchemaString, type,
- codec, syncMarker, fileSchemaString);
+ return new AvroSource<>(fileMetadata, getMinBundleSize(), start, end, mode);
}
@Override
@@ -284,64 +330,27 @@
}
@Override
- public AvroCoder<T> getDefaultOutputCoder() {
- if (coder == null) {
- coder = AvroCoder.of(type, internOrParseSchemaString(readSchemaString));
- }
- return coder;
- }
-
- public String getSchema() {
- return readSchemaString;
+ public Coder<T> getOutputCoder() {
+ return mode.getOutputCoder();
}
@VisibleForTesting
- Schema getReadSchema() {
- if (readSchemaString == null) {
- return null;
- }
-
- // If the schema has not been parsed, parse it.
- if (readSchema == null) {
- readSchema = internOrParseSchemaString(readSchemaString);
- }
- return readSchema;
+ @Nullable
+ String getReaderSchemaString() {
+ return mode.readerSchemaString;
}
- @VisibleForTesting
- Schema getFileSchema() {
- if (fileSchemaString == null) {
- return null;
- }
-
- // If the schema has not been parsed, parse it.
- if (fileSchema == null) {
- fileSchema = internOrParseSchemaString(fileSchemaString);
- }
- return fileSchema;
- }
-
- private byte[] getSyncMarker() {
- return syncMarker;
- }
-
- private String getCodec() {
- return codec;
- }
-
- /**
- * Avro file metadata.
- */
+ /** Avro file metadata. */
@VisibleForTesting
static class AvroMetadata {
- private byte[] syncMarker;
- private String codec;
- private String schemaString;
+ private final byte[] syncMarker;
+ private final String codec;
+ private final String schemaString;
AvroMetadata(byte[] syncMarker, String codec, String schemaString) {
this.syncMarker = checkNotNull(syncMarker, "syncMarker");
this.codec = checkNotNull(codec, "codec");
- this.schemaString = checkNotNull(schemaString, "schemaString");
+ this.schemaString = internSchemaString(checkNotNull(schemaString, "schemaString"));
}
/**
@@ -432,18 +441,6 @@
return new AvroMetadata(syncMarker, codec, schemaString);
}
- private DatumReader<T> createDatumReader() {
- Schema readSchema = getReadSchema();
- Schema fileSchema = getFileSchema();
- checkNotNull(readSchema, "No read schema has been initialized for source %s", this);
- checkNotNull(fileSchema, "No file schema has been initialized for source %s", this);
- if (type == GenericRecord.class) {
- return new GenericDatumReader<>(fileSchema, readSchema);
- } else {
- return new ReflectDatumReader<>(fileSchema, readSchema);
- }
- }
-
// A logical reference cache used to store schemas and schema strings to allow us to
// "intern" values and reduce the number of copies of equivalent objects.
private static final Map<String, Schema> schemaLogicalReferenceCache = new WeakHashMap<>();
@@ -479,23 +476,10 @@
switch (getMode()) {
case SINGLE_FILE_OR_SUBRANGE:
return new AvroSource<>(
- getSingleFileMetadata(),
- getMinBundleSize(),
- getStartOffset(),
- getEndOffset(),
- readSchemaString,
- type,
- codec,
- syncMarker,
- fileSchemaString);
+ getSingleFileMetadata(), getMinBundleSize(), getStartOffset(), getEndOffset(), mode);
case FILEPATTERN:
return new AvroSource<>(
- getFileOrPatternSpec(),
- getMinBundleSize(),
- readSchemaString,
- type,
- codec,
- syncMarker);
+ getFileOrPatternSpecProvider(), getEmptyMatchTreatment(), getMinBundleSize(), mode);
default:
throw new InvalidObjectException(
String.format("Unknown mode %s for AvroSource %s", getMode(), this));
@@ -509,6 +493,8 @@
*/
@Experimental(Experimental.Kind.SOURCE_SINK)
static class AvroBlock<T> extends Block<T> {
+ private final Mode<T> mode;
+
// The number of records in the block.
private final long numRecords;
@@ -519,7 +505,7 @@
private long currentRecordIndex = 0;
// A DatumReader to read records from the block.
- private final DatumReader<T> reader;
+ private final DatumReader<?> reader;
// A BinaryDecoder used by the reader to decode records.
private final BinaryDecoder decoder;
@@ -559,11 +545,25 @@
}
}
- AvroBlock(byte[] data, long numRecords, AvroSource<T> source) throws IOException {
+ AvroBlock(
+ byte[] data,
+ long numRecords,
+ Mode<T> mode,
+ String writerSchemaString,
+ String codec)
+ throws IOException {
+ this.mode = mode;
this.numRecords = numRecords;
- this.reader = source.createDatumReader();
- this.decoder =
- DecoderFactory.get().binaryDecoder(decodeAsInputStream(data, source.getCodec()), null);
+ checkNotNull(writerSchemaString, "writerSchemaString");
+ Schema writerSchema = internOrParseSchemaString(writerSchemaString);
+ Schema readerSchema =
+ internOrParseSchemaString(
+ MoreObjects.firstNonNull(mode.readerSchemaString, writerSchemaString));
+ this.reader =
+ (mode.type == GenericRecord.class)
+ ? new GenericDatumReader<T>(writerSchema, readerSchema)
+ : new ReflectDatumReader<T>(writerSchema, readerSchema);
+ this.decoder = DecoderFactory.get().binaryDecoder(decodeAsInputStream(data, codec), null);
}
@Override
@@ -576,7 +576,9 @@
if (currentRecordIndex >= numRecords) {
return false;
}
- currentRecord = reader.read(null, decoder);
+ Object record = reader.read(null, decoder);
+ currentRecord =
+ (mode.parseFn == null) ? ((T) record) : mode.parseFn.apply((GenericRecord) record);
currentRecordIndex++;
return true;
}
@@ -599,6 +601,8 @@
*/
@Experimental(Experimental.Kind.SOURCE_SINK)
public static class AvroReader<T> extends BlockBasedReader<T> {
+ private AvroMetadata metadata;
+
// The current block.
private AvroBlock<T> currentBlock;
@@ -672,10 +676,16 @@
"Only able to read %s/%s bytes in the block before EOF reached.",
bytesRead,
blockSize);
- currentBlock = new AvroBlock<>(data, numRecords, getCurrentSource());
+ currentBlock =
+ new AvroBlock<>(
+ data,
+ numRecords,
+ getCurrentSource().mode,
+ metadata.getSchemaString(),
+ metadata.getCodec());
// Read the end of this block, which MUST be a sync marker for correctness.
- byte[] syncMarker = getCurrentSource().getSyncMarker();
+ byte[] syncMarker = metadata.getSyncMarker();
byte[] readSyncMarker = new byte[syncMarker.length];
long syncMarkerOffset = startOfNextBlock + headerSize + blockSize;
bytesRead = IOUtils.readFully(stream, readSyncMarker);
@@ -746,7 +756,7 @@
private PushbackInputStream createStream(ReadableByteChannel channel) {
return new PushbackInputStream(
Channels.newInputStream(channel),
- getCurrentSource().getSyncMarker().length);
+ metadata.getSyncMarker().length);
}
// Postcondition: the stream is positioned at the beginning of the first block after the start
@@ -754,8 +764,15 @@
// currentBlockSizeBytes will be set to 0 indicating that the previous block was empty.
@Override
protected void startReading(ReadableByteChannel channel) throws IOException {
+ try {
+ metadata = readMetadataFromFile(getCurrentSource().getSingleFileMetadata().resourceId());
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Error reading metadata from file " + getCurrentSource().getSingleFileMetadata(), e);
+ }
+
long startOffset = getCurrentSource().getStartOffset();
- byte[] syncMarker = getCurrentSource().getSyncMarker();
+ byte[] syncMarker = metadata.getSyncMarker();
long syncMarkerLength = syncMarker.length;
if (startOffset != 0) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroUtils.java
new file mode 100644
index 0000000..65c5bf1
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import java.io.Serializable;
+import org.apache.avro.Schema;
+
+/** Helpers for working with Avro. */
+class AvroUtils {
+ /** Helper to get around the fact that {@link Schema} itself is not serializable. */
+ public static Supplier<Schema> serializableSchemaSupplier(String jsonSchema) {
+ return Suppliers.memoize(
+ Suppliers.compose(new JsonToSchema(), Suppliers.ofInstance(jsonSchema)));
+ }
+
+ private static class JsonToSchema implements Function<String, Schema>, Serializable {
+ @Override
+ public Schema apply(String input) {
+ return new Schema.Parser().parse(input);
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BlockBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BlockBasedSource.java
index cf6671e..ec4f4ad 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BlockBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BlockBasedSource.java
@@ -21,8 +21,10 @@
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
/**
@@ -62,13 +64,36 @@
public abstract class BlockBasedSource<T> extends FileBasedSource<T> {
/**
* Creates a {@code BlockBasedSource} based on a file name or pattern. Subclasses must call this
- * constructor when creating a {@code BlockBasedSource} for a file pattern. See
- * {@link FileBasedSource} for more information.
+ * constructor when creating a {@code BlockBasedSource} for a file pattern. See {@link
+ * FileBasedSource} for more information.
+ */
+ public BlockBasedSource(
+ String fileOrPatternSpec, EmptyMatchTreatment emptyMatchTreatment, long minBundleSize) {
+ this(StaticValueProvider.of(fileOrPatternSpec), emptyMatchTreatment, minBundleSize);
+ }
+
+ /**
+ * Like {@link #BlockBasedSource(String, EmptyMatchTreatment, long)} but with a default {@link
+ * EmptyMatchTreatment} of {@link EmptyMatchTreatment#DISALLOW}.
*/
public BlockBasedSource(String fileOrPatternSpec, long minBundleSize) {
- super(StaticValueProvider.of(fileOrPatternSpec), minBundleSize);
+ this(StaticValueProvider.of(fileOrPatternSpec), minBundleSize);
}
+ /** Like {@link #BlockBasedSource(String, long)}. */
+ public BlockBasedSource(ValueProvider<String> fileOrPatternSpec, long minBundleSize) {
+ this(fileOrPatternSpec, EmptyMatchTreatment.DISALLOW, minBundleSize);
+ }
+
+ /** Like {@link #BlockBasedSource(String, EmptyMatchTreatment, long)}. */
+ public BlockBasedSource(
+ ValueProvider<String> fileOrPatternSpec,
+ EmptyMatchTreatment emptyMatchTreatment,
+ long minBundleSize) {
+ super(fileOrPatternSpec, emptyMatchTreatment, minBundleSize);
+ }
+
+
/**
* Creates a {@code BlockBasedSource} for a single file. Subclasses must call this constructor
* when implementing {@link BlockBasedSource#createForSubrangeOfFile}. See documentation in
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
index c882447..80a03eb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
@@ -114,12 +114,8 @@
}
}));
}
- return read.apply("StripIds", ParDo.of(new ValueWithRecordId.StripIdsDoFn<T>()));
- }
-
- @Override
- protected Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
+ return read.apply("StripIds", ParDo.of(new ValueWithRecordId.StripIdsDoFn<T>()))
+ .setCoder(source.getOutputCoder());
}
@Override
@@ -211,8 +207,8 @@
}
@Override
- public Coder<ValueWithRecordId<T>> getDefaultOutputCoder() {
- return ValueWithRecordId.ValueWithRecordIdCoder.of(getSource().getDefaultOutputCoder());
+ public Coder<ValueWithRecordId<T>> getOutputCoder() {
+ return ValueWithRecordId.ValueWithRecordIdCoder.of(getSource().getOutputCoder());
}
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
index 6ab8dec..ae55d80 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
@@ -20,28 +20,17 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-import com.google.common.io.ByteStreams;
-import com.google.common.primitives.Ints;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.PushbackInputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.NoSuchElementException;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
import javax.annotation.concurrent.GuardedBy;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.joda.time.Instant;
/**
@@ -54,21 +43,20 @@
* FileBasedSource<T> mySource = ...;
* PCollection<T> collection = p.apply(Read.from(CompressedSource
* .from(mySource)
- * .withDecompression(CompressedSource.CompressionMode.GZIP)));
+ * .withCompression(Compression.GZIP)));
* } </pre>
*
- * <p>Supported compression algorithms are {@link CompressionMode#GZIP},
- * {@link CompressionMode#BZIP2}, {@link CompressionMode#ZIP} and {@link CompressionMode#DEFLATE}.
- * User-defined compression types are supported by implementing
+ * <p>Supported compression algorithms are {@link Compression#GZIP},
+ * {@link Compression#BZIP2}, {@link Compression#ZIP} and {@link Compression#DEFLATE}.
+ * User-defined compression types are supported by implementing a
* {@link DecompressingChannelFactory}.
*
* <p>By default, the compression algorithm is selected from those supported in
- * {@link CompressionMode} based on the file name provided to the source, namely
- * {@code ".bz2"} indicates {@link CompressionMode#BZIP2}, {@code ".gz"} indicates
- * {@link CompressionMode#GZIP}, {@code ".zip"} indicates {@link CompressionMode#ZIP} and
- * {@code ".deflate"} indicates {@link CompressionMode#DEFLATE}. If the file name does not match
- * any of the supported
- * algorithms, it is assumed to be uncompressed data.
+ * {@link Compression} based on the file name provided to the source, namely
+ * {@code ".bz2"} indicates {@link Compression#BZIP2}, {@code ".gz"} indicates
+ * {@link Compression#GZIP}, {@code ".zip"} indicates {@link Compression#ZIP} and
+ * {@code ".deflate"} indicates {@link Compression#DEFLATE}. If the file name does not match
+ * any of the supported algorithms, it is assumed to be uncompressed data.
*
* @param <T> The type to read from the compressed file.
*/
@@ -85,203 +73,75 @@
throws IOException;
}
- /**
- * Factory interface for creating channels that decompress the content of an underlying channel,
- * based on both the channel and the file name.
- */
- private interface FileNameBasedDecompressingChannelFactory
- extends DecompressingChannelFactory {
- /**
- * Given a channel, create a channel that decompresses the content read from the channel.
- */
- ReadableByteChannel createDecompressingChannel(String fileName, ReadableByteChannel channel)
- throws IOException;
-
- /**
- * Given a file name, returns true if the file name matches any supported compression
- * scheme.
- */
- boolean isCompressed(String fileName);
- }
-
- /**
- * Default compression types supported by the {@code CompressedSource}.
- */
+ /** @deprecated Use {@link Compression} instead */
+ @Deprecated
public enum CompressionMode implements DecompressingChannelFactory {
- /**
- * Reads a byte channel assuming it is compressed with gzip.
- */
- GZIP {
- @Override
- public boolean matches(String fileName) {
- return fileName.toLowerCase().endsWith(".gz");
- }
+ /** @see Compression#UNCOMPRESSED */
+ UNCOMPRESSED(Compression.UNCOMPRESSED),
- @Override
- public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
- throws IOException {
- // Determine if the input stream is gzipped. The input stream returned from the
- // GCS connector may already be decompressed; GCS does this based on the
- // content-encoding property.
- PushbackInputStream stream = new PushbackInputStream(Channels.newInputStream(channel), 2);
- byte[] headerBytes = new byte[2];
- int bytesRead = ByteStreams.read(
- stream /* source */, headerBytes /* dest */, 0 /* offset */, 2 /* len */);
- stream.unread(headerBytes, 0, bytesRead);
- if (bytesRead >= 2) {
- byte zero = 0x00;
- int header = Ints.fromBytes(zero, zero, headerBytes[1], headerBytes[0]);
- if (header == GZIPInputStream.GZIP_MAGIC) {
- return Channels.newChannel(new GzipCompressorInputStream(stream, true));
- }
- }
- return Channels.newChannel(stream);
- }
- },
+ /** @see Compression#AUTO */
+ AUTO(Compression.AUTO),
- /**
- * Reads a byte channel assuming it is compressed with bzip2.
- */
- BZIP2 {
- @Override
- public boolean matches(String fileName) {
- return fileName.toLowerCase().endsWith(".bz2");
- }
+ /** @see Compression#GZIP */
+ GZIP(Compression.GZIP),
- @Override
- public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
- throws IOException {
- return Channels.newChannel(
- new BZip2CompressorInputStream(Channels.newInputStream(channel)));
- }
- },
+ /** @see Compression#BZIP2 */
+ BZIP2(Compression.BZIP2),
- /**
- * Reads a byte channel assuming it is compressed with zip.
- * If the zip file contains multiple entries, files in the zip are concatenated all together.
- */
- ZIP {
- @Override
- public boolean matches(String fileName) {
- return fileName.toLowerCase().endsWith(".zip");
- }
+ /** @see Compression#ZIP */
+ ZIP(Compression.ZIP),
- public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
- throws IOException {
- FullZipInputStream zip = new FullZipInputStream(Channels.newInputStream(channel));
- return Channels.newChannel(zip);
- }
- },
+ /** @see Compression#DEFLATE */
+ DEFLATE(Compression.DEFLATE);
- /**
- * Reads a byte channel assuming it is compressed with deflate.
- */
- DEFLATE {
- @Override
- public boolean matches(String fileName) {
- return fileName.toLowerCase().endsWith(".deflate");
- }
+ private Compression canonical;
- public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
- throws IOException {
- return Channels.newChannel(
- new DeflateCompressorInputStream(Channels.newInputStream(channel)));
- }
- };
-
- /**
- * Extend of {@link ZipInputStream} to automatically read all entries in the zip.
- */
- private static class FullZipInputStream extends InputStream {
-
- private ZipInputStream zipInputStream;
- private ZipEntry currentEntry;
-
- public FullZipInputStream(InputStream is) throws IOException {
- super();
- zipInputStream = new ZipInputStream(is);
- currentEntry = zipInputStream.getNextEntry();
- }
-
- @Override
- public int read() throws IOException {
- int result = zipInputStream.read();
- while (result == -1) {
- currentEntry = zipInputStream.getNextEntry();
- if (currentEntry == null) {
- return -1;
- } else {
- result = zipInputStream.read();
- }
- }
- return result;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- int result = zipInputStream.read(b, off, len);
- while (result == -1) {
- currentEntry = zipInputStream.getNextEntry();
- if (currentEntry == null) {
- return -1;
- } else {
- result = zipInputStream.read(b, off, len);
- }
- }
- return result;
- }
-
+ CompressionMode(Compression canonical) {
+ this.canonical = canonical;
}
/**
* Returns {@code true} if the given file name implies that the contents are compressed
* according to the compression embodied by this factory.
*/
- public abstract boolean matches(String fileName);
-
- @Override
- public abstract ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
- throws IOException;
- }
-
- /**
- * Reads a byte channel detecting compression according to the file name. If the filename
- * is not any other known {@link CompressionMode}, it is presumed to be uncompressed.
- */
- private static class DecompressAccordingToFilename
- implements FileNameBasedDecompressingChannelFactory {
-
- @Override
- public ReadableByteChannel createDecompressingChannel(
- String fileName, ReadableByteChannel channel) throws IOException {
- for (CompressionMode type : CompressionMode.values()) {
- if (type.matches(fileName)) {
- return type.createDecompressingChannel(channel);
- }
- }
- // Uncompressed
- return channel;
+ public boolean matches(String fileName) {
+ return canonical.matches(fileName);
}
@Override
- public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel) {
- throw new UnsupportedOperationException(
- String.format("%s does not support createDecompressingChannel(%s) but only"
- + " createDecompressingChannel(%s,%s)",
- getClass().getSimpleName(),
- String.class.getSimpleName(),
- ReadableByteChannel.class.getSimpleName(),
- ReadableByteChannel.class.getSimpleName()));
+ public ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
+ throws IOException {
+ return canonical.readDecompressed(channel);
}
- @Override
- public boolean isCompressed(String fileName) {
- for (CompressionMode type : CompressionMode.values()) {
- if (type.matches(fileName)) {
- return true;
- }
+ /** Returns whether the file's extension matches of one of the known compression formats. */
+ public static boolean isCompressed(String filename) {
+ return Compression.AUTO.isCompressed(filename);
+ }
+
+ static DecompressingChannelFactory fromCanonical(Compression compression) {
+ switch (compression) {
+ case AUTO:
+ return AUTO;
+
+ case UNCOMPRESSED:
+ return UNCOMPRESSED;
+
+ case GZIP:
+ return GZIP;
+
+ case BZIP2:
+ return BZIP2;
+
+ case ZIP:
+ return ZIP;
+
+ case DEFLATE:
+ return DEFLATE;
+
+ default:
+ throw new IllegalArgumentException("Unsupported compression type: " + compression);
}
- return false;
}
}
@@ -294,7 +154,7 @@
* configured via {@link CompressedSource#withDecompression}.
*/
public static <T> CompressedSource<T> from(FileBasedSource<T> sourceDelegate) {
- return new CompressedSource<>(sourceDelegate, new DecompressAccordingToFilename());
+ return new CompressedSource<>(sourceDelegate, CompressionMode.AUTO);
}
/**
@@ -305,6 +165,11 @@
return new CompressedSource<>(this.sourceDelegate, channelFactory);
}
+ /** Like {@link #withDecompression} but takes a canonical {@link Compression}. */
+ public CompressedSource<T> withCompression(Compression compression) {
+ return withDecompression(CompressionMode.fromCanonical(compression));
+ }
+
/**
* Creates a {@code CompressedSource} from a delegate file based source and a decompressing
* channel factory.
@@ -365,12 +230,19 @@
* from the requested file name that the file is not compressed.
*/
@Override
- protected final boolean isSplittable() throws Exception {
- if (channelFactory instanceof FileNameBasedDecompressingChannelFactory) {
- FileNameBasedDecompressingChannelFactory fileNameBasedChannelFactory =
- (FileNameBasedDecompressingChannelFactory) channelFactory;
- return !fileNameBasedChannelFactory.isCompressed(getFileOrPatternSpec())
- && sourceDelegate.isSplittable();
+ protected final boolean isSplittable() {
+ try {
+ if (!sourceDelegate.isSplittable()) {
+ return false;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ if (channelFactory == CompressionMode.UNCOMPRESSED) {
+ return true;
+ }
+ if (channelFactory == CompressionMode.AUTO) {
+ return !Compression.AUTO.isCompressed(getFileOrPatternSpec());
}
return false;
}
@@ -385,12 +257,8 @@
*/
@Override
protected final FileBasedReader<T> createSingleFileReader(PipelineOptions options) {
- if (channelFactory instanceof FileNameBasedDecompressingChannelFactory) {
- FileNameBasedDecompressingChannelFactory fileNameBasedChannelFactory =
- (FileNameBasedDecompressingChannelFactory) channelFactory;
- if (!fileNameBasedChannelFactory.isCompressed(getFileOrPatternSpec())) {
- return sourceDelegate.createSingleFileReader(options);
- }
+ if (isSplittable()) {
+ return sourceDelegate.createSingleFileReader(options);
}
return new CompressedReader<T>(
this, sourceDelegate.createSingleFileReader(options));
@@ -416,11 +284,11 @@
}
/**
- * Returns the delegate source's default output coder.
+ * Returns the delegate source's output coder.
*/
@Override
- public final Coder<T> getDefaultOutputCoder() {
- return sourceDelegate.getDefaultOutputCoder();
+ public final Coder<T> getOutputCoder() {
+ return sourceDelegate.getOutputCoder();
}
public final DecompressingChannelFactory getChannelFactory() {
@@ -435,19 +303,19 @@
public static class CompressedReader<T> extends FileBasedReader<T> {
private final FileBasedReader<T> readerDelegate;
- private final CompressedSource<T> source;
private final Object progressLock = new Object();
@GuardedBy("progressLock")
private int numRecordsRead;
@GuardedBy("progressLock")
private CountingChannel channel;
+ private DecompressingChannelFactory channelFactory;
/**
* Create a {@code CompressedReader} from a {@code CompressedSource} and delegate reader.
*/
public CompressedReader(CompressedSource<T> source, FileBasedReader<T> readerDelegate) {
super(source);
- this.source = source;
+ this.channelFactory = source.getChannelFactory();
this.readerDelegate = readerDelegate;
}
@@ -537,14 +405,12 @@
channel = this.channel;
}
- if (source.getChannelFactory() instanceof FileNameBasedDecompressingChannelFactory) {
- FileNameBasedDecompressingChannelFactory channelFactory =
- (FileNameBasedDecompressingChannelFactory) source.getChannelFactory();
- readerDelegate.startReading(channelFactory.createDecompressingChannel(
- getCurrentSource().getFileOrPatternSpec(),
- channel));
+ if (channelFactory == CompressionMode.AUTO) {
+ readerDelegate.startReading(
+ Compression.detect(getCurrentSource().getFileOrPatternSpec())
+ .readDecompressed(channel));
} else {
- readerDelegate.startReading(source.getChannelFactory().createDecompressingChannel(
+ readerDelegate.startReading(channelFactory.createDecompressingChannel(
channel));
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java
new file mode 100644
index 0000000..bb40ed4
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.primitives.Ints;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
+import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+/** Various compression types for reading/writing files. */
+public enum Compression {
+ /**
+ * When reading a file, automatically determine the compression type based on filename extension.
+ * Not applicable when writing files.
+ */
+ AUTO("") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) {
+ throw new UnsupportedOperationException(
+ "Must resolve compression into a concrete value before calling readDecompressed()");
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) {
+ throw new UnsupportedOperationException("AUTO is applicable only to reading files");
+ }
+ },
+
+ /** No compression. */
+ UNCOMPRESSED("") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) {
+ return channel;
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) {
+ return channel;
+ }
+ },
+
+ /** GZip compression. */
+ GZIP(".gz", ".gz") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) throws IOException {
+ // Determine if the input stream is gzipped. The input stream returned from the
+ // GCS connector may already be decompressed; GCS does this based on the
+ // content-encoding property.
+ PushbackInputStream stream = new PushbackInputStream(Channels.newInputStream(channel), 2);
+ byte[] headerBytes = new byte[2];
+ int bytesRead =
+ ByteStreams.read(
+ stream /* source */, headerBytes /* dest */, 0 /* offset */, 2 /* len */);
+ stream.unread(headerBytes, 0, bytesRead);
+ if (bytesRead >= 2) {
+ byte zero = 0x00;
+ int header = Ints.fromBytes(zero, zero, headerBytes[1], headerBytes[0]);
+ if (header == GZIPInputStream.GZIP_MAGIC) {
+ return Channels.newChannel(new GzipCompressorInputStream(stream, true));
+ }
+ }
+ return Channels.newChannel(stream);
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) throws IOException {
+ return Channels.newChannel(new GZIPOutputStream(Channels.newOutputStream(channel), true));
+ }
+ },
+
+ /** BZip compression. */
+ BZIP2(".bz2", ".bz2") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) throws IOException {
+ return Channels.newChannel(
+ new BZip2CompressorInputStream(Channels.newInputStream(channel), true));
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) throws IOException {
+ return Channels.newChannel(
+ new BZip2CompressorOutputStream(Channels.newOutputStream(channel)));
+ }
+ },
+
+ /** Zip compression. */
+ ZIP(".zip", ".zip") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) throws IOException {
+ FullZipInputStream zip = new FullZipInputStream(Channels.newInputStream(channel));
+ return Channels.newChannel(zip);
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) throws IOException {
+ throw new UnsupportedOperationException("Writing ZIP files is currently unsupported");
+ }
+ },
+
+ /** Deflate compression. */
+ DEFLATE(".deflate", ".deflate", ".zlib") {
+ @Override
+ public ReadableByteChannel readDecompressed(ReadableByteChannel channel) throws IOException {
+ return Channels.newChannel(
+ new DeflateCompressorInputStream(Channels.newInputStream(channel)));
+ }
+
+ @Override
+ public WritableByteChannel writeCompressed(WritableByteChannel channel) throws IOException {
+ return Channels.newChannel(
+ new DeflateCompressorOutputStream(Channels.newOutputStream(channel)));
+ }
+ };
+
+ private final String suggestedSuffix;
+ private final List<String> detectedSuffixes;
+
+ Compression(String suggestedSuffix, String... detectedSuffixes) {
+ this.suggestedSuffix = suggestedSuffix;
+ this.detectedSuffixes = Arrays.asList(detectedSuffixes);
+ }
+
+ public String getSuggestedSuffix() {
+ return suggestedSuffix;
+ }
+
+ public boolean matches(String filename) {
+ for (String suffix : detectedSuffixes) {
+ if (filename.toLowerCase().endsWith(suffix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean isCompressed(String filename) {
+ Compression compression = this;
+ if (compression == AUTO) {
+ compression = detect(filename);
+ }
+ return compression != UNCOMPRESSED;
+ }
+
+ public static Compression detect(String filename) {
+ for (Compression value : values()) {
+ if (value.matches(filename)) {
+ return value;
+ }
+ }
+ return UNCOMPRESSED;
+ }
+
+ public abstract ReadableByteChannel readDecompressed(ReadableByteChannel channel)
+ throws IOException;
+
+ public abstract WritableByteChannel writeCompressed(WritableByteChannel channel)
+ throws IOException;
+
+ /** Concatenates all {@link ZipInputStream}s contained within the zip file. */
+ private static class FullZipInputStream extends InputStream {
+ private ZipInputStream zipInputStream;
+ private ZipEntry currentEntry;
+
+ public FullZipInputStream(InputStream is) throws IOException {
+ super();
+ zipInputStream = new ZipInputStream(is);
+ currentEntry = zipInputStream.getNextEntry();
+ }
+
+ @Override
+ public int read() throws IOException {
+ int result = zipInputStream.read();
+ while (result == -1) {
+ currentEntry = zipInputStream.getNextEntry();
+ if (currentEntry == null) {
+ return -1;
+ } else {
+ result = zipInputStream.read();
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int result = zipInputStream.read(b, off, len);
+ while (result == -1) {
+ currentEntry = zipInputStream.getNextEntry();
+ if (currentEntry == null) {
+ return -1;
+ } else {
+ result = zipInputStream.read(b, off, len);
+ }
+ }
+ return result;
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java
new file mode 100644
index 0000000..b006e26
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.io.BaseEncoding;
+import java.io.Serializable;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
+
+/** Always returns a constant {@link FilenamePolicy}, {@link Schema}, metadata, and codec. */
+class ConstantAvroDestination<UserT, OutputT>
+ extends DynamicAvroDestinations<UserT, Void, OutputT> {
+ private static class SchemaFunction implements Serializable, Function<String, Schema> {
+ @Nullable
+ @Override
+ public Schema apply(@Nullable String input) {
+ return new Schema.Parser().parse(input);
+ }
+ }
+
+ // This should be a multiple of 4 to not get a partial encoded byte.
+ private static final int METADATA_BYTES_MAX_LENGTH = 40;
+ private final FilenamePolicy filenamePolicy;
+ private final Supplier<Schema> schema;
+ private final Map<String, Object> metadata;
+ private final SerializableAvroCodecFactory codec;
+ private final SerializableFunction<UserT, OutputT> formatFunction;
+
+ private class Metadata implements HasDisplayData {
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ for (Map.Entry<String, Object> entry : metadata.entrySet()) {
+ DisplayData.Type type = DisplayData.inferType(entry.getValue());
+ if (type != null) {
+ builder.add(DisplayData.item(entry.getKey(), type, entry.getValue()));
+ } else {
+ String base64 = BaseEncoding.base64().encode((byte[]) entry.getValue());
+ String repr =
+ base64.length() <= METADATA_BYTES_MAX_LENGTH
+ ? base64
+ : base64.substring(0, METADATA_BYTES_MAX_LENGTH) + "...";
+ builder.add(DisplayData.item(entry.getKey(), repr));
+ }
+ }
+ }
+ }
+
+ public ConstantAvroDestination(
+ FilenamePolicy filenamePolicy,
+ Schema schema,
+ Map<String, Object> metadata,
+ CodecFactory codec,
+ SerializableFunction<UserT, OutputT> formatFunction) {
+ this.filenamePolicy = filenamePolicy;
+ this.schema = Suppliers.compose(new SchemaFunction(), Suppliers.ofInstance(schema.toString()));
+ this.metadata = metadata;
+ this.codec = new SerializableAvroCodecFactory(codec);
+ this.formatFunction = formatFunction;
+ }
+
+ @Override
+ public OutputT formatRecord(UserT record) {
+ return formatFunction.apply(record);
+ }
+
+ @Override
+ public Void getDestination(UserT element) {
+ return (Void) null;
+ }
+
+ @Override
+ public Void getDefaultDestination() {
+ return (Void) null;
+ }
+
+ @Override
+ public FilenamePolicy getFilenamePolicy(Void destination) {
+ return filenamePolicy;
+ }
+
+ @Override
+ public Schema getSchema(Void destination) {
+ return schema.get();
+ }
+
+ @Override
+ public Map<String, Object> getMetadata(Void destination) {
+ return metadata;
+ }
+
+ @Override
+ public CodecFactory getCodec(Void destination) {
+ return codec.getCodec();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ filenamePolicy.populateDisplayData(builder);
+ builder.add(DisplayData.item("schema", schema.get().toString()).withLabel("Record Schema"));
+ builder.addIfNotDefault(
+ DisplayData.item("codec", codec.getCodec().toString()).withLabel("Avro Compression Codec"),
+ AvroIO.TypedWrite.DEFAULT_SERIALIZABLE_CODEC.toString());
+ builder.include("Metadata", new Metadata());
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
index 81082e5..d56bb5a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
@@ -24,6 +24,7 @@
import java.io.IOException;
import java.util.List;
import java.util.NoSuchElementException;
+import java.util.Objects;
import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.DefaultCoder;
@@ -136,6 +137,16 @@
public Instant apply(Long input) {
return Instant.now();
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof NowTimestampFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
/**
@@ -177,9 +188,24 @@
}
@Override
- public Coder<Long> getDefaultOutputCoder() {
+ public Coder<Long> getOutputCoder() {
return VarLongCoder.of();
}
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof BoundedCountingSource)) {
+ return false;
+ }
+ BoundedCountingSource that = (BoundedCountingSource) other;
+ return this.getStartOffset() == that.getStartOffset()
+ && this.getEndOffset() == that.getEndOffset();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(this.getStartOffset(), (int) this.getEndOffset());
+ }
}
/**
@@ -335,12 +361,25 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<Long> getDefaultOutputCoder() {
+ public Coder<Long> getOutputCoder() {
return VarLongCoder.of();
}
+
+ public boolean equals(Object other) {
+ if (!(other instanceof UnboundedCountingSource)) {
+ return false;
+ }
+ UnboundedCountingSource that = (UnboundedCountingSource) other;
+ return this.start == that.start
+ && this.stride == that.stride
+ && this.elementsPerPeriod == that.elementsPerPeriod
+ && Objects.equals(this.period, that.period)
+ && Objects.equals(this.timestampFn, that.timestampFn);
+ }
+
+ public int hashCode() {
+ return Objects.hash(start, stride, elementsPerPeriod, period, timestampFn);
+ }
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
index 07bc2db..2f22e82 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
@@ -20,150 +20,376 @@
import static com.google.common.base.MoreObjects.firstNonNull;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider;
-import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
-import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
/**
- * A default {@link FilenamePolicy} for unwindowed files. This policy is constructed using three
- * parameters that together define the output name of a sharded file, in conjunction with the number
- * of shards and index of the particular file, using {@link #constructName}.
+ * A default {@link FilenamePolicy} for windowed and unwindowed files. This policy is constructed
+ * using three parameters that together define the output name of a sharded file, in conjunction
+ * with the number of shards, index of the particular file, current window and pane information,
+ * using {@link #constructName}.
*
- * <p>Most users of unwindowed files will use this {@link DefaultFilenamePolicy}. For more advanced
- * uses in generating different files for each window and other sharding controls, see the
- * {@code WriteOneFilePerWindow} example pipeline.
+ * <p>Most users will use this {@link DefaultFilenamePolicy}. For more advanced uses in generating
+ * different files for each window and other sharding controls, see the {@code
+ * WriteOneFilePerWindow} example pipeline.
*/
public final class DefaultFilenamePolicy extends FilenamePolicy {
- /** The default sharding name template used in {@link #constructUsingStandardParameters}. */
- public static final String DEFAULT_SHARD_TEMPLATE = ShardNameTemplate.INDEX_OF_MAX;
+ /** The default sharding name template. */
+ public static final String DEFAULT_UNWINDOWED_SHARD_TEMPLATE = ShardNameTemplate.INDEX_OF_MAX;
- // Pattern that matches shard placeholders within a shard template.
- private static final Pattern SHARD_FORMAT_RE = Pattern.compile("(S+|N+)");
+ /**
+ * The default windowed sharding name template used when writing windowed files. This is used as
+ * default in cases when user did not specify shard template to be used and there is a need to
+ * write windowed files. In cases when user does specify shard template to be used then provided
+ * template will be used for both windowed and non-windowed file names.
+ */
+ private static final String DEFAULT_WINDOWED_SHARD_TEMPLATE =
+ "W-P" + DEFAULT_UNWINDOWED_SHARD_TEMPLATE;
+ /*
+ * pattern for both windowed and non-windowed file names.
+ */
+ private static final Pattern SHARD_FORMAT_RE = Pattern.compile("(S+|N+|W|P)");
+
+ /**
+ * Encapsulates constructor parameters to {@link DefaultFilenamePolicy}.
+ *
+ * <p>This is used as the {@code DestinationT} argument to allow {@link DefaultFilenamePolicy}
+ * objects to be dynamically generated.
+ */
+ public static class Params implements Serializable {
+ private final ValueProvider<ResourceId> baseFilename;
+ private final String shardTemplate;
+ private final boolean explicitTemplate;
+ private final String suffix;
+
+ /**
+ * Construct a default Params object. The shard template will be set to the default {@link
+ * #DEFAULT_UNWINDOWED_SHARD_TEMPLATE} value.
+ */
+ public Params() {
+ this.baseFilename = null;
+ this.shardTemplate = DEFAULT_UNWINDOWED_SHARD_TEMPLATE;
+ this.suffix = "";
+ this.explicitTemplate = false;
+ }
+
+ private Params(
+ ValueProvider<ResourceId> baseFilename,
+ String shardTemplate,
+ String suffix,
+ boolean explicitTemplate) {
+ this.baseFilename = baseFilename;
+ this.shardTemplate = shardTemplate;
+ this.suffix = suffix;
+ this.explicitTemplate = explicitTemplate;
+ }
+
+ /**
+ * Specify that writes are windowed. This affects the default shard template, changing it to
+ * {@link #DEFAULT_WINDOWED_SHARD_TEMPLATE}.
+ */
+ public Params withWindowedWrites() {
+ String template = this.shardTemplate;
+ if (!explicitTemplate) {
+ template = DEFAULT_WINDOWED_SHARD_TEMPLATE;
+ }
+ return new Params(baseFilename, template, suffix, explicitTemplate);
+ }
+
+ /** Sets the base filename. */
+ public Params withBaseFilename(ResourceId baseFilename) {
+ return withBaseFilename(StaticValueProvider.of(baseFilename));
+ }
+
+ /** Like {@link #withBaseFilename(ResourceId)}, but takes in a {@link ValueProvider}. */
+ public Params withBaseFilename(ValueProvider<ResourceId> baseFilename) {
+ return new Params(baseFilename, shardTemplate, suffix, explicitTemplate);
+ }
+
+ /** Sets the shard template. */
+ public Params withShardTemplate(String shardTemplate) {
+ return new Params(baseFilename, shardTemplate, suffix, true);
+ }
+
+ /** Sets the suffix. */
+ public Params withSuffix(String suffix) {
+ return new Params(baseFilename, shardTemplate, suffix, explicitTemplate);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(baseFilename.get(), shardTemplate, suffix);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Params)) {
+ return false;
+ }
+ Params other = (Params) o;
+ return baseFilename.get().equals(other.baseFilename.get())
+ && shardTemplate.equals(other.shardTemplate)
+ && suffix.equals(other.suffix);
+ }
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("baseFilename", baseFilename)
+ .add("shardTemplate", shardTemplate)
+ .add("suffix", suffix)
+ .toString();
+ }
+ }
+
+ /** A Coder for {@link Params}. */
+ public static class ParamsCoder extends AtomicCoder<Params> {
+ private static final ParamsCoder INSTANCE = new ParamsCoder();
+ private Coder<String> stringCoder = StringUtf8Coder.of();
+
+ public static ParamsCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(Params value, OutputStream outStream) throws IOException {
+ if (value == null) {
+ throw new CoderException("cannot encode a null value");
+ }
+ stringCoder.encode(value.baseFilename.get().toString(), outStream);
+ stringCoder.encode(value.shardTemplate, outStream);
+ stringCoder.encode(value.suffix, outStream);
+ }
+
+ @Override
+ public Params decode(InputStream inStream) throws IOException {
+ ResourceId prefix =
+ FileBasedSink.convertToFileResourceIfPossible(stringCoder.decode(inStream));
+ String shardTemplate = stringCoder.decode(inStream);
+ String suffix = stringCoder.decode(inStream);
+ return new Params()
+ .withBaseFilename(prefix)
+ .withShardTemplate(shardTemplate)
+ .withSuffix(suffix);
+ }
+ }
+
+ private final Params params;
/**
* Constructs a new {@link DefaultFilenamePolicy}.
*
* @see DefaultFilenamePolicy for more information on the arguments to this function.
*/
@VisibleForTesting
- DefaultFilenamePolicy(ValueProvider<String> prefix, String shardTemplate, String suffix) {
- this.prefix = prefix;
- this.shardTemplate = shardTemplate;
- this.suffix = suffix;
+ DefaultFilenamePolicy(Params params) {
+ this.params = params;
}
/**
- * A helper function to construct a {@link DefaultFilenamePolicy} using the standard filename
- * parameters, namely a provided {@link ResourceId} for the output prefix, and possibly-null
- * shard name template and suffix.
+ * Construct a {@link DefaultFilenamePolicy}.
*
- * <p>Any filename component of the provided resource will be used as the filename prefix.
+ * <p>This is a shortcut for:
*
- * <p>If provided, the shard name template will be used; otherwise {@link #DEFAULT_SHARD_TEMPLATE}
- * will be used.
+ * <pre>{@code
+ * DefaultFilenamePolicy.fromParams(new Params()
+ * .withBaseFilename(baseFilename)
+ * .withShardTemplate(shardTemplate)
+ * .withSuffix(filenameSuffix)
+ * .withWindowedWrites())
+ * }</pre>
*
- * <p>If provided, the suffix will be used; otherwise the files will have an empty suffix.
+ * <p>Where the respective {@code with} methods are invoked only if the value is non-null or true.
*/
- public static DefaultFilenamePolicy constructUsingStandardParameters(
- ValueProvider<ResourceId> outputPrefix,
+ public static DefaultFilenamePolicy fromStandardParameters(
+ ValueProvider<ResourceId> baseFilename,
@Nullable String shardTemplate,
- @Nullable String filenameSuffix) {
- return new DefaultFilenamePolicy(
- NestedValueProvider.of(outputPrefix, new ExtractFilename()),
- firstNonNull(shardTemplate, DEFAULT_SHARD_TEMPLATE),
- firstNonNull(filenameSuffix, ""));
+ @Nullable String filenameSuffix,
+ boolean windowedWrites) {
+ Params params = new Params().withBaseFilename(baseFilename);
+ if (shardTemplate != null) {
+ params = params.withShardTemplate(shardTemplate);
+ }
+ if (filenameSuffix != null) {
+ params = params.withSuffix(filenameSuffix);
+ }
+ if (windowedWrites) {
+ params = params.withWindowedWrites();
+ }
+ return fromParams(params);
}
- private final ValueProvider<String> prefix;
- private final String shardTemplate;
- private final String suffix;
+ /** Construct a {@link DefaultFilenamePolicy} from a {@link Params} object. */
+ public static DefaultFilenamePolicy fromParams(Params params) {
+ return new DefaultFilenamePolicy(params);
+ }
/**
* Constructs a fully qualified name from components.
*
- * <p>The name is built from a prefix, shard template (with shard numbers
- * applied), and a suffix. All components are required, but may be empty
- * strings.
+ * <p>The name is built from a base filename, shard template (with shard numbers applied), and a
+ * suffix. All components are required, but may be empty strings.
*
- * <p>Within a shard template, repeating sequences of the letters "S" or "N"
- * are replaced with the shard number, or number of shards respectively. The
- * numbers are formatted with leading zeros to match the length of the
- * repeated sequence of letters.
+ * <p>Within a shard template, repeating sequences of the letters "S" or "N" are replaced with the
+ * shard number, or number of shards respectively. "P" is replaced with by stringification of
+ * current pane. "W" is replaced by stringification of current window.
*
- * <p>For example, if prefix = "output", shardTemplate = "-SSS-of-NNN", and
- * suffix = ".txt", with shardNum = 1 and numShards = 100, the following is
- * produced: "output-001-of-100.txt".
+ * <p>The numbers are formatted with leading zeros to match the length of the repeated sequence of
+ * letters.
+ *
+ * <p>For example, if baseFilename = "path/to/output", shardTemplate = "-SSS-of-NNN", and suffix =
+ * ".txt", with shardNum = 1 and numShards = 100, the following is produced:
+ * "path/to/output-001-of-100.txt".
*/
- public static String constructName(
- String prefix, String shardTemplate, String suffix, int shardNum, int numShards) {
+ static ResourceId constructName(
+ ResourceId baseFilename,
+ String shardTemplate,
+ String suffix,
+ int shardNum,
+ int numShards,
+ String paneStr,
+ String windowStr) {
+ String prefix = extractFilename(baseFilename);
// Matcher API works with StringBuffer, rather than StringBuilder.
StringBuffer sb = new StringBuffer();
sb.append(prefix);
Matcher m = SHARD_FORMAT_RE.matcher(shardTemplate);
while (m.find()) {
- boolean isShardNum = (m.group(1).charAt(0) == 'S');
+ boolean isCurrentShardNum = (m.group(1).charAt(0) == 'S');
+ boolean isNumberOfShards = (m.group(1).charAt(0) == 'N');
+ boolean isPane = (m.group(1).charAt(0) == 'P') && paneStr != null;
+ boolean isWindow = (m.group(1).charAt(0) == 'W') && windowStr != null;
char[] zeros = new char[m.end() - m.start()];
Arrays.fill(zeros, '0');
DecimalFormat df = new DecimalFormat(String.valueOf(zeros));
- String formatted = df.format(isShardNum ? shardNum : numShards);
- m.appendReplacement(sb, formatted);
+ if (isCurrentShardNum) {
+ String formatted = df.format(shardNum);
+ m.appendReplacement(sb, formatted);
+ } else if (isNumberOfShards) {
+ String formatted = df.format(numShards);
+ m.appendReplacement(sb, formatted);
+ } else if (isPane) {
+ m.appendReplacement(sb, paneStr);
+ } else if (isWindow) {
+ m.appendReplacement(sb, windowStr);
+ }
}
m.appendTail(sb);
sb.append(suffix);
- return sb.toString();
+ return baseFilename
+ .getCurrentDirectory()
+ .resolve(sb.toString(), StandardResolveOptions.RESOLVE_FILE);
}
@Override
@Nullable
- public ResourceId unwindowedFilename(ResourceId outputDirectory, Context context,
- String extension) {
- String filename =
- constructName(
- prefix.get(), shardTemplate, suffix, context.getShardNumber(), context.getNumShards())
- + extension;
- return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ public ResourceId unwindowedFilename(
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
+ return constructName(
+ params.baseFilename.get(),
+ params.shardTemplate,
+ params.suffix + outputFileHints.getSuggestedFilenameSuffix(),
+ shardNumber,
+ numShards,
+ null,
+ null);
}
@Override
- public ResourceId windowedFilename(ResourceId outputDirectory,
- WindowedContext c, String extension) {
- throw new UnsupportedOperationException("There is no default policy for windowed file"
- + " output. Please provide an explicit FilenamePolicy to generate filenames.");
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ String paneStr = paneInfoToString(paneInfo);
+ String windowStr = windowToString(window);
+ return constructName(
+ params.baseFilename.get(),
+ params.shardTemplate,
+ params.suffix + outputFileHints.getSuggestedFilenameSuffix(),
+ shardNumber,
+ numShards,
+ paneStr,
+ windowStr);
+ }
+
+ /*
+ * Since not all windows have toString() that is nice or is compatible to be a part of file name.
+ */
+ private String windowToString(BoundedWindow window) {
+ if (window instanceof GlobalWindow) {
+ return "GlobalWindow";
+ }
+ if (window instanceof IntervalWindow) {
+ IntervalWindow iw = (IntervalWindow) window;
+ return String.format("%s-%s", iw.start().toString(), iw.end().toString());
+ }
+ return window.toString();
+ }
+
+ private String paneInfoToString(PaneInfo paneInfo) {
+ String paneString = String.format("pane-%d", paneInfo.getIndex());
+ if (paneInfo.getTiming() == Timing.LATE) {
+ paneString = String.format("%s-late", paneString);
+ }
+ if (paneInfo.isLast()) {
+ paneString = String.format("%s-last", paneString);
+ }
+ return paneString;
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
- String filenamePattern;
- if (prefix.isAccessible()) {
- filenamePattern = String.format("%s%s%s", prefix.get(), shardTemplate, suffix);
- } else {
- filenamePattern = String.format("%s%s%s", prefix, shardTemplate, suffix);
- }
+ String displayBaseFilename =
+ params.baseFilename.isAccessible()
+ ? params.baseFilename.get().toString()
+ : ("(" + params.baseFilename + ")");
builder.add(
- DisplayData.item("filenamePattern", filenamePattern)
- .withLabel("Filename Pattern"));
+ DisplayData.item(
+ "filenamePattern",
+ String.format("%s%s%s", displayBaseFilename, params.shardTemplate, params.suffix))
+ .withLabel("Filename pattern"));
+ builder.add(
+ DisplayData.item("filePrefix", params.baseFilename).withLabel("Output File Prefix"));
+ builder.add(
+ DisplayData.item("shardNameTemplate", params.shardTemplate)
+ .withLabel("Output Shard Name Template"));
+ builder.add(DisplayData.item("fileSuffix", params.suffix).withLabel("Output file Suffix"));
}
- private static class ExtractFilename implements SerializableFunction<ResourceId, String> {
- @Override
- public String apply(ResourceId input) {
- if (input.isDirectory()) {
- return "";
- } else {
- return firstNonNull(input.getFilename(), "");
- }
+ private static String extractFilename(ResourceId input) {
+ if (input.isDirectory()) {
+ return "";
+ } else {
+ return firstNonNull(input.getFilename(), "");
}
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java
new file mode 100644
index 0000000..f4e8ee6
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations;
+
+/**
+ * A specialization of {@link DynamicDestinations} for {@link AvroIO}. In addition to dynamic file
+ * destinations, this allows specifying other AVRO properties (schema, metadata, codec) per
+ * destination.
+ */
+public abstract class DynamicAvroDestinations<UserT, DestinationT, OutputT>
+ extends DynamicDestinations<UserT, DestinationT, OutputT> {
+ /** Return an AVRO schema for a given destination. */
+ public abstract Schema getSchema(DestinationT destination);
+
+ /** Return AVRO file metadata for a given destination. */
+ public Map<String, Object> getMetadata(DestinationT destination) {
+ return ImmutableMap.<String, Object>of();
+ }
+
+ /** Return an AVRO codec for a given destination. */
+ public CodecFactory getCodec(DestinationT destination) {
+ return AvroIO.TypedWrite.DEFAULT_CODEC;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
new file mode 100644
index 0000000..b087bc5
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.ParamsCoder;
+import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+/** Some helper classes that derive from {@link FileBasedSink.DynamicDestinations}. */
+public class DynamicFileDestinations {
+ /** Always returns a constant {@link FilenamePolicy}. */
+ private static class ConstantFilenamePolicy<UserT, OutputT>
+ extends DynamicDestinations<UserT, Void, OutputT> {
+ private final FilenamePolicy filenamePolicy;
+ private final SerializableFunction<UserT, OutputT> formatFunction;
+
+ public ConstantFilenamePolicy(
+ FilenamePolicy filenamePolicy, SerializableFunction<UserT, OutputT> formatFunction) {
+ this.filenamePolicy = filenamePolicy;
+ this.formatFunction = formatFunction;
+ }
+
+ @Override
+ public OutputT formatRecord(UserT record) {
+ return formatFunction.apply(record);
+ }
+
+ @Override
+ public Void getDestination(UserT element) {
+ return (Void) null;
+ }
+
+ @Override
+ public Coder<Void> getDestinationCoder() {
+ return null;
+ }
+
+ @Override
+ public Void getDefaultDestination() {
+ return (Void) null;
+ }
+
+ @Override
+ public FilenamePolicy getFilenamePolicy(Void destination) {
+ return filenamePolicy;
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ checkState(filenamePolicy != null);
+ filenamePolicy.populateDisplayData(builder);
+ }
+ }
+
+ /**
+ * A base class for a {@link DynamicDestinations} object that returns differently-configured
+ * instances of {@link DefaultFilenamePolicy}.
+ */
+ private static class DefaultPolicyDestinations<UserT, OutputT>
+ extends DynamicDestinations<UserT, Params, OutputT> {
+ private final SerializableFunction<UserT, Params> destinationFunction;
+ private final Params emptyDestination;
+ private final SerializableFunction<UserT, OutputT> formatFunction;
+
+ public DefaultPolicyDestinations(
+ SerializableFunction<UserT, Params> destinationFunction,
+ Params emptyDestination,
+ SerializableFunction<UserT, OutputT> formatFunction) {
+ this.destinationFunction = destinationFunction;
+ this.emptyDestination = emptyDestination;
+ this.formatFunction = formatFunction;
+ }
+
+ @Override
+ public OutputT formatRecord(UserT record) {
+ return formatFunction.apply(record);
+ }
+
+ @Override
+ public Params getDestination(UserT element) {
+ return destinationFunction.apply(element);
+ }
+
+ @Override
+ public Params getDefaultDestination() {
+ return emptyDestination;
+ }
+
+ @Nullable
+ @Override
+ public Coder<Params> getDestinationCoder() {
+ return ParamsCoder.of();
+ }
+
+ @Override
+ public FilenamePolicy getFilenamePolicy(DefaultFilenamePolicy.Params params) {
+ return DefaultFilenamePolicy.fromParams(params);
+ }
+ }
+
+ /** Returns a {@link DynamicDestinations} that always returns the same {@link FilenamePolicy}. */
+ public static <UserT, OutputT> DynamicDestinations<UserT, Void, OutputT> constant(
+ FilenamePolicy filenamePolicy, SerializableFunction<UserT, OutputT> formatFunction) {
+ return new ConstantFilenamePolicy<>(filenamePolicy, formatFunction);
+ }
+
+ /**
+ * A specialization of {@link #constant(FilenamePolicy, SerializableFunction)} for the case where
+ * UserT and OutputT are the same type and the format function is the identity.
+ */
+ public static <UserT> DynamicDestinations<UserT, Void, UserT> constant(
+ FilenamePolicy filenamePolicy) {
+ return new ConstantFilenamePolicy<>(filenamePolicy, SerializableFunctions.<UserT>identity());
+ }
+
+ /**
+ * Returns a {@link DynamicDestinations} that returns instances of {@link DefaultFilenamePolicy}
+ * configured with the given {@link Params}.
+ */
+ public static <UserT, OutputT> DynamicDestinations<UserT, Params, OutputT> toDefaultPolicies(
+ SerializableFunction<UserT, Params> destinationFunction,
+ Params emptyDestination,
+ SerializableFunction<UserT, OutputT> formatFunction) {
+ return new DefaultPolicyDestinations<>(destinationFunction, emptyDestination, formatFunction);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 8102316..d577fea 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -23,40 +23,40 @@
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Verify.verifyNotNull;
import static org.apache.beam.sdk.io.WriteFiles.UNKNOWN_SHARDNUM;
+import static org.apache.beam.sdk.values.TypeDescriptors.extractFromTypeParameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
-import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.zip.GZIPOutputStream;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.NullableCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.Context;
-import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.WindowedContext;
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.MoveOptions.StandardMoveOptions;
@@ -66,6 +66,7 @@
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
@@ -73,8 +74,9 @@
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder;
import org.apache.beam.sdk.util.MimeTypes;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
-import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors.TypeVariableExtractor;
import org.joda.time.Instant;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
@@ -82,104 +84,106 @@
import org.slf4j.LoggerFactory;
/**
- * Abstract class for file-based output. An implementation of FileBasedSink writes file-based
- * output and defines the format of output files (how values are written, headers/footers, MIME
- * type, etc.).
+ * Abstract class for file-based output. An implementation of FileBasedSink writes file-based output
+ * and defines the format of output files (how values are written, headers/footers, MIME type,
+ * etc.).
*
* <p>At pipeline construction time, the methods of FileBasedSink are called to validate the sink
* and to create a {@link WriteOperation} that manages the process of writing to the sink.
*
* <p>The process of writing to file-based sink is as follows:
+ *
* <ol>
- * <li>An optional subclass-defined initialization,
- * <li>a parallel write of bundles to temporary files, and finally,
- * <li>these temporary files are renamed with final output filenames.
+ * <li>An optional subclass-defined initialization,
+ * <li>a parallel write of bundles to temporary files, and finally,
+ * <li>these temporary files are renamed with final output filenames.
* </ol>
*
* <p>In order to ensure fault-tolerance, a bundle may be executed multiple times (e.g., in the
* event of failure/retry or for redundancy). However, exactly one of these executions will have its
- * result passed to the finalize method. Each call to {@link Writer#openWindowed}
- * or {@link Writer#openUnwindowed} is passed a unique <i>bundle id</i> when it is called
- * by the WriteFiles transform, so even redundant or retried bundles will have a unique way of
- * identifying
- * their output.
+ * result passed to the finalize method. Each call to {@link Writer#openWindowed} or {@link
+ * Writer#openUnwindowed} is passed a unique <i>bundle id</i> when it is called by the WriteFiles
+ * transform, so even redundant or retried bundles will have a unique way of identifying their
+ * output.
*
* <p>The bundle id should be used to guarantee that a bundle's output is unique. This uniqueness
* guarantee is important; if a bundle is to be output to a file, for example, the name of the file
* will encode the unique bundle id to avoid conflicts with other writers.
*
- * {@link FileBasedSink} can take a custom {@link FilenamePolicy} object to determine output
- * filenames, and this policy object can be used to write windowed or triggered
- * PCollections into separate files per window pane. This allows file output from unbounded
- * PCollections, and also works for bounded PCollecctions.
+ * <p>{@link FileBasedSink} can take a custom {@link FilenamePolicy} object to determine output
+ * filenames, and this policy object can be used to write windowed or triggered PCollections into
+ * separate files per window pane. This allows file output from unbounded PCollections, and also
+ * works for bounded PCollecctions.
*
* <p>Supported file systems are those registered with {@link FileSystems}.
*
- * @param <T> the type of values written to the sink.
+ * @param <OutputT> the type of values written to the sink.
*/
@Experimental(Kind.FILESYSTEM)
-public abstract class FileBasedSink<T> implements Serializable, HasDisplayData {
+public abstract class FileBasedSink<UserT, DestinationT, OutputT>
+ implements Serializable, HasDisplayData {
private static final Logger LOG = LoggerFactory.getLogger(FileBasedSink.class);
- /**
- * Directly supported file output compression types.
- */
+ /** @deprecated use {@link Compression}. */
+ @Deprecated
public enum CompressionType implements WritableByteChannelFactory {
- /**
- * No compression, or any other transformation, will be used.
- */
- UNCOMPRESSED("", null) {
- @Override
- public WritableByteChannel create(WritableByteChannel channel) throws IOException {
- return channel;
- }
- },
- /**
- * Provides GZip output transformation.
- */
- GZIP(".gz", MimeTypes.BINARY) {
- @Override
- public WritableByteChannel create(WritableByteChannel channel) throws IOException {
- return Channels.newChannel(new GZIPOutputStream(Channels.newOutputStream(channel), true));
- }
- },
- /**
- * Provides BZip2 output transformation.
- */
- BZIP2(".bz2", MimeTypes.BINARY) {
- @Override
- public WritableByteChannel create(WritableByteChannel channel) throws IOException {
- return Channels
- .newChannel(new BZip2CompressorOutputStream(Channels.newOutputStream(channel)));
- }
- },
- /**
- * Provides deflate output transformation.
- */
- DEFLATE(".deflate", MimeTypes.BINARY) {
- @Override
- public WritableByteChannel create(WritableByteChannel channel) throws IOException {
- return Channels
- .newChannel(new DeflateCompressorOutputStream(Channels.newOutputStream(channel)));
- }
- };
+ /** @see Compression#UNCOMPRESSED */
+ UNCOMPRESSED(Compression.UNCOMPRESSED),
- private String filenameSuffix;
- @Nullable private String mimeType;
+ /** @see Compression#GZIP */
+ GZIP(Compression.GZIP),
- CompressionType(String suffix, @Nullable String mimeType) {
- this.filenameSuffix = suffix;
- this.mimeType = mimeType;
+ /** @see Compression#BZIP2 */
+ BZIP2(Compression.BZIP2),
+
+ /** @see Compression#DEFLATE */
+ DEFLATE(Compression.DEFLATE);
+
+ private Compression canonical;
+
+ CompressionType(Compression canonical) {
+ this.canonical = canonical;
}
@Override
- public String getFilenameSuffix() {
- return filenameSuffix;
+ public String getSuggestedFilenameSuffix() {
+ return canonical.getSuggestedSuffix();
}
@Override
- @Nullable public String getMimeType() {
- return mimeType;
+ @Nullable
+ public String getMimeType() {
+ return (canonical == Compression.UNCOMPRESSED) ? null : MimeTypes.BINARY;
+ }
+
+ @Override
+ public WritableByteChannel create(WritableByteChannel channel) throws IOException {
+ return canonical.writeCompressed(channel);
+ }
+
+ public static CompressionType fromCanonical(Compression canonical) {
+ switch(canonical) {
+ case AUTO:
+ throw new IllegalArgumentException("AUTO is not supported for writing");
+
+ case UNCOMPRESSED:
+ return UNCOMPRESSED;
+
+ case GZIP:
+ return GZIP;
+
+ case BZIP2:
+ return BZIP2;
+
+ case ZIP:
+ throw new IllegalArgumentException("ZIP is unsupported");
+
+ case DEFLATE:
+ return DEFLATE;
+
+ default:
+ throw new UnsupportedOperationException("Unsupported compression type: " + canonical);
+ }
}
}
@@ -205,132 +209,166 @@
}
}
+ private final DynamicDestinations<?, DestinationT, OutputT> dynamicDestinations;
+
/**
* The {@link WritableByteChannelFactory} that is used to wrap the raw data output to the
- * underlying channel. The default is to not compress the output using
- * {@link CompressionType#UNCOMPRESSED}.
+ * underlying channel. The default is to not compress the output using {@link
+ * Compression#UNCOMPRESSED}.
*/
private final WritableByteChannelFactory writableByteChannelFactory;
/**
- * A naming policy for output files.
+ * A class that allows value-dependent writes in {@link FileBasedSink}.
+ *
+ * <p>Users can define a custom type to represent destinations, and provide a mapping to turn this
+ * destination type into an instance of {@link FilenamePolicy}.
*/
+ @Experimental(Kind.FILESYSTEM)
+ public abstract static class DynamicDestinations<UserT, DestinationT, OutputT>
+ implements HasDisplayData, Serializable {
+ interface SideInputAccessor {
+ <SideInputT> SideInputT sideInput(PCollectionView<SideInputT> view);
+ }
+
+ private SideInputAccessor sideInputAccessor;
+
+ static class SideInputAccessorViaProcessContext implements SideInputAccessor {
+ private DoFn<?, ?>.ProcessContext processContext;
+
+ SideInputAccessorViaProcessContext(DoFn<?, ?>.ProcessContext processContext) {
+ this.processContext = processContext;
+ }
+
+ @Override
+ public <SideInputT> SideInputT sideInput(PCollectionView<SideInputT> view) {
+ return processContext.sideInput(view);
+ }
+ }
+
+ /**
+ * Override to specify that this object needs access to one or more side inputs. This side
+ * inputs must be globally windowed, as they will be accessed from the global window.
+ */
+ public List<PCollectionView<?>> getSideInputs() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the value of a given side input. The view must be present in {@link
+ * #getSideInputs()}.
+ */
+ protected final <SideInputT> SideInputT sideInput(PCollectionView<SideInputT> view) {
+ return sideInputAccessor.sideInput(view);
+ }
+
+ final void setSideInputAccessor(SideInputAccessor sideInputAccessor) {
+ this.sideInputAccessor = sideInputAccessor;
+ }
+
+ final void setSideInputAccessorFromProcessContext(DoFn<?, ?>.ProcessContext context) {
+ this.sideInputAccessor = new SideInputAccessorViaProcessContext(context);
+ }
+
+ /** Convert an input record type into the output type. */
+ public abstract OutputT formatRecord(UserT record);
+
+ /**
+ * Returns an object that represents at a high level the destination being written to. May not
+ * return null. A destination must have deterministic hash and equality methods defined.
+ */
+ public abstract DestinationT getDestination(UserT element);
+
+ /**
+ * Returns the default destination. This is used for collections that have no elements as the
+ * destination to write empty files to.
+ */
+ public abstract DestinationT getDefaultDestination();
+
+ /**
+ * Returns the coder for {@link DestinationT}. If this is not overridden, then the coder
+ * registry will be use to find a suitable coder. This must be a deterministic coder, as {@link
+ * DestinationT} will be used as a key type in a {@link
+ * org.apache.beam.sdk.transforms.GroupByKey}.
+ */
+ @Nullable
+ public Coder<DestinationT> getDestinationCoder() {
+ return null;
+ }
+
+ /** Converts a destination into a {@link FilenamePolicy}. May not return null. */
+ public abstract FilenamePolicy getFilenamePolicy(DestinationT destination);
+
+ /** Populates the display data. */
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {}
+
+ // Gets the destination coder. If the user does not provide one, try to find one in the coder
+ // registry. If no coder can be found, throws CannotProvideCoderException.
+ final Coder<DestinationT> getDestinationCoderWithDefault(CoderRegistry registry)
+ throws CannotProvideCoderException {
+ Coder<DestinationT> destinationCoder = getDestinationCoder();
+ if (destinationCoder != null) {
+ return destinationCoder;
+ }
+ // If dynamicDestinations doesn't provide a coder, try to find it in the coder registry.
+ @Nullable
+ TypeDescriptor<DestinationT> descriptor =
+ extractFromTypeParameters(
+ this,
+ DynamicDestinations.class,
+ new TypeVariableExtractor<
+ DynamicDestinations<UserT, DestinationT, OutputT>, DestinationT>() {});
+ try {
+ return registry.getCoder(descriptor);
+ } catch (CannotProvideCoderException e) {
+ throw new CannotProvideCoderException(
+ "Failed to infer coder for DestinationT from type "
+ + descriptor + ", please provide it explicitly by overriding getDestinationCoder()",
+ e);
+ }
+ }
+ }
+
+ /** A naming policy for output files. */
+ @Experimental(Kind.FILESYSTEM)
public abstract static class FilenamePolicy implements Serializable {
/**
- * Context used for generating a name based on shard number, and num shards.
- * The policy must produce unique filenames for unique {@link Context} objects.
- *
- * <p>Be careful about adding fields to this as existing strategies will not notice the new
- * fields, and may not produce unique filenames.
- */
- public static class Context {
- private int shardNumber;
- private int numShards;
-
-
- public Context(int shardNumber, int numShards) {
- this.shardNumber = shardNumber;
- this.numShards = numShards;
- }
-
- public int getShardNumber() {
- return shardNumber;
- }
-
-
- public int getNumShards() {
- return numShards;
- }
- }
-
- /**
- * Context used for generating a name based on window, pane, shard number, and num shards.
- * The policy must produce unique filenames for unique {@link WindowedContext} objects.
- *
- * <p>Be careful about adding fields to this as existing strategies will not notice the new
- * fields, and may not produce unique filenames.
- */
- public static class WindowedContext {
- private int shardNumber;
- private int numShards;
- private BoundedWindow window;
- private PaneInfo paneInfo;
-
- public WindowedContext(
- BoundedWindow window,
- PaneInfo paneInfo,
- int shardNumber,
- int numShards) {
- this.window = window;
- this.paneInfo = paneInfo;
- this.shardNumber = shardNumber;
- this.numShards = numShards;
- }
-
- public BoundedWindow getWindow() {
- return window;
- }
-
- public PaneInfo getPaneInfo() {
- return paneInfo;
- }
-
- public int getShardNumber() {
- return shardNumber;
- }
-
- public int getNumShards() {
- return numShards;
- }
- }
-
- /**
* When a sink has requested windowed or triggered output, this method will be invoked to return
* the file {@link ResourceId resource} to be created given the base output directory and a
- * (possibly empty) extension from {@link FileBasedSink} configuration
- * (e.g., {@link CompressionType}).
+ * {@link OutputFileHints} containing information about the file, including a suggested
+ * extension (e.g. coming from {@link Compression}).
*
- * <p>The {@link WindowedContext} object gives access to the window and pane,
- * as well as sharding information. The policy must return unique and consistent filenames
- * for different windows and panes.
+ * <p>The policy must return unique and consistent filenames for different windows and panes.
*/
@Experimental(Kind.FILESYSTEM)
public abstract ResourceId windowedFilename(
- ResourceId outputDirectory, WindowedContext c, String extension);
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints);
/**
* When a sink has not requested windowed or triggered output, this method will be invoked to
* return the file {@link ResourceId resource} to be created given the base output directory and
- * a (possibly empty) extension applied by additional {@link FileBasedSink} configuration
- * (e.g., {@link CompressionType}).
+ * a {@link OutputFileHints} containing information about the file, including a suggested (e.g.
+ * coming from {@link Compression}).
*
- * <p>The {@link Context} object only provides sharding information, which is used by the policy
- * to generate unique and consistent filenames.
+ * <p>The shardNumber and numShards parameters, should be used by the policy to generate unique
+ * and consistent filenames.
*/
@Experimental(Kind.FILESYSTEM)
- @Nullable public abstract ResourceId unwindowedFilename(
- ResourceId outputDirectory, Context c, String extension);
+ @Nullable
+ public abstract ResourceId unwindowedFilename(
+ int shardNumber, int numShards, OutputFileHints outputFileHints);
- /**
- * Populates the display data.
- */
- public void populateDisplayData(DisplayData.Builder builder) {
- }
+ /** Populates the display data. */
+ public void populateDisplayData(DisplayData.Builder builder) {}
}
- /** The policy used to generate names of files to be produced. */
- private final FilenamePolicy filenamePolicy;
/** The directory to which files will be written. */
- private final ValueProvider<ResourceId> baseOutputDirectoryProvider;
-
- /**
- * Construct a {@link FileBasedSink} with the given filename policy, producing uncompressed files.
- */
- @Experimental(Kind.FILESYSTEM)
- public FileBasedSink(
- ValueProvider<ResourceId> baseOutputDirectoryProvider, FilenamePolicy filenamePolicy) {
- this(baseOutputDirectoryProvider, filenamePolicy, CompressionType.UNCOMPRESSED);
- }
+ private final ValueProvider<ResourceId> tempDirectoryProvider;
private static class ExtractDirectory implements SerializableFunction<ResourceId, ResourceId> {
@Override
@@ -340,95 +378,100 @@
}
/**
- * Construct a {@link FileBasedSink} with the given filename policy and output channel type.
+ * Construct a {@link FileBasedSink} with the given temp directory, producing uncompressed files.
*/
@Experimental(Kind.FILESYSTEM)
public FileBasedSink(
- ValueProvider<ResourceId> baseOutputDirectoryProvider,
- FilenamePolicy filenamePolicy,
+ ValueProvider<ResourceId> tempDirectoryProvider,
+ DynamicDestinations<?, DestinationT, OutputT> dynamicDestinations) {
+ this(tempDirectoryProvider, dynamicDestinations, Compression.UNCOMPRESSED);
+ }
+
+ /** Construct a {@link FileBasedSink} with the given temp directory and output channel type. */
+ @Experimental(Kind.FILESYSTEM)
+ public FileBasedSink(
+ ValueProvider<ResourceId> tempDirectoryProvider,
+ DynamicDestinations<?, DestinationT, OutputT> dynamicDestinations,
WritableByteChannelFactory writableByteChannelFactory) {
- this.baseOutputDirectoryProvider =
- NestedValueProvider.of(baseOutputDirectoryProvider, new ExtractDirectory());
- this.filenamePolicy = filenamePolicy;
+ this.tempDirectoryProvider =
+ NestedValueProvider.of(tempDirectoryProvider, new ExtractDirectory());
+ this.dynamicDestinations = checkNotNull(dynamicDestinations);
this.writableByteChannelFactory = writableByteChannelFactory;
}
- /**
- * Returns the base directory inside which files will be written according to the configured
- * {@link FilenamePolicy}.
- */
+ /** Construct a {@link FileBasedSink} with the given temp directory and output channel type. */
@Experimental(Kind.FILESYSTEM)
- public ValueProvider<ResourceId> getBaseOutputDirectoryProvider() {
- return baseOutputDirectoryProvider;
+ public FileBasedSink(
+ ValueProvider<ResourceId> tempDirectoryProvider,
+ DynamicDestinations<?, DestinationT, OutputT> dynamicDestinations,
+ Compression compression) {
+ this(tempDirectoryProvider, dynamicDestinations, CompressionType.fromCanonical(compression));
+ }
+
+ /** Return the {@link DynamicDestinations} used. */
+ @SuppressWarnings("unchecked")
+ public DynamicDestinations<UserT, DestinationT, OutputT> getDynamicDestinations() {
+ return (DynamicDestinations<UserT, DestinationT, OutputT>) dynamicDestinations;
}
/**
- * Returns the policy by which files will be named inside of the base output directory. Note that
- * the {@link FilenamePolicy} may itself specify one or more inner directories before each output
- * file, say when writing windowed outputs in a {@code output/YYYY/MM/DD/file.txt} format.
+ * Returns the directory inside which temprary files will be written according to the configured
+ * {@link FilenamePolicy}.
*/
@Experimental(Kind.FILESYSTEM)
- public final FilenamePolicy getFilenamePolicy() {
- return filenamePolicy;
+ public ValueProvider<ResourceId> getTempDirectoryProvider() {
+ return tempDirectoryProvider;
}
public void validate(PipelineOptions options) {}
- /**
- * Return a subclass of {@link WriteOperation} that will manage the write
- * to the sink.
- */
- public abstract WriteOperation<T> createWriteOperation();
+ /** Return a subclass of {@link WriteOperation} that will manage the write to the sink. */
+ public abstract WriteOperation<DestinationT, OutputT> createWriteOperation();
public void populateDisplayData(DisplayData.Builder builder) {
- getFilenamePolicy().populateDisplayData(builder);
+ getDynamicDestinations().populateDisplayData(builder);
}
/**
* Abstract operation that manages the process of writing to {@link FileBasedSink}.
*
- * <p>The primary responsibilities of the WriteOperation is the management of output
- * files. During a write, {@link Writer}s write bundles to temporary file
- * locations. After the bundles have been written,
+ * <p>The primary responsibilities of the WriteOperation is the management of output files. During
+ * a write, {@link Writer}s write bundles to temporary file locations. After the bundles have been
+ * written,
+ *
* <ol>
- * <li>{@link WriteOperation#finalize} is given a list of the temporary
- * files containing the output bundles.
- * <li>During finalize, these temporary files are copied to final output locations and named
- * according to a file naming template.
- * <li>Finally, any temporary files that were created during the write are removed.
+ * <li>{@link WriteOperation#finalize} is given a list of the temporary files containing the
+ * output bundles.
+ * <li>During finalize, these temporary files are copied to final output locations and named
+ * according to a file naming template.
+ * <li>Finally, any temporary files that were created during the write are removed.
* </ol>
*
- * <p>Subclass implementations of WriteOperation must implement
- * {@link WriteOperation#createWriter} to return a concrete
- * FileBasedSinkWriter.
+ * <p>Subclass implementations of WriteOperation must implement {@link
+ * WriteOperation#createWriter} to return a concrete FileBasedSinkWriter.
*
- * <h2>Temporary and Output File Naming:</h2> During the write, bundles are written to temporary
- * files using the tempDirectory that can be provided via the constructor of
- * WriteOperation. These temporary files will be named
- * {@code {tempDirectory}/{bundleId}}, where bundleId is the unique id of the bundle.
- * For example, if tempDirectory is "gs://my-bucket/my_temp_output", the output for a
- * bundle with bundle id 15723 will be "gs://my-bucket/my_temp_output/15723".
+ * <h2>Temporary and Output File Naming:</h2>
*
- * <p>Final output files are written to baseOutputFilename with the format
- * {@code {baseOutputFilename}-0000i-of-0000n.{extension}} where n is the total number of bundles
- * written and extension is the file extension. Both baseOutputFilename and extension are required
- * constructor arguments.
+ * <p>During the write, bundles are written to temporary files using the tempDirectory that can be
+ * provided via the constructor of WriteOperation. These temporary files will be named {@code
+ * {tempDirectory}/{bundleId}}, where bundleId is the unique id of the bundle. For example, if
+ * tempDirectory is "gs://my-bucket/my_temp_output", the output for a bundle with bundle id 15723
+ * will be "gs://my-bucket/my_temp_output/15723".
*
- * <p>Subclass implementations can change the file naming template by supplying a value for
- * fileNamingTemplate.
+ * <p>Final output files are written to the location specified by the {@link FilenamePolicy}. If
+ * no filename policy is specified, then the {@link DefaultFilenamePolicy} will be used. The
+ * directory that the files are written to is determined by the {@link FilenamePolicy} instance.
*
* <p>Note that in the case of permanent failure of a bundle's write, no clean up of temporary
* files will occur.
*
* <p>If there are no elements in the PCollection being written, no output will be generated.
*
- * @param <T> the type of values written to the sink.
+ * @param <OutputT> the type of values written to the sink.
*/
- public abstract static class WriteOperation<T> implements Serializable {
- /**
- * The Sink that this WriteOperation will write to.
- */
- protected final FileBasedSink<T> sink;
+ public abstract static class WriteOperation<DestinationT, OutputT> implements Serializable {
+ /** The Sink that this WriteOperation will write to. */
+ protected final FileBasedSink<?, DestinationT, OutputT> sink;
/** Directory for temporary output files. */
protected final ValueProvider<ResourceId> tempDirectory;
@@ -445,17 +488,19 @@
}
/**
- * Constructs a WriteOperation using the default strategy for generating a temporary
- * directory from the base output filename.
+ * Constructs a WriteOperation using the default strategy for generating a temporary directory
+ * from the base output filename.
*
- * <p>Default is a uniquely named sibling of baseOutputFilename, e.g. if baseOutputFilename is
- * /path/to/foo, the temporary directory will be /path/to/temp-beam-foo-$date.
+ * <p>Default is a uniquely named subdirectory of the provided tempDirectory, e.g. if
+ * tempDirectory is /path/to/foo/, the temporary directory will be
+ * /path/to/foo/temp-beam-foo-$date.
*
* @param sink the FileBasedSink that will be used to configure this write operation.
*/
- public WriteOperation(FileBasedSink<T> sink) {
- this(sink, NestedValueProvider.of(
- sink.getBaseOutputDirectoryProvider(), new TemporaryDirectoryBuilder()));
+ public WriteOperation(FileBasedSink<?, DestinationT, OutputT> sink) {
+ this(
+ sink,
+ NestedValueProvider.of(sink.getTempDirectoryProvider(), new TemporaryDirectoryBuilder()));
}
private static class TemporaryDirectoryBuilder
@@ -471,10 +516,12 @@
private final Long tempId = TEMP_COUNT.getAndIncrement();
@Override
- public ResourceId apply(ResourceId baseOutputDirectory) {
+ public ResourceId apply(ResourceId tempDirectory) {
// Temp directory has a timestamp and a unique ID
String tempDirName = String.format(".temp-beam-%s-%s", timestamp, tempId);
- return baseOutputDirectory.resolve(tempDirName, StandardResolveOptions.RESOLVE_DIRECTORY);
+ return tempDirectory
+ .getCurrentDirectory()
+ .resolve(tempDirName, StandardResolveOptions.RESOLVE_DIRECTORY);
}
}
@@ -485,74 +532,82 @@
* @param tempDirectory the base directory to be used for temporary output files.
*/
@Experimental(Kind.FILESYSTEM)
- public WriteOperation(FileBasedSink<T> sink, ResourceId tempDirectory) {
+ public WriteOperation(FileBasedSink<?, DestinationT, OutputT> sink, ResourceId tempDirectory) {
this(sink, StaticValueProvider.of(tempDirectory));
}
private WriteOperation(
- FileBasedSink<T> sink, ValueProvider<ResourceId> tempDirectory) {
+ FileBasedSink<?, DestinationT, OutputT> sink, ValueProvider<ResourceId> tempDirectory) {
this.sink = sink;
this.tempDirectory = tempDirectory;
this.windowedWrites = false;
}
/**
- * Clients must implement to return a subclass of {@link Writer}. This
- * method must not mutate the state of the object.
+ * Clients must implement to return a subclass of {@link Writer}. This method must not mutate
+ * the state of the object.
*/
- public abstract Writer<T> createWriter() throws Exception;
+ public abstract Writer<DestinationT, OutputT> createWriter() throws Exception;
- /**
- * Indicates that the operation will be performing windowed writes.
- */
+ /** Indicates that the operation will be performing windowed writes. */
public void setWindowedWrites(boolean windowedWrites) {
this.windowedWrites = windowedWrites;
}
/**
- * Finalizes writing by copying temporary output files to their final location and optionally
- * removing temporary files.
+ * Finalizes writing by copying temporary output files to their final location.
*
* <p>Finalization may be overridden by subclass implementations to perform customized
- * finalization (e.g., initiating some operation on output bundles, merging them, etc.).
- * {@code writerResults} contains the filenames of written bundles.
+ * finalization (e.g., initiating some operation on output bundles, merging them, etc.). {@code
+ * writerResults} contains the filenames of written bundles.
*
* <p>If subclasses override this method, they must guarantee that its implementation is
* idempotent, as it may be executed multiple times in the case of failure or for redundancy. It
* is a best practice to attempt to try to make this method atomic.
*
+ * <p>Returns the map of temporary files generated to final filenames. Callers must call {@link
+ * #removeTemporaryFiles(Set)} to cleanup the temporary files.
+ *
* @param writerResults the results of writes (FileResult).
*/
- public void finalize(Iterable<FileResult> writerResults) throws Exception {
- // Collect names of temporary files and rename them.
+ public Map<ResourceId, ResourceId> finalize(Iterable<FileResult<DestinationT>> writerResults)
+ throws Exception {
+ // Collect names of temporary files and copies them.
Map<ResourceId, ResourceId> outputFilenames = buildOutputFilenames(writerResults);
copyToOutputFiles(outputFilenames);
+ return outputFilenames;
+ }
- // Optionally remove temporary files.
- // We remove the entire temporary directory, rather than specifically removing the files
- // from writerResults, because writerResults includes only successfully completed bundles,
- // and we'd like to clean up the failed ones too.
- // Note that due to GCS eventual consistency, matching files in the temp directory is also
- // currently non-perfect and may fail to delete some files.
- //
- // When windows or triggers are specified, files are generated incrementally so deleting
- // the entire directory in finalize is incorrect.
- removeTemporaryFiles(outputFilenames.keySet(), !windowedWrites);
+ /*
+ * Remove temporary files after finalization.
+ *
+ * <p>In the case where we are doing global-window, untriggered writes, we remove the entire
+ * temporary directory, rather than specifically removing the files from writerResults, because
+ * writerResults includes only successfully completed bundles, and we'd like to clean up the
+ * failed ones too. The reason we remove files here rather than in finalize is that finalize
+ * might be called multiple times (e.g. if the bundle contained multiple destinations), and
+ * deleting the entire directory can't be done until all calls to finalize.
+ *
+ * <p>When windows or triggers are specified, files are generated incrementally so deleting the
+ * entire directory in finalize is incorrect. If windowedWrites is true, we instead delete the
+ * files individually. This means that some temporary files generated by failed bundles might
+ * not be cleaned up. Note that {@link WriteFiles} does attempt clean up files if exceptions
+ * are thrown, however there are still some scenarios where temporary files might be left.
+ */
+ public void removeTemporaryFiles(Set<ResourceId> filenames) throws IOException {
+ removeTemporaryFiles(filenames, !windowedWrites);
}
@Experimental(Kind.FILESYSTEM)
protected final Map<ResourceId, ResourceId> buildOutputFilenames(
- Iterable<FileResult> writerResults) {
+ Iterable<FileResult<DestinationT>> writerResults) {
int numShards = Iterables.size(writerResults);
- Map<ResourceId, ResourceId> outputFilenames = new HashMap<>();
-
- FilenamePolicy policy = getSink().getFilenamePolicy();
- ResourceId baseOutputDir = getSink().getBaseOutputDirectoryProvider().get();
+ Map<ResourceId, ResourceId> outputFilenames = Maps.newHashMap();
// Either all results have a shard number set (if the sink is configured with a fixed
// number of shards), or they all don't (otherwise).
Boolean isShardNumberSetEverywhere = null;
- for (FileResult result : writerResults) {
+ for (FileResult<DestinationT> result : writerResults) {
boolean isShardNumberSetHere = (result.getShard() != UNKNOWN_SHARDNUM);
if (isShardNumberSetEverywhere == null) {
isShardNumberSetEverywhere = isShardNumberSetHere;
@@ -568,7 +623,7 @@
isShardNumberSetEverywhere = true;
}
- List<FileResult> resultsWithShardNumbers = Lists.newArrayList();
+ List<FileResult<DestinationT>> resultsWithShardNumbers = Lists.newArrayList();
if (isShardNumberSetEverywhere) {
resultsWithShardNumbers = Lists.newArrayList(writerResults);
} else {
@@ -577,36 +632,40 @@
// case of triggers, the list of FileResult objects in the Finalize iterable is not
// deterministic, and might change over retries. This breaks the assumption below that
// sorting the FileResult objects provides idempotency.
- List<FileResult> sortedByTempFilename =
+ List<FileResult<DestinationT>> sortedByTempFilename =
Ordering.from(
- new Comparator<FileResult>() {
- @Override
- public int compare(FileResult first, FileResult second) {
- String firstFilename = first.getTempFilename().toString();
- String secondFilename = second.getTempFilename().toString();
- return firstFilename.compareTo(secondFilename);
- }
- })
+ new Comparator<FileResult<DestinationT>>() {
+ @Override
+ public int compare(
+ FileResult<DestinationT> first, FileResult<DestinationT> second) {
+ String firstFilename = first.getTempFilename().toString();
+ String secondFilename = second.getTempFilename().toString();
+ return firstFilename.compareTo(secondFilename);
+ }
+ })
.sortedCopy(writerResults);
for (int i = 0; i < sortedByTempFilename.size(); i++) {
resultsWithShardNumbers.add(sortedByTempFilename.get(i).withShard(i));
}
}
- for (FileResult result : resultsWithShardNumbers) {
+ for (FileResult<DestinationT> result : resultsWithShardNumbers) {
checkArgument(
result.getShard() != UNKNOWN_SHARDNUM, "Should have set shard number on %s", result);
outputFilenames.put(
result.getTempFilename(),
result.getDestinationFile(
- policy, baseOutputDir, numShards, getSink().getExtension()));
+ getSink().getDynamicDestinations(),
+ numShards,
+ getSink().getWritableByteChannelFactory()));
}
int numDistinctShards = new HashSet<>(outputFilenames.values()).size();
- checkState(numDistinctShards == outputFilenames.size(),
- "Only generated %s distinct file names for %s files.",
- numDistinctShards, outputFilenames.size());
-
+ checkState(
+ numDistinctShards == outputFilenames.size(),
+ "Only generated %s distinct file names for %s files.",
+ numDistinctShards,
+ outputFilenames.size());
return outputFilenames;
}
@@ -615,18 +674,18 @@
*
* <p>Can be called from subclasses that override {@link WriteOperation#finalize}.
*
- * <p>Files will be named according to the file naming template. The order of the output files
- * will be the same as the sorted order of the input filenames. In other words, if the input
- * filenames are ["C", "A", "B"], baseOutputFilename is "file", the extension is ".txt", and
- * the fileNamingTemplate is "-SSS-of-NNN", the contents of A will be copied to
- * file-000-of-003.txt, the contents of B will be copied to file-001-of-003.txt, etc.
+ * <p>Files will be named according to the {@link FilenamePolicy}. The order of the output files
+ * will be the same as the sorted order of the input filenames. In other words (when using
+ * {@link DefaultFilenamePolicy}), if the input filenames are ["C", "A", "B"], baseFilename (int
+ * the policy) is "dir/file", the extension is ".txt", and the fileNamingTemplate is
+ * "-SSS-of-NNN", the contents of A will be copied to dir/file-000-of-003.txt, the contents of B
+ * will be copied to dir/file-001-of-003.txt, etc.
*
* @param filenames the filenames of temporary files.
*/
@VisibleForTesting
@Experimental(Kind.FILESYSTEM)
- final void copyToOutputFiles(Map<ResourceId, ResourceId> filenames)
- throws IOException {
+ final void copyToOutputFiles(Map<ResourceId, ResourceId> filenames) throws IOException {
int numFiles = filenames.size();
if (numFiles > 0) {
LOG.debug("Copying {} files.", numFiles);
@@ -670,8 +729,9 @@
// ignore the exception for now to avoid failing the pipeline.
if (shouldRemoveTemporaryDirectory) {
try {
- MatchResult singleMatch = Iterables.getOnlyElement(
- FileSystems.match(Collections.singletonList(tempDir.toString() + "*")));
+ MatchResult singleMatch =
+ Iterables.getOnlyElement(
+ FileSystems.match(Collections.singletonList(tempDir.toString() + "*")));
for (Metadata matchResult : singleMatch.metadata()) {
matches.add(matchResult.resourceId());
}
@@ -698,54 +758,45 @@
}
}
- /**
- * Returns the FileBasedSink for this write operation.
- */
- public FileBasedSink<T> getSink() {
+ /** Returns the FileBasedSink for this write operation. */
+ public FileBasedSink<?, DestinationT, OutputT> getSink() {
return sink;
}
@Override
public String toString() {
- String tempDirectoryStr =
- tempDirectory.isAccessible() ? tempDirectory.get().toString() : tempDirectory.toString();
return getClass().getSimpleName()
+ "{"
+ "tempDirectory="
- + tempDirectoryStr
+ + tempDirectory
+ ", windowedWrites="
+ windowedWrites
+ '}';
}
}
- /** Returns the extension that will be written to the produced files. */
- protected final String getExtension() {
- String extension = MoreObjects.firstNonNull(writableByteChannelFactory.getFilenameSuffix(), "");
- if (!extension.isEmpty() && !extension.startsWith(".")) {
- extension = "." + extension;
- }
- return extension;
+ /** Returns the {@link WritableByteChannelFactory} used. */
+ protected final WritableByteChannelFactory getWritableByteChannelFactory() {
+ return writableByteChannelFactory;
}
/**
- * Abstract writer that writes a bundle to a {@link FileBasedSink}. Subclass
- * implementations provide a method that can write a single value to a
- * {@link WritableByteChannel}.
+ * Abstract writer that writes a bundle to a {@link FileBasedSink}. Subclass implementations
+ * provide a method that can write a single value to a {@link WritableByteChannel}.
*
* <p>Subclass implementations may also override methods that write headers and footers before and
* after the values in a bundle, respectively, as well as provide a MIME type for the output
* channel.
*
- * <p>Multiple {@link Writer} instances may be created on the same worker, and therefore
- * any access to static members or methods should be thread safe.
+ * <p>Multiple {@link Writer} instances may be created on the same worker, and therefore any
+ * access to static members or methods should be thread safe.
*
- * @param <T> the type of values to write.
+ * @param <OutputT> the type of values to write.
*/
- public abstract static class Writer<T> {
+ public abstract static class Writer<DestinationT, OutputT> {
private static final Logger LOG = LoggerFactory.getLogger(Writer.class);
- private final WriteOperation<T> writeOperation;
+ private final WriteOperation<DestinationT, OutputT> writeOperation;
/** Unique id for this output bundle. */
private String id;
@@ -753,29 +804,26 @@
private BoundedWindow window;
private PaneInfo paneInfo;
private int shard = -1;
+ private DestinationT destination;
/** The output file for this bundle. May be null if opening failed. */
private @Nullable ResourceId outputFile;
- /**
- * The channel to write to.
- */
+ /** The channel to write to. */
private WritableByteChannel channel;
/**
* The MIME type used in the creation of the output channel (if the file system supports it).
*
- * <p>This is the default for the sink, but it may be overridden by a supplied
- * {@link WritableByteChannelFactory}. For example, {@link TextIO.Write} uses
- * {@link MimeTypes#TEXT} by default but if {@link CompressionType#BZIP2} is set then
- * the MIME type will be overridden to {@link MimeTypes#BINARY}.
+ * <p>This is the default for the sink, but it may be overridden by a supplied {@link
+ * WritableByteChannelFactory}. For example, {@link TextIO.Write} uses {@link MimeTypes#TEXT} by
+ * default but if {@link Compression#BZIP2} is set then the MIME type will be overridden to
+ * {@link MimeTypes#BINARY}.
*/
private final String mimeType;
- /**
- * Construct a new {@link Writer} that will produce files of the given MIME type.
- */
- public Writer(WriteOperation<T> writeOperation, String mimeType) {
+ /** Construct a new {@link Writer} that will produce files of the given MIME type. */
+ public Writer(WriteOperation<DestinationT, OutputT> writeOperation, String mimeType) {
checkNotNull(writeOperation);
this.writeOperation = writeOperation;
this.mimeType = mimeType;
@@ -795,14 +843,12 @@
*/
protected void writeHeader() throws Exception {}
- /**
- * Writes footer at the end of output files. Nothing by default; subclasses may override.
- */
+ /** Writes footer at the end of output files. Nothing by default; subclasses may override. */
protected void writeFooter() throws Exception {}
/**
- * Called after all calls to {@link #writeHeader}, {@link #write} and {@link #writeFooter}.
- * If any resources opened in the write processes need to be flushed, flush them here.
+ * Called after all calls to {@link #writeHeader}, {@link #write} and {@link #writeFooter}. If
+ * any resources opened in the write processes need to be flushed, flush them here.
*/
protected void finishWrite() throws Exception {}
@@ -818,28 +864,27 @@
* id populated for the case of static sharding. In cases where the runner is dynamically
* picking sharding, shard might be set to -1.
*/
- public final void openWindowed(String uId, BoundedWindow window, PaneInfo paneInfo, int shard)
+ public final void openWindowed(
+ String uId, BoundedWindow window, PaneInfo paneInfo, int shard, DestinationT destination)
throws Exception {
if (!getWriteOperation().windowedWrites) {
throw new IllegalStateException("openWindowed called a non-windowed sink.");
}
- open(uId, window, paneInfo, shard);
+ open(uId, window, paneInfo, shard, destination);
}
- /**
- * Called for each value in the bundle.
- */
- public abstract void write(T value) throws Exception;
+ /** Called for each value in the bundle. */
+ public abstract void write(OutputT value) throws Exception;
/**
- * Similar to {@link #openWindowed} however for the case where unwindowed writes were
- * requested.
+ * Similar to {@link #openWindowed} however for the case where unwindowed writes were requested.
*/
- public final void openUnwindowed(String uId, int shard) throws Exception {
+ public final void openUnwindowed(String uId, int shard, DestinationT destination)
+ throws Exception {
if (getWriteOperation().windowedWrites) {
throw new IllegalStateException("openUnwindowed called a windowed sink.");
}
- open(uId, null, null, shard);
+ open(uId, null, null, shard, destination);
}
// Helper function to close a channel, on exception cases.
@@ -855,14 +900,18 @@
}
}
- private void open(String uId,
- @Nullable BoundedWindow window,
- @Nullable PaneInfo paneInfo,
- int shard) throws Exception {
+ private void open(
+ String uId,
+ @Nullable BoundedWindow window,
+ @Nullable PaneInfo paneInfo,
+ int shard,
+ DestinationT destination)
+ throws Exception {
this.id = uId;
this.window = window;
this.paneInfo = paneInfo;
this.shard = shard;
+ this.destination = destination;
ResourceId tempDirectory = getWriteOperation().tempDirectory.get();
outputFile = tempDirectory.resolve(id, StandardResolveOptions.RESOLVE_FILE);
verifyNotNull(
@@ -908,7 +957,7 @@
}
/** Closes the channel and returns the bundle result. */
- public final FileResult close() throws Exception {
+ public final FileResult<DestinationT> close() throws Exception {
checkState(outputFile != null, "FileResult.close cannot be called with a null outputFile");
LOG.debug("Writing footer to {}.", outputFile);
@@ -929,7 +978,9 @@
checkState(
channel.isOpen(),
- "Channel %s to %s should only be closed by its owner: %s", channel, outputFile);
+ "Channel %s to %s should only be closed by its owner: %s",
+ channel,
+ outputFile);
LOG.debug("Closing channel to {}.", outputFile);
try {
@@ -938,35 +989,46 @@
throw new IOException(String.format("Failed closing channel to %s", outputFile), e);
}
- FileResult result = new FileResult(outputFile, shard, window, paneInfo);
+ FileResult<DestinationT> result =
+ new FileResult<>(outputFile, shard, window, paneInfo, destination);
LOG.debug("Result for bundle {}: {}", this.id, outputFile);
return result;
}
- /**
- * Return the WriteOperation that this Writer belongs to.
- */
- public WriteOperation<T> getWriteOperation() {
+ /** Return the WriteOperation that this Writer belongs to. */
+ public WriteOperation<DestinationT, OutputT> getWriteOperation() {
return writeOperation;
}
+
+ /** Return the user destination object for this writer. */
+ public DestinationT getDestination() {
+ return destination;
+ }
}
/**
- * Result of a single bundle write. Contains the filename produced by the bundle, and if known
- * the final output filename.
+ * Result of a single bundle write. Contains the filename produced by the bundle, and if known the
+ * final output filename.
*/
- public static final class FileResult {
+ public static final class FileResult<DestinationT> {
private final ResourceId tempFilename;
private final int shard;
private final BoundedWindow window;
private final PaneInfo paneInfo;
+ private final DestinationT destination;
@Experimental(Kind.FILESYSTEM)
- public FileResult(ResourceId tempFilename, int shard, BoundedWindow window, PaneInfo paneInfo) {
+ public FileResult(
+ ResourceId tempFilename,
+ int shard,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ DestinationT destination) {
this.tempFilename = tempFilename;
this.shard = shard;
this.window = window;
this.paneInfo = paneInfo;
+ this.destination = destination;
}
@Experimental(Kind.FILESYSTEM)
@@ -978,8 +1040,8 @@
return shard;
}
- public FileResult withShard(int shard) {
- return new FileResult(tempFilename, shard, window, paneInfo);
+ public FileResult<DestinationT> withShard(int shard) {
+ return new FileResult<>(tempFilename, shard, window, paneInfo, destination);
}
public BoundedWindow getWindow() {
@@ -990,17 +1052,23 @@
return paneInfo;
}
+ public DestinationT getDestination() {
+ return destination;
+ }
+
@Experimental(Kind.FILESYSTEM)
- public ResourceId getDestinationFile(FilenamePolicy policy, ResourceId outputDirectory,
- int numShards, String extension) {
+ public ResourceId getDestinationFile(
+ DynamicDestinations<?, DestinationT, ?> dynamicDestinations,
+ int numShards,
+ OutputFileHints outputFileHints) {
checkArgument(getShard() != UNKNOWN_SHARDNUM);
checkArgument(numShards > 0);
+ FilenamePolicy policy = dynamicDestinations.getFilenamePolicy(destination);
if (getWindow() != null) {
- return policy.windowedFilename(outputDirectory, new WindowedContext(
- getWindow(), getPaneInfo(), getShard(), numShards), extension);
+ return policy.windowedFilename(
+ getShard(), numShards, getWindow(), getPaneInfo(), outputFileHints);
} else {
- return policy.unwindowedFilename(outputDirectory, new Context(getShard(), numShards),
- extension);
+ return policy.unwindowedFilename(getShard(), numShards, outputFileHints);
}
}
@@ -1014,22 +1082,24 @@
}
}
- /**
- * A coder for {@link FileResult} objects.
- */
- public static final class FileResultCoder extends StructuredCoder<FileResult> {
+ /** A coder for {@link FileResult} objects. */
+ public static final class FileResultCoder<DestinationT>
+ extends StructuredCoder<FileResult<DestinationT>> {
private static final Coder<String> FILENAME_CODER = StringUtf8Coder.of();
private static final Coder<Integer> SHARD_CODER = VarIntCoder.of();
private static final Coder<PaneInfo> PANE_INFO_CODER = NullableCoder.of(PaneInfoCoder.INSTANCE);
-
private final Coder<BoundedWindow> windowCoder;
+ private final Coder<DestinationT> destinationCoder;
- protected FileResultCoder(Coder<BoundedWindow> windowCoder) {
+ protected FileResultCoder(
+ Coder<BoundedWindow> windowCoder, Coder<DestinationT> destinationCoder) {
this.windowCoder = NullableCoder.of(windowCoder);
+ this.destinationCoder = destinationCoder;
}
- public static FileResultCoder of(Coder<BoundedWindow> windowCoder) {
- return new FileResultCoder(windowCoder);
+ public static <DestinationT> FileResultCoder<DestinationT> of(
+ Coder<BoundedWindow> windowCoder, Coder<DestinationT> destinationCoder) {
+ return new FileResultCoder<>(windowCoder, destinationCoder);
}
@Override
@@ -1038,8 +1108,7 @@
}
@Override
- public void encode(FileResult value, OutputStream outStream)
- throws IOException {
+ public void encode(FileResult<DestinationT> value, OutputStream outStream) throws IOException {
if (value == null) {
throw new CoderException("cannot encode a null value");
}
@@ -1047,17 +1116,22 @@
windowCoder.encode(value.getWindow(), outStream);
PANE_INFO_CODER.encode(value.getPaneInfo(), outStream);
SHARD_CODER.encode(value.getShard(), outStream);
+ destinationCoder.encode(value.getDestination(), outStream);
}
@Override
- public FileResult decode(InputStream inStream)
- throws IOException {
+ public FileResult<DestinationT> decode(InputStream inStream) throws IOException {
String tempFilename = FILENAME_CODER.decode(inStream);
BoundedWindow window = windowCoder.decode(inStream);
PaneInfo paneInfo = PANE_INFO_CODER.decode(inStream);
int shard = SHARD_CODER.decode(inStream);
- return new FileResult(FileSystems.matchNewResource(tempFilename, false /* isDirectory */),
- shard, window, paneInfo);
+ DestinationT destination = destinationCoder.decode(inStream);
+ return new FileResult<>(
+ FileSystems.matchNewResource(tempFilename, false /* isDirectory */),
+ shard,
+ window,
+ paneInfo,
+ destination);
}
@Override
@@ -1066,41 +1140,48 @@
windowCoder.verifyDeterministic();
PANE_INFO_CODER.verifyDeterministic();
SHARD_CODER.verifyDeterministic();
+ destinationCoder.verifyDeterministic();
}
}
/**
- * Implementations create instances of {@link WritableByteChannel} used by {@link FileBasedSink}
- * and related classes to allow <em>decorating</em>, or otherwise transforming, the raw data that
- * would normally be written directly to the {@link WritableByteChannel} passed into
- * {@link WritableByteChannelFactory#create(WritableByteChannel)}.
- *
- * <p>Subclasses should override {@link #toString()} with something meaningful, as it is used when
- * building {@link DisplayData}.
+ * Provides hints about how to generate output files, such as a suggested filename suffix (e.g.
+ * based on the compression type), and the file MIME type.
*/
- public interface WritableByteChannelFactory extends Serializable {
- /**
- * @param channel the {@link WritableByteChannel} to wrap
- * @return the {@link WritableByteChannel} to be used during output
- */
- WritableByteChannel create(WritableByteChannel channel) throws IOException;
-
+ public interface OutputFileHints extends Serializable {
/**
* Returns the MIME type that should be used for the files that will hold the output data. May
* return {@code null} if this {@code WritableByteChannelFactory} does not meaningfully change
- * the MIME type (e.g., for {@link CompressionType#UNCOMPRESSED}).
+ * the MIME type (e.g., for {@link Compression#UNCOMPRESSED}).
*
* @see MimeTypes
* @see <a href=
- * 'http://www.iana.org/assignments/media-types/media-types.xhtml'>http://www.iana.org/assignments/media-types/media-types.xhtml</a>
+ * 'http://www.iana.org/assignments/media-types/media-types.xhtml'>http://www.iana.org/assignments/media-types/media-types.xhtml</a>
*/
@Nullable
String getMimeType();
/**
- * @return an optional filename suffix, eg, ".gz" is returned by {@link CompressionType#GZIP}
+ * @return an optional filename suffix, eg, ".gz" is returned for {@link Compression#GZIP}
*/
@Nullable
- String getFilenameSuffix();
+ String getSuggestedFilenameSuffix();
+ }
+
+ /**
+ * Implementations create instances of {@link WritableByteChannel} used by {@link FileBasedSink}
+ * and related classes to allow <em>decorating</em>, or otherwise transforming, the raw data that
+ * would normally be written directly to the {@link WritableByteChannel} passed into {@link
+ * WritableByteChannelFactory#create(WritableByteChannel)}.
+ *
+ * <p>Subclasses should override {@link #toString()} with something meaningful, as it is used when
+ * building {@link DisplayData}.
+ */
+ public interface WritableByteChannelFactory extends OutputFileHints {
+ /**
+ * @param channel the {@link WritableByteChannel} to wrap
+ * @return the {@link WritableByteChannel} to be used during output
+ */
+ WritableByteChannel create(WritableByteChannel channel) throws IOException;
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
index d4413c9..dabda84 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
@@ -23,19 +23,17 @@
import static com.google.common.base.Verify.verify;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import java.io.IOException;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SeekableByteChannel;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.io.fs.MatchResult.Status;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
@@ -68,6 +66,7 @@
private static final Logger LOG = LoggerFactory.getLogger(FileBasedSource.class);
private final ValueProvider<String> fileOrPatternSpec;
+ private final EmptyMatchTreatment emptyMatchTreatment;
@Nullable private MatchResult.Metadata singleFileMetadata;
private final Mode mode;
@@ -80,12 +79,25 @@
}
/**
- * Create a {@code FileBaseSource} based on a file or a file pattern specification.
+ * Create a {@code FileBaseSource} based on a file or a file pattern specification, with the given
+ * strategy for treating filepatterns that do not match any files.
+ */
+ protected FileBasedSource(
+ ValueProvider<String> fileOrPatternSpec,
+ EmptyMatchTreatment emptyMatchTreatment,
+ long minBundleSize) {
+ super(0, Long.MAX_VALUE, minBundleSize);
+ this.mode = Mode.FILEPATTERN;
+ this.emptyMatchTreatment = emptyMatchTreatment;
+ this.fileOrPatternSpec = fileOrPatternSpec;
+ }
+
+ /**
+ * Like {@link #FileBasedSource(ValueProvider, EmptyMatchTreatment, long)}, but uses the default
+ * value of {@link EmptyMatchTreatment#DISALLOW}.
*/
protected FileBasedSource(ValueProvider<String> fileOrPatternSpec, long minBundleSize) {
- super(0, Long.MAX_VALUE, minBundleSize);
- mode = Mode.FILEPATTERN;
- this.fileOrPatternSpec = fileOrPatternSpec;
+ this(fileOrPatternSpec, EmptyMatchTreatment.DISALLOW, minBundleSize);
}
/**
@@ -110,6 +122,9 @@
mode = Mode.SINGLE_FILE_OR_SUBRANGE;
this.singleFileMetadata = checkNotNull(fileMetadata, "fileMetadata");
this.fileOrPatternSpec = StaticValueProvider.of(fileMetadata.resourceId().toString());
+
+ // This field will be unused in this mode.
+ this.emptyMatchTreatment = null;
}
/**
@@ -139,6 +154,10 @@
return fileOrPatternSpec;
}
+ public final EmptyMatchTreatment getEmptyMatchTreatment() {
+ return emptyMatchTreatment;
+ }
+
public final Mode getMode() {
return mode;
}
@@ -196,22 +215,11 @@
// This implementation of method getEstimatedSizeBytes is provided to simplify subclasses. Here
// we perform the size estimation of files and file patterns using the interface provided by
// FileSystem.
- checkState(
- fileOrPatternSpec.isAccessible(),
- "Cannot estimate size of a FileBasedSource with inaccessible file pattern: {}.",
- fileOrPatternSpec);
String fileOrPattern = fileOrPatternSpec.get();
if (mode == Mode.FILEPATTERN) {
long totalSize = 0;
- List<MatchResult> inputs = FileSystems.match(Collections.singletonList(fileOrPattern));
- MatchResult result = Iterables.getOnlyElement(inputs);
- checkArgument(
- result.status() == Status.OK,
- "Error matching the pattern or glob %s: status %s",
- fileOrPattern,
- result.status());
- List<Metadata> allMatches = result.metadata();
+ List<Metadata> allMatches = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata();
for (Metadata metadata : allMatches) {
totalSize += metadata.sizeBytes();
}
@@ -232,10 +240,9 @@
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
if (mode == Mode.FILEPATTERN) {
- String patternDisplay = getFileOrPatternSpecProvider().isAccessible()
- ? getFileOrPatternSpecProvider().get()
- : getFileOrPatternSpecProvider().toString();
- builder.add(DisplayData.item("filePattern", patternDisplay).withLabel("File Pattern"));
+ builder.add(
+ DisplayData.item("filePattern", getFileOrPatternSpecProvider())
+ .withLabel("File Pattern"));
}
}
@@ -246,17 +253,12 @@
// split a FileBasedSource based on a file pattern to FileBasedSources based on full single
// files. For files that can be efficiently seeked, we further split FileBasedSources based on
// those files to FileBasedSources based on sub ranges of single files.
- checkState(
- fileOrPatternSpec.isAccessible(),
- "Cannot split a FileBasedSource without access to the file or pattern specification: {}.",
- fileOrPatternSpec);
String fileOrPattern = fileOrPatternSpec.get();
if (mode == Mode.FILEPATTERN) {
long startTime = System.currentTimeMillis();
- List<Metadata> expandedFiles = FileBasedSource.expandFilePattern(fileOrPattern);
- checkArgument(!expandedFiles.isEmpty(),
- "Unable to find any files matching %s", fileOrPattern);
+ List<Metadata> expandedFiles =
+ FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata();
List<FileBasedSource<T>> splitResults = new ArrayList<>(expandedFiles.size());
for (Metadata metadata : expandedFiles) {
FileBasedSource<T> split = createForSubrangeOfFile(metadata, 0, metadata.sizeBytes());
@@ -319,15 +321,13 @@
public final BoundedReader<T> createReader(PipelineOptions options) throws IOException {
// Validate the current source prior to creating a reader for it.
this.validate();
- checkState(
- fileOrPatternSpec.isAccessible(),
- "Cannot create a file reader without access to the file or pattern specification: {}.",
- fileOrPatternSpec);
String fileOrPattern = fileOrPatternSpec.get();
if (mode == Mode.FILEPATTERN) {
long startTime = System.currentTimeMillis();
- List<Metadata> fileMetadata = FileBasedSource.expandFilePattern(fileOrPattern);
+ List<Metadata> fileMetadata =
+ FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata();
+ LOG.info("Matched {} files for pattern {}", fileMetadata.size(), fileOrPattern);
List<FileBasedReader<T>> fileReaders = new ArrayList<>();
for (Metadata metadata : fileMetadata) {
long endOffset = metadata.sizeBytes();
@@ -349,13 +349,11 @@
@Override
public String toString() {
- String fileString = fileOrPatternSpec.isAccessible()
- ? fileOrPatternSpec.get() : fileOrPatternSpec.toString();
switch (mode) {
case FILEPATTERN:
- return fileString;
+ return fileOrPatternSpec.toString();
case SINGLE_FILE_OR_SUBRANGE:
- return fileString + " range " + super.toString();
+ return fileOrPatternSpec + " range " + super.toString();
default:
throw new IllegalStateException("Unexpected mode: " + mode);
}
@@ -389,13 +387,6 @@
return metadata.sizeBytes();
}
- private static List<Metadata> expandFilePattern(String fileOrPatternSpec) throws IOException {
- MatchResult matches =
- Iterables.getOnlyElement(FileSystems.match(Collections.singletonList(fileOrPatternSpec)));
- LOG.info("Matched {} files for pattern {}", matches.metadata().size(), fileOrPatternSpec);
- return ImmutableList.copyOf(matches.metadata());
- }
-
/**
* A {@link Source.Reader reader} that implements code common to readers of
* {@code FileBasedSource}s.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
new file mode 100644
index 0000000..a244c07
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.StandardCharsets;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.Watch;
+import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.util.StreamUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Transforms for working with files. Currently includes matching of filepatterns via {@link #match}
+ * and {@link #matchAll}, and reading matches via {@link #readMatches}.
+ */
+public class FileIO {
+ private static final Logger LOG = LoggerFactory.getLogger(FileIO.class);
+
+ /**
+ * Matches a filepattern using {@link FileSystems#match} and produces a collection of matched
+ * resources (both files and directories) as {@link MatchResult.Metadata}.
+ *
+ * <p>By default, matches the filepattern once and produces a bounded {@link PCollection}. To
+ * continuously watch the filepattern for new matches, use {@link MatchAll#continuously(Duration,
+ * TerminationCondition)} - this will produce an unbounded {@link PCollection}.
+ *
+ * <p>By default, a filepattern matching no resources is treated according to {@link
+ * EmptyMatchTreatment#DISALLOW}. To configure this behavior, use {@link
+ * Match#withEmptyMatchTreatment}.
+ */
+ public static Match match() {
+ return new AutoValue_FileIO_Match.Builder()
+ .setConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
+ .build();
+ }
+
+ /**
+ * Like {@link #match}, but matches each filepattern in a collection of filepatterns.
+ *
+ * <p>Resources are not deduplicated between filepatterns, i.e. if the same resource matches
+ * multiple filepatterns, it will be produced multiple times.
+ *
+ * <p>By default, a filepattern matching no resources is treated according to {@link
+ * EmptyMatchTreatment#ALLOW_IF_WILDCARD}. To configure this behavior, use {@link
+ * MatchAll#withEmptyMatchTreatment}.
+ */
+ public static MatchAll matchAll() {
+ return new AutoValue_FileIO_MatchAll.Builder()
+ .setConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
+ .build();
+ }
+
+ /**
+ * Converts each result of {@link #match} or {@link #matchAll} to a {@link ReadableFile} which can
+ * be used to read the contents of each file, optionally decompressing it.
+ */
+ public static ReadMatches readMatches() {
+ return new AutoValue_FileIO_ReadMatches.Builder()
+ .setCompression(Compression.AUTO)
+ .setDirectoryTreatment(ReadMatches.DirectoryTreatment.SKIP)
+ .build();
+ }
+
+ /** A utility class for accessing a potentially compressed file. */
+ public static final class ReadableFile {
+ private final MatchResult.Metadata metadata;
+ private final Compression compression;
+
+ ReadableFile(MatchResult.Metadata metadata, Compression compression) {
+ this.metadata = metadata;
+ this.compression = compression;
+ }
+
+ /** Returns the {@link MatchResult.Metadata} of the file. */
+ public MatchResult.Metadata getMetadata() {
+ return metadata;
+ }
+
+ /** Returns the method with which this file will be decompressed in {@link #open}. */
+ public Compression getCompression() {
+ return compression;
+ }
+
+ /**
+ * Returns a {@link ReadableByteChannel} reading the data from this file, potentially
+ * decompressing it using {@link #getCompression}.
+ */
+ public ReadableByteChannel open() throws IOException {
+ return compression.readDecompressed(FileSystems.open(metadata.resourceId()));
+ }
+
+ /**
+ * Returns a {@link SeekableByteChannel} equivalent to {@link #open}, but fails if this file is
+ * not {@link MatchResult.Metadata#isReadSeekEfficient seekable}.
+ */
+ public SeekableByteChannel openSeekable() throws IOException {
+ checkState(
+ getMetadata().isReadSeekEfficient(),
+ "The file %s is not seekable",
+ metadata.resourceId());
+ return ((SeekableByteChannel) open());
+ }
+
+ /** Returns the full contents of the file as bytes. */
+ public byte[] readFullyAsBytes() throws IOException {
+ return StreamUtils.getBytes(Channels.newInputStream(open()));
+ }
+
+ /** Returns the full contents of the file as a {@link String} decoded as UTF-8. */
+ public String readFullyAsUTF8String() throws IOException {
+ return new String(readFullyAsBytes(), StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public String toString() {
+ return "ReadableFile{metadata=" + metadata + ", compression=" + compression + '}';
+ }
+ }
+
+ /**
+ * Describes configuration for matching filepatterns, such as {@link EmptyMatchTreatment} and
+ * continuous watching for matching files.
+ */
+ @AutoValue
+ public abstract static class MatchConfiguration implements HasDisplayData, Serializable {
+ /** Creates a {@link MatchConfiguration} with the given {@link EmptyMatchTreatment}. */
+ public static MatchConfiguration create(EmptyMatchTreatment emptyMatchTreatment) {
+ return new AutoValue_FileIO_MatchConfiguration.Builder()
+ .setEmptyMatchTreatment(emptyMatchTreatment)
+ .build();
+ }
+
+ abstract EmptyMatchTreatment getEmptyMatchTreatment();
+
+ @Nullable
+ abstract Duration getWatchInterval();
+
+ @Nullable
+ abstract TerminationCondition<String, ?> getWatchTerminationCondition();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setEmptyMatchTreatment(EmptyMatchTreatment treatment);
+
+ abstract Builder setWatchInterval(Duration watchInterval);
+
+ abstract Builder setWatchTerminationCondition(TerminationCondition<String, ?> condition);
+
+ abstract MatchConfiguration build();
+ }
+
+ /** Sets the {@link EmptyMatchTreatment}. */
+ public MatchConfiguration withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return toBuilder().setEmptyMatchTreatment(treatment).build();
+ }
+
+ /**
+ * Continuously watches for new files at the given interval until the given termination
+ * condition is reached, where the input to the condition is the filepattern.
+ */
+ public MatchConfiguration continuously(
+ Duration interval, TerminationCondition<String, ?> condition) {
+ return toBuilder().setWatchInterval(interval).setWatchTerminationCondition(condition).build();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder
+ .add(
+ DisplayData.item("emptyMatchTreatment", getEmptyMatchTreatment().toString())
+ .withLabel("Treatment of filepatterns that match no files"))
+ .addIfNotNull(
+ DisplayData.item("watchForNewFilesInterval", getWatchInterval())
+ .withLabel("Interval to watch for new files"));
+ }
+ }
+
+ /** Implementation of {@link #match}. */
+ @AutoValue
+ public abstract static class Match extends PTransform<PBegin, PCollection<MatchResult.Metadata>> {
+ @Nullable
+ abstract ValueProvider<String> getFilepattern();
+
+ abstract MatchConfiguration getConfiguration();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setFilepattern(ValueProvider<String> filepattern);
+
+ abstract Builder setConfiguration(MatchConfiguration configuration);
+
+ abstract Match build();
+ }
+
+ /** Matches the given filepattern. */
+ public Match filepattern(String filepattern) {
+ return this.filepattern(ValueProvider.StaticValueProvider.of(filepattern));
+ }
+
+ /** Like {@link #filepattern(String)} but using a {@link ValueProvider}. */
+ public Match filepattern(ValueProvider<String> filepattern) {
+ return toBuilder().setFilepattern(filepattern).build();
+ }
+
+ /** Sets the {@link MatchConfiguration}. */
+ public Match withConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setConfiguration(configuration).build();
+ }
+
+ /** See {@link MatchConfiguration#withEmptyMatchTreatment(EmptyMatchTreatment)}. */
+ public Match withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withConfiguration(getConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /**
+ * See {@link MatchConfiguration#continuously}. The returned {@link PCollection} is unbounded.
+ *
+ * <p>This works only in runners supporting {@link Experimental.Kind#SPLITTABLE_DO_FN}.
+ */
+ @Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+ public Match continuously(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withConfiguration(getConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ @Override
+ public PCollection<MatchResult.Metadata> expand(PBegin input) {
+ return input
+ .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of()))
+ .apply("Via MatchAll", matchAll().withConfiguration(getConfiguration()));
+ }
+ }
+
+ /** Implementation of {@link #matchAll}. */
+ @AutoValue
+ public abstract static class MatchAll
+ extends PTransform<PCollection<String>, PCollection<MatchResult.Metadata>> {
+ abstract MatchConfiguration getConfiguration();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConfiguration(MatchConfiguration configuration);
+
+ abstract MatchAll build();
+ }
+
+ /** Like {@link Match#withConfiguration}. */
+ public MatchAll withConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setConfiguration(configuration).build();
+ }
+
+ /** Like {@link Match#withEmptyMatchTreatment}. */
+ public MatchAll withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withConfiguration(getConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /** Like {@link Match#continuously}. */
+ @Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+ public MatchAll continuously(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withConfiguration(getConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ @Override
+ public PCollection<MatchResult.Metadata> expand(PCollection<String> input) {
+ PCollection<MatchResult.Metadata> res;
+ if (getConfiguration().getWatchInterval() == null) {
+ res = input.apply(
+ "Match filepatterns",
+ ParDo.of(new MatchFn(getConfiguration().getEmptyMatchTreatment())));
+ } else {
+ res = input
+ .apply(
+ "Continuously match filepatterns",
+ Watch.growthOf(new MatchPollFn())
+ .withPollInterval(getConfiguration().getWatchInterval())
+ .withTerminationPerInput(getConfiguration().getWatchTerminationCondition()))
+ .apply(Values.<MatchResult.Metadata>create());
+ }
+ return res.apply(Reshuffle.<MatchResult.Metadata>viaRandomKey());
+ }
+
+ private static class MatchFn extends DoFn<String, MatchResult.Metadata> {
+ private final EmptyMatchTreatment emptyMatchTreatment;
+
+ public MatchFn(EmptyMatchTreatment emptyMatchTreatment) {
+ this.emptyMatchTreatment = emptyMatchTreatment;
+ }
+
+ @ProcessElement
+ public void process(ProcessContext c) throws Exception {
+ String filepattern = c.element();
+ MatchResult match = FileSystems.match(filepattern, emptyMatchTreatment);
+ LOG.info("Matched {} files for pattern {}", match.metadata().size(), filepattern);
+ for (MatchResult.Metadata metadata : match.metadata()) {
+ c.output(metadata);
+ }
+ }
+ }
+
+ private static class MatchPollFn extends Watch.Growth.PollFn<String, MatchResult.Metadata> {
+ @Override
+ public Watch.Growth.PollResult<MatchResult.Metadata> apply(String element, Context c)
+ throws Exception {
+ return Watch.Growth.PollResult.incomplete(
+ Instant.now(), FileSystems.match(element, EmptyMatchTreatment.ALLOW).metadata());
+ }
+ }
+ }
+
+ /** Implementation of {@link #readMatches}. */
+ @AutoValue
+ public abstract static class ReadMatches
+ extends PTransform<PCollection<MatchResult.Metadata>, PCollection<ReadableFile>> {
+ enum DirectoryTreatment {
+ SKIP,
+ PROHIBIT
+ }
+
+ abstract Compression getCompression();
+
+ abstract DirectoryTreatment getDirectoryTreatment();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setCompression(Compression compression);
+
+ abstract Builder setDirectoryTreatment(DirectoryTreatment directoryTreatment);
+
+ abstract ReadMatches build();
+ }
+
+ /** Reads files using the given {@link Compression}. Default is {@link Compression#AUTO}. */
+ public ReadMatches withCompression(Compression compression) {
+ checkArgument(compression != null, "compression can not be null");
+ return toBuilder().setCompression(compression).build();
+ }
+
+ /**
+ * Controls how to handle directories in the input {@link PCollection}. Default is {@link
+ * DirectoryTreatment#SKIP}.
+ */
+ public ReadMatches withDirectoryTreatment(DirectoryTreatment directoryTreatment) {
+ checkArgument(directoryTreatment != null, "directoryTreatment can not be null");
+ return toBuilder().setDirectoryTreatment(directoryTreatment).build();
+ }
+
+ @Override
+ public PCollection<ReadableFile> expand(PCollection<MatchResult.Metadata> input) {
+ return input.apply(ParDo.of(new ToReadableFileFn(this)));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("compression", getCompression().toString()));
+ builder.add(DisplayData.item("directoryTreatment", getDirectoryTreatment().toString()));
+ }
+
+ private static class ToReadableFileFn extends DoFn<MatchResult.Metadata, ReadableFile> {
+ private final ReadMatches spec;
+
+ private ToReadableFileFn(ReadMatches spec) {
+ this.spec = spec;
+ }
+
+ @ProcessElement
+ public void process(ProcessContext c) {
+ MatchResult.Metadata metadata = c.element();
+ if (metadata.resourceId().isDirectory()) {
+ switch (spec.getDirectoryTreatment()) {
+ case SKIP:
+ return;
+
+ case PROHIBIT:
+ throw new IllegalArgumentException(
+ "Trying to read " + metadata.resourceId() + " which is a directory");
+
+ default:
+ throw new UnsupportedOperationException(
+ "Unknown DirectoryTreatment: " + spec.getDirectoryTreatment());
+ }
+ }
+
+ Compression compression =
+ (spec.getCompression() == Compression.AUTO)
+ ? Compression.detect(metadata.resourceId().getFilename())
+ : spec.getCompression();
+ c.output(
+ new ReadableFile(
+ MatchResult.Metadata.builder()
+ .setResourceId(metadata.resourceId())
+ .setSizeBytes(metadata.sizeBytes())
+ .setIsReadSeekEfficient(
+ metadata.isReadSeekEfficient() && compression == Compression.UNCOMPRESSED)
+ .build(),
+ compression));
+ }
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 1aacc90..96394b8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -54,6 +54,7 @@
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.io.fs.CreateOptions;
import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.MatchResult.Status;
@@ -69,16 +70,23 @@
@Experimental(Kind.FILESYSTEM)
public class FileSystems {
- public static final String DEFAULT_SCHEME = "default";
+ public static final String DEFAULT_SCHEME = "file";
private static final Pattern FILE_SCHEME_PATTERN =
Pattern.compile("(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*):.*");
+ private static final Pattern GLOB_PATTERN =
+ Pattern.compile("[*?{}]");
private static final AtomicReference<Map<String, FileSystem>> SCHEME_TO_FILESYSTEM =
new AtomicReference<Map<String, FileSystem>>(
- ImmutableMap.<String, FileSystem>of("file", new LocalFileSystem()));
+ ImmutableMap.<String, FileSystem>of(DEFAULT_SCHEME, new LocalFileSystem()));
/********************************** METHODS FOR CLIENT **********************************/
+ /** Checks whether the given spec contains a glob wildcard character. */
+ public static boolean hasGlobWildcard(String spec) {
+ return GLOB_PATTERN.matcher(spec).find();
+ }
+
/**
* This is the entry point to convert user-provided specs to {@link ResourceId ResourceIds}.
* Callers should use {@link #match} to resolve users specs ambiguities before
@@ -99,6 +107,12 @@
* component of {@link ResourceId}. This allows SDK libraries to construct file system agnostic
* spec. {@link FileSystem FileSystems} can support additional patterns for user-provided specs.
*
+ * <p>In case the spec schemes don't match any known {@link FileSystem} implementations,
+ * FileSystems will attempt to use {@link LocalFileSystem} to resolve a path.
+ *
+ * <p>Specs that do not match any resources are treated according to
+ * {@link EmptyMatchTreatment#DISALLOW}.
+ *
* @return {@code List<MatchResult>} in the same order of the input specs.
*
* @throws IllegalArgumentException if specs are invalid -- empty or have different schemes.
@@ -111,6 +125,17 @@
return getFileSystemInternal(getOnlyScheme(specs)).match(specs);
}
+ /** Like {@link #match(List)}, but with a configurable {@link EmptyMatchTreatment}. */
+ public static List<MatchResult> match(List<String> specs, EmptyMatchTreatment emptyMatchTreatment)
+ throws IOException {
+ List<MatchResult> matches = getFileSystemInternal(getOnlyScheme(specs)).match(specs);
+ List<MatchResult> res = Lists.newArrayListWithExpectedSize(matches.size());
+ for (int i = 0; i < matches.size(); i++) {
+ res.add(maybeAdjustEmptyMatchResult(specs.get(i), matches.get(i), emptyMatchTreatment));
+ }
+ return res;
+ }
+
/**
* Like {@link #match(List)}, but for a single resource specification.
@@ -127,6 +152,30 @@
matches);
return matches.get(0);
}
+
+ /** Like {@link #match(String)}, but with a configurable {@link EmptyMatchTreatment}. */
+ public static MatchResult match(String spec, EmptyMatchTreatment emptyMatchTreatment)
+ throws IOException {
+ MatchResult res = match(spec);
+ return maybeAdjustEmptyMatchResult(spec, res, emptyMatchTreatment);
+ }
+
+ private static MatchResult maybeAdjustEmptyMatchResult(
+ String spec, MatchResult res, EmptyMatchTreatment emptyMatchTreatment)
+ throws IOException {
+ if (res.status() != Status.NOT_FOUND) {
+ return res;
+ }
+ boolean notFoundAllowed =
+ emptyMatchTreatment == EmptyMatchTreatment.ALLOW
+ || (FileSystems.hasGlobWildcard(spec)
+ && emptyMatchTreatment == EmptyMatchTreatment.ALLOW_IF_WILDCARD);
+ if (notFoundAllowed) {
+ return MatchResult.create(Status.OK, Collections.<Metadata>emptyList());
+ }
+ return res;
+ }
+
/**
* Returns the {@link Metadata} for a single file resource. Expects a resource specification
* {@code spec} that matches a single result.
@@ -176,7 +225,7 @@
.transform(new Function<ResourceId, String>() {
@Override
public String apply(@Nonnull ResourceId resourceId) {
- return resourceId.toString();
+ return resourceId.toString();
}})
.toList());
}
@@ -423,7 +472,7 @@
Matcher matcher = FILE_SCHEME_PATTERN.matcher(spec);
if (!matcher.matches()) {
- return "file";
+ return DEFAULT_SCHEME;
} else {
return matcher.group("scheme").toLowerCase();
}
@@ -440,21 +489,11 @@
if (rval != null) {
return rval;
}
- rval = schemeToFileSystem.get(DEFAULT_SCHEME);
- if (rval != null) {
- return rval;
- }
- throw new IllegalStateException("Unable to find registrar for " + scheme);
+ return schemeToFileSystem.get(DEFAULT_SCHEME);
}
/********************************** METHODS FOR REGISTRATION **********************************/
- /** @deprecated to be removed. */
- @Deprecated // for DataflowRunner backwards compatibility.
- public static void setDefaultConfigInWorkers(PipelineOptions options) {
- setDefaultPipelineOptions(options);
- }
-
/**
* Sets the default configuration in workers.
*
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java
index 189539f..854a8bb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java
@@ -18,7 +18,6 @@
package org.apache.beam.sdk.io;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
import com.google.auto.value.AutoValue;
import javax.annotation.Nullable;
@@ -127,7 +126,7 @@
numElements > 0,
"Number of elements in withRate must be positive, but was: %s",
numElements);
- checkNotNull(periodLength, "periodLength in withRate must be non-null");
+ checkArgument(periodLength != null, "periodLength can not be null");
return toBuilder().setElementsPerPeriod(numElements).setPeriod(periodLength).build();
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java
index b732bee..3891b91 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java
@@ -34,10 +34,12 @@
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -46,11 +48,32 @@
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.MatchResult.Status;
+import org.apache.commons.lang3.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* {@link FileSystem} implementation for local files.
+ *
+ * {@link #match} should interpret {@code spec} and resolve paths correctly according to OS being
+ * used. In order to do that specs should be defined in one of the below formats:
+ *
+ * <p>Linux/Mac:
+ * <ul>
+ * <li>pom.xml</li>
+ * <li>/Users/beam/Documents/pom.xml</li>
+ * <li>file:/Users/beam/Documents/pom.xml</li>
+ * <li>file:///Users/beam/Documents/pom.xml</li>
+ * </ul>
+ *
+ * <p>Windows OS:
+ * <ul>
+ * <li>pom.xml</li>
+ * <li>C:/Users/beam/Documents/pom.xml</li>
+ * <li>C:\\Users\\beam\\Documents\\pom.xml</li>
+ * <li>file:/C:/Users/beam/Documents/pom.xml</li>
+ * <li>file:///C:/Users/beam/Documents/pom.xml</li>
+ * </ul>
*/
class LocalFileSystem extends FileSystem<LocalResourceId> {
@@ -159,8 +182,12 @@
@Override
protected void delete(Collection<LocalResourceId> resourceIds) throws IOException {
for (LocalResourceId resourceId : resourceIds) {
- LOG.debug("Deleting file {}", resourceId);
- Files.delete(resourceId.getPath());
+ try {
+ Files.delete(resourceId.getPath());
+ } catch (NoSuchFileException e) {
+ LOG.info("Ignoring failed deletion of file {} which already does not exist: {}", resourceId,
+ e);
+ }
}
}
@@ -176,8 +203,20 @@
}
private MatchResult matchOne(String spec) throws IOException {
- File file = Paths.get(spec).toFile();
+ if (spec.toLowerCase().startsWith("file:")) {
+ spec = spec.substring("file:".length());
+ }
+ if (SystemUtils.IS_OS_WINDOWS) {
+ List<String> prefixes = Arrays.asList("///", "/");
+ for (String prefix : prefixes) {
+ if (spec.toLowerCase().startsWith(prefix)) {
+ spec = spec.substring(prefix.length());
+ }
+ }
+ }
+
+ File file = Paths.get(spec).toFile();
if (file.exists()) {
return MatchResult.create(Status.OK, ImmutableList.of(toMetadata(file)));
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
index 05f0d97..c3687a9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
+import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.io.range.OffsetRangeTracker;
import org.apache.beam.sdk.io.range.RangeTracker;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -110,8 +111,7 @@
@Override
public List<? extends OffsetBasedSource<T>> split(
long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
- // Split the range into bundles based on the desiredBundleSizeBytes. Final bundle is adjusted to
- // make sure that we do not end up with a too small bundle at the end. If the desired bundle
+ // Split the range into bundles based on the desiredBundleSizeBytes. If the desired bundle
// size is smaller than the minBundleSize of the source then minBundleSize will be used instead.
long desiredBundleSizeOffsetUnits = Math.max(
@@ -119,20 +119,10 @@
minBundleSize);
List<OffsetBasedSource<T>> subSources = new ArrayList<>();
- long start = startOffset;
- long maxEnd = Math.min(endOffset, getMaxEndOffset(options));
-
- while (start < maxEnd) {
- long end = start + desiredBundleSizeOffsetUnits;
- end = Math.min(end, maxEnd);
- // Avoid having a too small bundle at the end and ensure that we respect minBundleSize.
- long remaining = maxEnd - end;
- if ((remaining < desiredBundleSizeOffsetUnits / 4) || (remaining < minBundleSize)) {
- end = maxEnd;
- }
- subSources.add(createSourceForSubrange(start, end));
-
- start = end;
+ for (OffsetRange range :
+ new OffsetRange(startOffset, Math.min(endOffset, getMaxEndOffset(options)))
+ .split(desiredBundleSizeOffsetUnits, minBundleSize)) {
+ subSources.add(createSourceForSubrange(range.getFrom(), range.getTo()));
}
return subSources;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
index a07fca8..9b273f8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
@@ -18,7 +18,6 @@
package org.apache.beam.sdk.io;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.NameUtils;
@@ -95,17 +94,14 @@
}
@Override
- protected Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
- }
-
- @Override
public final PCollection<T> expand(PBegin input) {
source.validate();
- return PCollection.<T>createPrimitiveOutputInternal(input.getPipeline(),
- WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
- .setCoder(getDefaultOutputCoder());
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ source.getOutputCoder());
}
/**
@@ -163,16 +159,13 @@
}
@Override
- protected Coder<T> getDefaultOutputCoder() {
- return source.getDefaultOutputCoder();
- }
-
- @Override
public final PCollection<T> expand(PBegin input) {
source.validate();
-
- return PCollection.<T>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED);
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ IsBounded.UNBOUNDED,
+ source.getOutputCoder());
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadAllViaFileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadAllViaFileBasedSource.java
new file mode 100644
index 0000000..c53f405
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadAllViaFileBasedSource.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import java.io.IOException;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Reads each file in the input {@link PCollection} of {@link ReadableFile} using given parameters
+ * for splitting files into offset ranges and for creating a {@link FileBasedSource} for a file. The
+ * input {@link PCollection} must not contain {@link ResourceId#isDirectory directories}.
+ *
+ * <p>To obtain the collection of {@link ReadableFile} from a filepattern, use {@link
+ * FileIO#readMatches()}.
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class ReadAllViaFileBasedSource<T>
+ extends PTransform<PCollection<ReadableFile>, PCollection<T>> {
+ private final long desiredBundleSizeBytes;
+ private final SerializableFunction<String, ? extends FileBasedSource<T>> createSource;
+ private final Coder<T> coder;
+
+ public ReadAllViaFileBasedSource(
+ long desiredBundleSizeBytes,
+ SerializableFunction<String, ? extends FileBasedSource<T>> createSource,
+ Coder<T> coder) {
+ this.desiredBundleSizeBytes = desiredBundleSizeBytes;
+ this.createSource = createSource;
+ this.coder = coder;
+ }
+
+ @Override
+ public PCollection<T> expand(PCollection<ReadableFile> input) {
+ return input
+ .apply("Split into ranges", ParDo.of(new SplitIntoRangesFn(desiredBundleSizeBytes)))
+ .apply("Reshuffle", Reshuffle.<KV<ReadableFile, OffsetRange>>viaRandomKey())
+ .apply("Read ranges", ParDo.of(new ReadFileRangesFn<T>(createSource)))
+ .setCoder(coder);
+ }
+
+ private static class SplitIntoRangesFn extends DoFn<ReadableFile, KV<ReadableFile, OffsetRange>> {
+ private final long desiredBundleSizeBytes;
+
+ private SplitIntoRangesFn(long desiredBundleSizeBytes) {
+ this.desiredBundleSizeBytes = desiredBundleSizeBytes;
+ }
+
+ @ProcessElement
+ public void process(ProcessContext c) {
+ Metadata metadata = c.element().getMetadata();
+ if (!metadata.isReadSeekEfficient()) {
+ c.output(KV.of(c.element(), new OffsetRange(0, metadata.sizeBytes())));
+ return;
+ }
+ for (OffsetRange range :
+ new OffsetRange(0, metadata.sizeBytes()).split(desiredBundleSizeBytes, 0)) {
+ c.output(KV.of(c.element(), range));
+ }
+ }
+ }
+
+ private static class ReadFileRangesFn<T> extends DoFn<KV<ReadableFile, OffsetRange>, T> {
+ private final SerializableFunction<String, ? extends FileBasedSource<T>> createSource;
+
+ private ReadFileRangesFn(
+ SerializableFunction<String, ? extends FileBasedSource<T>> createSource) {
+ this.createSource = createSource;
+ }
+
+ @ProcessElement
+ public void process(ProcessContext c) throws IOException {
+ ReadableFile file = c.element().getKey();
+ OffsetRange range = c.element().getValue();
+ FileBasedSource<T> source =
+ CompressedSource.from(createSource.apply(file.getMetadata().resourceId().toString()))
+ .withCompression(file.getCompression());
+ try (BoundedSource.BoundedReader<T> reader =
+ source
+ .createForSubrangeOfFile(file.getMetadata(), range.getFrom(), range.getTo())
+ .createReader(c.getPipelineOptions())) {
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ c.output(reader.getCurrent());
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadableFileCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadableFileCoder.java
new file mode 100644
index 0000000..51bb83e
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadableFileCoder.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.io.fs.MatchResult;
+import org.apache.beam.sdk.io.fs.MetadataCoder;
+
+/** A {@link Coder} for {@link FileIO.ReadableFile}. */
+public class ReadableFileCoder extends AtomicCoder<FileIO.ReadableFile> {
+ private static final ReadableFileCoder INSTANCE = new ReadableFileCoder();
+
+ /** Returns the instance of {@link ReadableFileCoder}. */
+ public static ReadableFileCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(FileIO.ReadableFile value, OutputStream os) throws IOException {
+ MetadataCoder.of().encode(value.getMetadata(), os);
+ VarIntCoder.of().encode(value.getCompression().ordinal(), os);
+ }
+
+ @Override
+ public FileIO.ReadableFile decode(InputStream is) throws IOException {
+ MatchResult.Metadata metadata = MetadataCoder.of().decode(is);
+ Compression compression = Compression.values()[VarIntCoder.of().decode(is)];
+ return new FileIO.ReadableFile(metadata, compression);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
index 542d91c..f578715 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
@@ -59,12 +59,30 @@
* <p>It is recommended to use {@link com.google.common.base.Preconditions} for implementing
* this method.
*/
- public abstract void validate();
+ public void validate() {}
- /**
- * Returns the default {@code Coder} to use for the data read from this source.
- */
- public abstract Coder<T> getDefaultOutputCoder();
+ /** @deprecated Override {@link #getOutputCoder()} instead. */
+ @Deprecated
+ public Coder<T> getDefaultOutputCoder() {
+ // If the subclass doesn't override getDefaultOutputCoder(), hopefully it overrides the proper
+ // version - getOutputCoder(). Check that it does, before calling the method (if subclass
+ // doesn't override it, we'll call the default implementation and get infinite recursion).
+ try {
+ if (getClass().getMethod("getOutputCoder").getDeclaringClass().equals(Source.class)) {
+ throw new UnsupportedOperationException(
+ getClass() + " needs to override getOutputCoder().");
+ }
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ return getOutputCoder();
+ }
+
+ /** Returns the {@code Coder} to use for the data read from this source. */
+ public Coder<T> getOutputCoder() {
+ // Call the old method for compatibility.
+ return getDefaultOutputCoder();
+ }
/**
* {@inheritDoc}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index c274595..55287ca 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -35,8 +35,6 @@
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.Read.Bounded;
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -44,7 +42,6 @@
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.values.PBegin;
@@ -66,7 +63,7 @@
public static Read read() {
return new AutoValue_TFRecordIO_Read.Builder()
.setValidate(true)
- .setCompressionType(CompressionType.AUTO)
+ .setCompression(Compression.AUTO)
.build();
}
@@ -80,7 +77,7 @@
.setShardTemplate(null)
.setFilenameSuffix(null)
.setNumShards(0)
- .setCompressionType(CompressionType.NONE)
+ .setCompression(Compression.UNCOMPRESSED)
.build();
}
@@ -92,7 +89,7 @@
abstract boolean getValidate();
- abstract CompressionType getCompressionType();
+ abstract Compression getCompression();
abstract Builder toBuilder();
@@ -100,7 +97,7 @@
abstract static class Builder {
abstract Builder setFilepattern(ValueProvider<String> filepattern);
abstract Builder setValidate(boolean validate);
- abstract Builder setCompressionType(CompressionType compressionType);
+ abstract Builder setCompression(Compression compression);
abstract Read build();
}
@@ -136,18 +133,22 @@
return toBuilder().setValidate(false).build();
}
- /**
- * Returns a transform for reading TFRecord files that decompresses all input files
- * using the specified compression type.
- *
- * <p>If no compression type is specified, the default is
- * {@link TFRecordIO.CompressionType#AUTO}.
- * In this mode, the compression type of the file is determined by its extension
- * (e.g., {@code *.gz} is gzipped, {@code *.zlib} is zlib compressed, and all other
- * extensions are uncompressed).
- */
+ /** @deprecated Use {@link #withCompression}. */
+ @Deprecated
public Read withCompressionType(TFRecordIO.CompressionType compressionType) {
- return toBuilder().setCompressionType(compressionType).build();
+ return withCompression(compressionType.canonical);
+ }
+
+ /**
+ * Returns a transform for reading TFRecord files that decompresses all input files using the
+ * specified compression type.
+ *
+ * <p>If no compression type is specified, the default is {@link Compression#AUTO}. In this
+ * mode, the compression type of the file is determined by its extension via {@link
+ * Compression#detect(String)}.
+ */
+ public Read withCompression(Compression compression) {
+ return toBuilder().setCompression(compression).build();
}
@Override
@@ -171,52 +172,26 @@
}
}
- final Bounded<byte[]> read = org.apache.beam.sdk.io.Read.from(getSource());
- PCollection<byte[]> pcol = input.getPipeline().apply("Read", read);
- // Honor the default output coder that would have been used by this PTransform.
- pcol.setCoder(getDefaultOutputCoder());
- return pcol;
+ return input.apply("Read", org.apache.beam.sdk.io.Read.from(getSource()));
}
// Helper to create a source specific to the requested compression type.
protected FileBasedSource<byte[]> getSource() {
- switch (getCompressionType()) {
- case NONE:
- return new TFRecordSource(getFilepattern());
- case AUTO:
- return CompressedSource.from(new TFRecordSource(getFilepattern()));
- case GZIP:
- return
- CompressedSource.from(new TFRecordSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.GZIP);
- case ZLIB:
- return
- CompressedSource.from(new TFRecordSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.DEFLATE);
- default:
- throw new IllegalArgumentException("Unknown compression type: " + getCompressionType());
- }
+ return CompressedSource.from(new TFRecordSource(getFilepattern()))
+ .withCompression(getCompression());
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
-
- String filepatternDisplay = getFilepattern().isAccessible()
- ? getFilepattern().get() : getFilepattern().toString();
builder
- .add(DisplayData.item("compressionType", getCompressionType().toString())
+ .add(DisplayData.item("compressionType", getCompression().toString())
.withLabel("Compression Type"))
.addIfNotDefault(DisplayData.item("validation", getValidate())
.withLabel("Validation Enabled"), true)
- .addIfNotNull(DisplayData.item("filePattern", filepatternDisplay)
+ .addIfNotNull(DisplayData.item("filePattern", getFilepattern())
.withLabel("File Pattern"));
}
-
- @Override
- protected Coder<byte[]> getDefaultOutputCoder() {
- return ByteArrayCoder.of();
- }
}
/////////////////////////////////////////////////////////////////////////////
@@ -237,7 +212,7 @@
@Nullable abstract String getShardTemplate();
/** Option to indicate the output sink's compression type. Default is NONE. */
- abstract CompressionType getCompressionType();
+ abstract Compression getCompression();
abstract Builder toBuilder();
@@ -251,7 +226,7 @@
abstract Builder setNumShards(int numShards);
- abstract Builder setCompressionType(CompressionType compressionType);
+ abstract Builder setCompression(Compression compression);
abstract Write build();
}
@@ -340,46 +315,45 @@
return withNumShards(1).withShardNameTemplate("");
}
+ /** @deprecated use {@link #withCompression}. */
+ @Deprecated
+ public Write withCompressionType(CompressionType compressionType) {
+ return withCompression(compressionType.canonical);
+ }
+
/**
* Writes to output files using the specified compression type.
*
- * <p>If no compression type is specified, the default is
- * {@link TFRecordIO.CompressionType#NONE}.
- * See {@link TFRecordIO.Read#withCompressionType} for more details.
+ * <p>If no compression type is specified, the default is {@link Compression#UNCOMPRESSED}. See
+ * {@link TFRecordIO.Read#withCompression} for more details.
*/
- public Write withCompressionType(CompressionType compressionType) {
- return toBuilder().setCompressionType(compressionType).build();
+ public Write withCompression(Compression compression) {
+ return toBuilder().setCompression(compression).build();
}
@Override
public PDone expand(PCollection<byte[]> input) {
checkState(getOutputPrefix() != null,
"need to set the output prefix of a TFRecordIO.Write transform");
- WriteFiles<byte[]> write = WriteFiles.to(
+ WriteFiles<byte[], Void, byte[]> write =
+ WriteFiles.to(
new TFRecordSink(
getOutputPrefix(),
getShardTemplate(),
getFilenameSuffix(),
- getCompressionType()));
+ getCompression()));
if (getNumShards() > 0) {
write = write.withNumShards(getNumShards());
}
- return input.apply("Write", write);
+ input.apply("Write", write);
+ return PDone.in(input.getPipeline());
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
-
- String outputPrefixString = null;
- if (getOutputPrefix().isAccessible()) {
- ResourceId dir = getOutputPrefix().get();
- outputPrefixString = dir.toString();
- } else {
- outputPrefixString = getOutputPrefix().toString();
- }
builder
- .add(DisplayData.item("filePrefix", outputPrefixString)
+ .add(DisplayData.item("filePrefix", getOutputPrefix())
.withLabel("Output File Prefix"))
.addIfNotNull(DisplayData.item("fileSuffix", getFilenameSuffix())
.withLabel("Output File Suffix"))
@@ -387,50 +361,35 @@
.withLabel("Output Shard Name Template"))
.addIfNotDefault(DisplayData.item("numShards", getNumShards())
.withLabel("Maximum Output Shards"), 0)
- .add(DisplayData.item("compressionType", getCompressionType().toString())
+ .add(DisplayData.item("compressionType", getCompression().toString())
.withLabel("Compression Type"));
}
-
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
}
- /**
- * Possible TFRecord file compression types.
- */
+ /** @deprecated Use {@link Compression}. */
+ @Deprecated
public enum CompressionType {
- /**
- * Automatically determine the compression type based on filename extension.
- */
- AUTO(""),
- /**
- * Uncompressed.
- */
- NONE(""),
- /**
- * GZipped.
- */
- GZIP(".gz"),
- /**
- * ZLIB compressed.
- */
- ZLIB(".zlib");
+ /** @see Compression#AUTO */
+ AUTO(Compression.AUTO),
- private String filenameSuffix;
+ /** @see Compression#UNCOMPRESSED */
+ NONE(Compression.UNCOMPRESSED),
- CompressionType(String suffix) {
- this.filenameSuffix = suffix;
+ /** @see Compression#GZIP */
+ GZIP(Compression.GZIP),
+
+ /** @see Compression#DEFLATE */
+ ZLIB(Compression.DEFLATE);
+
+ private Compression canonical;
+
+ CompressionType(Compression canonical) {
+ this.canonical = canonical;
}
- /**
- * Determine if a given filename matches a compression type based on its extension.
- * @param filename the filename to match
- * @return true iff the filename ends with the compression type's known extension.
- */
+ /** @see Compression#matches */
public boolean matches(String filename) {
- return filename.toLowerCase().endsWith(filenameSuffix.toLowerCase());
+ return canonical.matches(filename);
}
}
@@ -445,11 +404,6 @@
@VisibleForTesting
static class TFRecordSource extends FileBasedSource<byte[]> {
@VisibleForTesting
- TFRecordSource(String fileSpec) {
- super(StaticValueProvider.of(fileSpec), 1L);
- }
-
- @VisibleForTesting
TFRecordSource(ValueProvider<String> fileSpec) {
super(fileSpec, Long.MAX_VALUE);
}
@@ -473,12 +427,12 @@
}
@Override
- public Coder<byte[]> getDefaultOutputCoder() {
+ public Coder<byte[]> getOutputCoder() {
return DEFAULT_BYTE_ARRAY_CODER;
}
@Override
- protected boolean isSplittable() throws Exception {
+ protected boolean isSplittable() {
// TFRecord files are not splittable
return false;
}
@@ -546,74 +500,46 @@
}
}
- /**
- * A {@link FileBasedSink} for TFRecord files. Produces TFRecord files.
- */
+ /** A {@link FileBasedSink} for TFRecord files. Produces TFRecord files. */
@VisibleForTesting
- static class TFRecordSink extends FileBasedSink<byte[]> {
+ static class TFRecordSink extends FileBasedSink<byte[], Void, byte[]> {
@VisibleForTesting
- TFRecordSink(ValueProvider<ResourceId> outputPrefix,
+ TFRecordSink(
+ ValueProvider<ResourceId> outputPrefix,
@Nullable String shardTemplate,
@Nullable String suffix,
- TFRecordIO.CompressionType compressionType) {
+ Compression compression) {
super(
outputPrefix,
- DefaultFilenamePolicy.constructUsingStandardParameters(
- outputPrefix, shardTemplate, suffix),
- writableByteChannelFactory(compressionType));
- }
-
- private static class ExtractDirectory implements SerializableFunction<ResourceId, ResourceId> {
- @Override
- public ResourceId apply(ResourceId input) {
- return input.getCurrentDirectory();
- }
+ DynamicFileDestinations.<byte[]>constant(
+ DefaultFilenamePolicy.fromStandardParameters(
+ outputPrefix, shardTemplate, suffix, false)),
+ compression);
}
@Override
- public WriteOperation<byte[]> createWriteOperation() {
+ public WriteOperation<Void, byte[]> createWriteOperation() {
return new TFRecordWriteOperation(this);
}
- private static WritableByteChannelFactory writableByteChannelFactory(
- TFRecordIO.CompressionType compressionType) {
- switch (compressionType) {
- case AUTO:
- throw new IllegalArgumentException("Unsupported compression type AUTO");
- case NONE:
- return CompressionType.UNCOMPRESSED;
- case GZIP:
- return CompressionType.GZIP;
- case ZLIB:
- return CompressionType.DEFLATE;
- }
- return CompressionType.UNCOMPRESSED;
- }
-
- /**
- * A {@link WriteOperation
- * WriteOperation} for TFRecord files.
- */
- private static class TFRecordWriteOperation extends WriteOperation<byte[]> {
+ /** A {@link WriteOperation WriteOperation} for TFRecord files. */
+ private static class TFRecordWriteOperation extends WriteOperation<Void, byte[]> {
private TFRecordWriteOperation(TFRecordSink sink) {
super(sink);
}
@Override
- public Writer<byte[]> createWriter() throws Exception {
+ public Writer<Void, byte[]> createWriter() throws Exception {
return new TFRecordWriter(this);
}
}
- /**
- * A {@link Writer Writer}
- * for TFRecord files.
- */
- private static class TFRecordWriter extends Writer<byte[]> {
+ /** A {@link Writer Writer} for TFRecord files. */
+ private static class TFRecordWriter extends Writer<Void, byte[]> {
private WritableByteChannel outChannel;
private TFRecordCodec codec;
- private TFRecordWriter(WriteOperation<byte[]> writeOperation) {
+ private TFRecordWriter(WriteOperation<Void, byte[]> writeOperation) {
super(writeOperation, MimeTypes.BINARY);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index 5c068ce..fb01634 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -20,40 +20,67 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment;
import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
+import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
-import org.apache.beam.sdk.io.Read.Bounded;
+import org.apache.beam.sdk.io.FileIO.MatchConfiguration;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
/**
* {@link PTransform}s for reading and writing text files.
*
+ * <h2>Reading text files</h2>
+ *
* <p>To read a {@link PCollection} from one or more text files, use {@code TextIO.read()} to
* instantiate a transform and use {@link TextIO.Read#from(String)} to specify the path of the
- * file(s) to be read.
+ * file(s) to be read. Alternatively, if the filenames to be read are themselves in a {@link
+ * PCollection}, apply {@link TextIO#readAll()} or {@link TextIO#readFiles}.
*
- * <p>{@link TextIO.Read} returns a {@link PCollection} of {@link String Strings}, each
- * corresponding to one line of an input UTF-8 text file (split into lines delimited by '\n', '\r',
- * or '\r\n').
+ * <p>{@link #read} returns a {@link PCollection} of {@link String Strings}, each corresponding to
+ * one line of an input UTF-8 text file (split into lines delimited by '\n', '\r', or '\r\n',
+ * or specified delimiter see {@link TextIO.Read#withDelimiter}).
*
- * <p>Example:
+ * <h3>Filepattern expansion and watching</h3>
+ *
+ * <p>By default, the filepatterns are expanded only once. {@link Read#watchForNewFiles} and {@link
+ * ReadAll#watchForNewFiles} allow streaming of new files matching the filepattern(s).
+ *
+ * <p>By default, {@link #read} prohibits filepatterns that match no files, and {@link #readAll}
+ * allows them in case the filepattern contains a glob wildcard character. Use {@link
+ * TextIO.Read#withEmptyMatchTreatment} and {@link TextIO.ReadAll#withEmptyMatchTreatment} to
+ * configure this behavior.
+ *
+ * <p>Example 1: reading a file or filepattern.
*
* <pre>{@code
* Pipeline p = ...;
@@ -62,20 +89,47 @@
* PCollection<String> lines = p.apply(TextIO.read().from("/local/path/to/file.txt"));
* }</pre>
*
+ * <p>Example 2: reading a PCollection of filenames.
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * // E.g. the filenames might be computed from other data in the pipeline, or
+ * // read from a data source.
+ * PCollection<String> filenames = ...;
+ *
+ * // Read all files in the collection.
+ * PCollection<String> lines = filenames.apply(TextIO.readAll());
+ * }</pre>
+ *
+ * <p>Example 3: streaming new files matching a filepattern.
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<String> lines = p.apply(TextIO.read()
+ * .from("/local/path/to/files/*")
+ * .watchForNewFiles(
+ * // Check for new files every minute
+ * Duration.standardMinutes(1),
+ * // Stop watching the filepattern if no new files appear within an hour
+ * afterTimeSinceNewOutput(Duration.standardHours(1))));
+ * }</pre>
+ *
+ * <h3>Reading a very large number of files</h3>
+ *
+ * <p>If it is known that the filepattern will match a very large number of files (e.g. tens of
+ * thousands or more), use {@link Read#withHintMatchesManyFiles} for better performance and
+ * scalability. Note that it may decrease performance if the filepattern matches only a small number
+ * of files.
+ *
+ * <h2>Writing text files</h2>
+ *
* <p>To write a {@link PCollection} to one or more text files, use {@code TextIO.write()}, using
* {@link TextIO.Write#to(String)} to specify the output prefix of the files to write.
*
- * <p>By default, all input is put into the global window before writing. If per-window writes are
- * desired - for example, when using a streaming runner -
- * {@link TextIO.Write#withWindowedWrites()} will cause windowing and triggering to be
- * preserved. When producing windowed writes, the number of output shards must be set explicitly
- * using {@link TextIO.Write#withNumShards(int)}; some runners may set this for you to a
- * runner-chosen value, so you may need not set it yourself. A {@link FilenamePolicy} must be
- * set, and unique windows and triggers must produce unique filenames.
- *
- * <p>Any existing files with the same names as generated output files will be overwritten.
- *
* <p>For example:
+ *
* <pre>{@code
* // A simple Write to a local file (only runs locally):
* PCollection<String> lines = ...;
@@ -83,9 +137,52 @@
*
* // Same as above, only with Gzip compression:
* PCollection<String> lines = ...;
- * lines.apply(TextIO.write().to("/path/to/file.txt"));
+ * lines.apply(TextIO.write().to("/path/to/file.txt"))
* .withSuffix(".txt")
- * .withWritableByteChannelFactory(FileBasedSink.CompressionType.GZIP));
+ * .withCompression(Compression.GZIP));
+ * }</pre>
+ *
+ * <p>Any existing files with the same names as generated output files will be overwritten.
+ *
+ * <p>If you want better control over how filenames are generated than the default policy allows, a
+ * custom {@link FilenamePolicy} can also be set using {@link TextIO.Write#to(FilenamePolicy)}.
+ *
+ * <h3>Writing windowed or unbounded data</h3>
+ *
+ * <p>By default, all input is put into the global window before writing. If per-window writes are
+ * desired - for example, when using a streaming runner - {@link TextIO.Write#withWindowedWrites()}
+ * will cause windowing and triggering to be preserved. When producing windowed writes with a
+ * streaming runner that supports triggers, the number of output shards must be set explicitly using
+ * {@link TextIO.Write#withNumShards(int)}; some runners may set this for you to a runner-chosen
+ * value, so you may need not set it yourself. If setting an explicit template using {@link
+ * TextIO.Write#withShardNameTemplate(String)}, make sure that the template contains placeholders
+ * for the window and the pane; W is expanded into the window text, and P into the pane; the default
+ * template will include both the window and the pane in the filename.
+ *
+ * <h3>Writing data to multiple destinations</h3>
+ *
+ * <p>TextIO also supports dynamic, value-dependent file destinations. The most general form of this
+ * is done via {@link TextIO.Write#to(DynamicDestinations)}. A {@link DynamicDestinations} class
+ * allows you to convert any input value into a custom destination object, and map that destination
+ * object to a {@link FilenamePolicy}. This allows using different filename policies (or more
+ * commonly, differently-configured instances of the same policy) based on the input record. Often
+ * this is used in conjunction with {@link TextIO#writeCustomType}, which allows your {@link
+ * DynamicDestinations} object to examine the input type and takes a format function to convert that
+ * type to a string for writing.
+ *
+ * <p>A convenience shortcut is provided for the case where the default naming policy is used, but
+ * different configurations of this policy are wanted based on the input record. Default naming
+ * policies can be configured using the {@link DefaultFilenamePolicy.Params} object.
+ *
+ * <pre>{@code
+ * PCollection<UserEvent>> lines = ...;
+ * lines.apply(TextIO.<UserEvent>writeCustomType(new FormatEvent())
+ * .to(new SerializableFunction<UserEvent, Params>() {
+ * public String apply(UserEvent value) {
+ * return new Params().withBaseFilename(baseDirectory + "/" + value.country());
+ * }
+ * }),
+ * new Params().withBaseFilename(baseDirectory + "/empty");
* }</pre>
*/
public class TextIO {
@@ -94,7 +191,41 @@
* {@link PCollection} containing one element for each line of the input files.
*/
public static Read read() {
- return new AutoValue_TextIO_Read.Builder().setCompressionType(CompressionType.AUTO).build();
+ return new AutoValue_TextIO_Read.Builder()
+ .setCompression(Compression.AUTO)
+ .setHintMatchesManyFiles(false)
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
+ .build();
+ }
+
+ /**
+ * A {@link PTransform} that works like {@link #read}, but reads each file in a {@link
+ * PCollection} of filepatterns.
+ *
+ * <p>Can be applied to both bounded and unbounded {@link PCollection PCollections}, so this is
+ * suitable for reading a {@link PCollection} of filepatterns arriving as a stream. However, every
+ * filepattern is expanded once at the moment it is processed, rather than watched for new files
+ * matching the filepattern to appear. Likewise, every file is read once, rather than watched for
+ * new entries.
+ */
+ public static ReadAll readAll() {
+ return new AutoValue_TextIO_ReadAll.Builder()
+ .setCompression(Compression.AUTO)
+ .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
+ .build();
+ }
+
+ /**
+ * Like {@link #read}, but reads each file in a {@link PCollection} of {@link
+ * FileIO.ReadableFile}, returned by {@link FileIO#readMatches}.
+ */
+ public static ReadFiles readFiles() {
+ return new AutoValue_TextIO_ReadFiles.Builder()
+ // 64MB is a reasonable value that allows to amortize the cost of opening files,
+ // but is not so large as to exhaust a typical runner's maximum amount of output per
+ // ProcessElement call.
+ .setDesiredBundleSizeBytes(64 * 1024 * 1024L)
+ .build();
}
/**
@@ -103,11 +234,32 @@
* line.
*/
public static Write write() {
- return new AutoValue_TextIO_Write.Builder()
+ return new TextIO.Write();
+ }
+
+ /**
+ * A {@link PTransform} that writes a {@link PCollection} to a text file (or multiple text files
+ * matching a sharding pattern), with each element of the input collection encoded into its own
+ * line.
+ *
+ * <p>This version allows you to apply {@link TextIO} writes to a PCollection of a custom type
+ * {@link UserT}. A format mechanism that converts the input type {@link UserT} to the String that
+ * will be written to the file must be specified. If using a custom {@link DynamicDestinations}
+ * object this is done using {@link DynamicDestinations#formatRecord}, otherwise the {@link
+ * TypedWrite#withFormatFunction} can be used to specify a format function.
+ *
+ * <p>The advantage of using a custom type is that is it allows a user-provided {@link
+ * DynamicDestinations} object, set via {@link Write#to(DynamicDestinations)} to examine the
+ * custom type when choosing a destination.
+ */
+ public static <UserT> TypedWrite<UserT, Void> writeCustomType() {
+ return new AutoValue_TextIO_TypedWrite.Builder<UserT, Void>()
.setFilenamePrefix(null)
+ .setTempDirectory(null)
.setShardTemplate(null)
.setFilenameSuffix(null)
.setFilenamePolicy(null)
+ .setDynamicDestinations(null)
.setWritableByteChannelFactory(FileBasedSink.CompressionType.UNCOMPRESSED)
.setWindowedWrites(false)
.setNumShards(0)
@@ -117,15 +269,28 @@
/** Implementation of {@link #read}. */
@AutoValue
public abstract static class Read extends PTransform<PBegin, PCollection<String>> {
- @Nullable abstract ValueProvider<String> getFilepattern();
- abstract CompressionType getCompressionType();
+ @Nullable
+ abstract ValueProvider<String> getFilepattern();
+
+ abstract MatchConfiguration getMatchConfiguration();
+
+ abstract boolean getHintMatchesManyFiles();
+
+ abstract Compression getCompression();
+
+ @SuppressWarnings("mutable") // this returns an array that can be mutated by the caller
+ @Nullable
+ abstract byte[] getDelimiter();
abstract Builder toBuilder();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setFilepattern(ValueProvider<String> filepattern);
- abstract Builder setCompressionType(CompressionType compressionType);
+ abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration);
+ abstract Builder setHintMatchesManyFiles(boolean hintManyFiles);
+ abstract Builder setCompression(Compression compression);
+ abstract Builder setDelimiter(byte[] delimiter);
abstract Read build();
}
@@ -139,100 +304,292 @@
*
* <p>Standard <a href="http://docs.oracle.com/javase/tutorial/essential/io/find.html" >Java
* Filesystem glob patterns</a> ("*", "?", "[..]") are supported.
+ *
+ * <p>If it is known that the filepattern will match a very large number of files (at least tens
+ * of thousands), use {@link #withHintMatchesManyFiles} for better performance and scalability.
*/
public Read from(String filepattern) {
- checkNotNull(filepattern, "Filepattern cannot be empty.");
+ checkArgument(filepattern != null, "filepattern can not be null");
return from(StaticValueProvider.of(filepattern));
}
/** Same as {@code from(filepattern)}, but accepting a {@link ValueProvider}. */
public Read from(ValueProvider<String> filepattern) {
- checkNotNull(filepattern, "Filepattern cannot be empty.");
+ checkArgument(filepattern != null, "filepattern can not be null");
return toBuilder().setFilepattern(filepattern).build();
}
- /**
- * Returns a new transform for reading from text files that's like this one but
- * reads from input sources using the specified compression type.
- *
- * <p>If no compression type is specified, the default is {@link TextIO.CompressionType#AUTO}.
- */
+ /** Sets the {@link MatchConfiguration}. */
+ public Read withMatchConfiguration(MatchConfiguration matchConfiguration) {
+ return toBuilder().setMatchConfiguration(matchConfiguration).build();
+ }
+
+ /** @deprecated Use {@link #withCompression}. */
+ @Deprecated
public Read withCompressionType(TextIO.CompressionType compressionType) {
- return toBuilder().setCompressionType(compressionType).build();
+ return withCompression(compressionType.canonical);
+ }
+
+ /**
+ * Reads from input sources using the specified compression type.
+ *
+ * <p>If no compression type is specified, the default is {@link Compression#AUTO}.
+ */
+ public Read withCompression(Compression compression) {
+ return toBuilder().setCompression(compression).build();
+ }
+
+ /**
+ * See {@link MatchConfiguration#continuously}.
+ *
+ * <p>This works only in runners supporting {@link Kind#SPLITTABLE_DO_FN}.
+ */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public Read watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ /**
+ * Hints that the filepattern specified in {@link #from(String)} matches a very large number of
+ * files.
+ *
+ * <p>This hint may cause a runner to execute the transform differently, in a way that improves
+ * performance for this case, but it may worsen performance if the filepattern matches only
+ * a small number of files (e.g., in a runner that supports dynamic work rebalancing, it will
+ * happen less efficiently within individual files).
+ */
+ public Read withHintMatchesManyFiles() {
+ return toBuilder().setHintMatchesManyFiles(true).build();
+ }
+
+ /** See {@link MatchConfiguration#withEmptyMatchTreatment}. */
+ public Read withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /**
+ * Set the custom delimiter to be used in place of the default ones ('\r', '\n' or '\r\n').
+ */
+ public Read withDelimiter(byte[] delimiter) {
+ checkArgument(delimiter != null, "delimiter can not be null");
+ checkArgument(!isSelfOverlapping(delimiter), "delimiter must not self-overlap");
+ return toBuilder().setDelimiter(delimiter).build();
+ }
+
+ static boolean isSelfOverlapping(byte[] s) {
+ // s self-overlaps if v exists such as s = vu = wv with u and w non empty
+ for (int i = 1; i < s.length - 1; ++i) {
+ if (ByteBuffer.wrap(s, 0, i).equals(ByteBuffer.wrap(s, s.length - i, i))) {
+ return true;
+ }
+ }
+ return false;
}
@Override
public PCollection<String> expand(PBegin input) {
- if (getFilepattern() == null) {
- throw new IllegalStateException("need to set the filepattern of a TextIO.Read transform");
+ checkNotNull(getFilepattern(), "need to set the filepattern of a TextIO.Read transform");
+ if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) {
+ return input.apply("Read", org.apache.beam.sdk.io.Read.from(getSource()));
}
-
- final Bounded<String> read = org.apache.beam.sdk.io.Read.from(getSource());
- PCollection<String> pcol = input.getPipeline().apply("Read", read);
- // Honor the default output coder that would have been used by this PTransform.
- pcol.setCoder(getDefaultOutputCoder());
- return pcol;
+ // All other cases go through ReadAll.
+ return input
+ .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of()))
+ .apply(
+ "Via ReadAll",
+ readAll()
+ .withCompression(getCompression())
+ .withMatchConfiguration(getMatchConfiguration())
+ .withDelimiter(getDelimiter()));
}
// Helper to create a source specific to the requested compression type.
protected FileBasedSource<String> getSource() {
- switch (getCompressionType()) {
- case UNCOMPRESSED:
- return new TextSource(getFilepattern());
- case AUTO:
- return CompressedSource.from(new TextSource(getFilepattern()));
- case BZIP2:
- return
- CompressedSource.from(new TextSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.BZIP2);
- case GZIP:
- return
- CompressedSource.from(new TextSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.GZIP);
- case ZIP:
- return
- CompressedSource.from(new TextSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.ZIP);
- case DEFLATE:
- return
- CompressedSource.from(new TextSource(getFilepattern()))
- .withDecompression(CompressedSource.CompressionMode.DEFLATE);
- default:
- throw new IllegalArgumentException("Unknown compression type: " + getFilepattern());
- }
+ return CompressedSource.from(
+ new TextSource(
+ getFilepattern(),
+ getMatchConfiguration().getEmptyMatchTreatment(),
+ getDelimiter()))
+ .withCompression(getCompression());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder
+ .add(
+ DisplayData.item("compressionType", getCompression().toString())
+ .withLabel("Compression Type"))
+ .addIfNotNull(
+ DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"))
+ .include("matchConfiguration", getMatchConfiguration())
+ .addIfNotNull(
+ DisplayData.item("delimiter", Arrays.toString(getDelimiter()))
+ .withLabel("Custom delimiter to split records"));
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+
+ /** Implementation of {@link #readAll}. */
+ @AutoValue
+ public abstract static class ReadAll
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ abstract MatchConfiguration getMatchConfiguration();
+
+ abstract Compression getCompression();
+
+ @SuppressWarnings("mutable") // this returns an array that can be mutated by the caller
+ @Nullable
+ abstract byte[] getDelimiter();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration);
+ abstract Builder setCompression(Compression compression);
+ abstract Builder setDelimiter(byte[] delimiter);
+ abstract ReadAll build();
+ }
+
+ /** Sets the {@link MatchConfiguration}. */
+ public ReadAll withMatchConfiguration(MatchConfiguration configuration) {
+ return toBuilder().setMatchConfiguration(configuration).build();
+ }
+
+ /** @deprecated Use {@link #withCompression}. */
+ @Deprecated
+ public ReadAll withCompressionType(TextIO.CompressionType compressionType) {
+ return withCompression(compressionType.canonical);
+ }
+
+ /**
+ * Reads from input sources using the specified compression type.
+ *
+ * <p>If no compression type is specified, the default is {@link Compression#AUTO}.
+ */
+ public ReadAll withCompression(Compression compression) {
+ return toBuilder().setCompression(compression).build();
+ }
+
+ /** Same as {@link Read#withEmptyMatchTreatment}. */
+ public ReadAll withEmptyMatchTreatment(EmptyMatchTreatment treatment) {
+ return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment));
+ }
+
+ /** Same as {@link Read#watchForNewFiles(Duration, TerminationCondition)}. */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ public ReadAll watchForNewFiles(
+ Duration pollInterval, TerminationCondition<String, ?> terminationCondition) {
+ return withMatchConfiguration(
+ getMatchConfiguration().continuously(pollInterval, terminationCondition));
+ }
+
+ ReadAll withDelimiter(byte[] delimiter) {
+ return toBuilder().setDelimiter(delimiter).build();
+ }
+
+ @Override
+ public PCollection<String> expand(PCollection<String> input) {
+ return input
+ .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration()))
+ .apply(
+ FileIO.readMatches()
+ .withCompression(getCompression())
+ .withDirectoryTreatment(DirectoryTreatment.PROHIBIT))
+ .apply(readFiles().withDelimiter(getDelimiter()));
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- String filepatternDisplay = getFilepattern().isAccessible()
- ? getFilepattern().get() : getFilepattern().toString();
builder
- .add(DisplayData.item("compressionType", getCompressionType().toString())
- .withLabel("Compression Type"))
- .addIfNotNull(DisplayData.item("filePattern", filepatternDisplay)
- .withLabel("File Pattern"));
+ .add(
+ DisplayData.item("compressionType", getCompression().toString())
+ .withLabel("Compression Type"))
+ .addIfNotNull(
+ DisplayData.item("delimiter", Arrays.toString(getDelimiter()))
+ .withLabel("Custom delimiter to split records"))
+ .include("matchConfiguration", getMatchConfiguration());
+ }
+
+ }
+
+ /** Implementation of {@link #readFiles}. */
+ @AutoValue
+ public abstract static class ReadFiles
+ extends PTransform<PCollection<FileIO.ReadableFile>, PCollection<String>> {
+ abstract long getDesiredBundleSizeBytes();
+
+ @SuppressWarnings("mutable") // this returns an array that can be mutated by the caller
+ @Nullable
+ abstract byte[] getDelimiter();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setDesiredBundleSizeBytes(long desiredBundleSizeBytes);
+ abstract Builder setDelimiter(byte[] delimiter);
+ abstract ReadFiles build();
+ }
+
+ @VisibleForTesting
+ ReadFiles withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
+ }
+
+ /** Like {@link Read#withDelimiter}. */
+ public ReadFiles withDelimiter(byte[] delimiter) {
+ return toBuilder().setDelimiter(delimiter).build();
}
@Override
- protected Coder<String> getDefaultOutputCoder() {
- return StringUtf8Coder.of();
+ public PCollection<String> expand(PCollection<FileIO.ReadableFile> input) {
+ return input.apply(
+ "Read all via FileBasedSource",
+ new ReadAllViaFileBasedSource<>(
+ getDesiredBundleSizeBytes(),
+ new CreateTextSourceFn(getDelimiter()),
+ StringUtf8Coder.of()));
+ }
+
+ private static class CreateTextSourceFn
+ implements SerializableFunction<String, FileBasedSource<String>> {
+ private byte[] delimiter;
+
+ private CreateTextSourceFn(byte[] delimiter) {
+ this.delimiter = delimiter;
+ }
+
+ @Override
+ public FileBasedSource<String> apply(String input) {
+ return new TextSource(
+ StaticValueProvider.of(input), EmptyMatchTreatment.DISALLOW, delimiter);
+ }
}
}
-
- /////////////////////////////////////////////////////////////////////////////
+ // ///////////////////////////////////////////////////////////////////////////
/** Implementation of {@link #write}. */
@AutoValue
- public abstract static class Write extends PTransform<PCollection<String>, PDone> {
+ public abstract static class TypedWrite<UserT, DestinationT>
+ extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>> {
/** The prefix of each file written, combined with suffix and shardTemplate. */
@Nullable abstract ValueProvider<ResourceId> getFilenamePrefix();
/** The suffix of each file written, combined with prefix and shardTemplate. */
@Nullable abstract String getFilenameSuffix();
+ /** The base directory used for generating temporary files. */
+ @Nullable
+ abstract ValueProvider<ResourceId> getTempDirectory();
+
/** An optional header to add to each file. */
@Nullable abstract String getHeader();
@@ -248,6 +605,22 @@
/** A policy for naming output files. */
@Nullable abstract FilenamePolicy getFilenamePolicy();
+ /** Allows for value-dependent {@link DynamicDestinations} to be vended. */
+ @Nullable
+ abstract DynamicDestinations<UserT, DestinationT, String> getDynamicDestinations();
+
+ /** A destination function for using {@link DefaultFilenamePolicy}. */
+ @Nullable
+ abstract SerializableFunction<UserT, Params> getDestinationFunction();
+
+ /** A default destination for empty PCollections. */
+ @Nullable
+ abstract Params getEmptyDestination();
+
+ /** A function that converts UserT to a String, for writing to the file. */
+ @Nullable
+ abstract SerializableFunction<UserT, String> getFormatFunction();
+
/** Whether to write windowed output files. */
abstract boolean getWindowedWrites();
@@ -257,66 +630,77 @@
*/
abstract WritableByteChannelFactory getWritableByteChannelFactory();
- abstract Builder toBuilder();
+ abstract Builder<UserT, DestinationT> toBuilder();
@AutoValue.Builder
- abstract static class Builder {
- abstract Builder setFilenamePrefix(ValueProvider<ResourceId> filenamePrefix);
- abstract Builder setShardTemplate(@Nullable String shardTemplate);
- abstract Builder setFilenameSuffix(@Nullable String filenameSuffix);
- abstract Builder setHeader(@Nullable String header);
- abstract Builder setFooter(@Nullable String footer);
- abstract Builder setFilenamePolicy(@Nullable FilenamePolicy filenamePolicy);
- abstract Builder setNumShards(int numShards);
- abstract Builder setWindowedWrites(boolean windowedWrites);
- abstract Builder setWritableByteChannelFactory(
+ abstract static class Builder<UserT, DestinationT> {
+ abstract Builder<UserT, DestinationT> setFilenamePrefix(
+ ValueProvider<ResourceId> filenamePrefix);
+
+ abstract Builder<UserT, DestinationT> setTempDirectory(
+ ValueProvider<ResourceId> tempDirectory);
+
+ abstract Builder<UserT, DestinationT> setShardTemplate(@Nullable String shardTemplate);
+
+ abstract Builder<UserT, DestinationT> setFilenameSuffix(@Nullable String filenameSuffix);
+
+ abstract Builder<UserT, DestinationT> setHeader(@Nullable String header);
+
+ abstract Builder<UserT, DestinationT> setFooter(@Nullable String footer);
+
+ abstract Builder<UserT, DestinationT> setFilenamePolicy(
+ @Nullable FilenamePolicy filenamePolicy);
+
+ abstract Builder<UserT, DestinationT> setDynamicDestinations(
+ @Nullable DynamicDestinations<UserT, DestinationT, String> dynamicDestinations);
+
+ abstract Builder<UserT, DestinationT> setDestinationFunction(
+ @Nullable SerializableFunction<UserT, Params> destinationFunction);
+
+ abstract Builder<UserT, DestinationT> setEmptyDestination(Params emptyDestination);
+
+ abstract Builder<UserT, DestinationT> setFormatFunction(
+ SerializableFunction<UserT, String> formatFunction);
+
+ abstract Builder<UserT, DestinationT> setNumShards(int numShards);
+
+ abstract Builder<UserT, DestinationT> setWindowedWrites(boolean windowedWrites);
+
+ abstract Builder<UserT, DestinationT> setWritableByteChannelFactory(
WritableByteChannelFactory writableByteChannelFactory);
- abstract Write build();
+ abstract TypedWrite<UserT, DestinationT> build();
}
/**
- * Writes to text files with the given prefix. The given {@code prefix} can reference any
- * {@link FileSystem} on the classpath.
- *
- * <p>The name of the output files will be determined by the {@link FilenamePolicy} used.
+ * Writes to text files with the given prefix. The given {@code prefix} can reference any {@link
+ * FileSystem} on the classpath. This prefix is used by the {@link DefaultFilenamePolicy} to
+ * generate filenames.
*
* <p>By default, a {@link DefaultFilenamePolicy} will be used built using the specified prefix
- * to define the base output directory and file prefix, a shard identifier (see
- * {@link #withNumShards(int)}), and a common suffix (if supplied using
- * {@link #withSuffix(String)}).
+ * to define the base output directory and file prefix, a shard identifier (see {@link
+ * #withNumShards(int)}), and a common suffix (if supplied using {@link #withSuffix(String)}).
*
- * <p>This default policy can be overridden using {@link #withFilenamePolicy(FilenamePolicy)},
- * in which case {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should
- * not be set.
+ * <p>This default policy can be overridden using {@link #to(FilenamePolicy)}, in which case
+ * {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should not be set.
+ * Custom filename policies do not automatically see this prefix - you should explicitly pass
+ * the prefix into your {@link FilenamePolicy} object if you need this.
+ *
+ * <p>If {@link #withTempDirectory} has not been called, this filename prefix will be used to
+ * infer a directory for temporary files.
*/
- public Write to(String filenamePrefix) {
+ public TypedWrite<UserT, DestinationT> to(String filenamePrefix) {
return to(FileBasedSink.convertToFileResourceIfPossible(filenamePrefix));
}
- /**
- * Writes to text files with prefix from the given resource.
- *
- * <p>The name of the output files will be determined by the {@link FilenamePolicy} used.
- *
- * <p>By default, a {@link DefaultFilenamePolicy} will be used built using the specified prefix
- * to define the base output directory and file prefix, a shard identifier (see
- * {@link #withNumShards(int)}), and a common suffix (if supplied using
- * {@link #withSuffix(String)}).
- *
- * <p>This default policy can be overridden using {@link #withFilenamePolicy(FilenamePolicy)},
- * in which case {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should
- * not be set.
- */
+ /** Like {@link #to(String)}. */
@Experimental(Kind.FILESYSTEM)
- public Write to(ResourceId filenamePrefix) {
+ public TypedWrite<UserT, DestinationT> to(ResourceId filenamePrefix) {
return toResource(StaticValueProvider.of(filenamePrefix));
}
- /**
- * Like {@link #to(String)}.
- */
- public Write to(ValueProvider<String> outputPrefix) {
+ /** Like {@link #to(String)}. */
+ public TypedWrite<UserT, DestinationT> to(ValueProvider<String> outputPrefix) {
return toResource(NestedValueProvider.of(outputPrefix,
new SerializableFunction<String, ResourceId>() {
@Override
@@ -327,43 +711,92 @@
}
/**
- * Like {@link #to(ResourceId)}.
+ * Writes to files named according to the given {@link FileBasedSink.FilenamePolicy}. A
+ * directory for temporary files must be specified using {@link #withTempDirectory}.
*/
+ public TypedWrite<UserT, DestinationT> to(FilenamePolicy filenamePolicy) {
+ return toBuilder().setFilenamePolicy(filenamePolicy).build();
+ }
+
+ /**
+ * Use a {@link DynamicDestinations} object to vend {@link FilenamePolicy} objects. These
+ * objects can examine the input record when creating a {@link FilenamePolicy}. A directory for
+ * temporary files must be specified using {@link #withTempDirectory}.
+ */
+ public <NewDestinationT> TypedWrite<UserT, NewDestinationT> to(
+ DynamicDestinations<UserT, DestinationT, String> dynamicDestinations) {
+ return (TypedWrite) toBuilder().setDynamicDestinations(dynamicDestinations).build();
+ }
+
+ /**
+ * Write to dynamic destinations using the default filename policy. The destinationFunction maps
+ * the input record to a {@link DefaultFilenamePolicy.Params} object that specifies where the
+ * records should be written (base filename, file suffix, and shard template). The
+ * emptyDestination parameter specified where empty files should be written for when the written
+ * {@link PCollection} is empty.
+ */
+ public TypedWrite<UserT, Params> to(
+ SerializableFunction<UserT, Params> destinationFunction, Params emptyDestination) {
+ return (TypedWrite) toBuilder()
+ .setDestinationFunction(destinationFunction)
+ .setEmptyDestination(emptyDestination)
+ .build();
+ }
+
+ /** Like {@link #to(ResourceId)}. */
@Experimental(Kind.FILESYSTEM)
- public Write toResource(ValueProvider<ResourceId> filenamePrefix) {
+ public TypedWrite<UserT, DestinationT> toResource(ValueProvider<ResourceId> filenamePrefix) {
return toBuilder().setFilenamePrefix(filenamePrefix).build();
}
/**
+ * Specifies a format function to convert {@link UserT} to the output type. If {@link
+ * #to(DynamicDestinations)} is used, {@link DynamicDestinations#formatRecord(Object)} must be
+ * used instead.
+ */
+ public TypedWrite<UserT, DestinationT> withFormatFunction(
+ SerializableFunction<UserT, String> formatFunction) {
+ return toBuilder().setFormatFunction(formatFunction).build();
+ }
+
+ /** Set the base directory used to generate temporary files. */
+ @Experimental(Kind.FILESYSTEM)
+ public TypedWrite<UserT, DestinationT> withTempDirectory(
+ ValueProvider<ResourceId> tempDirectory) {
+ return toBuilder().setTempDirectory(tempDirectory).build();
+ }
+
+ /** Set the base directory used to generate temporary files. */
+ @Experimental(Kind.FILESYSTEM)
+ public TypedWrite<UserT, DestinationT> withTempDirectory(ResourceId tempDirectory) {
+ return withTempDirectory(StaticValueProvider.of(tempDirectory));
+ }
+
+ /**
* Uses the given {@link ShardNameTemplate} for naming output files. This option may only be
- * used when {@link #withFilenamePolicy(FilenamePolicy)} has not been configured.
+ * used when using one of the default filename-prefix to() overrides - i.e. not when using
+ * either {@link #to(FilenamePolicy)} or {@link #to(DynamicDestinations)}.
*
* <p>See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are
* used.
*/
- public Write withShardNameTemplate(String shardTemplate) {
+ public TypedWrite<UserT, DestinationT> withShardNameTemplate(String shardTemplate) {
return toBuilder().setShardTemplate(shardTemplate).build();
}
/**
- * Configures the filename suffix for written files. This option may only be used when
- * {@link #withFilenamePolicy(FilenamePolicy)} has not been configured.
+ * Configures the filename suffix for written files. This option may only be used when using one
+ * of the default filename-prefix to() overrides - i.e. not when using either {@link
+ * #to(FilenamePolicy)} or {@link #to(DynamicDestinations)}.
*
* <p>See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are
* used.
*/
- public Write withSuffix(String filenameSuffix) {
+ public TypedWrite<UserT, DestinationT> withSuffix(String filenameSuffix) {
return toBuilder().setFilenameSuffix(filenameSuffix).build();
}
/**
- * Configures the {@link FileBasedSink.FilenamePolicy} that will be used to name written files.
- */
- public Write withFilenamePolicy(FilenamePolicy filenamePolicy) {
- return toBuilder().setFilenamePolicy(filenamePolicy).build();
- }
-
- /**
* Configures the number of output shards produced overall (when using unwindowed writes) or
* per-window (when using windowed writes).
*
@@ -373,14 +806,13 @@
*
* @param numShards the number of shards to use, or 0 to let the system decide.
*/
- public Write withNumShards(int numShards) {
+ public TypedWrite<UserT, DestinationT> withNumShards(int numShards) {
checkArgument(numShards >= 0);
return toBuilder().setNumShards(numShards).build();
}
/**
- * Forces a single file as output and empty shard name template. This option is only compatible
- * with unwindowed writes.
+ * Forces a single file as output and empty shard name template.
*
* <p>For unwindowed writes, constraining the number of shards is likely to reduce the
* performance of a pipeline. Setting this value is not recommended unless you require a
@@ -388,7 +820,7 @@
*
* <p>This is equivalent to {@code .withNumShards(1).withShardNameTemplate("")}
*/
- public Write withoutSharding() {
+ public TypedWrite<UserT, DestinationT> withoutSharding() {
return withNumShards(1).withShardNameTemplate("");
}
@@ -397,7 +829,7 @@
*
* <p>A {@code null} value will clear any previously configured header.
*/
- public Write withHeader(@Nullable String header) {
+ public TypedWrite<UserT, DestinationT> withHeader(@Nullable String header) {
return toBuilder().setHeader(header).build();
}
@@ -406,47 +838,110 @@
*
* <p>A {@code null} value will clear any previously configured footer.
*/
- public Write withFooter(@Nullable String footer) {
+ public TypedWrite<UserT, DestinationT> withFooter(@Nullable String footer) {
return toBuilder().setFooter(footer).build();
}
/**
- * Returns a transform for writing to text files like this one but that has the given
- * {@link WritableByteChannelFactory} to be used by the {@link FileBasedSink} during output.
- * The default is value is {@link FileBasedSink.CompressionType#UNCOMPRESSED}.
+ * Returns a transform for writing to text files like this one but that has the given {@link
+ * WritableByteChannelFactory} to be used by the {@link FileBasedSink} during output. The
+ * default is value is {@link Compression#UNCOMPRESSED}.
*
* <p>A {@code null} value will reset the value to the default value mentioned above.
*/
- public Write withWritableByteChannelFactory(
+ public TypedWrite<UserT, DestinationT> withWritableByteChannelFactory(
WritableByteChannelFactory writableByteChannelFactory) {
return toBuilder().setWritableByteChannelFactory(writableByteChannelFactory).build();
}
- public Write withWindowedWrites() {
+ /**
+ * Returns a transform for writing to text files like this one but that compresses output using
+ * the given {@link Compression}. The default value is {@link Compression#UNCOMPRESSED}.
+ */
+ public TypedWrite<UserT, DestinationT> withCompression(Compression compression) {
+ checkArgument(compression != null, "compression can not be null");
+ return withWritableByteChannelFactory(
+ FileBasedSink.CompressionType.fromCanonical(compression));
+ }
+
+ /**
+ * Preserves windowing of input elements and writes them to files based on the element's window.
+ *
+ * <p>If using {@link #to(FileBasedSink.FilenamePolicy)}. Filenames will be generated using
+ * {@link FilenamePolicy#windowedFilename}. See also {@link WriteFiles#withWindowedWrites()}.
+ */
+ public TypedWrite<UserT, DestinationT> withWindowedWrites() {
return toBuilder().setWindowedWrites(true).build();
}
- @Override
- public PDone expand(PCollection<String> input) {
- checkState(getFilenamePrefix() != null,
- "Need to set the filename prefix of a TextIO.Write transform.");
- checkState(
- (getFilenamePolicy() == null)
- || (getShardTemplate() == null && getFilenameSuffix() == null),
- "Cannot set a filename policy and also a filename template or suffix.");
- checkState(!getWindowedWrites() || (getFilenamePolicy() != null),
- "When using windowed writes, a filename policy must be set via withFilenamePolicy().");
-
- FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
- if (usedFilenamePolicy == null) {
- usedFilenamePolicy = DefaultFilenamePolicy.constructUsingStandardParameters(
- getFilenamePrefix(), getShardTemplate(), getFilenameSuffix());
+ private DynamicDestinations<UserT, DestinationT, String> resolveDynamicDestinations() {
+ DynamicDestinations<UserT, DestinationT, String> dynamicDestinations =
+ getDynamicDestinations();
+ if (dynamicDestinations == null) {
+ if (getDestinationFunction() != null) {
+ // In this case, DestinationT == Params
+ dynamicDestinations =
+ (DynamicDestinations)
+ DynamicFileDestinations.toDefaultPolicies(
+ getDestinationFunction(), getEmptyDestination(), getFormatFunction());
+ } else {
+ // In this case, DestinationT == Void
+ FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
+ if (usedFilenamePolicy == null) {
+ usedFilenamePolicy =
+ DefaultFilenamePolicy.fromStandardParameters(
+ getFilenamePrefix(),
+ getShardTemplate(),
+ getFilenameSuffix(),
+ getWindowedWrites());
+ }
+ dynamicDestinations =
+ (DynamicDestinations)
+ DynamicFileDestinations.constant(usedFilenamePolicy, getFormatFunction());
+ }
}
- WriteFiles<String> write =
+ return dynamicDestinations;
+ }
+
+ @Override
+ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
+ checkState(
+ getFilenamePrefix() != null || getTempDirectory() != null,
+ "Need to set either the filename prefix or the tempDirectory of a TextIO.Write "
+ + "transform.");
+
+ List<?> allToArgs =
+ Lists.newArrayList(
+ getFilenamePolicy(),
+ getDynamicDestinations(),
+ getFilenamePrefix(),
+ getDestinationFunction());
+ checkArgument(
+ 1 == Iterables.size(Iterables.filter(allToArgs, Predicates.notNull())),
+ "Exactly one of filename policy, dynamic destinations, filename prefix, or destination "
+ + "function must be set");
+
+ if (getDynamicDestinations() != null) {
+ checkArgument(
+ getFormatFunction() == null,
+ "A format function should not be specified "
+ + "with DynamicDestinations. Use DynamicDestinations.formatRecord instead");
+ }
+ if (getFilenamePolicy() != null || getDynamicDestinations() != null) {
+ checkState(
+ getShardTemplate() == null && getFilenameSuffix() == null,
+ "shardTemplate and filenameSuffix should only be used with the default "
+ + "filename policy");
+ }
+ ValueProvider<ResourceId> tempDirectory = getTempDirectory();
+ if (tempDirectory == null) {
+ tempDirectory = getFilenamePrefix();
+ }
+ WriteFiles<UserT, DestinationT, String> write =
WriteFiles.to(
- new TextSink(
- getFilenamePrefix(),
- usedFilenamePolicy,
+ new TextSink<>(
+ tempDirectory,
+ resolveDynamicDestinations(),
getHeader(),
getFooter(),
getWritableByteChannelFactory()));
@@ -463,77 +958,202 @@
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- String prefixString = "";
- if (getFilenamePrefix() != null) {
- prefixString = getFilenamePrefix().isAccessible()
- ? getFilenamePrefix().get().toString() : getFilenamePrefix().toString();
- }
+ resolveDynamicDestinations().populateDisplayData(builder);
builder
- .addIfNotNull(DisplayData.item("filePrefix", prefixString)
- .withLabel("Output File Prefix"))
- .addIfNotNull(DisplayData.item("fileSuffix", getFilenameSuffix())
- .withLabel("Output File Suffix"))
- .addIfNotNull(DisplayData.item("shardNameTemplate", getShardTemplate())
- .withLabel("Output Shard Name Template"))
- .addIfNotDefault(DisplayData.item("numShards", getNumShards())
- .withLabel("Maximum Output Shards"), 0)
- .addIfNotNull(DisplayData.item("fileHeader", getHeader())
- .withLabel("File Header"))
- .addIfNotNull(DisplayData.item("fileFooter", getFooter())
- .withLabel("File Footer"))
- .add(DisplayData
- .item("writableByteChannelFactory", getWritableByteChannelFactory().toString())
- .withLabel("Compression/Transformation Type"));
- }
-
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
+ .addIfNotDefault(
+ DisplayData.item("numShards", getNumShards()).withLabel("Maximum Output Shards"), 0)
+ .addIfNotNull(
+ DisplayData.item("tempDirectory", getTempDirectory())
+ .withLabel("Directory for temporary files"))
+ .addIfNotNull(DisplayData.item("fileHeader", getHeader()).withLabel("File Header"))
+ .addIfNotNull(DisplayData.item("fileFooter", getFooter()).withLabel("File Footer"))
+ .add(
+ DisplayData.item(
+ "writableByteChannelFactory", getWritableByteChannelFactory().toString())
+ .withLabel("Compression/Transformation Type"));
}
}
/**
- * Possible text file compression types.
+ * This class is used as the default return value of {@link TextIO#write()}.
+ *
+ * <p>All methods in this class delegate to the appropriate method of {@link TextIO.TypedWrite}.
+ * This class exists for backwards compatibility, and will be removed in Beam 3.0.
*/
- public enum CompressionType {
- /**
- * Automatically determine the compression type based on filename extension.
- */
- AUTO(""),
- /**
- * Uncompressed (i.e., may be split).
- */
- UNCOMPRESSED(""),
- /**
- * GZipped.
- */
- GZIP(".gz"),
- /**
- * BZipped.
- */
- BZIP2(".bz2"),
- /**
- * Zipped.
- */
- ZIP(".zip"),
- /**
- * Deflate compressed.
- */
- DEFLATE(".deflate");
+ public static class Write extends PTransform<PCollection<String>, PDone> {
+ @VisibleForTesting TypedWrite<String, ?> inner;
- private String filenameSuffix;
+ Write() {
+ this(TextIO.<String>writeCustomType());
+ }
- CompressionType(String suffix) {
- this.filenameSuffix = suffix;
+ Write(TypedWrite<String, ?> inner) {
+ this.inner = inner;
+ }
+
+ /** See {@link TypedWrite#to(String)}. */
+ public Write to(String filenamePrefix) {
+ return new Write(
+ inner.to(filenamePrefix).withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#to(ResourceId)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write to(ResourceId filenamePrefix) {
+ return new Write(
+ inner.to(filenamePrefix).withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#to(ValueProvider)}. */
+ public Write to(ValueProvider<String> outputPrefix) {
+ return new Write(
+ inner.to(outputPrefix).withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#toResource(ValueProvider)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write toResource(ValueProvider<ResourceId> filenamePrefix) {
+ return new Write(
+ inner
+ .toResource(filenamePrefix)
+ .withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#to(FilenamePolicy)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write to(FilenamePolicy filenamePolicy) {
+ return new Write(
+ inner.to(filenamePolicy).withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#to(DynamicDestinations)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write to(DynamicDestinations<String, ?, String> dynamicDestinations) {
+ return new Write(
+ inner.to((DynamicDestinations) dynamicDestinations).withFormatFunction(null));
+ }
+
+ /** See {@link TypedWrite#to(SerializableFunction, Params)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write to(
+ SerializableFunction<String, Params> destinationFunction, Params emptyDestination) {
+ return new Write(
+ inner
+ .to(destinationFunction, emptyDestination)
+ .withFormatFunction(SerializableFunctions.<String>identity()));
+ }
+
+ /** See {@link TypedWrite#withTempDirectory(ValueProvider)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write withTempDirectory(ValueProvider<ResourceId> tempDirectory) {
+ return new Write(inner.withTempDirectory(tempDirectory));
+ }
+
+ /** See {@link TypedWrite#withTempDirectory(ResourceId)}. */
+ @Experimental(Kind.FILESYSTEM)
+ public Write withTempDirectory(ResourceId tempDirectory) {
+ return new Write(inner.withTempDirectory(tempDirectory));
+ }
+
+ /** See {@link TypedWrite#withShardNameTemplate(String)}. */
+ public Write withShardNameTemplate(String shardTemplate) {
+ return new Write(inner.withShardNameTemplate(shardTemplate));
+ }
+
+ /** See {@link TypedWrite#withSuffix(String)}. */
+ public Write withSuffix(String filenameSuffix) {
+ return new Write(inner.withSuffix(filenameSuffix));
+ }
+
+ /** See {@link TypedWrite#withNumShards(int)}. */
+ public Write withNumShards(int numShards) {
+ return new Write(inner.withNumShards(numShards));
+ }
+
+ /** See {@link TypedWrite#withoutSharding()}. */
+ public Write withoutSharding() {
+ return new Write(inner.withoutSharding());
+ }
+
+ /** See {@link TypedWrite#withHeader(String)}. */
+ public Write withHeader(@Nullable String header) {
+ return new Write(inner.withHeader(header));
+ }
+
+ /** See {@link TypedWrite#withFooter(String)}. */
+ public Write withFooter(@Nullable String footer) {
+ return new Write(inner.withFooter(footer));
+ }
+
+ /** See {@link TypedWrite#withWritableByteChannelFactory(WritableByteChannelFactory)}. */
+ public Write withWritableByteChannelFactory(
+ WritableByteChannelFactory writableByteChannelFactory) {
+ return new Write(inner.withWritableByteChannelFactory(writableByteChannelFactory));
+ }
+
+ /** See {@link TypedWrite#withWindowedWrites}. */
+ public Write withWindowedWrites() {
+ return new Write(inner.withWindowedWrites());
}
/**
- * Determine if a given filename matches a compression type based on its extension.
- * @param filename the filename to match
- * @return true iff the filename ends with the compression type's known extension.
+ * Specify that output filenames are wanted.
+ *
+ * <p>The nested {@link TypedWrite}transform always has access to output filenames, however due
+ * to backwards-compatibility concerns, {@link Write} cannot return them. This method simply
+ * returns the inner {@link TypedWrite} transform which has {@link WriteFilesResult} as its
+ * output type, allowing access to output files.
+ *
+ * <p>The supplied {@code DestinationT} type must be: the same as that supplied in {@link
+ * #to(DynamicDestinations)} if that method was used; {@link Params} if {@link
+ * #to(SerializableFunction, Params)} was used, or {@code Void} otherwise.
*/
+ public <DestinationT> TypedWrite<String, DestinationT> withOutputFilenames() {
+ return (TypedWrite) inner;
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ inner.populateDisplayData(builder);
+ }
+
+ @Override
+ public PDone expand(PCollection<String> input) {
+ inner.expand(input);
+ return PDone.in(input.getPipeline());
+ }
+ }
+
+ /** @deprecated Use {@link Compression}. */
+ @Deprecated
+ public enum CompressionType {
+ /** @see Compression#AUTO */
+ AUTO(Compression.AUTO),
+
+ /** @see Compression#UNCOMPRESSED */
+ UNCOMPRESSED(Compression.UNCOMPRESSED),
+
+ /** @see Compression#GZIP */
+ GZIP(Compression.GZIP),
+
+ /** @see Compression#BZIP2 */
+ BZIP2(Compression.BZIP2),
+
+ /** @see Compression#ZIP */
+ ZIP(Compression.ZIP),
+
+ /** @see Compression#ZIP */
+ DEFLATE(Compression.DEFLATE);
+
+ private Compression canonical;
+
+ CompressionType(Compression canonical) {
+ this.canonical = canonical;
+ }
+
+ /** @see Compression#matches */
public boolean matches(String filename) {
- return filename.toLowerCase().endsWith(filenameSuffix.toLowerCase());
+ return canonical.matches(filename);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
index 511d697..387e0ac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
@@ -34,27 +34,29 @@
* '\n'} represented in {@code UTF-8} format as the record separator. Each record (including the
* last) is terminated.
*/
-class TextSink extends FileBasedSink<String> {
+class TextSink<UserT, DestinationT> extends FileBasedSink<UserT, DestinationT, String> {
@Nullable private final String header;
@Nullable private final String footer;
TextSink(
ValueProvider<ResourceId> baseOutputFilename,
- FilenamePolicy filenamePolicy,
+ DynamicDestinations<UserT, DestinationT, String> dynamicDestinations,
@Nullable String header,
@Nullable String footer,
WritableByteChannelFactory writableByteChannelFactory) {
- super(baseOutputFilename, filenamePolicy, writableByteChannelFactory);
+ super(baseOutputFilename, dynamicDestinations, writableByteChannelFactory);
this.header = header;
this.footer = footer;
}
+
@Override
- public WriteOperation<String> createWriteOperation() {
- return new TextWriteOperation(this, header, footer);
+ public WriteOperation<DestinationT, String> createWriteOperation() {
+ return new TextWriteOperation<>(this, header, footer);
}
/** A {@link WriteOperation WriteOperation} for text files. */
- private static class TextWriteOperation extends WriteOperation<String> {
+ private static class TextWriteOperation<DestinationT>
+ extends WriteOperation<DestinationT, String> {
@Nullable private final String header;
@Nullable private final String footer;
@@ -65,20 +67,20 @@
}
@Override
- public Writer<String> createWriter() throws Exception {
- return new TextWriter(this, header, footer);
+ public Writer<DestinationT, String> createWriter() throws Exception {
+ return new TextWriter<>(this, header, footer);
}
}
/** A {@link Writer Writer} for text files. */
- private static class TextWriter extends Writer<String> {
+ private static class TextWriter<DestinationT> extends Writer<DestinationT, String> {
private static final String NEWLINE = "\n";
@Nullable private final String header;
@Nullable private final String footer;
private OutputStreamWriter out;
public TextWriter(
- WriteOperation<String> writeOperation,
+ WriteOperation<DestinationT, String> writeOperation,
@Nullable String header,
@Nullable String footer) {
super(writeOperation, MimeTypes.TEXT);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java
index 4d9fa77..f3e4f77 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java
@@ -28,6 +28,7 @@
import java.util.NoSuchElementException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
@@ -47,12 +48,17 @@
*/
@VisibleForTesting
class TextSource extends FileBasedSource<String> {
- TextSource(ValueProvider<String> fileSpec) {
- super(fileSpec, 1L);
+ byte[] delimiter;
+
+ TextSource(ValueProvider<String> fileSpec, EmptyMatchTreatment emptyMatchTreatment,
+ byte[] delimiter) {
+ super(fileSpec, emptyMatchTreatment, 1L);
+ this.delimiter = delimiter;
}
- private TextSource(MatchResult.Metadata metadata, long start, long end) {
+ private TextSource(MatchResult.Metadata metadata, long start, long end, byte[] delimiter) {
super(metadata, 1L, start, end);
+ this.delimiter = delimiter;
}
@Override
@@ -60,22 +66,23 @@
MatchResult.Metadata metadata,
long start,
long end) {
- return new TextSource(metadata, start, end);
+ return new TextSource(metadata, start, end, delimiter);
+
}
@Override
protected FileBasedReader<String> createSingleFileReader(PipelineOptions options) {
- return new TextBasedReader(this);
+ return new TextBasedReader(this, delimiter);
}
@Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
/**
* A {@link FileBasedReader FileBasedReader}
- * which can decode records delimited by newline characters.
+ * which can decode records delimited by delimiter characters.
*
* <p>See {@link TextSource} for further details.
*/
@@ -84,18 +91,20 @@
private static final int READ_BUFFER_SIZE = 8192;
private final ByteBuffer readBuffer = ByteBuffer.allocate(READ_BUFFER_SIZE);
private ByteString buffer;
- private int startOfSeparatorInBuffer;
- private int endOfSeparatorInBuffer;
+ private int startOfDelimiterInBuffer;
+ private int endOfDelimiterInBuffer;
private long startOfRecord;
private volatile long startOfNextRecord;
private volatile boolean eof;
private volatile boolean elementIsPresent;
private String currentValue;
private ReadableByteChannel inChannel;
+ private byte[] delimiter;
- private TextBasedReader(TextSource source) {
+ private TextBasedReader(TextSource source, byte[] delimiter) {
super(source);
buffer = ByteString.EMPTY;
+ this.delimiter = delimiter;
}
@Override
@@ -126,18 +135,24 @@
protected void startReading(ReadableByteChannel channel) throws IOException {
this.inChannel = channel;
// If the first offset is greater than zero, we need to skip bytes until we see our
- // first separator.
- if (getCurrentSource().getStartOffset() > 0) {
+ // first delimiter.
+ long startOffset = getCurrentSource().getStartOffset();
+ if (startOffset > 0) {
checkState(channel instanceof SeekableByteChannel,
"%s only supports reading from a SeekableByteChannel when given a start offset"
+ " greater than 0.", TextSource.class.getSimpleName());
- long requiredPosition = getCurrentSource().getStartOffset() - 1;
+ long requiredPosition = startOffset - 1;
+ if (delimiter != null && startOffset >= delimiter.length) {
+ // we need to move back the offset of at worse delimiter.size to be sure to see
+ // all the bytes of the delimiter in the call to findDelimiterBounds() below
+ requiredPosition = startOffset - delimiter.length;
+ }
((SeekableByteChannel) channel).position(requiredPosition);
- findSeparatorBounds();
- buffer = buffer.substring(endOfSeparatorInBuffer);
- startOfNextRecord = requiredPosition + endOfSeparatorInBuffer;
- endOfSeparatorInBuffer = 0;
- startOfSeparatorInBuffer = 0;
+ findDelimiterBounds();
+ buffer = buffer.substring(endOfDelimiterInBuffer);
+ startOfNextRecord = requiredPosition + endOfDelimiterInBuffer;
+ endOfDelimiterInBuffer = 0;
+ startOfDelimiterInBuffer = 0;
}
}
@@ -151,37 +166,60 @@
* | element bytes | delimiter bytes | unconsumed bytes |
* ------------------------------------------------------
* 0 start of end of buffer
- * separator separator size
+ * delimiter delimiter size
* in buffer in buffer
* }</pre>
*/
- private void findSeparatorBounds() throws IOException {
+ private void findDelimiterBounds() throws IOException {
int bytePositionInBuffer = 0;
while (true) {
if (!tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + 1)) {
- startOfSeparatorInBuffer = endOfSeparatorInBuffer = bytePositionInBuffer;
+ startOfDelimiterInBuffer = endOfDelimiterInBuffer = bytePositionInBuffer;
break;
}
byte currentByte = buffer.byteAt(bytePositionInBuffer);
- if (currentByte == '\n') {
- startOfSeparatorInBuffer = bytePositionInBuffer;
- endOfSeparatorInBuffer = startOfSeparatorInBuffer + 1;
- break;
- } else if (currentByte == '\r') {
- startOfSeparatorInBuffer = bytePositionInBuffer;
- endOfSeparatorInBuffer = startOfSeparatorInBuffer + 1;
+ if (delimiter == null) {
+ // default delimiter
+ if (currentByte == '\n') {
+ startOfDelimiterInBuffer = bytePositionInBuffer;
+ endOfDelimiterInBuffer = startOfDelimiterInBuffer + 1;
+ break;
+ } else if (currentByte == '\r') {
+ startOfDelimiterInBuffer = bytePositionInBuffer;
+ endOfDelimiterInBuffer = startOfDelimiterInBuffer + 1;
- if (tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + 2)) {
- currentByte = buffer.byteAt(bytePositionInBuffer + 1);
- if (currentByte == '\n') {
- endOfSeparatorInBuffer += 1;
+ if (tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + 2)) {
+ currentByte = buffer.byteAt(bytePositionInBuffer + 1);
+ if (currentByte == '\n') {
+ endOfDelimiterInBuffer += 1;
+ }
+ }
+ break;
+ }
+ } else {
+ // user defined delimiter
+ int i = 0;
+ // initialize delimiter not found
+ startOfDelimiterInBuffer = endOfDelimiterInBuffer = bytePositionInBuffer;
+ while ((i <= delimiter.length - 1) && (currentByte == delimiter[i])) {
+ // read next byte
+ i++;
+ if (tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + i + 1)) {
+ currentByte = buffer.byteAt(bytePositionInBuffer + i);
+ } else {
+ // corner case: delimiter truncated at the end of the file
+ startOfDelimiterInBuffer = endOfDelimiterInBuffer = bytePositionInBuffer;
+ break;
}
}
- break;
+ if (i == delimiter.length) {
+ // all bytes of delimiter found
+ endOfDelimiterInBuffer = bytePositionInBuffer + i;
+ break;
+ }
}
-
// Move to the next byte in buffer.
bytePositionInBuffer += 1;
}
@@ -190,7 +228,7 @@
@Override
protected boolean readNextRecord() throws IOException {
startOfRecord = startOfNextRecord;
- findSeparatorBounds();
+ findDelimiterBounds();
// If we have reached EOF file and consumed all of the buffer then we know
// that there are no more records.
@@ -200,21 +238,21 @@
}
decodeCurrentElement();
- startOfNextRecord = startOfRecord + endOfSeparatorInBuffer;
+ startOfNextRecord = startOfRecord + endOfDelimiterInBuffer;
return true;
}
/**
* Decodes the current element updating the buffer to only contain the unconsumed bytes.
*
- * <p>This invalidates the currently stored {@code startOfSeparatorInBuffer} and
- * {@code endOfSeparatorInBuffer}.
+ * <p>This invalidates the currently stored {@code startOfDelimiterInBuffer} and
+ * {@code endOfDelimiterInBuffer}.
*/
private void decodeCurrentElement() throws IOException {
- ByteString dataToDecode = buffer.substring(0, startOfSeparatorInBuffer);
+ ByteString dataToDecode = buffer.substring(0, startOfDelimiterInBuffer);
currentValue = dataToDecode.toStringUtf8();
elementIsPresent = true;
- buffer = buffer.substring(endOfSeparatorInBuffer);
+ buffer = buffer.substring(endOfDelimiterInBuffer);
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index 0786e5d..72ce5d0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -15,33 +15,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.beam.sdk.io;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import com.google.common.base.Objects;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.hash.Hashing;
+import java.io.IOException;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ShardedKeyCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.FileBasedSink.FileResult;
import org.apache.beam.sdk.io.FileBasedSink.FileResultCoder;
import org.apache.beam.sdk.io.FileBasedSink.WriteOperation;
import org.apache.beam.sdk.io.FileBasedSink.Writer;
+import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
@@ -53,11 +70,19 @@
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.ShardedKey;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,13 +91,12 @@
* global initialization of a sink, followed by a parallel write, and ends with a sequential
* finalization of the write. The output of a write is {@link PDone}.
*
- * <p>By default, every bundle in the input {@link PCollection} will be processed by a
- * {@link WriteOperation}, so the number of output
- * will vary based on runner behavior, though at least 1 output will always be produced. The
- * exact parallelism of the write stage can be controlled using {@link WriteFiles#withNumShards},
- * typically used to control how many files are produced or to globally limit the number of
- * workers connecting to an external service. However, this option can often hurt performance: it
- * adds an additional {@link GroupByKey} to the pipeline.
+ * <p>By default, every bundle in the input {@link PCollection} will be processed by a {@link
+ * WriteOperation}, so the number of output will vary based on runner behavior, though at least 1
+ * output will always be produced. The exact parallelism of the write stage can be controlled using
+ * {@link WriteFiles#withNumShards}, typically used to control how many files are produced or to
+ * globally limit the number of workers connecting to an external service. However, this option can
+ * often hurt performance: it adds an additional {@link GroupByKey} to the pipeline.
*
* <p>Example usage with runner-determined sharding:
*
@@ -83,44 +107,76 @@
* <pre>{@code p.apply(WriteFiles.to(new MySink(...)).withNumShards(3));}</pre>
*/
@Experimental(Experimental.Kind.SOURCE_SINK)
-public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
+public class WriteFiles<UserT, DestinationT, OutputT>
+ extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>> {
private static final Logger LOG = LoggerFactory.getLogger(WriteFiles.class);
+ // The maximum number of file writers to keep open in a single bundle at a time, since file
+ // writers default to 64mb buffers. This comes into play when writing per-window files.
+ // The first 20 files from a single WriteFiles transform will write files inline in the
+ // transform. Anything beyond that might be shuffled.
+ // Keep in mind that specific runners may decide to run multiple bundles in parallel, based on
+ // their own policy.
+ private static final int DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE = 20;
+
+ // When we spill records, shard the output keys to prevent hotspots.
+ // We could consider making this a parameter.
+ private static final int SPILLED_RECORD_SHARDING_FACTOR = 10;
+
static final int UNKNOWN_SHARDNUM = -1;
- private FileBasedSink<T> sink;
- private WriteOperation<T> writeOperation;
+ private FileBasedSink<UserT, DestinationT, OutputT> sink;
+ private WriteOperation<DestinationT, OutputT> writeOperation;
// This allows the number of shards to be dynamically computed based on the input
// PCollection.
- @Nullable
- private final PTransform<PCollection<T>, PCollectionView<Integer>> computeNumShards;
+ @Nullable private final PTransform<PCollection<UserT>, PCollectionView<Integer>> computeNumShards;
// We don't use a side input for static sharding, as we want this value to be updatable
// when a pipeline is updated.
@Nullable
private final ValueProvider<Integer> numShardsProvider;
- private boolean windowedWrites;
+ private final boolean windowedWrites;
+ private int maxNumWritersPerBundle;
+ // This is the set of side inputs used by this transform. This is usually populated by the users's
+ // DynamicDestinations object.
+ private final List<PCollectionView<?>> sideInputs;
/**
* Creates a {@link WriteFiles} transform that writes to the given {@link FileBasedSink}, letting
* the runner control how many different shards are produced.
*/
- public static <T> WriteFiles<T> to(FileBasedSink<T> sink) {
- checkNotNull(sink, "sink");
- return new WriteFiles<>(sink, null /* runner-determined sharding */, null, false);
+ public static <UserT, DestinationT, OutputT> WriteFiles<UserT, DestinationT, OutputT> to(
+ FileBasedSink<UserT, DestinationT, OutputT> sink) {
+ checkArgument(sink != null, "sink can not be null");
+ return new WriteFiles<>(
+ sink,
+ null /* runner-determined sharding */,
+ null,
+ false,
+ DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE,
+ sink.getDynamicDestinations().getSideInputs());
}
private WriteFiles(
- FileBasedSink<T> sink,
- @Nullable PTransform<PCollection<T>, PCollectionView<Integer>> computeNumShards,
+ FileBasedSink<UserT, DestinationT, OutputT> sink,
+ @Nullable PTransform<PCollection<UserT>, PCollectionView<Integer>> computeNumShards,
@Nullable ValueProvider<Integer> numShardsProvider,
- boolean windowedWrites) {
+ boolean windowedWrites,
+ int maxNumWritersPerBundle,
+ List<PCollectionView<?>> sideInputs) {
this.sink = sink;
this.computeNumShards = computeNumShards;
this.numShardsProvider = numShardsProvider;
this.windowedWrites = windowedWrites;
+ this.maxNumWritersPerBundle = maxNumWritersPerBundle;
+ this.sideInputs = sideInputs;
}
@Override
- public PDone expand(PCollection<T> input) {
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return PCollectionViews.toAdditionalInputs(sideInputs);
+ }
+
+ @Override
+ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
if (input.isBounded() == IsBounded.UNBOUNDED) {
checkArgument(windowedWrites,
"Must use windowed writes when applying %s to an unbounded PCollection",
@@ -151,29 +207,32 @@
.include("sink", sink);
if (getSharding() != null) {
builder.include("sharding", getSharding());
- } else if (getNumShards() != null) {
- String numShards = getNumShards().isAccessible()
- ? getNumShards().get().toString() : getNumShards().toString();
- builder.add(DisplayData.item("numShards", numShards)
+ } else {
+ builder.addIfNotNull(DisplayData.item("numShards", getNumShards())
.withLabel("Fixed Number of Shards"));
}
}
- /**
- * Returns the {@link FileBasedSink} associated with this PTransform.
- */
- public FileBasedSink<T> getSink() {
+ /** Returns the {@link FileBasedSink} associated with this PTransform. */
+ public FileBasedSink<UserT, DestinationT, OutputT> getSink() {
return sink;
}
/**
+ * Returns whether or not to perform windowed writes.
+ */
+ public boolean isWindowedWrites() {
+ return windowedWrites;
+ }
+
+ /**
* Gets the {@link PTransform} that will be used to determine sharding. This can be either a
* static number of shards (as following a call to {@link #withNumShards(int)}), dynamic (by
* {@link #withSharding(PTransform)}), or runner-determined (by {@link
* #withRunnerDeterminedSharding()}.
*/
@Nullable
- public PTransform<PCollection<T>, PCollectionView<Integer>> getSharding() {
+ public PTransform<PCollection<UserT>, PCollectionView<Integer>> getSharding() {
return computeNumShards;
}
@@ -191,7 +250,7 @@
* <p>A value less than or equal to 0 will be equivalent to the default behavior of
* runner-determined sharding.
*/
- public WriteFiles<T> withNumShards(int numShards) {
+ public WriteFiles<UserT, DestinationT, OutputT> withNumShards(int numShards) {
if (numShards > 0) {
return withNumShards(StaticValueProvider.of(numShards));
}
@@ -205,8 +264,38 @@
* <p>This option should be used sparingly as it can hurt performance. See {@link WriteFiles} for
* more information.
*/
- public WriteFiles<T> withNumShards(ValueProvider<Integer> numShardsProvider) {
- return new WriteFiles<>(sink, null, numShardsProvider, windowedWrites);
+ public WriteFiles<UserT, DestinationT, OutputT> withNumShards(
+ ValueProvider<Integer> numShardsProvider) {
+ return new WriteFiles<>(
+ sink,
+ computeNumShards,
+ numShardsProvider,
+ windowedWrites,
+ maxNumWritersPerBundle,
+ sideInputs);
+ }
+
+ /** Set the maximum number of writers created in a bundle before spilling to shuffle. */
+ public WriteFiles<UserT, DestinationT, OutputT> withMaxNumWritersPerBundle(
+ int maxNumWritersPerBundle) {
+ return new WriteFiles<>(
+ sink,
+ computeNumShards,
+ numShardsProvider,
+ windowedWrites,
+ maxNumWritersPerBundle,
+ sideInputs);
+ }
+
+ public WriteFiles<UserT, DestinationT, OutputT> withSideInputs(
+ List<PCollectionView<?>> sideInputs) {
+ return new WriteFiles<>(
+ sink,
+ computeNumShards,
+ numShardsProvider,
+ windowedWrites,
+ maxNumWritersPerBundle,
+ sideInputs);
}
/**
@@ -216,81 +305,167 @@
* <p>This option should be used sparingly as it can hurt performance. See {@link WriteFiles} for
* more information.
*/
- public WriteFiles<T> withSharding(PTransform<PCollection<T>, PCollectionView<Integer>> sharding) {
- checkNotNull(
- sharding, "Cannot provide null sharding. Use withRunnerDeterminedSharding() instead");
- return new WriteFiles<>(sink, sharding, null, windowedWrites);
+ public WriteFiles<UserT, DestinationT, OutputT> withSharding(
+ PTransform<PCollection<UserT>, PCollectionView<Integer>> sharding) {
+ checkArgument(
+ sharding != null, "sharding can not be null. Use withRunnerDeterminedSharding() instead.");
+ return new WriteFiles<>(
+ sink, sharding, null, windowedWrites, maxNumWritersPerBundle, sideInputs);
}
/**
* Returns a new {@link WriteFiles} that will write to the current {@link FileBasedSink} with
* runner-determined sharding.
*/
- public WriteFiles<T> withRunnerDeterminedSharding() {
- return new WriteFiles<>(sink, null, null, windowedWrites);
+ public WriteFiles<UserT, DestinationT, OutputT> withRunnerDeterminedSharding() {
+ return new WriteFiles<>(sink, null, null, windowedWrites, maxNumWritersPerBundle, sideInputs);
}
/**
* Returns a new {@link WriteFiles} that writes preserves windowing on it's input.
*
- * <p>If this option is not specified, windowing and triggering are replaced by
- * {@link GlobalWindows} and {@link DefaultTrigger}.
+ * <p>If this option is not specified, windowing and triggering are replaced by {@link
+ * GlobalWindows} and {@link DefaultTrigger}.
*
- * <p>If there is no data for a window, no output shards will be generated for that window.
- * If a window triggers multiple times, then more than a single output shard might be
- * generated multiple times; it's up to the sink implementation to keep these output shards
- * unique.
+ * <p>If there is no data for a window, no output shards will be generated for that window. If a
+ * window triggers multiple times, then more than a single output shard might be generated
+ * multiple times; it's up to the sink implementation to keep these output shards unique.
*
- * <p>This option can only be used if {@link #withNumShards(int)} is also set to a
- * positive value.
+ * <p>This option can only be used if {@link #withNumShards(int)} is also set to a positive value.
*/
- public WriteFiles<T> withWindowedWrites() {
- return new WriteFiles<>(sink, computeNumShards, numShardsProvider, true);
+ public WriteFiles<UserT, DestinationT, OutputT> withWindowedWrites() {
+ return new WriteFiles<>(
+ sink, computeNumShards, numShardsProvider, true, maxNumWritersPerBundle, sideInputs);
+ }
+
+ private static class WriterKey<DestinationT> {
+ private final BoundedWindow window;
+ private final PaneInfo paneInfo;
+ private final DestinationT destination;
+
+ WriterKey(BoundedWindow window, PaneInfo paneInfo, DestinationT destination) {
+ this.window = window;
+ this.paneInfo = paneInfo;
+ this.destination = destination;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof WriterKey)) {
+ return false;
+ }
+ WriterKey other = (WriterKey) o;
+ return Objects.equal(window, other.window)
+ && Objects.equal(paneInfo, other.paneInfo)
+ && Objects.equal(destination, other.destination);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(window, paneInfo, destination);
+ }
+ }
+
+ // Hash the destination in a manner that we can then use as a key in a GBK. Since Java's
+ // hashCode isn't guaranteed to be stable across machines, we instead serialize the destination
+ // and use murmur3_32 to hash it. We enforce that destinationCoder must be deterministic, so
+ // this can be used as a key.
+ private static <DestinationT> int hashDestination(
+ DestinationT destination, Coder<DestinationT> destinationCoder) throws IOException {
+ return Hashing.murmur3_32()
+ .hashBytes(CoderUtils.encodeToByteArray(destinationCoder, destination))
+ .asInt();
}
/**
- * Writes all the elements in a bundle using a {@link Writer} produced by the
- * {@link WriteOperation} associated with the {@link FileBasedSink} with windowed writes enabled.
+ * Writes all the elements in a bundle using a {@link Writer} produced by the {@link
+ * WriteOperation} associated with the {@link FileBasedSink}.
*/
- private class WriteWindowedBundles extends DoFn<T, FileResult> {
- private Map<KV<BoundedWindow, PaneInfo>, Writer<T>> windowedWriters;
+ private class WriteBundles extends DoFn<UserT, FileResult<DestinationT>> {
+ private final TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittenRecordsTag;
+ private final Coder<DestinationT> destinationCoder;
+ private final boolean windowedWrites;
+
+ private Map<WriterKey<DestinationT>, Writer<DestinationT, OutputT>> writers;
+ private int spilledShardNum = UNKNOWN_SHARDNUM;
+
+ WriteBundles(
+ boolean windowedWrites,
+ TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittenRecordsTag,
+ Coder<DestinationT> destinationCoder) {
+ this.windowedWrites = windowedWrites;
+ this.unwrittenRecordsTag = unwrittenRecordsTag;
+ this.destinationCoder = destinationCoder;
+ }
@StartBundle
public void startBundle(StartBundleContext c) {
// Reset state in case of reuse. We need to make sure that each bundle gets unique writers.
- windowedWriters = Maps.newHashMap();
+ writers = Maps.newHashMap();
}
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
+ sink.getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
PaneInfo paneInfo = c.pane();
- Writer<T> writer;
// If we are doing windowed writes, we need to ensure that we have separate files for
- // data in different windows/panes.
- KV<BoundedWindow, PaneInfo> key = KV.of(window, paneInfo);
- writer = windowedWriters.get(key);
+ // data in different windows/panes. Similar for dynamic writes, make sure that different
+ // destinations go to different writers.
+ // In the case of unwindowed writes, the window and the pane will always be the same, and
+ // the map will only have a single element.
+ DestinationT destination = sink.getDynamicDestinations().getDestination(c.element());
+ WriterKey<DestinationT> key = new WriterKey<>(window, c.pane(), destination);
+ Writer<DestinationT, OutputT> writer = writers.get(key);
if (writer == null) {
- String uuid = UUID.randomUUID().toString();
- LOG.info(
- "Opening writer {} for write operation {}, window {} pane {}",
- uuid,
- writeOperation,
- window,
- paneInfo);
- writer = writeOperation.createWriter();
- writer.openWindowed(uuid, window, paneInfo, UNKNOWN_SHARDNUM);
- windowedWriters.put(key, writer);
- LOG.debug("Done opening writer");
+ if (writers.size() <= maxNumWritersPerBundle) {
+ String uuid = UUID.randomUUID().toString();
+ LOG.info(
+ "Opening writer {} for write operation {}, window {} pane {} destination {}",
+ uuid,
+ writeOperation,
+ window,
+ paneInfo,
+ destination);
+ writer = writeOperation.createWriter();
+ if (windowedWrites) {
+ writer.openWindowed(uuid, window, paneInfo, UNKNOWN_SHARDNUM, destination);
+ } else {
+ writer.openUnwindowed(uuid, UNKNOWN_SHARDNUM, destination);
+ }
+ writers.put(key, writer);
+ LOG.debug("Done opening writer");
+ } else {
+ if (spilledShardNum == UNKNOWN_SHARDNUM) {
+ // Cache the random value so we only call ThreadLocalRandom once per DoFn instance.
+ spilledShardNum = ThreadLocalRandom.current().nextInt(SPILLED_RECORD_SHARDING_FACTOR);
+ } else {
+ spilledShardNum = (spilledShardNum + 1) % SPILLED_RECORD_SHARDING_FACTOR;
+ }
+ c.output(
+ unwrittenRecordsTag,
+ KV.of(
+ ShardedKey.of(hashDestination(destination, destinationCoder), spilledShardNum),
+ c.element()));
+ return;
+ }
}
-
- writeOrClose(writer, c.element());
+ writeOrClose(writer, getSink().getDynamicDestinations().formatRecord(c.element()));
}
@FinishBundle
public void finishBundle(FinishBundleContext c) throws Exception {
- for (Map.Entry<KV<BoundedWindow, PaneInfo>, Writer<T>> entry : windowedWriters.entrySet()) {
- FileResult result = entry.getValue().close();
- BoundedWindow window = entry.getKey().getKey();
+ for (Map.Entry<WriterKey<DestinationT>, Writer<DestinationT, OutputT>> entry :
+ writers.entrySet()) {
+ Writer<DestinationT, OutputT> writer = entry.getValue();
+ FileResult<DestinationT> result;
+ try {
+ result = writer.close();
+ } catch (Exception e) {
+ // If anything goes wrong, make sure to delete the temporary file.
+ writer.cleanup();
+ throw e;
+ }
+ BoundedWindow window = entry.getKey().window;
c.output(result, window.maxTimestamp(), window);
}
}
@@ -301,82 +476,63 @@
}
}
- /**
- * Writes all the elements in a bundle using a {@link Writer} produced by the
- * {@link WriteOperation} associated with the {@link FileBasedSink} with windowed writes disabled.
- */
- private class WriteUnwindowedBundles extends DoFn<T, FileResult> {
- // Writer that will write the records in this bundle. Lazily
- // initialized in processElement.
- private Writer<T> writer = null;
- private BoundedWindow window = null;
+ enum ShardAssignment { ASSIGN_IN_FINALIZE, ASSIGN_WHEN_WRITING }
- @StartBundle
- public void startBundle(StartBundleContext c) {
- // Reset state in case of reuse. We need to make sure that each bundle gets unique writers.
- writer = null;
+ /*
+ * Like {@link WriteBundles}, but where the elements for each shard have been collected into a
+ * single iterable.
+ */
+ private class WriteShardedBundles
+ extends DoFn<KV<ShardedKey<Integer>, Iterable<UserT>>, FileResult<DestinationT>> {
+ ShardAssignment shardNumberAssignment;
+ WriteShardedBundles(ShardAssignment shardNumberAssignment) {
+ this.shardNumberAssignment = shardNumberAssignment;
}
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
- // Cache a single writer for the bundle.
- if (writer == null) {
- LOG.info("Opening writer for write operation {}", writeOperation);
- writer = writeOperation.createWriter();
- writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
- LOG.debug("Done opening writer");
- }
- this.window = window;
- writeOrClose(this.writer, c.element());
- }
-
- @FinishBundle
- public void finishBundle(FinishBundleContext c) throws Exception {
- if (writer == null) {
- return;
- }
- FileResult result = writer.close();
- c.output(result, window.maxTimestamp(), window);
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- builder.delegate(WriteFiles.this);
- }
- }
-
- /**
- * Like {@link WriteWindowedBundles} and {@link WriteUnwindowedBundles}, but where the elements
- * for each shard have been collected into a single iterable.
- */
- private class WriteShardedBundles extends DoFn<KV<Integer, Iterable<T>>, FileResult> {
- @ProcessElement
- public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
- // In a sharded write, single input element represents one shard. We can open and close
- // the writer in each call to processElement.
- LOG.info("Opening writer for write operation {}", writeOperation);
- Writer<T> writer = writeOperation.createWriter();
- if (windowedWrites) {
- writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), c.element().getKey());
- } else {
- writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
- }
- LOG.debug("Done opening writer");
-
- try {
- for (T t : c.element().getValue()) {
- writeOrClose(writer, t);
+ sink.getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
+ // Since we key by a 32-bit hash of the destination, there might be multiple destinations
+ // in this iterable. The number of destinations is generally very small (1000s or less), so
+ // there will rarely be hash collisions.
+ Map<DestinationT, Writer<DestinationT, OutputT>> writers = Maps.newHashMap();
+ for (UserT input : c.element().getValue()) {
+ DestinationT destination = sink.getDynamicDestinations().getDestination(input);
+ Writer<DestinationT, OutputT> writer = writers.get(destination);
+ if (writer == null) {
+ LOG.debug("Opening writer for write operation {}", writeOperation);
+ writer = writeOperation.createWriter();
+ if (windowedWrites) {
+ int shardNumber =
+ shardNumberAssignment == ShardAssignment.ASSIGN_WHEN_WRITING
+ ? c.element().getKey().getShardNumber()
+ : UNKNOWN_SHARDNUM;
+ writer.openWindowed(
+ UUID.randomUUID().toString(), window, c.pane(), shardNumber, destination);
+ } else {
+ writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, destination);
+ }
+ LOG.debug("Done opening writer");
+ writers.put(destination, writer);
}
-
- // Close the writer; if this throws let the error propagate.
- FileResult result = writer.close();
- c.output(result);
- } catch (Exception e) {
- // If anything goes wrong, make sure to delete the temporary file.
- writer.cleanup();
- throw e;
+ writeOrClose(writer, getSink().getDynamicDestinations().formatRecord(input));
}
- }
+
+ // Close all writers.
+ for (Map.Entry<DestinationT, Writer<DestinationT, OutputT>> entry : writers.entrySet()) {
+ Writer<DestinationT, OutputT> writer = entry.getValue();
+ FileResult<DestinationT> result;
+ try {
+ // Close the writer; if this throws let the error propagate.
+ result = writer.close();
+ c.output(result);
+ } catch (Exception e) {
+ // If anything goes wrong, make sure to delete the temporary file.
+ writer.cleanup();
+ throw e;
+ }
+ }
+ }
@Override
public void populateDisplayData(DisplayData.Builder builder) {
@@ -384,12 +540,15 @@
}
}
- private static <T> void writeOrClose(Writer<T> writer, T t) throws Exception {
+ private static <DestinationT, OutputT> void writeOrClose(
+ Writer<DestinationT, OutputT> writer, OutputT t) throws Exception {
try {
writer.write(t);
} catch (Exception e) {
try {
writer.close();
+ // If anything goes wrong, make sure to delete the temporary file.
+ writer.cleanup();
} catch (Exception closeException) {
if (closeException instanceof InterruptedException) {
// Do not silently ignore interrupted state.
@@ -402,20 +561,25 @@
}
}
- private static class ApplyShardingKey<T> extends DoFn<T, KV<Integer, T>> {
+ private class ApplyShardingKey extends DoFn<UserT, KV<ShardedKey<Integer>, UserT>> {
private final PCollectionView<Integer> numShardsView;
private final ValueProvider<Integer> numShardsProvider;
+ private final Coder<DestinationT> destinationCoder;
+
private int shardNumber;
- ApplyShardingKey(PCollectionView<Integer> numShardsView,
- ValueProvider<Integer> numShardsProvider) {
+ ApplyShardingKey(
+ PCollectionView<Integer> numShardsView,
+ ValueProvider<Integer> numShardsProvider,
+ Coder<DestinationT> destinationCoder) {
+ this.destinationCoder = destinationCoder;
this.numShardsView = numShardsView;
this.numShardsProvider = numShardsProvider;
shardNumber = UNKNOWN_SHARDNUM;
}
@ProcessElement
- public void processElement(ProcessContext context) {
+ public void processElement(ProcessContext context) throws IOException {
final int shardCount;
if (numShardsView != null) {
shardCount = context.sideInput(numShardsView);
@@ -435,119 +599,214 @@
} else {
shardNumber = (shardNumber + 1) % shardCount;
}
- context.output(KV.of(shardNumber, context.element()));
+ // We avoid using destination itself as a sharding key, because destination is often large.
+ // e.g. when using {@link DefaultFilenamePolicy}, the destination contains the entire path
+ // to the file. Often most of the path is constant across all destinations, just the path
+ // suffix is appended by the destination function. Instead we key by a 32-bit hash (carefully
+ // chosen to be guaranteed stable), and call getDestination again in the next ParDo to resolve
+ // the destinations. This does mean that multiple destinations might end up on the same shard,
+ // however the number of collisions should be small, so there's no need to worry about memory
+ // issues.
+ DestinationT destination = sink.getDynamicDestinations().getDestination(context.element());
+ context.output(
+ KV.of(
+ ShardedKey.of(hashDestination(destination, destinationCoder), shardNumber),
+ context.element()));
}
}
+ Multimap<DestinationT, FileResult<DestinationT>> perDestinationResults(
+ Iterable<FileResult<DestinationT>> results) {
+ Multimap<DestinationT, FileResult<DestinationT>> perDestination = ArrayListMultimap.create();
+ for (FileResult<DestinationT> result : results) {
+ perDestination.put(result.getDestination(), result);
+ }
+ return perDestination;
+ }
+
/**
* A write is performed as sequence of three {@link ParDo}'s.
*
- * <p>This singleton collection containing the WriteOperation is then used as a side
- * input to a ParDo over the PCollection of elements to write. In this bundle-writing phase,
- * {@link WriteOperation#createWriter} is called to obtain a {@link Writer}.
- * {@link Writer#open} and {@link Writer#close} are called in
- * {@link DoFn.StartBundle} and {@link DoFn.FinishBundle}, respectively, and
- * {@link Writer#write} method is called for every element in the bundle. The output
- * of this ParDo is a PCollection of <i>writer result</i> objects (see {@link FileBasedSink}
- * for a description of writer results)-one for each bundle.
+ * <p>This singleton collection containing the WriteOperation is then used as a side input to a
+ * ParDo over the PCollection of elements to write. In this bundle-writing phase, {@link
+ * WriteOperation#createWriter} is called to obtain a {@link Writer}. {@link Writer#open} and
+ * {@link Writer#close} are called in {@link DoFn.StartBundle} and {@link DoFn.FinishBundle},
+ * respectively, and {@link Writer#write} method is called for every element in the bundle. The
+ * output of this ParDo is a PCollection of <i>writer result</i> objects (see {@link
+ * FileBasedSink} for a description of writer results)-one for each bundle.
*
* <p>The final do-once ParDo uses a singleton collection asinput and the collection of writer
- * results as a side-input. In this ParDo, {@link WriteOperation#finalize} is called
- * to finalize the write.
+ * results as a side-input. In this ParDo, {@link WriteOperation#finalize} is called to finalize
+ * the write.
*
- * <p>If the write of any element in the PCollection fails, {@link Writer#close} will be
- * called before the exception that caused the write to fail is propagated and the write result
- * will be discarded.
+ * <p>If the write of any element in the PCollection fails, {@link Writer#close} will be called
+ * before the exception that caused the write to fail is propagated and the write result will be
+ * discarded.
*
* <p>Since the {@link WriteOperation} is serialized after the initialization ParDo and
* deserialized in the bundle-writing and finalization phases, any state change to the
- * WriteOperation object that occurs during initialization is visible in the latter
- * phases. However, the WriteOperation is not serialized after the bundle-writing
- * phase. This is why implementations should guarantee that
- * {@link WriteOperation#createWriter} does not mutate WriteOperation).
+ * WriteOperation object that occurs during initialization is visible in the latter phases.
+ * However, the WriteOperation is not serialized after the bundle-writing phase. This is why
+ * implementations should guarantee that {@link WriteOperation#createWriter} does not mutate
+ * WriteOperation).
*/
- private PDone createWrite(PCollection<T> input) {
+ private WriteFilesResult<DestinationT> createWrite(PCollection<UserT> input) {
Pipeline p = input.getPipeline();
if (!windowedWrites) {
// Re-window the data into the global window and remove any existing triggers.
input =
input.apply(
- Window.<T>into(new GlobalWindows())
+ Window.<UserT>into(new GlobalWindows())
.triggering(DefaultTrigger.of())
.discardingFiredPanes());
}
-
// Perform the per-bundle writes as a ParDo on the input PCollection (with the
// WriteOperation as a side input) and collect the results of the writes in a
// PCollection. There is a dependency between this ParDo and the first (the
// WriteOperation PCollection as a side input), so this will happen after the
// initial ParDo.
- PCollection<FileResult> results;
+ PCollection<FileResult<DestinationT>> results;
final PCollectionView<Integer> numShardsView;
+ @SuppressWarnings("unchecked")
Coder<BoundedWindow> shardedWindowCoder =
(Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder();
+ final Coder<DestinationT> destinationCoder;
+ try {
+ destinationCoder =
+ sink.getDynamicDestinations()
+ .getDestinationCoderWithDefault(input.getPipeline().getCoderRegistry());
+ destinationCoder.verifyDeterministic();
+ } catch (CannotProvideCoderException | NonDeterministicException e) {
+ throw new RuntimeException(e);
+ }
+
if (computeNumShards == null && numShardsProvider == null) {
numShardsView = null;
- results =
+ TupleTag<FileResult<DestinationT>> writtenRecordsTag =
+ new TupleTag<>("writtenRecordsTag");
+ TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittedRecordsTag =
+ new TupleTag<>("unwrittenRecordsTag");
+ String writeName = windowedWrites ? "WriteWindowedBundles" : "WriteBundles";
+ PCollectionTuple writeTuple =
input.apply(
- "WriteBundles",
- ParDo.of(windowedWrites ? new WriteWindowedBundles() : new WriteUnwindowedBundles()));
+ writeName,
+ ParDo.of(new WriteBundles(windowedWrites, unwrittedRecordsTag, destinationCoder))
+ .withSideInputs(sideInputs)
+ .withOutputTags(writtenRecordsTag, TupleTagList.of(unwrittedRecordsTag)));
+ PCollection<FileResult<DestinationT>> writtenBundleFiles =
+ writeTuple
+ .get(writtenRecordsTag)
+ .setCoder(FileResultCoder.of(shardedWindowCoder, destinationCoder));
+ // Any "spilled" elements are written using WriteShardedBundles. Assign shard numbers in
+ // finalize to stay consistent with what WriteWindowedBundles does.
+ PCollection<FileResult<DestinationT>> writtenGroupedFiles =
+ writeTuple
+ .get(unwrittedRecordsTag)
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(VarIntCoder.of()), input.getCoder()))
+ .apply("GroupUnwritten", GroupByKey.<ShardedKey<Integer>, UserT>create())
+ .apply(
+ "WriteUnwritten",
+ ParDo.of(new WriteShardedBundles(ShardAssignment.ASSIGN_IN_FINALIZE))
+ .withSideInputs(sideInputs))
+ .setCoder(FileResultCoder.of(shardedWindowCoder, destinationCoder));
+ results =
+ PCollectionList.of(writtenBundleFiles)
+ .and(writtenGroupedFiles)
+ .apply(Flatten.<FileResult<DestinationT>>pCollections());
} else {
- List<PCollectionView<?>> sideInputs = Lists.newArrayList();
+ List<PCollectionView<?>> shardingSideInputs = Lists.newArrayList();
if (computeNumShards != null) {
numShardsView = input.apply(computeNumShards);
- sideInputs.add(numShardsView);
+ shardingSideInputs.add(numShardsView);
} else {
numShardsView = null;
}
-
- PCollection<KV<Integer, Iterable<T>>> sharded =
+ PCollection<KV<ShardedKey<Integer>, Iterable<UserT>>> sharded =
input
- .apply("ApplyShardLabel", ParDo.of(
- new ApplyShardingKey<T>(numShardsView,
- (numShardsView != null) ? null : numShardsProvider))
- .withSideInputs(sideInputs))
- .apply("GroupIntoShards", GroupByKey.<Integer, T>create());
+ .apply(
+ "ApplyShardLabel",
+ ParDo.of(
+ new ApplyShardingKey(
+ numShardsView,
+ (numShardsView != null) ? null : numShardsProvider,
+ destinationCoder))
+ .withSideInputs(shardingSideInputs))
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(VarIntCoder.of()), input.getCoder()))
+ .apply("GroupIntoShards", GroupByKey.<ShardedKey<Integer>, UserT>create());
shardedWindowCoder =
(Coder<BoundedWindow>) sharded.getWindowingStrategy().getWindowFn().windowCoder();
-
- results = sharded.apply("WriteShardedBundles", ParDo.of(new WriteShardedBundles()));
+ // Since this path might be used by streaming runners processing triggers, it's important
+ // to assign shard numbers here so that they are deterministic. The ASSIGN_IN_FINALIZE
+ // strategy works by sorting all FileResult objects and assigning them numbers, which is not
+ // guaranteed to work well when processing triggers - if the finalize step retries it might
+ // see a different Iterable of FileResult objects, and it will assign different shard numbers.
+ results =
+ sharded.apply(
+ "WriteShardedBundles",
+ ParDo.of(new WriteShardedBundles(ShardAssignment.ASSIGN_WHEN_WRITING))
+ .withSideInputs(sideInputs));
}
- results.setCoder(FileResultCoder.of(shardedWindowCoder));
+ results.setCoder(FileResultCoder.of(shardedWindowCoder, destinationCoder));
+ PCollection<KV<DestinationT, String>> outputFilenames;
if (windowedWrites) {
// When processing streaming windowed writes, results will arrive multiple times. This
// means we can't share the below implementation that turns the results into a side input,
// as new data arriving into a side input does not trigger the listening DoFn. Instead
// we aggregate the result set using a singleton GroupByKey, so the DoFn will be triggered
// whenever new data arrives.
- PCollection<KV<Void, FileResult>> keyedResults =
- results.apply("AttachSingletonKey", WithKeys.<Void, FileResult>of((Void) null));
- keyedResults.setCoder(KvCoder.of(VoidCoder.of(),
- FileResultCoder.of(shardedWindowCoder)));
+ PCollection<KV<Void, FileResult<DestinationT>>> keyedResults =
+ results.apply(
+ "AttachSingletonKey", WithKeys.<Void, FileResult<DestinationT>>of((Void) null));
+ keyedResults.setCoder(
+ KvCoder.of(VoidCoder.of(), FileResultCoder.of(shardedWindowCoder, destinationCoder)));
// Is the continuation trigger sufficient?
- keyedResults
- .apply("FinalizeGroupByKey", GroupByKey.<Void, FileResult>create())
- .apply("Finalize", ParDo.of(new DoFn<KV<Void, Iterable<FileResult>>, Integer>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- LOG.info("Finalizing write operation {}.", writeOperation);
- List<FileResult> results = Lists.newArrayList(c.element().getValue());
- writeOperation.finalize(results);
- LOG.debug("Done finalizing write operation");
- }
- }));
+ outputFilenames =
+ keyedResults
+ .apply("FinalizeGroupByKey", GroupByKey.<Void, FileResult<DestinationT>>create())
+ .apply(
+ "FinalizeWindowed",
+ ParDo.of(
+ new DoFn<
+ KV<Void, Iterable<FileResult<DestinationT>>>,
+ KV<DestinationT, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Set<ResourceId> tempFiles = Sets.newHashSet();
+ Multimap<DestinationT, FileResult<DestinationT>> results =
+ perDestinationResults(c.element().getValue());
+ for (Map.Entry<DestinationT, Collection<FileResult<DestinationT>>> entry :
+ results.asMap().entrySet()) {
+ LOG.info(
+ "Finalizing write operation {} for destination {} num shards: {}.",
+ writeOperation,
+ entry.getKey(),
+ entry.getValue().size());
+ Map<ResourceId, ResourceId> finalizeMap =
+ writeOperation.finalize(entry.getValue());
+ tempFiles.addAll(finalizeMap.keySet());
+ for (ResourceId outputFile : finalizeMap.values()) {
+ c.output(KV.of(entry.getKey(), outputFile.toString()));
+ }
+ LOG.debug("Done finalizing write operation for {}.", entry.getKey());
+ }
+ writeOperation.removeTemporaryFiles(tempFiles);
+ LOG.debug("Removed temporary files for {}.", writeOperation);
+ }
+ }))
+ .setCoder(KvCoder.of(destinationCoder, StringUtf8Coder.of()));
} else {
- final PCollectionView<Iterable<FileResult>> resultsView =
- results.apply(View.<FileResult>asIterable());
- ImmutableList.Builder<PCollectionView<?>> sideInputs =
+ final PCollectionView<Iterable<FileResult<DestinationT>>> resultsView =
+ results.apply(View.<FileResult<DestinationT>>asIterable());
+ ImmutableList.Builder<PCollectionView<?>> finalizeSideInputs =
ImmutableList.<PCollectionView<?>>builder().add(resultsView);
if (numShardsView != null) {
- sideInputs.add(numShardsView);
+ finalizeSideInputs.add(numShardsView);
}
+ finalizeSideInputs.addAll(sideInputs);
// Finalize the write in another do-once ParDo on the singleton collection containing the
// Writer. The results from the per-bundle writes are given as an Iterable side input.
@@ -558,42 +817,103 @@
// set numShards, then all shards will be written out as empty files. For this reason we
// use a side input here.
PCollection<Void> singletonCollection = p.apply(Create.of((Void) null));
- singletonCollection
- .apply("Finalize", ParDo.of(new DoFn<Void, Integer>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- LOG.info("Finalizing write operation {}.", writeOperation);
- List<FileResult> results = Lists.newArrayList(c.sideInput(resultsView));
- LOG.debug("Side input initialized to finalize write operation {}.", writeOperation);
-
- // We must always output at least 1 shard, and honor user-specified numShards if
- // set.
- int minShardsNeeded;
- if (numShardsView != null) {
- minShardsNeeded = c.sideInput(numShardsView);
- } else if (numShardsProvider != null) {
- minShardsNeeded = numShardsProvider.get();
- } else {
- minShardsNeeded = 1;
- }
- int extraShardsNeeded = minShardsNeeded - results.size();
- if (extraShardsNeeded > 0) {
- LOG.info(
- "Creating {} empty output shards in addition to {} written for a total of {}.",
- extraShardsNeeded, results.size(), minShardsNeeded);
- for (int i = 0; i < extraShardsNeeded; ++i) {
- Writer<T> writer = writeOperation.createWriter();
- writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
- FileResult emptyWrite = writer.close();
- results.add(emptyWrite);
- }
- LOG.debug("Done creating extra shards.");
- }
- writeOperation.finalize(results);
- LOG.debug("Done finalizing write operation {}", writeOperation);
- }
- }).withSideInputs(sideInputs.build()));
+ outputFilenames = singletonCollection.apply(
+ "FinalizeUnwindowed",
+ ParDo.of(
+ new DoFn<Void, KV<DestinationT, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ sink.getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
+ // We must always output at least 1 shard, and honor user-specified numShards
+ // if set.
+ int minShardsNeeded;
+ if (numShardsView != null) {
+ minShardsNeeded = c.sideInput(numShardsView);
+ } else if (numShardsProvider != null) {
+ minShardsNeeded = numShardsProvider.get();
+ } else {
+ minShardsNeeded = 1;
+ }
+ Set<ResourceId> tempFiles = Sets.newHashSet();
+ Multimap<DestinationT, FileResult<DestinationT>> perDestination =
+ perDestinationResults(c.sideInput(resultsView));
+ for (Map.Entry<DestinationT, Collection<FileResult<DestinationT>>> entry :
+ perDestination.asMap().entrySet()) {
+ Map<ResourceId, ResourceId> finalizeMap = Maps.newHashMap();
+ finalizeMap.putAll(
+ finalizeForDestinationFillEmptyShards(
+ entry.getKey(), entry.getValue(), minShardsNeeded));
+ tempFiles.addAll(finalizeMap.keySet());
+ for (ResourceId outputFile :finalizeMap.values()) {
+ c.output(KV.of(entry.getKey(), outputFile.toString()));
+ }
+ }
+ if (perDestination.isEmpty()) {
+ // If there is no input at all, write empty files to the default
+ // destination.
+ Map<ResourceId, ResourceId> finalizeMap = Maps.newHashMap();
+ DestinationT destination =
+ getSink().getDynamicDestinations().getDefaultDestination();
+ finalizeMap.putAll(
+ finalizeForDestinationFillEmptyShards(
+ destination,
+ Lists.<FileResult<DestinationT>>newArrayList(),
+ minShardsNeeded));
+ tempFiles.addAll(finalizeMap.keySet());
+ for (ResourceId outputFile :finalizeMap.values()) {
+ c.output(KV.of(destination, outputFile.toString()));
+ }
+ }
+ writeOperation.removeTemporaryFiles(tempFiles);
+ }
+ })
+ .withSideInputs(finalizeSideInputs.build()))
+ .setCoder(KvCoder.of(destinationCoder, StringUtf8Coder.of()));
}
- return PDone.in(input.getPipeline());
+
+ TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag =
+ new TupleTag<>("perDestinationOutputFilenames");
+ return WriteFilesResult.in(
+ input.getPipeline(),
+ perDestinationOutputFilenamesTag,
+ outputFilenames);
+ }
+
+ /**
+ * Finalize a list of files for a single destination. If a minimum number of shards is needed,
+ * this function will generate empty files for this destination to ensure that all shards are
+ * generated.
+ */
+ private Map<ResourceId, ResourceId> finalizeForDestinationFillEmptyShards(
+ DestinationT destination, Collection<FileResult<DestinationT>> results, int minShardsNeeded)
+ throws Exception {
+ checkState(!windowedWrites);
+
+ LOG.info(
+ "Finalizing write operation {} for destination {} num shards {}.",
+ writeOperation,
+ destination,
+ results.size());
+ int extraShardsNeeded = minShardsNeeded - results.size();
+ if (extraShardsNeeded > 0) {
+ LOG.info(
+ "Creating {} empty output shards in addition to {} written "
+ + "for a total of {} for destination {}.",
+ extraShardsNeeded,
+ results.size(),
+ minShardsNeeded,
+ destination);
+ for (int i = 0; i < extraShardsNeeded; ++i) {
+ Writer<DestinationT, OutputT> writer = writeOperation.createWriter();
+ // Currently this code path is only called in the unwindowed case.
+ writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, destination);
+ FileResult<DestinationT> emptyWrite = writer.close();
+ results.add(emptyWrite);
+ }
+ LOG.debug("Done creating extra shards for {}.", destination);
+ }
+ Map<ResourceId, ResourceId> finalizeMap = writeOperation.finalize(results);
+ LOG.debug("Done finalizing write operation {} for destination {}", writeOperation, destination);
+ return finalizeMap;
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java
new file mode 100644
index 0000000..77e9b9d
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** The result of a {@link WriteFiles} transform. */
+public class WriteFilesResult<DestinationT> implements POutput {
+ private final Pipeline pipeline;
+ private final TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag;
+ private final PCollection<KV<DestinationT, String>> perDestinationOutputFilenames;
+
+ private WriteFilesResult(
+ Pipeline pipeline,
+ TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag,
+ PCollection<KV<DestinationT, String>> perDestinationOutputFilenames) {
+ this.pipeline = pipeline;
+ this.perDestinationOutputFilenamesTag = perDestinationOutputFilenamesTag;
+ this.perDestinationOutputFilenames = perDestinationOutputFilenames;
+ }
+
+ static <DestinationT> WriteFilesResult<DestinationT> in(
+ Pipeline pipeline,
+ TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag,
+ PCollection<KV<DestinationT, String>> perDestinationOutputFilenames) {
+ return new WriteFilesResult<>(
+ pipeline,
+ perDestinationOutputFilenamesTag,
+ perDestinationOutputFilenames);
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ return ImmutableMap.<TupleTag<?>, PValue>of(
+ perDestinationOutputFilenamesTag,
+ perDestinationOutputFilenames);
+ }
+
+ @Override
+ public Pipeline getPipeline() {
+ return pipeline;
+ }
+
+ @Override
+ public void finishSpecifyingOutput(
+ String transformName, PInput input, PTransform<?, ?> transform) {}
+
+ /**
+ * Returns a {@link PCollection} of all output filenames generated by this {@link WriteFiles}
+ * organized by user destination type.
+ */
+ public PCollection<KV<DestinationT, String>> getPerDestinationOutputFilenames() {
+ return perDestinationOutputFilenames;
+ }
+}
+
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/EmptyMatchTreatment.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/EmptyMatchTreatment.java
new file mode 100644
index 0000000..8e12993
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/EmptyMatchTreatment.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.fs;
+
+import org.apache.beam.sdk.io.fs.MatchResult.Status;
+
+/**
+ * Options for allowing or disallowing filepatterns that match no resources in {@link
+ * org.apache.beam.sdk.io.FileSystems#match}.
+ */
+public enum EmptyMatchTreatment {
+ /**
+ * Filepatterns matching no resources are allowed. For such a filepattern, {@link
+ * MatchResult#status} will be {@link Status#OK} and {@link MatchResult#metadata} will return an
+ * empty list.
+ */
+ ALLOW,
+
+ /**
+ * Filepatterns matching no resources are disallowed. For such a filepattern, {@link
+ * MatchResult#status} will be {@link Status#NOT_FOUND} and {@link MatchResult#metadata} will
+ * throw a {@link java.io.FileNotFoundException}.
+ */
+ DISALLOW,
+
+ /**
+ * Filepatterns matching no resources are allowed if the filepattern contains a glob wildcard
+ * character, and disallowed otherwise (i.e. if the filepattern specifies a single file).
+ */
+ ALLOW_IF_WILDCARD
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java
index 642c049..aa80b96 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.List;
+import org.apache.beam.sdk.io.FileSystems;
/**
* The result of {@link org.apache.beam.sdk.io.FileSystem#match}.
@@ -78,7 +79,9 @@
public abstract Status status();
/**
- * {@link Metadata} of matched files.
+ * {@link Metadata} of matched files. Note that if {@link #status()} is {@link Status#NOT_FOUND},
+ * this may either throw a {@link java.io.FileNotFoundException} or return an empty list,
+ * depending on the {@link EmptyMatchTreatment} used in the {@link FileSystems#match} call.
*/
public abstract List<Metadata> metadata() throws IOException;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MetadataCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MetadataCoder.java
new file mode 100644
index 0000000..5c9c4d7
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MetadataCoder.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.fs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+
+/** A {@link Coder} for {@link Metadata}. */
+public class MetadataCoder extends AtomicCoder<Metadata> {
+ private static final ResourceIdCoder RESOURCE_ID_CODER = ResourceIdCoder.of();
+ private static final VarIntCoder INT_CODER = VarIntCoder.of();
+ private static final VarLongCoder LONG_CODER = VarLongCoder.of();
+
+ /** Creates a {@link MetadataCoder}. */
+ public static MetadataCoder of() {
+ return new MetadataCoder();
+ }
+
+ @Override
+ public void encode(Metadata value, OutputStream os) throws IOException {
+ RESOURCE_ID_CODER.encode(value.resourceId(), os);
+ INT_CODER.encode(value.isReadSeekEfficient() ? 1 : 0, os);
+ LONG_CODER.encode(value.sizeBytes(), os);
+ }
+
+ @Override
+ public Metadata decode(InputStream is) throws IOException {
+ ResourceId resourceId = RESOURCE_ID_CODER.decode(is);
+ boolean isReadSeekEfficient = INT_CODER.decode(is) == 1;
+ long sizeBytes = LONG_CODER.decode(is);
+ return Metadata.builder()
+ .setResourceId(resourceId)
+ .setIsReadSeekEfficient(isReadSeekEfficient)
+ .setSizeBytes(sizeBytes)
+ .build();
+ }
+
+ @Override
+ public boolean consistentWithEquals() {
+ return true;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdCoder.java
new file mode 100644
index 0000000..d7649c0
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdCoder.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.fs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.BooleanCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.FileSystems;
+
+/** A {@link Coder} for {@link ResourceId}. */
+public class ResourceIdCoder extends AtomicCoder<ResourceId> {
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+ private static final Coder<Boolean> BOOL_CODER = BooleanCoder.of();
+
+ /** Creates a {@link ResourceIdCoder}. */
+ public static ResourceIdCoder of() {
+ return new ResourceIdCoder();
+ }
+
+ @Override
+ public void encode(ResourceId value, OutputStream os) throws IOException {
+ STRING_CODER.encode(value.toString(), os);
+ BOOL_CODER.encode(value.isDirectory(), os);
+ }
+
+ @Override
+ public ResourceId decode(InputStream is) throws IOException {
+ String spec = STRING_CODER.decode(is);
+ boolean isDirectory = BOOL_CODER.decode(is);
+ return FileSystems.matchNewResource(spec, isDirectory);
+ }
+
+ @Override
+ public boolean consistentWithEquals() {
+ return true;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java
index 3fc8e32..dd6d009 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java
@@ -24,7 +24,7 @@
* from existing storage:
* <pre>{@code
* PCollection<TableRow> inputData = pipeline.apply(
- * BigQueryIO.read().from("clouddataflow-readonly:samples.weather_stations"));
+ * BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
* }</pre>
* and {@code Write} transforms that persist PCollections to external storage:
* <pre> {@code
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
index 99717a4..509e434 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
@@ -71,6 +71,10 @@
"Trying to return record which is before the last-returned record");
if (position == null) {
+ LOG.info(
+ "Adjusting range start from {} to {} as position of first returned record",
+ range.getStartKey(),
+ recordStart);
range = range.withStartKey(recordStart);
}
position = recordStart;
@@ -87,6 +91,15 @@
@Override
public synchronized boolean trySplitAtPosition(ByteKey splitPosition) {
+ // Sanity check.
+ if (!range.containsKey(splitPosition)) {
+ LOG.warn(
+ "{}: Rejecting split request at {} because it is not within the range.",
+ this,
+ splitPosition);
+ return false;
+ }
+
// Unstarted.
if (position == null) {
LOG.warn(
@@ -106,15 +119,6 @@
return false;
}
- // Sanity check.
- if (!range.containsKey(splitPosition)) {
- LOG.warn(
- "{}: Rejecting split request at {} because it is not within the range.",
- this,
- splitPosition);
- return false;
- }
-
range = range.withEndKey(splitPosition);
return true;
}
@@ -123,7 +127,12 @@
public synchronized double getFractionConsumed() {
if (position == null) {
return 0;
+ } else if (done) {
+ return 1.0;
+ } else if (position.compareTo(range.getEndKey()) >= 0) {
+ return 1.0;
}
+
return range.estimateFractionForKey(position);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java
new file mode 100644
index 0000000..d3bff37
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.range;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
+
+/** A restriction represented by a range of integers [from, to). */
+public class OffsetRange
+ implements Serializable,
+ HasDefaultTracker<
+ OffsetRange, org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker> {
+ private final long from;
+ private final long to;
+
+ public OffsetRange(long from, long to) {
+ checkArgument(from <= to, "Malformed range [%s, %s)", from, to);
+ this.from = from;
+ this.to = to;
+ }
+
+ public long getFrom() {
+ return from;
+ }
+
+ public long getTo() {
+ return to;
+ }
+
+ @Override
+ public org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker newTracker() {
+ return new org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker(this);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + from + ", " + to + ')';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ OffsetRange that = (OffsetRange) o;
+
+ if (from != that.from) {
+ return false;
+ }
+ return to == that.to;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (from ^ (from >>> 32));
+ result = 31 * result + (int) (to ^ (to >>> 32));
+ return result;
+ }
+
+ public List<OffsetRange> split(long desiredNumOffsetsPerSplit, long minNumOffsetPerSplit) {
+ List<OffsetRange> res = new ArrayList<>();
+ long start = getFrom();
+ long maxEnd = getTo();
+
+ while (start < maxEnd) {
+ long end = start + desiredNumOffsetsPerSplit;
+ end = Math.min(end, maxEnd);
+ // Avoid having a too small range at the end and ensure that we respect minNumOffsetPerSplit.
+ long remaining = maxEnd - end;
+ if ((remaining < desiredNumOffsetsPerSplit / 4) || (remaining < minNumOffsetPerSplit)) {
+ end = maxEnd;
+ }
+ res.add(new OffsetRange(start, end));
+ start = end;
+ }
+ return res;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java
index 51e2b1a..7b4b331 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java
@@ -26,6 +26,9 @@
/**
* A {@link RangeTracker} for non-negative positions of type {@code long}.
+ *
+ * <p>Not to be confused with {@link
+ * org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker}.
*/
public class OffsetRangeTracker implements RangeTracker<Long> {
private static final Logger LOG = LoggerFactory.getLogger(OffsetRangeTracker.class);
@@ -53,6 +56,8 @@
this.stopOffset = stopOffset;
}
+ private OffsetRangeTracker() { }
+
public synchronized boolean isStarted() {
// done => started: handles the case when the reader was empty.
return (offsetOfLastSplitPoint != -1) || done;
@@ -259,11 +264,19 @@
*/
@VisibleForTesting
OffsetRangeTracker copy() {
- OffsetRangeTracker res = new OffsetRangeTracker(startOffset, stopOffset);
- res.offsetOfLastSplitPoint = this.offsetOfLastSplitPoint;
- res.lastRecordStart = this.lastRecordStart;
- res.done = this.done;
- res.splitPointsSeen = this.splitPointsSeen;
- return res;
+ synchronized (this) {
+ OffsetRangeTracker res = new OffsetRangeTracker();
+ // This synchronized is not really necessary, because there's no concurrent access to "res",
+ // however it is necessary to prevent findbugs from complaining about unsynchronized access.
+ synchronized (res) {
+ res.startOffset = this.startOffset;
+ res.stopOffset = this.stopOffset;
+ res.offsetOfLastSplitPoint = this.offsetOfLastSplitPoint;
+ res.lastRecordStart = this.lastRecordStart;
+ res.done = this.done;
+ res.splitPointsSeen = this.splitPointsSeen;
+ }
+ return res;
+ }
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java
index 3c77043..6c88fa2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java
@@ -17,7 +17,10 @@
*/
package org.apache.beam.sdk.metrics;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.auto.value.AutoValue;
+import com.google.common.base.Strings;
import java.io.Serializable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -38,10 +41,14 @@
public abstract String name();
public static MetricName named(String namespace, String name) {
+ checkArgument(!Strings.isNullOrEmpty(namespace), "Metric namespace must be non-empty");
+ checkArgument(!Strings.isNullOrEmpty(name), "Metric name must be non-empty");
return new AutoValue_MetricName(namespace, name);
}
public static MetricName named(Class<?> namespace, String name) {
+ checkArgument(namespace != null, "Metric namespace must be non-null");
+ checkArgument(!Strings.isNullOrEmpty(name), "Metric name must be non-empty");
return new AutoValue_MetricName(namespace.getName(), name);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
index 62b0806..361c75f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
@@ -23,7 +23,8 @@
import org.apache.beam.sdk.annotations.Experimental.Kind;
/**
- * Holds the metrics for a single step and unit-of-commit (bundle).
+ * Holds the metrics for a single step. Each of the methods should return an implementation of the
+ * appropriate metrics interface for the "current" step.
*/
@Experimental(Kind.METRICS)
public interface MetricsContainer extends Serializable {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/package-info.java
index f71dc7a..a391b98 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/package-info.java
@@ -25,4 +25,8 @@
* <p>Runners should look at {@link org.apache.beam.sdk.metrics.MetricsContainer} for details on
* how to support metrics.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.metrics;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
index 3375dc7..39debb5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
@@ -33,6 +33,8 @@
.add(PipelineOptions.class)
.add(ApplicationNameOptions.class)
.add(StreamingOptions.class)
+ .add(ExperimentalOptions.class)
+ .add(SdkHarnessOptions.class)
.build();
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java
new file mode 100644
index 0000000..cb5c41c
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.options;
+
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+
+/**
+ * Apache Beam provides a number of experimental features that can
+ * be enabled with this flag. If executing against a managed service, please contact the
+ * service owners before enabling any experiments.
+ */
+@Experimental
+@Hidden
+public interface ExperimentalOptions extends PipelineOptions {
+ @Description("[Experimental] Apache Beam provides a number of experimental features that can "
+ + "be enabled with this flag. If executing against a managed service, please contact the "
+ + "service owners before enabling any experiments.")
+ @Nullable
+ List<String> getExperiments();
+ void setExperiments(@Nullable List<String> value);
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index 9a4d25a..77117b6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -35,6 +35,7 @@
import org.apache.beam.sdk.options.ProxyInvocationHandler.Serializer;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.util.ReleaseInfo;
import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
@@ -176,7 +177,12 @@
*
* <h2>Serialization Of PipelineOptions</h2>
*
- * {@link PipelineRunner}s require support for options to be serialized. Each property
+ * {@link PipelineOptions} is intentionally <i>not</i> marked {@link java.io.Serializable}, in order
+ * to discourage pipeline authors from capturing {@link PipelineOptions} at pipeline construction
+ * time, because a pipeline may be saved as a template and run with a different set of options
+ * than the ones it was constructed with. See {@link Pipeline#run(PipelineOptions)}.
+ *
+ * <p>However, {@link PipelineRunner}s require support for options to be serialized. Each property
* within {@link PipelineOptions} must be able to be serialized using Jackson's
* {@link ObjectMapper} or the getter method for the property annotated with
* {@link JsonIgnore @JsonIgnore}.
@@ -348,4 +354,39 @@
return NEXT_ID.getAndIncrement();
}
}
+
+ /**
+ * A user agent string as per RFC2616, describing the pipeline to external services.
+ *
+ * <p>https://www.ietf.org/rfc/rfc2616.txt
+ *
+ * <p>It should follow the BNF Form:
+ * <pre><code>
+ * user agent = 1*(product | comment)
+ * product = token ["/" product-version]
+ * product-version = token
+ * </code></pre>
+ * Where a token is a series of characters without a separator.
+ *
+ * <p>The string defaults to {@code [name]/[version]} based on the properties of the Apache Beam
+ * release.
+ */
+ @Description("A user agent string describing the pipeline to external services."
+ + " The format should follow RFC2616. This option defaults to \"[name]/[version]\""
+ + " where name and version are properties of the Apache Beam release.")
+ @Default.InstanceFactory(UserAgentFactory.class)
+ String getUserAgent();
+ void setUserAgent(String userAgent);
+
+ /**
+ * Returns a user agent string constructed from {@link ReleaseInfo#getName()} and
+ * {@link ReleaseInfo#getVersion()}, in the format {@code [name]/[version]}.
+ */
+ class UserAgentFactory implements DefaultValueFactory<String> {
+ @Override
+ public String create(PipelineOptions options) {
+ ReleaseInfo info = ReleaseInfo.getReleaseInfo();
+ return String.format("%s/%s", info.getName(), info.getVersion()).replace(" ", "_");
+ }
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
index c0990cb..ad6979e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
@@ -63,6 +63,7 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -184,18 +185,20 @@
private final String[] args;
private final boolean validation;
private final boolean strictParsing;
+ private final boolean isCli;
// Do not allow direct instantiation
private Builder() {
- this(null, false, true);
+ this(null, false, true, false);
}
private Builder(String[] args, boolean validation,
- boolean strictParsing) {
+ boolean strictParsing, boolean isCli) {
this.defaultAppName = findCallersClassName();
this.args = args;
this.validation = validation;
this.strictParsing = strictParsing;
+ this.isCli = isCli;
}
/**
@@ -237,7 +240,7 @@
*/
public Builder fromArgs(String... args) {
checkNotNull(args, "Arguments should not be null.");
- return new Builder(args, validation, strictParsing);
+ return new Builder(args, validation, strictParsing, true);
}
/**
@@ -247,7 +250,7 @@
* validation.
*/
public Builder withValidation() {
- return new Builder(args, true, strictParsing);
+ return new Builder(args, true, strictParsing, isCli);
}
/**
@@ -255,7 +258,7 @@
* arguments.
*/
public Builder withoutStrictParsing() {
- return new Builder(args, validation, false);
+ return new Builder(args, validation, false, isCli);
}
/**
@@ -300,7 +303,11 @@
}
if (validation) {
- PipelineOptionsValidator.validate(klass, t);
+ if (isCli) {
+ PipelineOptionsValidator.validateCli(klass, t);
+ } else {
+ PipelineOptionsValidator.validate(klass, t);
+ }
}
return t;
}
@@ -575,6 +582,8 @@
/**
* Validates that the interface conforms to the following:
* <ul>
+ * <li>Every inherited interface of {@code iface} must extend PipelineOptions except for
+ * PipelineOptions itself.
* <li>Any property with the same name must have the same return type for all derived
* interfaces of {@link PipelineOptions}.
* <li>Every bean property of any interface derived from {@link PipelineOptions} must have a
@@ -596,6 +605,10 @@
Class<T> iface, Set<Class<? extends PipelineOptions>> validatedPipelineOptionsInterfaces) {
checkArgument(iface.isInterface(), "Only interface types are supported.");
+ // Validate that every inherited interface must extend PipelineOptions except for
+ // PipelineOptions itself.
+ validateInheritedInterfacesExtendPipelineOptions(iface);
+
@SuppressWarnings("unchecked")
Set<Class<? extends PipelineOptions>> combinedPipelineOptionsInterfaces =
FluentIterable.from(validatedPipelineOptionsInterfaces).append(iface).toSet();
@@ -1252,6 +1265,44 @@
iface.getName());
}
+ private static void checkInheritedFrom(Class<?> checkClass, Class fromClass,
+ Set<Class<?>> nonPipelineOptions) {
+ if (checkClass.equals(fromClass)) {
+ return;
+ }
+
+ if (checkClass.getInterfaces().length == 0) {
+ nonPipelineOptions.add(checkClass);
+ return;
+ }
+
+ for (Class<?> klass : checkClass.getInterfaces()) {
+ checkInheritedFrom(klass, fromClass, nonPipelineOptions);
+ }
+ }
+
+ private static void throwNonPipelineOptions(Class<?> klass,
+ Set<Class<?>> nonPipelineOptionsClasses) {
+ StringBuilder errorBuilder = new StringBuilder(String.format(
+ "All inherited interfaces of [%s] should inherit from the PipelineOptions interface. "
+ + "The following inherited interfaces do not:",
+ klass.getName()));
+
+ for (Class<?> invalidKlass : nonPipelineOptionsClasses) {
+ errorBuilder.append(String.format("%n - %s", invalidKlass.getName()));
+ }
+ throw new IllegalArgumentException(errorBuilder.toString());
+ }
+
+ private static void validateInheritedInterfacesExtendPipelineOptions(Class<?> klass) {
+ Set<Class<?>> nonPipelineOptionsClasses = new LinkedHashSet<>();
+ checkInheritedFrom(klass, PipelineOptions.class, nonPipelineOptionsClasses);
+
+ if (!nonPipelineOptionsClasses.isEmpty()) {
+ throwNonPipelineOptions(klass, nonPipelineOptionsClasses);
+ }
+ }
+
private static class MultipleDefinitions {
private Method method;
private SortedSet<Method> collidingMethods;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java
index bd54ec3..fcffd74 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java
@@ -43,9 +43,29 @@
*
* @param klass The interface to fetch validation criteria from.
* @param options The {@link PipelineOptions} to validate.
- * @return The type
+ * @return Validated options.
*/
public static <T extends PipelineOptions> T validate(Class<T> klass, PipelineOptions options) {
+ return validate(klass, options, false);
+ }
+
+ /**
+ * Validates that the passed {@link PipelineOptions} from command line interface (CLI)
+ * conforms to all the validation criteria from the passed in interface.
+ *
+ * <p>Note that the interface requested must conform to the validation criteria specified on
+ * {@link PipelineOptions#as(Class)}.
+ *
+ * @param klass The interface to fetch validation criteria from.
+ * @param options The {@link PipelineOptions} to validate.
+ * @return Validated options.
+ */
+ public static <T extends PipelineOptions> T validateCli(Class<T> klass, PipelineOptions options) {
+ return validate(klass, options, true);
+ }
+
+ private static <T extends PipelineOptions> T validate(Class<T> klass, PipelineOptions options,
+ boolean isCli) {
checkNotNull(klass);
checkNotNull(options);
checkArgument(Proxy.isProxyClass(options.getClass()));
@@ -67,9 +87,15 @@
requiredGroups.put(requiredGroup, method);
}
} else {
- checkArgument(handler.invoke(asClassOptions, method, null) != null,
- "Missing required value for [%s, \"%s\"]. ",
- method, getDescription(method));
+ if (isCli) {
+ checkArgument(handler.invoke(asClassOptions, method, null) != null,
+ "Missing required value for [--%s, \"%s\"]. ",
+ handler.getOptionName(method), getDescription(method));
+ } else {
+ checkArgument(handler.invoke(asClassOptions, method, null) != null,
+ "Missing required value for [%s, \"%s\"]. ",
+ method, getDescription(method));
+ }
}
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
index eda21a8..926a7b9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
@@ -45,6 +45,8 @@
import com.google.common.collect.MutableClassToInstanceMap;
import java.beans.PropertyDescriptor;
import java.io.IOException;
+import java.io.NotSerializableException;
+import java.io.Serializable;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
@@ -87,7 +89,7 @@
* {@link PipelineOptions#as(Class)}.
*/
@ThreadSafe
-class ProxyInvocationHandler implements InvocationHandler {
+class ProxyInvocationHandler implements InvocationHandler, Serializable {
/**
* No two instances of this class are considered equivalent hence we generate a random hash code.
*/
@@ -164,6 +166,21 @@
+ Arrays.toString(args) + "].");
}
+ public String getOptionName(Method method) {
+ return gettersToPropertyNames.get(method.getName());
+ }
+
+ private void writeObject(java.io.ObjectOutputStream stream)
+ throws IOException {
+ throw new NotSerializableException(
+ "PipelineOptions objects are not serializable and should not be embedded into transforms "
+ + "(did you capture a PipelineOptions object in a field or in an anonymous class?). "
+ + "Instead, if you're using a DoFn, access PipelineOptions at runtime "
+ + "via ProcessContext/StartBundleContext/FinishBundleContext.getPipelineOptions(), "
+ + "or pre-extract necessary fields from PipelineOptions "
+ + "at pipeline construction time.");
+ }
+
/**
* Track whether options values are explicitly set, or retrieved from defaults.
*/
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
new file mode 100644
index 0000000..5f5dd6e
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.options;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.annotations.Experimental;
+
+/**
+ * Options that are used to control configuration of the SDK harness.
+ */
+@Experimental
+@Description("Options that are used to control configuration of the SDK harness.")
+public interface SdkHarnessOptions extends PipelineOptions {
+ /**
+ * The set of log levels that can be used in the SDK harness.
+ */
+ enum LogLevel {
+ /** Special level used to turn off logging. */
+ OFF,
+
+ /** LogLevel for logging error messages. */
+ ERROR,
+
+ /** LogLevel for logging warning messages. */
+ WARN,
+
+ /** LogLevel for logging informational messages. */
+ INFO,
+
+ /** LogLevel for logging diagnostic messages. */
+ DEBUG,
+
+ /** LogLevel for logging tracing messages. */
+ TRACE
+ }
+
+ /**
+ * This option controls the default log level of all loggers without a log level override.
+ */
+ @Description("Controls the default log level of all loggers without a log level override.")
+ @Default.Enum("INFO")
+ LogLevel getDefaultSdkHarnessLogLevel();
+ void setDefaultSdkHarnessLogLevel(LogLevel logLevel);
+
+ /**
+ * This option controls the log levels for specifically named loggers.
+ *
+ * <p>Later options with equivalent names override earlier options.
+ *
+ * <p>See {@link SdkHarnessLogLevelOverrides} for more information on how to configure logging
+ * on a per {@link Class}, {@link Package}, or name basis. If used from the command line,
+ * the expected format is {"Name":"LogLevel",...}, further details on
+ * {@link SdkHarnessLogLevelOverrides#from}.
+ */
+ @Description("This option controls the log levels for specifically named loggers. "
+ + "The expected format is {\"Name\":\"LogLevel\",...}. The SDK harness supports a logging "
+ + "hierarchy based off of names that are '.' separated. For example, by specifying the value "
+ + "{\"a.b.c.Foo\":\"DEBUG\"}, the logger for the class 'a.b.c.Foo' will be configured to "
+ + "output logs at the DEBUG level. Similarly, by specifying the value {\"a.b.c\":\"WARN\"}, "
+ + "all loggers underneath the 'a.b.c' package will be configured to output logs at the WARN "
+ + "level. System.out and System.err levels are configured via loggers of the corresponding "
+ + "name. Also, note that when multiple overrides are specified, the exact name followed by "
+ + "the closest parent takes precedence.")
+ SdkHarnessLogLevelOverrides getSdkHarnessLogLevelOverrides();
+ void setSdkHarnessLogLevelOverrides(SdkHarnessLogLevelOverrides value);
+
+ /**
+ * Defines a log level override for a specific class, package, or name.
+ *
+ * <p>The SDK harness supports a logging hierarchy based off of names that are "."
+ * separated. It is a common pattern to have the logger for a given class share the same name as
+ * the class itself. Given the classes {@code a.b.c.Foo}, {@code a.b.c.Xyz}, and {@code a.b.Bar},
+ * with loggers named {@code "a.b.c.Foo"}, {@code "a.b.c.Xyz"}, and {@code "a.b.Bar"}
+ * respectively, we can override the log levels:
+ * <ul>
+ * <li>for {@code Foo} by specifying the name {@code "a.b.c.Foo"} or the {@link Class}
+ * representing {@code a.b.c.Foo}.
+ * <li>for {@code Foo}, {@code Xyz}, and {@code Bar} by specifying the name {@code "a.b"} or
+ * the {@link Package} representing {@code a.b}.
+ * <li>for {@code Foo} and {@code Bar} by specifying both of their names or classes.
+ * </ul>
+ *
+ * <p>{@code System.out} and {@code System.err} messages are configured via loggers of the
+ * corresponding name. Note that by specifying multiple overrides, the exact name followed by the
+ * closest parent takes precedence.
+ */
+ class SdkHarnessLogLevelOverrides extends HashMap<String, LogLevel> {
+ /**
+ * Overrides the default log level for the passed in class.
+ *
+ * <p>This is equivalent to calling
+ * {@link #addOverrideForName(String, LogLevel)}
+ * and passing in the {@link Class#getName() class name}.
+ */
+ public SdkHarnessLogLevelOverrides addOverrideForClass(Class<?> klass, LogLevel logLevel) {
+ checkNotNull(klass, "Expected class to be not null.");
+ addOverrideForName(klass.getName(), logLevel);
+ return this;
+ }
+
+ /**
+ * Overrides the default log level for the passed in package.
+ *
+ * <p>This is equivalent to calling
+ * {@link #addOverrideForName(String, LogLevel)}
+ * and passing in the {@link Package#getName() package name}.
+ */
+ public SdkHarnessLogLevelOverrides addOverrideForPackage(Package pkg, LogLevel logLevel) {
+ checkNotNull(pkg, "Expected package to be not null.");
+ addOverrideForName(pkg.getName(), logLevel);
+ return this;
+ }
+
+ /**
+ * Overrides the default log logLevel for the passed in name.
+ *
+ * <p>Note that because of the hierarchical nature of logger names, this will
+ * override the log logLevel of all loggers that have the passed in name or
+ * a parent logger that has the passed in name.
+ */
+ public SdkHarnessLogLevelOverrides addOverrideForName(String name, LogLevel logLevel) {
+ checkNotNull(name, "Expected name to be not null.");
+ checkNotNull(logLevel,
+ "Expected logLevel to be one of %s.", Arrays.toString(LogLevel.values()));
+ put(name, logLevel);
+ return this;
+ }
+
+ /**
+ * Expects a map keyed by logger {@code Name}s with values representing {@code LogLevel}s.
+ * The {@code Name} generally represents the fully qualified Java
+ * {@link Class#getName() class name}, or fully qualified Java
+ * {@link Package#getName() package name}, or custom logger name. The {@code LogLevel}
+ * represents the log level and must be one of {@link LogLevel}.
+ */
+ @JsonCreator
+ public static SdkHarnessLogLevelOverrides from(Map<String, String> values) {
+ checkNotNull(values, "Expected values to be not null.");
+ SdkHarnessLogLevelOverrides overrides = new SdkHarnessLogLevelOverrides();
+ for (Map.Entry<String, String> entry : values.entrySet()) {
+ try {
+ overrides.addOverrideForName(entry.getKey(), LogLevel.valueOf(entry.getValue()));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(String.format(
+ "Unsupported log level '%s' requested for %s. Must be one of %s.",
+ entry.getValue(), entry.getKey(), Arrays.toString(LogLevel.values())));
+ }
+
+ }
+ return overrides;
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
index c7f1e09..3e6a24b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
@@ -41,25 +41,35 @@
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.PCollection;
/**
* A {@link ValueProvider} abstracts the notion of fetching a value that may or may not be currently
* available.
*
* <p>This can be used to parameterize transforms that only read values in at runtime, for example.
+ *
+ * <p>A common task is to create a {@link PCollection} containing the value of this
+ * {@link ValueProvider} regardless of whether it's accessible at construction time or not.
+ * For that, use {@link Create#ofProvider}.
+ *
+ * <p>For unit-testing a transform against a {@link ValueProvider} that only becomes available
+ * at runtime, use {@link TestPipeline#newProvider}.
*/
@JsonSerialize(using = ValueProvider.Serializer.class)
@JsonDeserialize(using = ValueProvider.Deserializer.class)
public interface ValueProvider<T> extends Serializable {
/**
- * Return the value wrapped by this {@link ValueProvider}.
+ * Returns the runtime value wrapped by this {@link ValueProvider} in case it is {@link
+ * #isAccessible}, otherwise fails.
*/
T get();
/**
- * Whether the contents of this {@link ValueProvider} is available to
- * routines that run at graph construction time.
+ * Whether the contents of this {@link ValueProvider} is currently available via {@link #get}.
*/
boolean isAccessible();
@@ -95,9 +105,7 @@
@Override
public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("value", value)
- .toString();
+ return String.valueOf(value);
}
}
@@ -154,8 +162,12 @@
@Override
public String toString() {
+ if (isAccessible()) {
+ return String.valueOf(get());
+ }
return MoreObjects.toStringHelper(this)
.add("value", value)
+ .add("translator", translator.getClass().getSimpleName())
.toString();
}
}
@@ -220,7 +232,8 @@
public T get() {
PipelineOptions options = optionsMap.get(optionsId);
if (options == null) {
- throw new RuntimeException("Not called from a runtime context.");
+ throw new IllegalStateException(
+ "Value only available at runtime, but accessed from a non-runtime context: " + this);
}
try {
Method method = klass.getMethod(methodName);
@@ -243,8 +256,7 @@
@Override
public boolean isAccessible() {
- PipelineOptions options = optionsMap.get(optionsId);
- return options != null;
+ return optionsMap.get(optionsId) != null;
}
/**
@@ -256,10 +268,12 @@
@Override
public String toString() {
+ if (isAccessible()) {
+ return String.valueOf(get());
+ }
return MoreObjects.toStringHelper(this)
.add("propertyName", propertyName)
.add("default", defaultValue)
- .add("value", isAccessible() ? get() : null)
.toString();
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
index 1cc46fe..9345462 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
@@ -19,41 +19,38 @@
import static com.google.common.base.Preconditions.checkNotNull;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.util.Map;
-import org.apache.beam.sdk.util.common.ReflectHelpers;
+import org.apache.beam.sdk.testing.TestPipeline;
-/**
- * Utilities for working with the {@link ValueProvider} interface.
- */
-class ValueProviders {
+/** Utilities for working with the {@link ValueProvider} interface. */
+public class ValueProviders {
private ValueProviders() {}
/**
- * Given {@code serializedOptions} as a JSON-serialized {@link PipelineOptions}, updates
- * the values according to the provided values in {@code runtimeValues}.
+ * Given {@code serializedOptions} as a JSON-serialized {@link PipelineOptions}, updates the
+ * values according to the provided values in {@code runtimeValues}.
+ *
+ * @deprecated Use {@link TestPipeline#newProvider} for testing {@link ValueProvider} code.
*/
+ @Deprecated
public static String updateSerializedOptions(
String serializedOptions, Map<String, String> runtimeValues) {
- ObjectMapper mapper = new ObjectMapper().registerModules(
- ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
ObjectNode root, options;
try {
- root = mapper.readValue(serializedOptions, ObjectNode.class);
+ root = PipelineOptionsFactory.MAPPER.readValue(serializedOptions, ObjectNode.class);
options = (ObjectNode) root.get("options");
checkNotNull(options, "Unable to locate 'options' in %s", serializedOptions);
} catch (IOException e) {
- throw new RuntimeException(
- String.format("Unable to parse %s", serializedOptions), e);
+ throw new RuntimeException(String.format("Unable to parse %s", serializedOptions), e);
}
for (Map.Entry<String, String> entry : runtimeValues.entrySet()) {
options.put(entry.getKey(), entry.getValue());
}
try {
- return mapper.writeValueAsString(root);
+ return PipelineOptionsFactory.MAPPER.writeValueAsString(root);
} catch (IOException e) {
throw new RuntimeException("Unable to parse re-serialize options", e);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
index 995bcb9..7593807 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
@@ -31,4 +31,8 @@
* where and how it should run after pipeline construction is complete.
*
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index fac558b..6c99bb7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -24,10 +24,12 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -39,6 +41,7 @@
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
@@ -56,7 +59,6 @@
public class TransformHierarchy {
private static final Logger LOG = LoggerFactory.getLogger(TransformHierarchy.class);
- private final Pipeline pipeline;
private final Node root;
private final Map<Node, PInput> unexpandedInputs;
private final Map<POutput, Node> producers;
@@ -65,12 +67,11 @@
// Maintain a stack based on the enclosing nodes
private Node current;
- public TransformHierarchy(Pipeline pipeline) {
- this.pipeline = pipeline;
+ public TransformHierarchy() {
producers = new HashMap<>();
producerInput = new HashMap<>();
unexpandedInputs = new HashMap<>();
- root = new Node(null, null, "", null);
+ root = new Node();
current = root;
}
@@ -98,6 +99,48 @@
return current;
}
+ @Internal
+ public Node pushFinalizedNode(
+ String name,
+ Map<TupleTag<?>, PValue> inputs,
+ PTransform<?, ?> transform,
+ Map<TupleTag<?>, PValue> outputs) {
+ checkNotNull(
+ transform, "A %s must be provided for all Nodes", PTransform.class.getSimpleName());
+ checkNotNull(
+ name, "A name must be provided for all %s Nodes", PTransform.class.getSimpleName());
+ checkNotNull(
+ inputs, "An input must be provided for all %s Nodes", PTransform.class.getSimpleName());
+ Node node = new Node(current, transform, name, inputs, outputs);
+ node.finishedSpecifying = true;
+ current.addComposite(node);
+ current = node;
+ return current;
+ }
+
+ @Internal
+ public Node addFinalizedPrimitiveNode(
+ String name,
+ Map<TupleTag<?>, PValue> inputs,
+ PTransform<?, ?> transform,
+ Map<TupleTag<?>, PValue> outputs) {
+ checkNotNull(
+ transform, "A %s must be provided for all Nodes", PTransform.class.getSimpleName());
+ checkNotNull(
+ name, "A name must be provided for all %s Nodes", PTransform.class.getSimpleName());
+ checkNotNull(
+ inputs, "Inputs must be provided for all %s Nodes", PTransform.class.getSimpleName());
+ checkNotNull(
+ outputs, "Outputs must be provided for all %s Nodes", PTransform.class.getSimpleName());
+ Node node = new Node(current, transform, name, inputs, outputs);
+ node.finishedSpecifying = true;
+ for (PValue output : outputs.values()) {
+ producers.put(output, node);
+ }
+ current.addComposite(node);
+ return node;
+ }
+
public Node replaceNode(Node existing, PInput input, PTransform<?, ?> transform) {
checkNotNull(existing);
checkNotNull(input);
@@ -145,14 +188,6 @@
Node producerNode = getProducer(inputValue);
PInput input = producerInput.remove(inputValue);
inputValue.finishSpecifying(input, producerNode.getTransform());
- checkState(
- producers.get(inputValue) != null,
- "Producer unknown for input %s",
- inputValue);
- checkState(
- producers.get(inputValue) != null,
- "Producer unknown for input %s",
- inputValue);
}
}
@@ -167,7 +202,7 @@
* nodes.
*/
public void setOutput(POutput output) {
- for (PValue value : output.expand().values()) {
+ for (PCollection<?> value : fullyExpand(output).values()) {
if (!producers.containsKey(value)) {
producers.put(value, current);
value.finishSpecifyingOutput(
@@ -201,13 +236,13 @@
}
Node getProducer(PValue produced) {
- return producers.get(produced);
+ return checkNotNull(producers.get(produced), "No producer found for %s", produced);
}
public Set<PValue> visit(PipelineVisitor visitor) {
finishSpecifying();
Set<PValue> visitedValues = new HashSet<>();
- root.visit(visitor, visitedValues);
+ root.visit(visitor, visitedValues, new HashSet<Node>(), new HashSet<Node>());
return visitedValues;
}
@@ -228,16 +263,59 @@
return current;
}
+ private Map<TupleTag<?>, PCollection<?>> fullyExpand(POutput output) {
+ Map<TupleTag<?>, PCollection<?>> result = new LinkedHashMap<>();
+ for (Map.Entry<TupleTag<?>, PValue> value : output.expand().entrySet()) {
+ if (value.getValue() instanceof PCollection) {
+ PCollection<?> previous = result.put(value.getKey(), (PCollection<?>) value.getValue());
+ checkArgument(
+ previous == null,
+ "Found conflicting %ss in flattened expansion of %s: %s maps to %s and %s",
+ output,
+ TupleTag.class.getSimpleName(),
+ value.getKey(),
+ previous,
+ value.getValue());
+ } else {
+ if (value.getValue().expand().size() == 1
+ && Iterables.getOnlyElement(value.getValue().expand().values())
+ .equals(value.getValue())) {
+ throw new IllegalStateException(
+ String.format(
+ "Non %s %s that expands into itself %s",
+ PCollection.class.getSimpleName(),
+ PValue.class.getSimpleName(),
+ value.getValue()));
+ }
+ for (Map.Entry<TupleTag<?>, PCollection<?>> valueComponent :
+ fullyExpand(value.getValue()).entrySet()) {
+ PCollection<?> previous = result.put(valueComponent.getKey(), valueComponent.getValue());
+ checkArgument(
+ previous == null,
+ "Found conflicting %ss in flattened expansion of %s: %s maps to %s and %s",
+ output,
+ TupleTag.class.getSimpleName(),
+ valueComponent.getKey(),
+ previous,
+ valueComponent.getValue());
+ }
+ }
+ }
+ return result;
+ }
+
/**
* Provides internal tracking of transform relationships with helper methods
* for initialization and ordered visitation.
*/
public class Node {
- private final Node enclosingNode;
+ // null for the root node, otherwise the enclosing node
+ @Nullable private final Node enclosingNode;
+
// The PTransform for this node, which may be a composite PTransform.
// The root of a TransformHierarchy is represented as a Node
// with a null transform field.
- private final PTransform<?, ?> transform;
+ @Nullable private final PTransform<?, ?> transform;
private final String fullName;
@@ -248,37 +326,79 @@
private final Map<TupleTag<?>, PValue> inputs;
// TODO: track which outputs need to be exported to parent.
- // Output of the transform, in expanded form.
- private Map<TupleTag<?>, PValue> outputs;
+ // Output of the transform, in expanded form. Null if not yet set.
+ @Nullable private Map<TupleTag<?>, PValue> outputs;
@VisibleForTesting
boolean finishedSpecifying = false;
/**
+ * Creates the root-level node. The root level node has a null enclosing node, a null transform,
+ * an empty map of inputs, an empty map of outputs, and a name equal to the empty string.
+ */
+ private Node() {
+ this.enclosingNode = null;
+ this.transform = null;
+ this.fullName = "";
+ this.inputs = Collections.emptyMap();
+ this.outputs = Collections.emptyMap();
+ }
+
+ /**
* Creates a new Node with the given parent and transform.
*
- * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
- * nodes.
- *
* @param enclosingNode the composite node containing this node
* @param transform the PTransform tracked by this node
* @param fullName the fully qualified name of the transform
* @param input the unexpanded input to the transform
*/
private Node(
- @Nullable Node enclosingNode,
- @Nullable PTransform<?, ?> transform,
+ Node enclosingNode,
+ PTransform<?, ?> transform,
String fullName,
- @Nullable PInput input) {
+ PInput input) {
this.enclosingNode = enclosingNode;
this.transform = transform;
this.fullName = fullName;
- this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
+ ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
+ inputs.putAll(input.expand());
+ inputs.putAll(transform.getAdditionalInputs());
+ this.inputs = inputs.build();
+ }
+
+ /**
+ * Creates a new {@link Node} with the given parent and transform, where inputs and outputs
+ * are already known.
+ *
+ * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
+ * nodes.
+ *
+ * @param enclosingNode the composite node containing this node
+ * @param transform the PTransform tracked by this node
+ * @param fullName the fully qualified name of the transform
+ * @param inputs the expanded inputs to the transform
+ * @param outputs the expanded outputs of the transform
+ */
+ private Node(
+ @Nullable Node enclosingNode,
+ @Nullable PTransform<?, ?> transform,
+ String fullName,
+ @Nullable Map<TupleTag<?>, PValue> inputs,
+ @Nullable Map<TupleTag<?>, PValue> outputs) {
+ this.enclosingNode = enclosingNode;
+ this.transform = transform;
+ this.fullName = fullName;
+ this.inputs = inputs == null ? Collections.<TupleTag<?>, PValue>emptyMap() : inputs;
+ this.outputs = outputs == null ? Collections.<TupleTag<?>, PValue>emptyMap() : outputs;
}
/**
* Returns the transform associated with this transform node.
+ *
+ * @return {@code null} if and only if this is the root node of the graph, which has no
+ * associated transform
*/
+ @Nullable
public PTransform<?, ?> getTransform() {
return transform;
}
@@ -354,9 +474,9 @@
return fullName;
}
- /** Returns the transform input, in unexpanded form. */
+ /** Returns the transform input, in fully expanded form. */
public Map<TupleTag<?>, PValue> getInputs() {
- return inputs == null ? Collections.<TupleTag<?>, PValue>emptyMap() : inputs;
+ return inputs;
}
/**
@@ -453,7 +573,7 @@
/**
* Returns the {@link AppliedPTransform} representing this {@link Node}.
*/
- public AppliedPTransform<?, ?, ?> toAppliedPTransform() {
+ public AppliedPTransform<?, ?, ?> toAppliedPTransform(Pipeline pipeline) {
return AppliedPTransform.of(
getFullName(), inputs, outputs, (PTransform) getTransform(), pipeline);
}
@@ -461,10 +581,60 @@
/**
* Visit the transform node.
*
+ * <p>The visit proceeds in the following order:
+ *
+ * <ul>
+ * <li>Visit all input {@link PValue PValues} returned by the flattened expansion of {@link
+ * Node#getInputs()}.
+ * <li>If the node is a composite:
+ * <ul>
+ * <li>Enter the node via {@link PipelineVisitor#enterCompositeTransform(Node)}.
+ * <li>If the result of {@link PipelineVisitor#enterCompositeTransform(Node)} was {@link
+ * CompositeBehavior#ENTER_TRANSFORM}, visit each child node of this {@link Node}.
+ * <li>Leave the node via {@link PipelineVisitor#leaveCompositeTransform(Node)}.
+ * </ul>
+ * <li>If the node is a primitive, visit it via {@link
+ * PipelineVisitor#visitPrimitiveTransform(Node)}.
+ * <li>Visit each {@link PValue} that was output by this node.
+ * </ul>
+ *
+ * <p>Additionally, the following ordering restrictions are observed:
+ *
+ * <ul>
+ * <li>A {@link Node} will be visited after its enclosing node has been entered and before its
+ * enclosing node has been left
+ * <li>A {@link Node} will not be visited if any enclosing {@link Node} has returned {@link
+ * CompositeBehavior#DO_NOT_ENTER_TRANSFORM} from the call to {@link
+ * PipelineVisitor#enterCompositeTransform(Node)}.
+ * <li>A {@link PValue} will only be visited after the {@link Node} that originally produced
+ * it has been visited.
+ * </ul>
+ *
* <p>Provides an ordered visit of the input values, the primitive transform (or child nodes for
* composite transforms), then the output values.
*/
- private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
+ private void visit(
+ PipelineVisitor visitor,
+ Set<PValue> visitedValues,
+ Set<Node> visitedNodes,
+ Set<Node> skippedComposites) {
+ if (getEnclosingNode() != null && !visitedNodes.contains(getEnclosingNode())) {
+ // Recursively enter all enclosing nodes, as appropriate.
+ getEnclosingNode().visit(visitor, visitedValues, visitedNodes, skippedComposites);
+ }
+ // These checks occur after visiting the enclosing node to ensure that if this node has been
+ // visited while visiting the enclosing node the node is not revisited, or, if an enclosing
+ // Node is skipped, this node is also skipped.
+ if (!visitedNodes.add(this)) {
+ LOG.debug("Not revisiting previously visited node {}", this);
+ return;
+ } else if (childNodeOf(skippedComposites)) {
+ // This node is a child of a node that has been passed over via CompositeBehavior, and
+ // should also be skipped. All child nodes of a skipped composite should always be skipped.
+ LOG.debug("Not revisiting Node {} which is a child of a previously passed composite", this);
+ return;
+ }
+
if (!finishedSpecifying) {
finishSpecifying();
}
@@ -472,22 +642,31 @@
if (!isRootNode()) {
// Visit inputs.
for (PValue inputValue : inputs.values()) {
+ Node valueProducer = getProducer(inputValue);
+ if (!visitedNodes.contains(valueProducer)) {
+ valueProducer.visit(visitor, visitedValues, visitedNodes, skippedComposites);
+ }
if (visitedValues.add(inputValue)) {
- visitor.visitValue(inputValue, getProducer(inputValue));
+ LOG.debug("Visiting input value {}", inputValue);
+ visitor.visitValue(inputValue, valueProducer);
}
}
}
if (isCompositeNode()) {
+ LOG.debug("Visiting composite node {}", this);
PipelineVisitor.CompositeBehavior recurse = visitor.enterCompositeTransform(this);
if (recurse.equals(CompositeBehavior.ENTER_TRANSFORM)) {
for (Node child : parts) {
- child.visit(visitor, visitedValues);
+ child.visit(visitor, visitedValues, visitedNodes, skippedComposites);
}
+ } else {
+ skippedComposites.add(this);
}
visitor.leaveCompositeTransform(this);
} else {
+ LOG.debug("Visiting primitive node {}", this);
visitor.visitPrimitiveTransform(this);
}
@@ -496,12 +675,24 @@
// Visit outputs.
for (PValue pValue : outputs.values()) {
if (visitedValues.add(pValue)) {
+ LOG.debug("Visiting output value {}", pValue);
visitor.visitValue(pValue, this);
}
}
}
}
+ private boolean childNodeOf(Set<Node> nodes) {
+ if (isRootNode()) {
+ return false;
+ }
+ Node parent = this.getEnclosingNode();
+ while (!parent.isRootNode() && !nodes.contains(parent)) {
+ parent = parent.getEnclosingNode();
+ }
+ return nodes.contains(parent);
+ }
+
/**
* Finish specifying a transform.
*
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
index 2726936..cd28c64 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
@@ -20,4 +20,8 @@
* <p>Internals for use by runners.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.runners;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
index 76d3e32..a4af6eb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.state;
+import javax.annotation.Nonnull;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -31,6 +32,11 @@
*/
@Experimental(Kind.STATE)
public interface BagState<T> extends GroupingState<T, Iterable<T>> {
+
+ @Override
+ @Nonnull
+ Iterable<T> read();
+
@Override
BagState<T> readLater();
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
index 94a36d3..5cf4229 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.state;
+import javax.annotation.Nonnull;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.transforms.Combine.CombineFn;
@@ -35,6 +36,10 @@
@Experimental(Kind.STATE)
public interface CombiningState<InputT, AccumT, OutputT> extends GroupingState<InputT, OutputT> {
+ @Override
+ @Nonnull
+ OutputT read();
+
/**
* Read the merged accumulator for this state cell. It is implied that reading the state involves
* reading the accumulator, so {@link #readLater} is sufficient to prefetch for this.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
index 9c4c23e..8f244d4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
@@ -33,10 +33,18 @@
*/
@Experimental(Kind.STATE)
public interface GroupingState<InputT, OutputT> extends ReadableState<OutputT>, State {
- /** Add a value to the buffer. */
+ /**
+ * Add a value to the buffer.
+ *
+ * <p>Elements added will not be reflected in {@code OutputT} objects returned by
+ * previous calls to {@link #read}.
+ */
void add(InputT value);
- /** Return true if this state is empty. */
+ /**
+ * Returns a {@link ReadableState} whose {@link #read} method will return true if this state is
+ * empty at the point when that {@link #read} call returns.
+ */
ReadableState<Boolean> isEmpty();
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
index 17ea332..8b89d7b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
@@ -33,7 +33,13 @@
@Experimental(Kind.STATE)
public interface MapState<K, V> extends State {
- /** Associates the specified value with the specified key in this state. */
+ /**
+ * Associates the specified value with the specified key in this state.
+ *
+ * <p>Changes will not be reflected in the results returned by
+ * previous calls to {@link ReadableState#read} on the results any of the reading methods
+ * ({@link #get}, {@link #keys}, {@link #values}, and {@link #entries}).
+ */
void put(K key, V value);
/**
@@ -44,10 +50,20 @@
*
* <p>If the specified key is not already associated with a value (or is mapped to {@code null})
* associates it with the given value and returns {@code null}, else returns the current value.
+ *
+ * <p>Changes will not be reflected in the results returned by
+ * previous calls to {@link ReadableState#read} on the results any of the reading methods
+ * ({@link #get}, {@link #keys}, {@link #values}, and {@link #entries}).
*/
ReadableState<V> putIfAbsent(K key, V value);
- /** Remove the mapping for a key from this map if it is present. */
+ /**
+ * Remove the mapping for a key from this map if it is present.
+ *
+ * <p>Changes will not be reflected in the results returned by
+ * previous calls to {@link ReadableState#read} on the results any of the reading methods
+ * ({@link #get}, {@link #keys}, {@link #values}, and {@link #entries}).
+ */
void remove(K key);
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
index 70703ce..dec064a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.state;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -36,7 +37,12 @@
* <p>If there will be many calls to {@link #read} for different state in short succession, you
* should first call {@link #readLater} for all of them so the reads can potentially be batched
* (depending on the underlying implementation}.
+ *
+ * <p>The returned object should be independent of the underlying state. Any direct modification
+ * of the returned object should not modify state without going through the appropriate state
+ * interface, and modification to the state should not be mirrored in the returned object.
*/
+ @Nullable
T read();
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
index 6977a97..94d76a7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.state;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
/**
@@ -28,9 +29,10 @@
/**
* A {@link ReadableState} constructed from a constant value, hence immediately available.
*/
- public static <T> ReadableState<T> immediate(final T value) {
+ public static <T> ReadableState<T> immediate(@Nullable final T value) {
return new ReadableState<T>() {
@Override
+ @Nullable
public T read() {
return value;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
index fd339b2..d94c5c1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
@@ -36,10 +36,18 @@
/**
* Ensures a value is a member of the set, returning {@code true} if it was added and {@code
* false} otherwise.
+ *
+ * <p>Elements added will not be reflected in {@code OutputT} objects returned by
+ * previous calls to {@link #read}.
*/
ReadableState<Boolean> addIfAbsent(T t);
- /** Removes the specified element from this set if it is present. */
+ /**
+ * Removes the specified element from this set if it is present.
+ *
+ * <p>Changes will not be reflected in {@code OutputT} objects returned by
+ * previous calls to {@link #read}.
+ */
void remove(T t);
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
index b0412bf..0443f25 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
@@ -22,6 +22,7 @@
import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.Combine;
/**
* A specification of a persistent state cell. This includes information necessary to encode the
@@ -43,6 +44,14 @@
/**
* <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
*
+ * <p>Perform case analysis on this {@link StateSpec} using the provided {@link Cases}.
+ */
+ @Internal
+ <ResultT> ResultT match(Cases<ResultT> cases);
+
+ /**
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
* <p>Given {code coders} are inferred from type arguments defined for this class. Coders which
* are already set should take precedence over offered coders.
*
@@ -60,4 +69,48 @@
*/
@Internal
void finishSpecifying();
+
+ /**
+ * Cases for doing a "switch" on the type of {@link StateSpec}.
+ */
+ interface Cases<ResultT> {
+ ResultT dispatchValue(Coder<?> valueCoder);
+ ResultT dispatchBag(Coder<?> elementCoder);
+ ResultT dispatchCombining(Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder);
+ ResultT dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder);
+ ResultT dispatchSet(Coder<?> elementCoder);
+
+ /**
+ * A base class for a visitor with a default method for cases it is not interested in.
+ */
+ abstract class WithDefault<ResultT> implements Cases<ResultT> {
+
+ protected abstract ResultT dispatchDefault();
+
+ @Override
+ public ResultT dispatchValue(Coder<?> valueCoder) {
+ return dispatchDefault();
+ }
+
+ @Override
+ public ResultT dispatchBag(Coder<?> elementCoder) {
+ return dispatchDefault();
+ }
+
+ @Override
+ public ResultT dispatchCombining(Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder) {
+ return dispatchDefault();
+ }
+
+ @Override
+ public ResultT dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder) {
+ return dispatchDefault();
+ }
+
+ @Override
+ public ResultT dispatchSet(Coder<?> elementCoder) {
+ return dispatchDefault();
+ }
+ }
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 7b71384..360d9d3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -76,7 +76,9 @@
}
/**
- * Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link
+ * <b>For internal use only; no backwards compatibility guarantees</b>
+ *
+ * <p>Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link
* CombineFnWithContext} to automatically merge multiple values of type {@code InputT} into a
* single resulting {@code OutputT}.
*
@@ -84,6 +86,7 @@
*
* @see #combining(Coder, CombineFnWithContext)
*/
+ @Internal
public static <InputT, AccumT, OutputT>
StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
@@ -105,11 +108,14 @@
}
/**
- * Identical to {@link #combining(CombineFnWithContext)}, but with an accumulator coder explicitly
- * supplied.
+ * <b>For internal use only; no backwards compatibility guarantees</b>
+ *
+ * <p>Identical to {@link #combining(CombineFnWithContext)}, but with an accumulator coder
+ * explicitly supplied.
*
* <p>If automatic coder inference fails, use this method.
*/
+ @Internal
public static <InputT, AccumT, OutputT>
StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
@@ -125,7 +131,7 @@
* @see #bag(Coder)
*/
public static <T> StateSpec<BagState<T>> bag() {
- return bag(null);
+ return new BagStateSpec<>(null);
}
/**
@@ -145,7 +151,7 @@
* @see #set(Coder)
*/
public static <T> StateSpec<SetState<T>> set() {
- return set(null);
+ return new SetStateSpec<>(null);
}
/**
@@ -272,6 +278,11 @@
return visitor.bindValue(id, this, coder);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ return cases.dispatchValue(coder);
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -336,6 +347,11 @@
return visitor.bindCombining(id, this, accumCoder, combineFn);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ return cases.dispatchCombining(combineFn, accumCoder);
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -407,6 +423,14 @@
return visitor.bindCombiningWithContext(id, this, accumCoder, combineFn);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s is for internal use only and does not support case dispatch",
+ getClass().getSimpleName()));
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -474,6 +498,11 @@
return visitor.bindBag(id, this, elemCoder);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ return cases.dispatchBag(elemCoder);
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -530,6 +559,11 @@
return visitor.bindMap(id, this, keyCoder, valueCoder);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ return cases.dispatchMap(keyCoder, valueCoder);
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -594,6 +628,11 @@
return visitor.bindSet(id, this, elemCoder);
}
+ @Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ return cases.dispatchSet(elemCoder);
+ }
+
@SuppressWarnings("unchecked")
@Override
public void offerCoders(Coder[] coders) {
@@ -658,6 +697,14 @@
}
@Override
+ public <ResultT> ResultT match(Cases<ResultT> cases) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s is for internal use only and does not support case dispatch",
+ getClass().getSimpleName()));
+ }
+
+ @Override
public void offerCoders(Coder[] coders) {
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
index d8b8e92..01570f0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
@@ -19,4 +19,8 @@
/**
* Classes and interfaces for interacting with state.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.state;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CombineFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CombineFnTester.java
new file mode 100644
index 0000000..efd2af3
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CombineFnTester.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.testing;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.hamcrest.Matcher;
+
+/**
+ * Utilities for testing {@link CombineFn CombineFns}. Ensures that the {@link CombineFn} gives
+ * correct results across various permutations and shardings of the input.
+ */
+public class CombineFnTester {
+ /**
+ * Tests that the the {@link CombineFn}, when applied to the provided input, produces the provided
+ * output. Tests a variety of permutations of the input.
+ */
+ public static <InputT, AccumT, OutputT> void testCombineFn(
+ CombineFn<InputT, AccumT, OutputT> fn, List<InputT> input, final OutputT expected) {
+ testCombineFn(fn, input, is(expected));
+ Collections.shuffle(input);
+ testCombineFn(fn, input, is(expected));
+ }
+
+ public static <InputT, AccumT, OutputT> void testCombineFn(
+ CombineFn<InputT, AccumT, OutputT> fn, List<InputT> input, Matcher<? super OutputT> matcher) {
+ int size = input.size();
+ checkCombineFnShardsMultipleOrders(fn, Collections.singletonList(input), matcher);
+ checkCombineFnShardsMultipleOrders(fn, shardEvenly(input, 2), matcher);
+ if (size > 4) {
+ checkCombineFnShardsMultipleOrders(fn, shardEvenly(input, size / 2), matcher);
+ checkCombineFnShardsMultipleOrders(
+ fn, shardEvenly(input, (int) (size / Math.sqrt(size))), matcher);
+ }
+ checkCombineFnShardsMultipleOrders(fn, shardExponentially(input, 1.4), matcher);
+ checkCombineFnShardsMultipleOrders(fn, shardExponentially(input, 2), matcher);
+ checkCombineFnShardsMultipleOrders(fn, shardExponentially(input, Math.E), matcher);
+ }
+
+ private static <InputT, AccumT, OutputT> void checkCombineFnShardsMultipleOrders(
+ CombineFn<InputT, AccumT, OutputT> fn,
+ List<? extends Iterable<InputT>> shards,
+ Matcher<? super OutputT> matcher) {
+ checkCombineFnShardsSingleMerge(fn, shards, matcher);
+ checkCombineFnShardsWithEmptyAccumulators(fn, shards, matcher);
+ checkCombineFnShardsIncrementalMerging(fn, shards, matcher);
+ Collections.shuffle(shards);
+ checkCombineFnShardsSingleMerge(fn, shards, matcher);
+ checkCombineFnShardsWithEmptyAccumulators(fn, shards, matcher);
+ checkCombineFnShardsIncrementalMerging(fn, shards, matcher);
+ }
+
+ private static <InputT, AccumT, OutputT> void checkCombineFnShardsSingleMerge(
+ CombineFn<InputT, AccumT, OutputT> fn,
+ Iterable<? extends Iterable<InputT>> shards,
+ Matcher<? super OutputT> matcher) {
+ List<AccumT> accumulators = combineInputs(fn, shards);
+ AccumT merged = fn.mergeAccumulators(accumulators);
+ assertThat(fn.extractOutput(merged), matcher);
+ }
+
+ private static <InputT, AccumT, OutputT> void checkCombineFnShardsWithEmptyAccumulators(
+ CombineFn<InputT, AccumT, OutputT> fn,
+ Iterable<? extends Iterable<InputT>> shards,
+ Matcher<? super OutputT> matcher) {
+ List<AccumT> accumulators = combineInputs(fn, shards);
+ accumulators.add(0, fn.createAccumulator());
+ accumulators.add(fn.createAccumulator());
+ AccumT merged = fn.mergeAccumulators(accumulators);
+ assertThat(fn.extractOutput(merged), matcher);
+ }
+
+ private static <InputT, AccumT, OutputT> void checkCombineFnShardsIncrementalMerging(
+ CombineFn<InputT, AccumT, OutputT> fn,
+ List<? extends Iterable<InputT>> shards,
+ Matcher<? super OutputT> matcher) {
+ AccumT accumulator = null;
+ for (AccumT inputAccum : combineInputs(fn, shards)) {
+ if (accumulator == null) {
+ accumulator = inputAccum;
+ } else {
+ accumulator = fn.mergeAccumulators(Arrays.asList(accumulator, inputAccum));
+ }
+ }
+ assertThat(fn.extractOutput(accumulator), matcher);
+ }
+
+ private static <InputT, AccumT, OutputT> List<AccumT> combineInputs(
+ CombineFn<InputT, AccumT, OutputT> fn, Iterable<? extends Iterable<InputT>> shards) {
+ List<AccumT> accumulators = new ArrayList<>();
+ int maybeCompact = 0;
+ for (Iterable<InputT> shard : shards) {
+ AccumT accumulator = fn.createAccumulator();
+ for (InputT elem : shard) {
+ accumulator = fn.addInput(accumulator, elem);
+ }
+ if (maybeCompact++ % 2 == 0) {
+ accumulator = fn.compact(accumulator);
+ }
+ accumulators.add(accumulator);
+ }
+ return accumulators;
+ }
+
+ private static <T> List<List<T>> shardEvenly(List<T> input, int numShards) {
+ List<List<T>> shards = new ArrayList<>(numShards);
+ for (int i = 0; i < numShards; i++) {
+ shards.add(input.subList(i * input.size() / numShards,
+ (i + 1) * input.size() / numShards));
+ }
+ return shards;
+ }
+
+ private static <T> List<List<T>> shardExponentially(
+ List<T> input, double base) {
+ assert base > 1.0;
+ List<List<T>> shards = new ArrayList<>();
+ int end = input.size();
+ while (end > 0) {
+ int start = (int) (end / base);
+ shards.add(input.subList(start, end));
+ end = start;
+ }
+ return shards;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
index 5ed0525..e798841 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
@@ -29,6 +29,7 @@
import java.util.List;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.util.FluentBackoff;
import org.apache.beam.sdk.util.NumberedShardedFile;
@@ -71,9 +72,11 @@
Pattern.compile("(?x) \\S* (?<shardnum> \\d+) -of- (?<numshards> \\d+)");
private final String expectedChecksum;
- private String actualChecksum;
private final ShardedFile shardedFile;
+ /** Access via {@link #getActualChecksum()}. */
+ @Nullable private String actualChecksum;
+
/**
* Constructor that uses default shard template.
*
@@ -123,20 +126,30 @@
@Override
public boolean matchesSafely(PipelineResult pipelineResult) {
- // Load output data
- List<String> outputs;
- try {
- outputs = shardedFile.readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff());
- } catch (Exception e) {
- throw new RuntimeException(
- String.format("Failed to read from: %s", shardedFile), e);
+ return getActualChecksum().equals(expectedChecksum);
+ }
+
+ /**
+ * Computes a checksum of the sharded file specified in the constructor. Not safe to call until
+ * the writing is complete.
+ */
+ private String getActualChecksum() {
+ if (actualChecksum == null) {
+ // Load output data
+ List<String> outputs;
+ try {
+ outputs = shardedFile.readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff());
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format("Failed to read from: %s", shardedFile), e);
+ }
+
+ // Verify outputs. Checksum is computed using SHA-1 algorithm
+ actualChecksum = computeHash(outputs);
+ LOG.debug("Generated checksum: {}", actualChecksum);
}
- // Verify outputs. Checksum is computed using SHA-1 algorithm
- actualChecksum = computeHash(outputs);
- LOG.debug("Generated checksum: {}", actualChecksum);
-
- return actualChecksum.equals(expectedChecksum);
+ return actualChecksum;
}
private String computeHash(@Nonnull List<String> strs) {
@@ -163,7 +176,7 @@
public void describeMismatchSafely(PipelineResult pResult, Description description) {
description
.appendText("was (")
- .appendText(actualChecksum)
+ .appendText(getActualChecksum())
.appendText(")");
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java
new file mode 100644
index 0000000..384b298
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.testing;
+
+/**
+ * Category tags for tests which validate that a Beam runner can handle keys up to a given size.
+ */
+public interface LargeKeys {
+ /**
+ * Tests if a runner supports 10KB keys.
+ */
+ public interface Above10KB {}
+
+ /**
+ * Tests if a runner supports 100KB keys.
+ */
+ public interface Above100KB extends Above10KB {}
+
+ /**
+ * Tests if a runner supports 1MB keys.
+ */
+ public interface Above1MB extends Above100KB {}
+
+ /**
+ * Tests if a runner supports 10MB keys.
+ */
+ public interface Above10MB extends Above1MB {}
+
+ /**
+ * Tests if a runner supports 100MB keys.
+ */
+ public interface Above100MB extends Above10MB {}
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
index 6e2b8c6..aed38dc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
@@ -31,7 +31,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.PipelineRunner;
@@ -74,8 +74,6 @@
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.joda.time.Duration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* An assertion on the contents of a {@link PCollection} incorporated into the pipeline. Such an
@@ -105,8 +103,6 @@
* <p>JUnit and Hamcrest must be linked in by any code that uses PAssert.
*/
public class PAssert {
-
- private static final Logger LOG = LoggerFactory.getLogger(PAssert.class);
public static final String SUCCESS_COUNTER = "PAssertSuccess";
public static final String FAILURE_COUNTER = "PAssertFailure";
private static final Counter successCounter = Metrics.counter(
@@ -170,10 +166,6 @@
return new PAssertionSite(message, new Throwable().getStackTrace());
}
- PAssertionSite() {
- this(null, new StackTraceElement[0]);
- }
-
PAssertionSite(String message, StackTraceElement[] creationStackTrace) {
this.message = message;
this.creationStackTrace = creationStackTrace;
@@ -381,15 +373,6 @@
*/
public static <T> IterableAssert<T> thatSingletonIterable(
String reason, PCollection<? extends Iterable<T>> actual) {
-
- try {
- } catch (NoSuchElementException | IllegalArgumentException exc) {
- throw new IllegalArgumentException(
- "PAssert.<T>thatSingletonIterable requires a PCollection<Iterable<T>>"
- + " with a Coder<Iterable<T>> where getCoderArguments() yields a"
- + " single Coder<T> to apply to the elements.");
- }
-
@SuppressWarnings("unchecked") // Safe covariant cast
PCollection<Iterable<T>> actualIterables = (PCollection<Iterable<T>>) actual;
@@ -487,7 +470,7 @@
private final PAssertionSite site;
public PCollectionContentsAssert(PCollection<T> actual, PAssertionSite site) {
- this(actual, IntoGlobalWindow.<T>of(), PaneExtractors.<T>allPanes(), site);
+ this(actual, IntoGlobalWindow.of(), PaneExtractors.<T>allPanes(), site);
}
public PCollectionContentsAssert(
@@ -533,7 +516,7 @@
Coder<BoundedWindow> windowCoder =
(Coder) actual.getWindowingStrategy().getWindowFn().windowCoder();
return new PCollectionContentsAssert<>(
- actual, IntoStaticWindows.<T>of(windowCoder, window), paneExtractor, site);
+ actual, IntoStaticWindows.of(windowCoder, window), paneExtractor, site);
}
/**
@@ -581,7 +564,7 @@
@SafeVarargs
final PCollectionContentsAssert<T> containsInAnyOrder(
SerializableMatcher<? super T>... elementMatchers) {
- return satisfies(SerializableMatchers.<T>containsInAnyOrder(elementMatchers));
+ return satisfies(SerializableMatchers.containsInAnyOrder(elementMatchers));
}
/**
@@ -592,7 +575,7 @@
private PCollectionContentsAssert<T> satisfies(
AssertRelation<Iterable<T>, Iterable<T>> relation, Iterable<T> expectedElements) {
return satisfies(
- new CheckRelationAgainstExpected<Iterable<T>>(
+ new CheckRelationAgainstExpected<>(
relation, expectedElements, IterableCoder.of(actual.getCoder())));
}
@@ -623,6 +606,7 @@
}
@Override
+ @Nullable
public Void apply(T actual) {
assertThat(actual, matcher);
return null;
@@ -668,7 +652,10 @@
public PCollectionSingletonIterableAssert(
PCollection<Iterable<T>> actual, PAssertionSite site) {
this(
- actual, IntoGlobalWindow.<Iterable<T>>of(), PaneExtractors.<Iterable<T>>onlyPane(), site);
+ actual,
+ IntoGlobalWindow.of(),
+ PaneExtractors.<Iterable<T>>allPanes(),
+ site);
}
public PCollectionSingletonIterableAssert(
@@ -721,7 +708,7 @@
Coder<BoundedWindow> windowCoder =
(Coder) actual.getWindowingStrategy().getWindowFn().windowCoder();
return new PCollectionSingletonIterableAssert<>(
- actual, IntoStaticWindows.<Iterable<T>>of(windowCoder, window), paneExtractor, site);
+ actual, IntoStaticWindows.of(windowCoder, window), paneExtractor, site);
}
@Override
@@ -753,7 +740,7 @@
private PCollectionSingletonIterableAssert<T> satisfies(
AssertRelation<Iterable<T>, Iterable<T>> relation, Iterable<T> expectedElements) {
return satisfies(
- new CheckRelationAgainstExpected<Iterable<T>>(
+ new CheckRelationAgainstExpected<>(
relation, expectedElements, IterableCoder.of(elementCoder)));
}
}
@@ -777,8 +764,12 @@
Coder<ViewT> coder,
PAssertionSite site) {
this(
- actual, view, IntoGlobalWindow.<ElemT>of(), PaneExtractors.<ElemT>onlyPane(), coder, site
- );
+ actual,
+ view,
+ IntoGlobalWindow.of(),
+ PaneExtractors.<ElemT>allPanes(),
+ coder,
+ site);
}
private PCollectionViewAssert(
@@ -798,7 +789,7 @@
@Override
public PCollectionViewAssert<ElemT, ViewT> inOnlyPane(BoundedWindow window) {
- return inPane(window, PaneExtractors.<ElemT>onlyPane());
+ return inPane(window, PaneExtractors.<ElemT>onlyPane(site));
}
@Override
@@ -841,7 +832,7 @@
.getPipeline()
.apply(
"PAssert$" + (assertCount++),
- new OneSideInputAssert<ViewT>(
+ new OneSideInputAssert<>(
CreateActual.from(actual, rewindowActuals, paneExtractor, view),
rewindowActuals.<Integer>windowDummy(),
checkerFn,
@@ -857,12 +848,13 @@
*/
private PCollectionViewAssert<ElemT, ViewT> satisfies(
AssertRelation<ViewT, ViewT> relation, final ViewT expectedValue) {
- return satisfies(new CheckRelationAgainstExpected<ViewT>(relation, expectedValue, coder));
+ return satisfies(new CheckRelationAgainstExpected<>(relation, expectedValue, coder));
}
/**
- * Always throws an {@link UnsupportedOperationException}: users are probably looking for
- * {@link #isEqualTo}.
+ * @throws UnsupportedOperationException always
+ * @deprecated {@link Object#equals(Object)} is not supported on PAssert objects. If you meant
+ * to test object equality, use {@link #isEqualTo} instead.
*/
@Deprecated
@Override
@@ -1223,11 +1215,7 @@
@ProcessElement
public void processElement(ProcessContext c) {
- try {
- c.output(doChecks(site, c.element(), checkerFn));
- } catch (Throwable t) {
- throw t;
- }
+ c.output(doChecks(site, c.element(), checkerFn));
}
}
@@ -1261,13 +1249,11 @@
PAssertionSite site,
ActualT actualContents,
SerializableFunction<ActualT, Void> checkerFn) {
- SuccessOrFailure result = SuccessOrFailure.success();
try {
checkerFn.apply(actualContents);
+ return SuccessOrFailure.success();
} catch (Throwable t) {
- result = SuccessOrFailure.failure(site, t.getMessage());
- } finally {
- return result;
+ return SuccessOrFailure.failure(site, t);
}
}
@@ -1285,6 +1271,7 @@
}
@Override
+ @Nullable
public Void apply(T actual) {
assertThat(actual, equalTo(expected));
return null;
@@ -1303,6 +1290,7 @@
}
@Override
+ @Nullable
public Void apply(T actual) {
assertThat(actual, not(equalTo(expected)));
return null;
@@ -1328,10 +1316,11 @@
}
public AssertContainsInAnyOrder(Iterable<T> expected) {
- this(Lists.<T>newArrayList(expected));
+ this(Lists.newArrayList(expected));
}
@Override
+ @Nullable
public Void apply(Iterable<T> actual) {
assertThat(actual, containsInAnyOrder(expected));
return null;
@@ -1355,7 +1344,7 @@
private static class AssertIsEqualToRelation<T> implements AssertRelation<T, T> {
@Override
public SerializableFunction<T, Void> assertFor(T expected) {
- return new AssertIsEqualTo<T>(expected);
+ return new AssertIsEqualTo<>(expected);
}
}
@@ -1365,7 +1354,7 @@
private static class AssertNotEqualToRelation<T> implements AssertRelation<T, T> {
@Override
public SerializableFunction<T, Void> assertFor(T expected) {
- return new AssertNotEqualTo<T>(expected);
+ return new AssertNotEqualTo<>(expected);
}
}
@@ -1377,7 +1366,7 @@
implements AssertRelation<Iterable<T>, Iterable<T>> {
@Override
public SerializableFunction<Iterable<T>, Void> assertFor(Iterable<T> expectedElements) {
- return new AssertContainsInAnyOrder<T>(expectedElements);
+ return new AssertContainsInAnyOrder<>(expectedElements);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PaneExtractors.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PaneExtractors.java
index f88efcb..8ff35f3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PaneExtractors.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PaneExtractors.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.sdk.testing;
-import static com.google.common.base.Preconditions.checkState;
-
import java.util.ArrayList;
import java.util.List;
import org.apache.beam.sdk.transforms.PTransform;
@@ -42,8 +40,9 @@
private PaneExtractors() {
}
- static <T> SimpleFunction<Iterable<ValueInSingleWindow<T>>, Iterable<T>> onlyPane() {
- return new ExtractOnlyPane<>();
+ static <T> SimpleFunction<Iterable<ValueInSingleWindow<T>>, Iterable<T>> onlyPane(
+ PAssert.PAssertionSite site) {
+ return new ExtractOnlyPane<>(site);
}
static <T> SimpleFunction<Iterable<ValueInSingleWindow<T>>, Iterable<T>> onTimePane() {
@@ -68,15 +67,23 @@
private static class ExtractOnlyPane<T>
extends SimpleFunction<Iterable<ValueInSingleWindow<T>>, Iterable<T>> {
+ private final PAssert.PAssertionSite site;
+
+ private ExtractOnlyPane(PAssert.PAssertionSite site) {
+ this.site = site;
+ }
+
@Override
public Iterable<T> apply(Iterable<ValueInSingleWindow<T>> input) {
List<T> outputs = new ArrayList<>();
for (ValueInSingleWindow<T> value : input) {
- checkState(value.getPane().isFirst() && value.getPane().isLast(),
- "Expected elements to be produced by a trigger that fires at most once, but got"
- + "a value in a pane that is %s. Actual Pane Info: %s",
- value.getPane().isFirst() ? "not the last pane" : "not the first pane",
- value.getPane());
+ if (!value.getPane().isFirst() || !value.getPane().isLast()) {
+ throw site.wrap(
+ String.format(
+ "Expected elements to be produced by a trigger that fires at most once, but got "
+ + "a value %s in a pane that is %s.",
+ value, value.getPane().isFirst() ? "not the last pane" : "not the first pane"));
+ }
outputs.add(value.getValue());
}
return outputs;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java
index cde0b94..a324bdd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java
@@ -27,6 +27,7 @@
import static org.junit.Assert.assertTrue;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -139,6 +140,16 @@
}
}
+ public static <T> List<T> readFromSplitsOfSource(
+ BoundedSource<T> source, long desiredBundleSizeBytes, PipelineOptions options)
+ throws Exception {
+ List<T> res = Lists.newArrayList();
+ for (BoundedSource<T> split : source.split(desiredBundleSizeBytes, options)) {
+ res.addAll(readFromSource(split, options));
+ }
+ return res;
+ }
+
/**
* Reads all elements from the given unstarted {@link Source.Reader}.
*/
@@ -212,7 +223,7 @@
List<? extends BoundedSource<T>> sources,
PipelineOptions options)
throws Exception {
- Coder<T> coder = referenceSource.getDefaultOutputCoder();
+ Coder<T> coder = referenceSource.getOutputCoder();
List<T> referenceRecords = readFromSource(referenceSource, options);
List<T> bundleRecords = new ArrayList<>();
for (BoundedSource<T> source : sources) {
@@ -221,7 +232,7 @@
+ source
+ " is not compatible with Coder type for referenceSource "
+ referenceSource,
- source.getDefaultOutputCoder(),
+ source.getOutputCoder(),
equalTo(coder));
List<T> elems = readFromSource(source, options);
bundleRecords.addAll(elems);
@@ -239,7 +250,7 @@
*/
public static <T> void assertUnstartedReaderReadsSameAsItsSource(
BoundedSource.BoundedReader<T> reader, PipelineOptions options) throws Exception {
- Coder<T> coder = reader.getCurrentSource().getDefaultOutputCoder();
+ Coder<T> coder = reader.getCurrentSource().getOutputCoder();
List<T> expected = readFromUnstartedReader(reader);
List<T> actual = readFromSource(reader.getCurrentSource(), options);
List<ReadableStructuralValue<T>> expectedStructural = createStructuralValues(coder, expected);
@@ -415,7 +426,7 @@
source,
primary,
residual);
- Coder<T> coder = primary.getDefaultOutputCoder();
+ Coder<T> coder = primary.getOutputCoder();
List<ReadableStructuralValue<T>> primaryValues =
createStructuralValues(coder, primaryItems);
List<ReadableStructuralValue<T>> currentValues =
@@ -728,8 +739,8 @@
}
@Override
- public Coder<T> getDefaultOutputCoder() {
- return boundedSource.getDefaultOutputCoder();
+ public Coder<T> getOutputCoder() {
+ return boundedSource.getOutputCoder();
}
private static class UnsplittableReader<T> extends BoundedReader<T> {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
index fde1669..eba6978 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
@@ -26,6 +26,7 @@
import java.util.Objects;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
@@ -97,6 +98,17 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same window supplier are compatible.",
+ StaticWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public Coder<BoundedWindow> windowCoder() {
return coder;
}
@@ -114,4 +126,9 @@
}
};
}
+
+ @Override
+ public boolean assignsToOneWindow() {
+ return true;
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StreamingIT.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StreamingIT.java
index 427b908..475372d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StreamingIT.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StreamingIT.java
@@ -29,6 +29,10 @@
* StreamingPipeline.main(...);
* }
* </code></pre>
+ *
+ * @deprecated tests which use unbounded PCollections should be in the category
+ * {@link UsesUnboundedPCollections}. Beyond that, it is up to the runner and test configuration
+ * to decide whether to run in streaming mode.
*/
@Deprecated
public interface StreamingIT {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
index 04e3c35..bac4eb3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
@@ -22,31 +22,27 @@
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.util.SerializableThrowable;
/**
* Output of {@link PAssert}. Passed to a conclude function to act upon.
*/
@DefaultCoder(SerializableCoder.class)
public final class SuccessOrFailure implements Serializable {
- // TODO Add a SerializableThrowable. instead of relying on PAssertionSite.(BEAM-1898)
private final boolean isSuccess;
@Nullable
private final PAssert.PAssertionSite site;
@Nullable
- private final String message;
-
- private SuccessOrFailure() {
- this(true, null, null);
- }
+ private final SerializableThrowable throwable;
private SuccessOrFailure(
boolean isSuccess,
@Nullable PAssert.PAssertionSite site,
- @Nullable String message) {
+ @Nullable Throwable throwable) {
this.isSuccess = isSuccess;
this.site = site;
- this.message = message;
+ this.throwable = new SerializableThrowable(throwable);
}
public boolean isSuccess() {
@@ -55,7 +51,7 @@
@Nullable
public AssertionError assertionError() {
- return site == null ? null : site.wrap(message);
+ return site == null ? null : site.wrap(throwable.getThrowable());
}
public static SuccessOrFailure success() {
@@ -63,19 +59,15 @@
}
public static SuccessOrFailure failure(@Nullable PAssert.PAssertionSite site,
- @Nullable String message) {
- return new SuccessOrFailure(false, site, message);
- }
-
- public static SuccessOrFailure failure(@Nullable PAssert.PAssertionSite site) {
- return new SuccessOrFailure(false, site, null);
+ @Nullable Throwable t) {
+ return new SuccessOrFailure(false, site, t);
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("isSuccess", isSuccess())
- .addValue(message)
+ .addValue(throwable)
.omitNullValues()
.toString();
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index 9206e04..f2729e9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -31,17 +31,19 @@
import com.google.common.base.Predicates;
import com.google.common.base.Strings;
import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
import java.io.IOException;
-import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.UUID;
import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.metrics.MetricNameFilter;
import org.apache.beam.sdk.metrics.MetricResult;
@@ -51,7 +53,10 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
@@ -137,7 +142,8 @@
private static class PipelineAbandonedNodeEnforcement extends PipelineRunEnforcement {
- private List<TransformHierarchy.Node> runVisitedNodes;
+ // Null until the pipeline has been run
+ @Nullable private List<TransformHierarchy.Node> runVisitedNodes;
private final Predicate<TransformHierarchy.Node> isPAssertNode =
new Predicate<TransformHierarchy.Node>() {
@@ -167,6 +173,7 @@
private PipelineAbandonedNodeEnforcement(final TestPipeline pipeline) {
super(pipeline);
+ runVisitedNodes = null;
}
private List<TransformHierarchy.Node> recordPipelineNodes(final Pipeline pipeline) {
@@ -307,6 +314,7 @@
@Override
public void evaluate() throws Throwable {
+ options.as(ApplicationNameOptions.class).setAppName(getAppName(description));
setDeducedEnforcementLevel();
@@ -329,6 +337,11 @@
* testing.
*/
public PipelineResult run() {
+ return run(getOptions());
+ }
+
+ /** Like {@link #run} but with the given potentially modified options. */
+ public PipelineResult run(PipelineOptions options) {
checkState(
enforcement.isPresent(),
"Is your TestPipeline declaration missing a @Rule annotation? Usage: "
@@ -337,7 +350,12 @@
final PipelineResult pipelineResult;
try {
enforcement.get().beforePipelineExecution();
- pipelineResult = super.run();
+ PipelineOptions updatedOptions =
+ MAPPER.convertValue(MAPPER.valueToTree(options), PipelineOptions.class);
+ updatedOptions
+ .as(TestValueProviderOptions.class)
+ .setProviderRuntimeValues(StaticValueProvider.of(providerRuntimeValues));
+ pipelineResult = super.run(updatedOptions);
verifyPAssertsSucceeded(this, pipelineResult);
} catch (RuntimeException exc) {
Throwable cause = exc.getCause();
@@ -354,6 +372,41 @@
return pipelineResult;
}
+ /** Implementation detail of {@link #newProvider}, do not use. */
+ @Internal
+ public interface TestValueProviderOptions extends PipelineOptions {
+ ValueProvider<Map<String, Object>> getProviderRuntimeValues();
+ void setProviderRuntimeValues(ValueProvider<Map<String, Object>> runtimeValues);
+ }
+
+ /**
+ * Returns a new {@link ValueProvider} that is inaccessible before {@link #run}, but will be
+ * accessible while the pipeline runs.
+ */
+ public <T> ValueProvider<T> newProvider(T runtimeValue) {
+ String uuid = UUID.randomUUID().toString();
+ providerRuntimeValues.put(uuid, runtimeValue);
+ return ValueProvider.NestedValueProvider.of(
+ options.as(TestValueProviderOptions.class).getProviderRuntimeValues(),
+ new GetFromRuntimeValues<T>(uuid));
+ }
+
+ private final Map<String, Object> providerRuntimeValues = Maps.newHashMap();
+
+ private static class GetFromRuntimeValues<T>
+ implements SerializableFunction<Map<String, Object>, T> {
+ private final String key;
+
+ private GetFromRuntimeValues(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public T apply(Map<String, Object> input) {
+ return (T) input.get(key);
+ }
+ }
+
/**
* Enables the abandoned node detection. Abandoned nodes are <code>PTransforms</code>, <code>
* PAsserts</code> included, that were not executed by the pipeline runner. Abandoned nodes are
@@ -402,7 +455,6 @@
MAPPER.readValue(beamTestPipelineOptions, String[].class))
.as(TestPipelineOptions.class);
- options.as(ApplicationNameOptions.class).setAppName(getAppName());
// If no options were specified, set some reasonable defaults
if (Strings.isNullOrEmpty(beamTestPipelineOptions)) {
// If there are no provided options, check to see if a dummy runner should be used.
@@ -450,56 +502,17 @@
}
}
- /** Returns the class + method name of the test, or a default name. */
- private static String getAppName() {
- Optional<StackTraceElement> stackTraceElement = findCallersStackTrace();
- if (stackTraceElement.isPresent()) {
- String methodName = stackTraceElement.get().getMethodName();
- String className = stackTraceElement.get().getClassName();
- if (className.contains(".")) {
- className = className.substring(className.lastIndexOf(".") + 1);
- }
- return className + "-" + methodName;
+ /** Returns the class + method name of the test. */
+ private String getAppName(Description description) {
+ String methodName = description.getMethodName();
+ Class<?> testClass = description.getTestClass();
+ if (testClass.isMemberClass()) {
+ return String.format(
+ "%s$%s-%s",
+ testClass.getEnclosingClass().getSimpleName(), testClass.getSimpleName(), methodName);
+ } else {
+ return String.format("%s-%s", testClass.getSimpleName(), methodName);
}
- return "UnitTest";
- }
-
- /** Returns the {@link StackTraceElement} of the calling class. */
- private static Optional<StackTraceElement> findCallersStackTrace() {
- Iterator<StackTraceElement> elements =
- Iterators.forArray(Thread.currentThread().getStackTrace());
- // First find the TestPipeline class in the stack trace.
- while (elements.hasNext()) {
- StackTraceElement next = elements.next();
- if (TestPipeline.class.getName().equals(next.getClassName())) {
- break;
- }
- }
- // Then find the first instance after that is not the TestPipeline
- Optional<StackTraceElement> firstInstanceAfterTestPipeline = Optional.absent();
- while (elements.hasNext()) {
- StackTraceElement next = elements.next();
- if (!TestPipeline.class.getName().equals(next.getClassName())) {
- if (!firstInstanceAfterTestPipeline.isPresent()) {
- firstInstanceAfterTestPipeline = Optional.of(next);
- }
- try {
- Class<?> nextClass = Class.forName(next.getClassName());
- for (Method method : nextClass.getMethods()) {
- if (method.getName().equals(next.getMethodName())) {
- if (method.isAnnotationPresent(org.junit.Test.class)) {
- return Optional.of(next);
- } else if (method.isAnnotationPresent(org.junit.Before.class)) {
- break;
- }
- }
- }
- } catch (Throwable t) {
- break;
- }
- }
- }
- return firstInstanceAfterTestPipeline;
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
index d41b9ef..45f4413 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
@@ -24,8 +24,10 @@
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import java.util.List;
+import java.util.Objects;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -86,8 +88,8 @@
/**
* Adds the specified elements to the source with timestamp equal to the current watermark.
*
- * @return A {@link TestStream.Builder} like this one that will add the provided elements
- * after all earlier events have completed.
+ * @return A {@link TestStream.Builder} like this one that will add the provided elements after
+ * all earlier events have completed.
*/
@SafeVarargs
public final Builder<T> addElements(T element, T... elements) {
@@ -103,8 +105,8 @@
/**
* Adds the specified elements to the source with the provided timestamps.
*
- * @return A {@link TestStream.Builder} like this one that will add the provided elements
- * after all earlier events have completed.
+ * @return A {@link TestStream.Builder} like this one that will add the provided elements after
+ * all earlier events have completed.
*/
@SafeVarargs
public final Builder<T> addElements(
@@ -136,7 +138,7 @@
* BoundedWindow#TIMESTAMP_MAX_VALUE} or beyond.
*
* @return A {@link TestStream.Builder} like this one that will advance the watermark to the
- * specified point after all earlier events have completed.
+ * specified point after all earlier events have completed.
*/
public Builder<T> advanceWatermarkTo(Instant newWatermark) {
checkArgument(
@@ -146,10 +148,11 @@
"The Watermark cannot progress beyond the maximum. Got: %s. Maximum: %s",
newWatermark,
BoundedWindow.TIMESTAMP_MAX_VALUE);
- ImmutableList<Event<T>> newEvents = ImmutableList.<Event<T>>builder()
- .addAll(events)
- .add(WatermarkEvent.<T>advanceTo(newWatermark))
- .build();
+ ImmutableList<Event<T>> newEvents =
+ ImmutableList.<Event<T>>builder()
+ .addAll(events)
+ .add(WatermarkEvent.<T>advanceTo(newWatermark))
+ .build();
return new Builder<T>(coder, newEvents, newWatermark);
}
@@ -157,7 +160,7 @@
* Advance the processing time by the specified amount.
*
* @return A {@link TestStream.Builder} like this one that will advance the processing time by
- * the specified amount after all earlier events have completed.
+ * the specified amount after all earlier events have completed.
*/
public Builder<T> advanceProcessingTime(Duration amount) {
checkArgument(
@@ -194,9 +197,7 @@
EventType getType();
}
- /**
- * The types of {@link Event} that are supported by {@link TestStream}.
- */
+ /** The types of {@link Event} that are supported by {@link TestStream}. */
public enum EventType {
ELEMENT,
WATERMARK,
@@ -213,7 +214,11 @@
return add(ImmutableList.<TimestampedValue<T>>builder().add(element).add(elements).build());
}
- static <T> Event<T> add(Iterable<TimestampedValue<T>> elements) {
+ /**
+ * <b>For internal use only: no backwards compatibility guarantees.</b>
+ */
+ @Internal
+ public static <T> Event<T> add(Iterable<TimestampedValue<T>> elements) {
return new AutoValue_TestStream_ElementEvent<>(EventType.ELEMENT, elements);
}
}
@@ -223,7 +228,11 @@
public abstract static class WatermarkEvent<T> implements Event<T> {
public abstract Instant getWatermark();
- static <T> Event<T> advanceTo(Instant newWatermark) {
+ /**
+ * <b>For internal use only: no backwards compatibility guarantees.</b>
+ */
+ @Internal
+ public static <T> Event<T> advanceTo(Instant newWatermark) {
return new AutoValue_TestStream_WatermarkEvent<>(EventType.WATERMARK, newWatermark);
}
}
@@ -233,16 +242,19 @@
public abstract static class ProcessingTimeEvent<T> implements Event<T> {
public abstract Duration getProcessingTimeAdvance();
- static <T> Event<T> advanceBy(Duration amount) {
+ /**
+ * <b>For internal use only: no backwards compatibility guarantees.</b>
+ */
+ @Internal
+ public static <T> Event<T> advanceBy(Duration amount) {
return new AutoValue_TestStream_ProcessingTimeEvent<>(EventType.PROCESSING_TIME, amount);
}
}
@Override
public PCollection<T> expand(PBegin input) {
- return PCollection.<T>createPrimitiveOutputInternal(
- input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
- .setCoder(coder);
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, coder);
}
public Coder<T> getValueCoder() {
@@ -257,4 +269,31 @@
public List<Event<T>> getEvents() {
return events;
}
+
+ /**
+ * <b>For internal use only. No backwards-compatibility guarantees.</b>
+ *
+ * <p>Builder a test stream directly from events. No validation is performed on
+ * watermark monotonicity, etc. This is assumed to be a previously-serialized
+ * {@link TestStream} transform that is correct by construction.
+ */
+ @Internal
+ public static <T> TestStream<T> fromRawEvents(Coder<T> coder, List<Event<T>> events) {
+ return new TestStream<>(coder, events);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof TestStream)) {
+ return false;
+ }
+ TestStream<?> that = (TestStream<?>) other;
+
+ return getValueCoder().equals(that.getValueCoder()) && getEvents().equals(that.getEvents());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(TestStream.class, getValueCoder(), getEvents());
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesCustomWindowMerging.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesCustomWindowMerging.java
new file mode 100644
index 0000000..fc40e02
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesCustomWindowMerging.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.testing;
+
+/**
+ * Category tag for validation tests which utilize custom window merging.
+ */
+public interface UsesCustomWindowMerging {}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
index e8c2f8d..7fa1056 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
@@ -40,6 +40,7 @@
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.TimestampedValue;
import org.joda.time.Instant;
import org.joda.time.ReadableInstant;
@@ -67,14 +68,28 @@
public static <T, W extends BoundedWindow> Map<W, Set<String>> runWindowFn(
WindowFn<T, W> windowFn,
List<Long> timestamps) throws Exception {
+ List<TimestampedValue<T>> timestampedValues = new ArrayList<>();
+ for (Long timestamp : timestamps){
+ timestampedValues.add(TimestampedValue.of((T) null, new Instant(timestamp)));
+ }
+ return runWindowFnWithValue(windowFn, timestampedValues);
+ }
- final TestWindowSet<W, String> windowSet = new TestWindowSet<W, String>();
- for (final Long timestamp : timestamps) {
- for (W window : windowFn.assignWindows(
- new TestAssignContext<T, W>(new Instant(timestamp), windowFn))) {
- windowSet.put(window, timestampValue(timestamp));
+ /**
+ * Runs the {@link WindowFn} over the provided input, returning a map
+ * of windows to the timestamps in those windows. This version allows to pass a list of
+ * {@link TimestampedValue} in case the values are used to assign windows.
+ */
+ public static <T, W extends BoundedWindow> Map<W, Set<String>> runWindowFnWithValue(
+ WindowFn<T, W> windowFn,
+ List<TimestampedValue<T>> timestampedValues) throws Exception {
+
+ final TestWindowSet<W, String> windowSet = new TestWindowSet<>();
+ for (final TimestampedValue<T> element : timestampedValues) {
+ for (W window : assignedWindowsWithValue(windowFn, element)) {
+ windowSet.put(window, timestampValue(element.getTimestamp().getMillis()));
}
- windowFn.mergeWindows(new TestMergeContext<T, W>(windowSet, windowFn));
+ windowFn.mergeWindows(new TestMergeContext<>(windowSet, windowFn));
}
Map<W, Set<String>> actual = new HashMap<>();
for (W window : windowSet.windows()) {
@@ -83,9 +98,23 @@
return actual;
}
+ /**
+ * runs {@link WindowFn#assignWindows(WindowFn.AssignContext)}.
+ */
public static <T, W extends BoundedWindow> Collection<W> assignedWindows(
WindowFn<T, W> windowFn, long timestamp) throws Exception {
- return windowFn.assignWindows(new TestAssignContext<T, W>(new Instant(timestamp), windowFn));
+ return assignedWindowsWithValue(windowFn,
+ TimestampedValue.of((T) null, new Instant(timestamp)));
+ }
+
+ /**
+ * runs {@link WindowFn#assignWindows(WindowFn.AssignContext)}. This version allows passing
+ * a {@link TimestampedValue} in case the value is needed to assign windows.
+ */
+ public static <T, W extends BoundedWindow> Collection<W> assignedWindowsWithValue(
+ WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
+ return windowFn.assignWindows(
+ new TestAssignContext<>(timestampedValue, windowFn));
}
private static String timestampValue(long timestamp) {
@@ -97,21 +126,21 @@
*/
private static class TestAssignContext<T, W extends BoundedWindow>
extends WindowFn<T, W>.AssignContext {
- private Instant timestamp;
+ private TimestampedValue<T> timestampedValue;
- public TestAssignContext(Instant timestamp, WindowFn<T, W> windowFn) {
+ public TestAssignContext(TimestampedValue<T> timestampedValue, WindowFn<T, W> windowFn) {
windowFn.super();
- this.timestamp = timestamp;
+ this.timestampedValue = timestampedValue;
}
@Override
public T element() {
- return null;
+ return timestampedValue.getValue();
}
@Override
public Instant timestamp() {
- return timestamp;
+ return timestampedValue.getTimestamp();
}
@Override
@@ -197,9 +226,20 @@
*/
public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimes(
WindowFn<T, W> windowFn, long timestamp) throws Exception {
- Collection<W> windows = WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp);
+ validateNonInterferingOutputTimesWithValue(windowFn,
+ TimestampedValue.of((T) null, new Instant(timestamp)));
+ }
+ /**
+ * Assigns the given {@code timestampedValue} to windows using the specified {@code windowFn}, and
+ * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the
+ * proper bound. This version allows passing a {@link TimestampedValue}
+ * in case the value is needed to assign windows.
+ */
+ public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimesWithValue(
+ WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
+ Collection<W> windows = assignedWindowsWithValue(windowFn, timestampedValue);
- Instant instant = new Instant(timestamp);
+ Instant instant = timestampedValue.getTimestamp();
for (W window : windows) {
Instant outputTimestamp = windowFn.getOutputTime(instant, window);
assertFalse("getOutputTime must be greater than or equal to input timestamp",
@@ -209,6 +249,7 @@
}
}
+
/**
* Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and
* verifies that result of {@link WindowFn#getOutputTime windowFn.getOutputTime} for later windows
@@ -220,7 +261,24 @@
*/
public static <T, W extends BoundedWindow> void validateGetOutputTimestamp(
WindowFn<T, W> windowFn, long timestamp) throws Exception {
- Collection<W> windows = WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp);
+ validateGetOutputTimestampWithValue(windowFn,
+ TimestampedValue.of((T) null, new Instant(timestamp)));
+ }
+
+
+ /**
+ * Assigns the given {@code timestampedValue} to windows using the specified {@code windowFn}, and
+ * verifies that result of {@link WindowFn#getOutputTime windowFn.getOutputTime} for later windows
+ * (as defined by {@code maxTimestamp} won't prevent the watermark from passing the end of earlier
+ * windows.
+ *
+ * <p>This verifies that overlapping windows don't interfere at all. Depending on the
+ * {@code windowFn} this may be stricter than desired. This version allows passing
+ * a {@link TimestampedValue} in case the value is needed to assign windows.
+ */
+ public static <T, W extends BoundedWindow> void validateGetOutputTimestampWithValue(
+ WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
+ Collection<W> windows = assignedWindowsWithValue(windowFn, timestampedValue);
List<W> sortedWindows = new ArrayList<>(windows);
Collections.sort(sortedWindows, new Comparator<BoundedWindow>() {
@Override
@@ -229,7 +287,7 @@
}
});
- Instant instant = new Instant(timestamp);
+ Instant instant = timestampedValue.getTimestamp();
Instant endOfPrevious = null;
for (W window : sortedWindows) {
Instant outputTimestamp = windowFn.getOutputTime(instant, window);
@@ -252,6 +310,7 @@
}
}
+
/**
* Verifies that later-ending merged windows from any of the timestamps hold up output of
* earlier-ending windows, using the provided {@link WindowFn} and {@link TimestampCombiner}.
@@ -269,15 +328,45 @@
TimestampCombiner timestampCombiner,
List<List<Long>> timestampsPerWindow) throws Exception {
+ List<List<TimestampedValue<T>>> timestampValuesPerWindow = new ArrayList<>();
+ for (List<Long> timestamps : timestampsPerWindow){
+ List<TimestampedValue<T>> timestampedValues = new ArrayList<>();
+ for (Long timestamp : timestamps){
+ TimestampedValue<T> tv = TimestampedValue.of(null, new Instant(timestamp));
+ timestampedValues.add(tv);
+ }
+ timestampValuesPerWindow.add(timestampedValues);
+ }
+ validateGetOutputTimestampsWithValue(windowFn, timestampCombiner, timestampValuesPerWindow);
+ }
+
+ /**
+ * Verifies that later-ending merged windows from any of the timestampValues hold up output of
+ * earlier-ending windows, using the provided {@link WindowFn} and {@link TimestampCombiner}.
+ *
+ * <p>Given a list of lists of timestampValues, where each list is expected to merge into a single
+ * window with end times in ascending order, assigns and merges windows for each list (as though
+ * each were a separate key/user session). Then combines each timestamp in the list according to
+ * the provided {@link TimestampCombiner}.
+ *
+ * <p>Verifies that a overlapping windows do not hold each other up via the watermark.
+ * This version allows passing {@link TimestampedValue} in case
+ * the value is needed to assign windows.
+ */
+ public static <T, W extends IntervalWindow>
+ void validateGetOutputTimestampsWithValue(
+ WindowFn<T, W> windowFn,
+ TimestampCombiner timestampCombiner,
+ List<List<TimestampedValue<T>>> timestampValuesPerWindow) throws Exception {
+
// Assign windows to each timestamp, then merge them, storing the merged windows in
- // a list in corresponding order to timestampsPerWindow
+ // a list in corresponding order to timestampValuesPerWindow
final List<W> windows = new ArrayList<>();
- for (List<Long> timestampsForWindow : timestampsPerWindow) {
+ for (List<TimestampedValue<T>> timestampValuesForWindow : timestampValuesPerWindow) {
final Set<W> windowsToMerge = new HashSet<>();
- for (long timestamp : timestampsForWindow) {
- windowsToMerge.addAll(
- WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp));
+ for (TimestampedValue<T> element : timestampValuesForWindow) {
+ windowsToMerge.addAll(assignedWindowsWithValue(windowFn, element));
}
windowFn.mergeWindows(windowFn.new MergeContext() {
@@ -293,16 +382,16 @@
});
}
- // Map every list of input timestamps to an output timestamp
+ // Map every list of input timestampValues timestamps to an output timestamp
final List<Instant> combinedOutputTimestamps = new ArrayList<>();
- for (int i = 0; i < timestampsPerWindow.size(); ++i) {
- List<Long> timestampsForWindow = timestampsPerWindow.get(i);
+ for (int i = 0; i < timestampValuesPerWindow.size(); ++i) {
+ List<TimestampedValue<T>> timestampValuesForWindow = timestampValuesPerWindow.get(i);
W window = windows.get(i);
List<Instant> outputInstants = new ArrayList<>();
- for (long inputTimestamp : timestampsForWindow) {
+ for (TimestampedValue<T> element : timestampValuesForWindow) {
outputInstants.add(
- assignOutputTime(timestampCombiner, new Instant(inputTimestamp), window));
+ assignOutputTime(timestampCombiner, new Instant(element.getTimestamp()), window));
}
combinedOutputTimestamps.add(combineOutputTimes(timestampCombiner, outputInstants));
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java
index 96091ef..953dd27 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java
@@ -22,6 +22,7 @@
import com.google.common.collect.ImmutableSet;
import java.io.Serializable;
import java.util.Collection;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -36,7 +37,8 @@
private final Coder<? extends BoundedWindow> coder;
private final Collection<byte[]> encodedWindows;
- private transient Collection<BoundedWindow> windows;
+ /** Access via {@link #get()}.*/
+ @Nullable private transient Collection<BoundedWindow> windows;
public static <W extends BoundedWindow> WindowSupplier of(Coder<W> coder, Iterable<W> windows) {
ImmutableSet.Builder<byte[]> windowsBuilder = ImmutableSet.builder();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/package-info.java
index e66677d..6a28529 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/package-info.java
@@ -19,4 +19,9 @@
* Defines utilities for unit testing Apache Beam pipelines. The tests for the {@code PTransform}s
* and examples included in the Apache Beam SDK provide examples of using these utilities.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.testing;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index d12d193..ff370243 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -283,8 +283,8 @@
* Like {@link #create(int, Comparator)}, but sorts values using their natural ordering.
*/
public static <T extends Comparable<T>>
- ApproximateQuantilesCombineFn<T, Top.Largest<T>> create(int numQuantiles) {
- return create(numQuantiles, new Top.Largest<T>());
+ ApproximateQuantilesCombineFn<T, Top.Natural<T>> create(int numQuantiles) {
+ return create(numQuantiles, new Top.Natural<T>());
}
/**
@@ -341,7 +341,7 @@
b++;
}
b--;
- int k = Math.max(2, (int) Math.ceil(maxNumElements / (1 << (b - 1))));
+ int k = Math.max(2, (int) Math.ceil(maxNumElements / (float) (1 << (b - 1))));
return new ApproximateQuantilesCombineFn<T, ComparatorT>(
numQuantiles, compareFn, k, b, maxNumElements);
}
@@ -366,6 +366,14 @@
.add(DisplayData.item("comparer", compareFn.getClass())
.withLabel("Record Comparer"));
}
+
+ int getNumBuffers() {
+ return numBuffers;
+ }
+
+ int getBufferSize() {
+ return bufferSize;
+ }
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java
index 5d38206..98c971d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java
@@ -26,7 +26,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
-import org.apache.avro.reflect.Nullable;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.CoderException;
@@ -455,7 +455,7 @@
}
private static void populateDisplayData(
- DisplayData.Builder builder, long sampleSize, Double maxEstimationError) {
+ DisplayData.Builder builder, long sampleSize, @Nullable Double maxEstimationError) {
builder
.add(DisplayData.item("sampleSize", sampleSize)
.withLabel("Sample Size"))
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 9e1cc71..3c5b55b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -20,7 +20,6 @@
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import java.io.IOException;
import java.io.InputStream;
@@ -32,6 +31,7 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
@@ -441,6 +441,7 @@
/**
* Returns the value that should be used for the combine of the empty set.
*/
+ @Nullable
public V identity() {
return null;
}
@@ -507,7 +508,7 @@
* <p>Used only as a private accumulator class.
*/
public static class Holder<V> {
- private V value;
+ @Nullable private V value;
private boolean present;
private Holder() { }
private Holder(V value) {
@@ -1122,11 +1123,7 @@
*/
@Override
public Map<TupleTag<?>, PValue> getAdditionalInputs() {
- ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder();
- for (PCollectionView<?> sideInput : sideInputs) {
- additionalInputs.put(sideInput.getTagInternal(), sideInput.getPCollection());
- }
- return additionalInputs.build();
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
/**
@@ -1277,14 +1274,15 @@
public PCollectionView<OutputT> expand(PCollection<InputT> input) {
PCollection<OutputT> combined =
input.apply(Combine.<InputT, OutputT>globally(fn).withoutDefaults().withFanout(fanout));
- return combined.apply(
- CreatePCollectionView.<OutputT, OutputT>of(
- PCollectionViews.singletonView(
- combined,
- input.getWindowingStrategy(),
- insertDefault,
- insertDefault ? fn.defaultValue() : null,
- combined.getCoder())));
+ PCollectionView<OutputT> view =
+ PCollectionViews.singletonView(
+ combined,
+ input.getWindowingStrategy(),
+ insertDefault,
+ insertDefault ? fn.defaultValue() : null,
+ combined.getCoder());
+ combined.apply(CreatePCollectionView.<OutputT, OutputT>of(view));
+ return view;
}
public int getFanout() {
@@ -1420,7 +1418,6 @@
* Returns a {@code CombineFn} that uses the given
* {@code SerializableFunction} to combine values.
*/
- @Deprecated
public static <V> SimpleCombineFn<V> of(
SerializableFunction<Iterable<V>, V> combiner) {
return new SimpleCombineFn<>(combiner);
@@ -1577,11 +1574,7 @@
*/
@Override
public Map<TupleTag<?>, PValue> getAdditionalInputs() {
- ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder();
- for (PCollectionView<?> sideInput : sideInputs) {
- additionalInputs.put(sideInput.getTagInternal(), sideInput.getPCollection());
- }
- return additionalInputs.build();
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
@Override
@@ -1954,10 +1947,10 @@
* the hot and cold key paths.
*/
private static class InputOrAccum<InputT, AccumT> {
- public final InputT input;
- public final AccumT accum;
+ @Nullable public final InputT input;
+ @Nullable public final AccumT accum;
- private InputOrAccum(InputT input, AccumT aggr) {
+ private InputOrAccum(@Nullable InputT input, @Nullable AccumT aggr) {
this.input = input;
this.accum = aggr;
}
@@ -2165,8 +2158,13 @@
}).withSideInputs(sideInputs));
try {
- Coder<KV<K, OutputT>> outputCoder = getDefaultOutputCoder(input);
- output.setCoder(outputCoder);
+ KvCoder<K, InputT> kvCoder = getKvCoder(input.getCoder());
+ @SuppressWarnings("unchecked")
+ Coder<OutputT> outputValueCoder =
+ ((GlobalCombineFn<InputT, ?, OutputT>) fn)
+ .getDefaultOutputCoder(
+ input.getPipeline().getCoderRegistry(), kvCoder.getValueCoder());
+ output.setCoder(KvCoder.of(kvCoder.getKeyCoder(), outputValueCoder));
} catch (CannotProvideCoderException exc) {
// let coder inference happen later, if it can
}
@@ -2209,19 +2207,6 @@
}
@Override
- public Coder<KV<K, OutputT>> getDefaultOutputCoder(
- PCollection<? extends KV<K, ? extends Iterable<InputT>>> input)
- throws CannotProvideCoderException {
- KvCoder<K, InputT> kvCoder = getKvCoder(input.getCoder());
- @SuppressWarnings("unchecked")
- Coder<OutputT> outputValueCoder =
- ((GlobalCombineFn<InputT, ?, OutputT>) fn)
- .getDefaultOutputCoder(
- input.getPipeline().getCoderRegistry(), kvCoder.getValueCoder());
- return KvCoder.of(kvCoder.getKeyCoder(), outputValueCoder);
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
Combine.populateDisplayData(builder, fn, fnDisplayData);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index c619783..02cb884 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -31,6 +31,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
@@ -176,6 +177,7 @@
* <p>It is an error to request a non-exist tuple tag from the {@link CoCombineResult}.
*/
@SuppressWarnings("unchecked")
+ @Nullable
public <V> V get(TupleTag<V> tag) {
checkArgument(
valuesMap.keySet().contains(tag), "TupleTag " + tag + " is not in the CoCombineResult");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java
new file mode 100644
index 0000000..fb732cf
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import com.google.common.base.MoreObjects;
+import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/** Pair of a bit of user code (a "closure") and the {@link Requirements} needed to run it. */
+@Experimental(Kind.CONTEXTFUL)
+public final class Contextful<ClosureT> implements Serializable {
+ private final ClosureT closure;
+ private final Requirements requirements;
+
+ private Contextful(ClosureT closure, Requirements requirements) {
+ this.closure = closure;
+ this.requirements = requirements;
+ }
+
+ /** Returns the closure. */
+ public ClosureT getClosure() {
+ return closure;
+ }
+
+ /** Returns the requirements needed to run the closure. */
+ public Requirements getRequirements() {
+ return requirements;
+ }
+
+ /** Constructs a pair of the given closure and its requirements. */
+ public static <ClosureT> Contextful<ClosureT> of(ClosureT closure, Requirements requirements) {
+ return new Contextful<>(closure, requirements);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("closure", closure)
+ .add("requirements", requirements)
+ .toString();
+ }
+
+ /**
+ * A function from an input to an output that may additionally access {@link Context} when
+ * computing the result.
+ */
+ public interface Fn<InputT, OutputT> extends Serializable {
+ /**
+ * Invokes the function on the given input with the given context. The function may use the
+ * context only for the capabilities declared in the {@link Contextful#getRequirements} of the
+ * enclosing {@link Contextful}.
+ */
+ OutputT apply(InputT element, Context c) throws Exception;
+
+ /** An accessor for additional capabilities available in {@link #apply}. */
+ abstract class Context {
+ /**
+ * Accesses the given side input. The window in which it is accessed is unspecified, depends
+ * on usage by the enclosing {@link PTransform}, and must be documented by that transform.
+ */
+ public <T> T sideInput(PCollectionView<T> view) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Convenience wrapper for creating a {@link Context} from a {@link DoFn.ProcessContext}, to
+ * support the common case when a {@link PTransform} is invoking the {@link
+ * Contextful#getClosure() closure} from inside a {@link DoFn}.
+ */
+ public static <InputT> Context wrapProcessContext(final DoFn<InputT, ?>.ProcessContext c) {
+ return new ContextFromProcessContext<>(c);
+ }
+
+ private static class ContextFromProcessContext<InputT> extends Context {
+ private final DoFn<InputT, ?>.ProcessContext c;
+
+ ContextFromProcessContext(DoFn<InputT, ?>.ProcessContext c) {
+ this.c = c;
+ }
+
+ @Override
+ public <T> T sideInput(PCollectionView<T> view) {
+ return c.sideInput(view);
+ }
+ }
+ }
+ }
+
+ /**
+ * Wraps a {@link SerializableFunction} as a {@link Contextful} of {@link Fn} with empty {@link
+ * Requirements}.
+ */
+ public static <InputT, OutputT> Contextful<Fn<InputT, OutputT>> fn(
+ final SerializableFunction<InputT, OutputT> fn) {
+ return new Contextful<Fn<InputT, OutputT>>(
+ new Fn<InputT, OutputT>() {
+ @Override
+ public OutputT apply(InputT element, Context c) throws Exception {
+ return fn.apply(element);
+ }
+ },
+ Requirements.empty());
+ }
+
+ /** Same with {@link #of} but with better type inference behavior for the case of {@link Fn}. */
+ public static <InputT, OutputT> Contextful<Fn<InputT, OutputT>> fn(
+ final Fn<InputT, OutputT> fn, Requirements requirements) {
+ return of(fn, requirements);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
index b405dd1..ee24b3f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
@@ -195,5 +195,15 @@
}
};
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && getClass().equals(other.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
index 7af8fb8..7f5920c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.transforms;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
@@ -52,6 +53,7 @@
import org.apache.beam.sdk.io.OffsetBasedSource.OffsetBasedReader;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
@@ -135,7 +137,7 @@
* Otherwise, use {@link Create.Values#withCoder} to set the coder explicitly.
*/
@SafeVarargs
- public static <T> Values<T> of(T elem, T... elems) {
+ public static <T> Values<T> of(@Nullable T elem, @Nullable T... elems) {
// This can't be an ImmutableList, as it may accept nulls
List<T> input = new ArrayList<>(elems.length + 1);
input.add(elem);
@@ -200,6 +202,14 @@
}
/**
+ * Returns an {@link OfValueProvider} transform that produces a {@link PCollection}
+ * of a single element provided by the given {@link ValueProvider}.
+ */
+ public static <T> OfValueProvider<T> ofProvider(ValueProvider<T> provider, Coder<T> coder) {
+ return new OfValueProvider<>(provider, coder);
+ }
+
+ /**
* Returns a new {@link Create.TimestampedValues} transform that produces a
* {@link PCollection} containing the elements of the provided {@code Iterable}
* with the specified timestamps.
@@ -305,29 +315,25 @@
@Override
public PCollection<T> expand(PBegin input) {
+ Coder<T> coder;
try {
- Coder<T> coder = getDefaultOutputCoder(input);
- try {
- CreateSource<T> source = CreateSource.fromIterable(elems, coder);
- return input.getPipeline().apply(Read.from(source));
- } catch (IOException e) {
- throw new RuntimeException(
- String.format("Unable to apply Create %s using Coder %s.", this, coder), e);
- }
+ CoderRegistry registry = input.getPipeline().getCoderRegistry();
+ coder =
+ this.coder.isPresent()
+ ? this.coder.get()
+ : typeDescriptor.isPresent()
+ ? registry.getCoder(typeDescriptor.get())
+ : getDefaultCreateCoder(registry, elems);
} catch (CannotProvideCoderException e) {
throw new IllegalArgumentException("Unable to infer a coder and no Coder was specified. "
+ "Please set a coder by invoking Create.withCoder() explicitly.", e);
}
- }
-
- @Override
- public Coder<T> getDefaultOutputCoder(PBegin input) throws CannotProvideCoderException {
- if (coder.isPresent()) {
- return coder.get();
- } else if (typeDescriptor.isPresent()) {
- return input.getPipeline().getCoderRegistry().getCoder(typeDescriptor.get());
- } else {
- return getDefaultCreateCoder(input.getPipeline().getCoderRegistry(), elems);
+ try {
+ CreateSource<T> source = CreateSource.fromIterable(elems, coder);
+ return input.getPipeline().apply(Read.from(source));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Unable to apply Create %s using Coder %s.", this, coder), e);
}
}
@@ -398,10 +404,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<T> getDefaultOutputCoder() {
+ public Coder<T> getOutputCoder() {
return coder;
}
@@ -485,6 +488,38 @@
/////////////////////////////////////////////////////////////////////////////
+ /** Implementation of {@link #ofProvider}. */
+ public static class OfValueProvider<T> extends PTransform<PBegin, PCollection<T>> {
+ private final ValueProvider<T> provider;
+ private final Coder<T> coder;
+
+ private OfValueProvider(ValueProvider<T> provider, Coder<T> coder) {
+ this.provider = checkNotNull(provider, "provider");
+ this.coder = checkNotNull(coder, "coder");
+ }
+
+ @Override
+ public PCollection<T> expand(PBegin input) {
+ if (provider.isAccessible()) {
+ Values<T> values = Create.of(provider.get());
+ return input.apply(values.withCoder(coder));
+ }
+ return input
+ .apply(Create.of((Void) null))
+ .apply(
+ MapElements.via(
+ new SimpleFunction<Void, T>() {
+ @Override
+ public T apply(Void input) {
+ return provider.get();
+ }
+ }))
+ .setCoder(coder);
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+
/**
* A {@code PTransform} that creates a {@code PCollection} whose elements have
* associated timestamps.
@@ -528,7 +563,23 @@
@Override
public PCollection<T> expand(PBegin input) {
try {
- Coder<T> coder = getDefaultOutputCoder(input);
+ Coder<T> coder;
+ if (elementCoder.isPresent()) {
+ coder = elementCoder.get();
+ } else if (typeDescriptor.isPresent()) {
+ coder = input.getPipeline().getCoderRegistry().getCoder(typeDescriptor.get());
+ } else {
+ Iterable<T> rawElements =
+ Iterables.transform(
+ timestampedElements,
+ new Function<TimestampedValue<T>, T>() {
+ @Override
+ public T apply(TimestampedValue<T> timestampedValue) {
+ return timestampedValue.getValue();
+ }
+ });
+ coder = getDefaultCreateCoder(input.getPipeline().getCoderRegistry(), rawElements);
+ }
PCollection<TimestampedValue<T>> intermediate = Pipeline.applyTransform(input,
Create.of(timestampedElements).withCoder(TimestampedValueCoder.of(coder)));
@@ -568,26 +619,6 @@
c.outputWithTimestamp(c.element().getValue(), c.element().getTimestamp());
}
}
-
- @Override
- public Coder<T> getDefaultOutputCoder(PBegin input) throws CannotProvideCoderException {
- if (elementCoder.isPresent()) {
- return elementCoder.get();
- } else if (typeDescriptor.isPresent()) {
- return input.getPipeline().getCoderRegistry().getCoder(typeDescriptor.get());
- } else {
- Iterable<T> rawElements =
- Iterables.transform(
- timestampedElements,
- new Function<TimestampedValue<T>, T>() {
- @Override
- public T apply(TimestampedValue<T> input) {
- return input.getValue();
- }
- });
- return getDefaultCreateCoder(input.getPipeline().getCoderRegistry(), rawElements);
- }
- }
}
private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry registry, Iterable<T> elems)
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Distinct.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Distinct.java
index 2d08cee..a0ddd14 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Distinct.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Distinct.java
@@ -17,9 +17,16 @@
*/
package org.apache.beam.sdk.transforms;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* {@code Distinct<T>} takes a {@code PCollection<T>} and
@@ -59,6 +66,8 @@
*/
public class Distinct<T> extends PTransform<PCollection<T>,
PCollection<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(Distinct.class);
+
/**
* Returns a {@code Distinct<T>} {@code PTransform}.
*
@@ -66,7 +75,7 @@
* {@code PCollection}s
*/
public static <T> Distinct<T> create() {
- return new Distinct<T>();
+ return new Distinct<>();
}
/**
@@ -78,26 +87,49 @@
*/
public static <T, IdT> WithRepresentativeValues<T, IdT> withRepresentativeValueFn(
SerializableFunction<T, IdT> fn) {
- return new WithRepresentativeValues<T, IdT>(fn, null);
+ return new WithRepresentativeValues<>(fn, null);
+ }
+
+ private static <T, W extends BoundedWindow> void validateWindowStrategy(
+ WindowingStrategy<T, W> strategy) {
+ if (!strategy.getWindowFn().isNonMerging()
+ && (!strategy.getTrigger().getClass().equals(DefaultTrigger.class)
+ || strategy.getAllowedLateness().isLongerThan(Duration.ZERO))) {
+ throw new UnsupportedOperationException(String.format(
+ "%s does not support non-merging windowing strategies, except when using the default "
+ + "trigger and zero allowed lateness.", Distinct.class.getSimpleName()));
+ }
}
@Override
public PCollection<T> expand(PCollection<T> in) {
- return in
- .apply("CreateIndex", MapElements.via(new SimpleFunction<T, KV<T, Void>>() {
- @Override
- public KV<T, Void> apply(T element) {
- return KV.of(element, (Void) null);
- }
- }))
- .apply(Combine.<T, Void>perKey(
- new SerializableFunction<Iterable<Void>, Void>() {
+ validateWindowStrategy(in.getWindowingStrategy());
+ PCollection<KV<T, Void>> combined =
+ in.apply("KeyByElement", MapElements.via(
+ new SimpleFunction<T, KV<T, Void>>() {
@Override
- public Void apply(Iterable<Void> iter) {
- return null; // ignore input
- }
+ public KV<T, Void> apply(T element) {
+ return KV.of(element, (Void) null);
+ }
}))
- .apply(Keys.<T>create());
+ .apply("DropValues",
+ Combine.<T, Void>perKey(
+ new SerializableFunction<Iterable<Void>, Void>() {
+ @Override
+ @Nullable
+ public Void apply(Iterable<Void> iter) {
+ return null; // ignore input
+ }
+ }));
+ return combined.apply("ExtractFirstKey", ParDo.of(new DoFn<KV<T, Void>, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (c.pane().isFirst()) {
+ // Only output the key if it's the first time it's been seen.
+ c.output(c.element().getKey());
+ }
+ }
+ }));
}
/**
@@ -120,22 +152,32 @@
this.representativeType = representativeType;
}
+
@Override
public PCollection<T> expand(PCollection<T> in) {
+ validateWindowStrategy(in.getWindowingStrategy());
WithKeys<IdT, T> withKeys = WithKeys.of(fn);
if (representativeType != null) {
withKeys = withKeys.withKeyType(representativeType);
}
- return in
- .apply(withKeys)
- .apply(Combine.<IdT, T, T>perKey(
+ PCollection<KV<IdT, T>> combined = in
+ .apply("KeyByRepresentativeValue", withKeys)
+ .apply("OneValuePerKey", Combine.<IdT, T, T>perKey(
new Combine.BinaryCombineFn<T>() {
@Override
public T apply(T left, T right) {
return left;
}
- }))
- .apply(Values.<T>create());
+ }));
+ return combined.apply("KeepFirstPane", ParDo.of(new DoFn<KV<IdT, T>, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ // Only output the value if it's the first time it's been seen.
+ if (c.pane().isFirst()) {
+ c.output(c.element().getValue());
+ }
+ }
+ }));
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index e711ac2..3e023db 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.transforms;
+import com.google.auto.value.AutoValue;
import java.io.Serializable;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
@@ -385,7 +386,7 @@
* <pre><code>{@literal new DoFn<KV<Key, Foo>, Baz>()} {
*
* {@literal @StateId("my-state-id")}
- * {@literal private final StateSpec<K, ValueState<MyState>>} myStateSpec =
+ * {@literal private final StateSpec<ValueState<MyState>>} myStateSpec =
* StateSpecs.value(new MyStateCoder());
*
* {@literal @ProcessElement}
@@ -523,12 +524,15 @@
* <li>It must return {@code void}.
* </ul>
*
- * <h2>Splittable DoFn's (WARNING: work in progress, do not use)</h2>
+ * <h2>Splittable DoFn's</h2>
*
* <p>A {@link DoFn} is <i>splittable</i> if its {@link ProcessElement} method has a parameter
* whose type is a subtype of {@link RestrictionTracker}. This is an advanced feature and an
- * overwhelming majority of users will never need to write a splittable {@link DoFn}. Right now
- * the implementation of this feature is in progress and it's not ready for any use.
+ * overwhelming majority of users will never need to write a splittable {@link DoFn}.
+ *
+ * <p>Not all runners support Splittable DoFn. See the
+ * <a href="https://beam.apache.org/documentation/runners/capability-matrix/">capability
+ * matrix</a>.
*
* <p>See <a href="https://s.apache.org/splittable-do-fn">the proposal</a> for an overview of the
* involved concepts (<i>splittable DoFn</i>, <i>restriction</i>, <i>restriction tracker</i>).
@@ -545,16 +549,18 @@
* returned by {@link GetInitialRestriction} implements {@link HasDefaultTracker}.
* <li>It <i>may</i> define a {@link GetRestrictionCoder} method.
* <li>The type of restrictions used by all of these methods must be the same.
+ * <li>Its {@link ProcessElement} method <i>may</i> return a {@link ProcessContinuation} to
+ * indicate whether there is more work to be done for the current element.
* <li>Its {@link ProcessElement} method <i>must not</i> use any extra context parameters, such as
* {@link BoundedWindow}.
* <li>The {@link DoFn} itself <i>may</i> be annotated with {@link BoundedPerElement} or
* {@link UnboundedPerElement}, but not both at the same time. If it's not annotated with
- * either of these, it's assumed to be {@link BoundedPerElement}.
+ * either of these, it's assumed to be {@link BoundedPerElement} if its {@link
+ * ProcessElement} method returns {@code void} and {@link UnboundedPerElement} if it
+ * returns a {@link ProcessContinuation}.
* </ul>
*
* <p>A non-splittable {@link DoFn} <i>must not</i> define any of these methods.
- *
- * <p>More documentation will be added when the feature becomes ready for general usage.
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@@ -677,9 +683,55 @@
@Experimental(Kind.SPLITTABLE_DO_FN)
public @interface UnboundedPerElement {}
+ // This can't be put into ProcessContinuation itself due to the following problem:
+ // http://ternarysearch.blogspot.com/2013/07/static-initialization-deadlock.html
+ private static final ProcessContinuation PROCESS_CONTINUATION_STOP =
+ new AutoValue_DoFn_ProcessContinuation(false, Duration.ZERO);
+
+ /**
+ * When used as a return value of {@link ProcessElement}, indicates whether there is more work to
+ * be done for the current element.
+ *
+ * <p>If the {@link ProcessElement} call completes because of a failed {@code tryClaim()} call
+ * on the {@link RestrictionTracker}, then the call MUST return {@link #stop()}.
+ */
+ @Experimental(Kind.SPLITTABLE_DO_FN)
+ @AutoValue
+ public abstract static class ProcessContinuation {
+ /** Indicates that there is no more work to be done for the current element. */
+ public static ProcessContinuation stop() {
+ return PROCESS_CONTINUATION_STOP;
+ }
+
+ /** Indicates that there is more work to be done for the current element. */
+ public static ProcessContinuation resume() {
+ return new AutoValue_DoFn_ProcessContinuation(true, Duration.ZERO);
+ }
+
+ /**
+ * If false, the {@link DoFn} promises that there is no more work remaining for the current
+ * element, so the runner should not resume the {@link ProcessElement} call.
+ */
+ public abstract boolean shouldResume();
+
+ /**
+ * A minimum duration that should elapse between the end of this {@link ProcessElement} call and
+ * the {@link ProcessElement} call continuing processing of the same element. By default, zero.
+ */
+ public abstract Duration resumeDelay();
+
+ /** Builder method to set the value of {@link #resumeDelay()}. */
+ public ProcessContinuation withResumeDelay(Duration resumeDelay) {
+ return new AutoValue_DoFn_ProcessContinuation(shouldResume(), resumeDelay);
+ }
+ }
+
/**
* Finalize the {@link DoFn} construction to prepare for processing.
* This method should be called by runners before any processing methods.
+ *
+ * @deprecated use {@link Setup} or {@link StartBundle} instead. This method will be removed in a
+ * future release.
*/
@Deprecated
public final void prepareForProcessing() {}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 8a03f3c..6168710 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -30,6 +30,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -290,6 +292,11 @@
}
@Override
+ public PipelineOptions pipelineOptions() {
+ return getPipelineOptions();
+ }
+
+ @Override
public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(
DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
@@ -509,17 +516,17 @@
private <T> List<ValueInSingleWindow<T>> getImmutableOutput(TupleTag<T> tag) {
@SuppressWarnings({"unchecked", "rawtypes"})
- List<ValueInSingleWindow<T>> elems = (List) outputs.get(tag);
+ List<ValueInSingleWindow<T>> elems = (List) getOutputs().get(tag);
return ImmutableList.copyOf(
MoreObjects.firstNonNull(elems, Collections.<ValueInSingleWindow<T>>emptyList()));
}
@SuppressWarnings({"unchecked", "rawtypes"})
public <T> List<ValueInSingleWindow<T>> getMutableOutput(TupleTag<T> tag) {
- List<ValueInSingleWindow<T>> outputList = (List) outputs.get(tag);
+ List<ValueInSingleWindow<T>> outputList = (List) getOutputs().get(tag);
if (outputList == null) {
outputList = new ArrayList<>();
- outputs.put(tag, (List) outputList);
+ getOutputs().put(tag, (List) outputList);
}
return outputList;
}
@@ -546,11 +553,6 @@
fn.super();
}
- private void throwUnsupportedOutputFromBundleMethods() {
- throw new UnsupportedOperationException(
- "DoFnTester doesn't support output from bundle methods");
- }
-
@Override
public PipelineOptions getPipelineOptions() {
return options;
@@ -559,12 +561,13 @@
@Override
public void output(
OutputT output, Instant timestamp, BoundedWindow window) {
- throwUnsupportedOutputFromBundleMethods();
+ output(mainOutputTag, output, timestamp, window);
}
@Override
public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
- throwUnsupportedOutputFromBundleMethods();
+ getMutableOutput(tag)
+ .add(ValueInSingleWindow.of(output, timestamp, window, PaneInfo.NO_FIRING));
}
}
@@ -642,12 +645,6 @@
getMutableOutput(tag)
.add(ValueInSingleWindow.of(output, timestamp, element.getWindow(), element.getPane()));
}
-
- private void throwUnsupportedOutputFromBundleMethods() {
- throw new UnsupportedOperationException(
- "DoFnTester doesn't support output from bundle methods");
- }
-
}
@Override
@@ -693,11 +690,12 @@
private TupleTag<OutputT> mainOutputTag = new TupleTag<>();
/** The original DoFn under test, if started. */
- private DoFn<InputT, OutputT> fn;
- private DoFnInvoker<InputT, OutputT> fnInvoker;
+ @Nullable private DoFn<InputT, OutputT> fn;
- /** The outputs from the {@link DoFn} under test. */
- private Map<TupleTag<?>, List<ValueInSingleWindow<?>>> outputs;
+ @Nullable private DoFnInvoker<InputT, OutputT> fnInvoker;
+
+ /** The outputs from the {@link DoFn} under test. Access via {@link #getOutputs()}. */
+ @CheckForNull private Map<TupleTag<?>, List<ValueInSingleWindow<?>>> outputs;
/** The state of processing of the {@link DoFn} under test. */
private State state = State.UNINITIALIZED;
@@ -709,12 +707,14 @@
param.match(
new DoFnSignature.Parameter.Cases.WithDefault<Void>() {
@Override
+ @Nullable
public Void dispatch(DoFnSignature.Parameter.ProcessContextParameter p) {
// ProcessContext parameter is obviously supported.
return null;
}
@Override
+ @Nullable
public Void dispatch(DoFnSignature.Parameter.WindowParameter p) {
// We also support the BoundedWindow parameter.
return null;
@@ -743,6 +743,12 @@
}
fnInvoker = DoFnInvokers.invokerFor(fn);
fnInvoker.invokeSetup();
- outputs = new HashMap<>();
+ }
+
+ private Map getOutputs() {
+ if (outputs == null) {
+ outputs = new HashMap<>();
+ }
+ return outputs;
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Filter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Filter.java
index d0314eb..2fd12de 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Filter.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Filter.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.transforms;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PCollection;
@@ -229,19 +228,18 @@
@Override
public PCollection<T> expand(PCollection<T> input) {
- return input.apply(ParDo.of(new DoFn<T, T>() {
- @ProcessElement
- public void processElement(ProcessContext c) {
- if (predicate.apply(c.element())) {
- c.output(c.element());
- }
- }
- }));
- }
-
- @Override
- protected Coder<T> getDefaultOutputCoder(PCollection<T> input) {
- return input.getCoder();
+ return input
+ .apply(
+ ParDo.of(
+ new DoFn<T, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (predicate.apply(c.element())) {
+ c.output(c.element());
+ }
+ }
+ }))
+ .setCoder(input.getCoder());
}
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java
index a8a94f9..193bb6e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java
@@ -17,11 +17,14 @@
*/
package org.apache.beam.sdk.transforms;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
-import java.lang.reflect.ParameterizedType;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.transforms.Contextful.Fn;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.TypeDescriptors;
@@ -32,30 +35,20 @@
*/
public class FlatMapElements<InputT, OutputT>
extends PTransform<PCollection<? extends InputT>, PCollection<OutputT>> {
- /**
- * Temporarily stores the argument of {@link #into(TypeDescriptor)} until combined with the
- * argument of {@link #via(SerializableFunction)} into the fully-specified {@link #fn}. Stays null
- * if constructed using {@link #via(SimpleFunction)} directly.
- */
- @Nullable
- private final transient TypeDescriptor<Iterable<OutputT>> outputType;
-
- /**
- * Non-null on a fully specified transform - is null only when constructed using {@link
- * #into(TypeDescriptor)}, until the fn is specified using {@link #via(SerializableFunction)}.
- */
- @Nullable
- private final SimpleFunction<InputT, Iterable<OutputT>> fn;
- private final DisplayData.ItemSpec<?> fnClassDisplayData;
+ @Nullable private final transient TypeDescriptor<InputT> inputType;
+ @Nullable private final transient TypeDescriptor<OutputT> outputType;
+ @Nullable private final transient Object originalFnForDisplayData;
+ @Nullable private final Contextful<Fn<InputT, Iterable<OutputT>>> fn;
private FlatMapElements(
- @Nullable SimpleFunction<InputT, Iterable<OutputT>> fn,
- @Nullable TypeDescriptor<Iterable<OutputT>> outputType,
- @Nullable Class<?> fnClass) {
+ @Nullable Contextful<Fn<InputT, Iterable<OutputT>>> fn,
+ @Nullable Object originalFnForDisplayData,
+ @Nullable TypeDescriptor<InputT> inputType,
+ TypeDescriptor<OutputT> outputType) {
this.fn = fn;
+ this.originalFnForDisplayData = originalFnForDisplayData;
+ this.inputType = inputType;
this.outputType = outputType;
- this.fnClassDisplayData = DisplayData.item("flatMapFn", fnClass).withLabel("FlatMap Function");
-
}
/**
@@ -82,7 +75,14 @@
*/
public static <InputT, OutputT> FlatMapElements<InputT, OutputT>
via(SimpleFunction<? super InputT, ? extends Iterable<OutputT>> fn) {
- return new FlatMapElements(fn, null, fn.getClass());
+ Contextful<Fn<InputT, Iterable<OutputT>>> wrapped = (Contextful) Contextful.fn(fn);
+ TypeDescriptor<OutputT> outputType =
+ TypeDescriptors.extractFromTypeParameters(
+ (TypeDescriptor<Iterable<OutputT>>) fn.getOutputTypeDescriptor(),
+ Iterable.class,
+ new TypeDescriptors.TypeVariableExtractor<Iterable<OutputT>, OutputT>() {});
+ TypeDescriptor<InputT> inputType = (TypeDescriptor<InputT>) fn.getInputTypeDescriptor();
+ return new FlatMapElements<>(wrapped, fn, inputType, outputType);
}
/**
@@ -91,7 +91,7 @@
*/
public static <OutputT> FlatMapElements<?, OutputT>
into(final TypeDescriptor<OutputT> outputType) {
- return new FlatMapElements<>(null, TypeDescriptors.iterables(outputType), null);
+ return new FlatMapElements<>(null, null, null, outputType);
}
/**
@@ -112,73 +112,65 @@
*/
public <NewInputT> FlatMapElements<NewInputT, OutputT>
via(SerializableFunction<NewInputT, ? extends Iterable<OutputT>> fn) {
- return new FlatMapElements(
- SimpleFunction.fromSerializableFunctionWithOutputType(fn, (TypeDescriptor) outputType),
- null,
- fn.getClass());
+ return new FlatMapElements<>(
+ (Contextful) Contextful.fn(fn), fn, TypeDescriptors.inputOf(fn), outputType);
+ }
+
+ /** Like {@link #via(SerializableFunction)}, but allows access to additional context. */
+ @Experimental(Experimental.Kind.CONTEXTFUL)
+ public <NewInputT> FlatMapElements<NewInputT, OutputT> via(
+ Contextful<Fn<NewInputT, Iterable<OutputT>>> fn) {
+ return new FlatMapElements<>(
+ fn, fn.getClosure(), TypeDescriptors.inputOf(fn.getClosure()), outputType);
}
@Override
public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
- checkNotNull(fn, "Must specify a function on FlatMapElements using .via()");
+ checkArgument(fn != null, ".via() is required");
return input.apply(
"FlatMap",
ParDo.of(
- new DoFn<InputT, OutputT>() {
- private static final long serialVersionUID = 0L;
+ new DoFn<InputT, OutputT>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Iterable<OutputT> res =
+ fn.getClosure().apply(c.element(), Fn.Context.wrapProcessContext(c));
+ for (OutputT output : res) {
+ c.output(output);
+ }
+ }
- @ProcessElement
- public void processElement(ProcessContext c) {
- for (OutputT element : fn.apply(c.element())) {
- c.output(element);
- }
- }
+ @Override
+ public TypeDescriptor<InputT> getInputTypeDescriptor() {
+ return inputType;
+ }
- @Override
- public TypeDescriptor<InputT> getInputTypeDescriptor() {
- return fn.getInputTypeDescriptor();
- }
+ @Override
+ public TypeDescriptor<OutputT> getOutputTypeDescriptor() {
+ checkState(
+ outputType != null,
+ "%s output type descriptor was null; "
+ + "this probably means that getOutputTypeDescriptor() was called after "
+ + "serialization/deserialization, but it is only available prior to "
+ + "serialization, for constructing a pipeline and inferring coders",
+ FlatMapElements.class.getSimpleName());
+ return outputType;
+ }
- @Override
- public TypeDescriptor<OutputT> getOutputTypeDescriptor() {
- @SuppressWarnings({"rawtypes", "unchecked"}) // safe by static typing
- TypeDescriptor<Iterable<?>> iterableType =
- (TypeDescriptor) fn.getOutputTypeDescriptor();
-
- @SuppressWarnings("unchecked") // safe by correctness of getIterableElementType
- TypeDescriptor<OutputT> outputType =
- (TypeDescriptor<OutputT>) getIterableElementType(iterableType);
-
- return outputType;
- }
- }));
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.delegate(FlatMapElements.this);
+ }
+ })
+ .withSideInputs(fn.getRequirements().getSideInputs()));
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- builder
- .include("flatMapFn", fn)
- .add(fnClassDisplayData);
- }
-
- /**
- * Does a best-effort job of getting the best {@link TypeDescriptor} for the type of the
- * elements contained in the iterable described by the given {@link TypeDescriptor}.
- */
- private static TypeDescriptor<?> getIterableElementType(
- TypeDescriptor<Iterable<?>> iterableTypeDescriptor) {
-
- // If a rawtype was used, the type token may be for Object, not a subtype of Iterable.
- // In this case, we rely on static typing of the function elsewhere to ensure it is
- // at least some kind of iterable, and grossly overapproximate the element type to be Object.
- if (!iterableTypeDescriptor.isSubtypeOf(new TypeDescriptor<Iterable<?>>() {})) {
- return new TypeDescriptor<Object>() {};
+ builder.add(DisplayData.item("class", originalFnForDisplayData.getClass()));
+ if (originalFnForDisplayData instanceof HasDisplayData) {
+ builder.include("fn", (HasDisplayData) originalFnForDisplayData);
}
-
- // Otherwise we can do the proper thing and get the actual type parameter.
- ParameterizedType iterableType =
- (ParameterizedType) iterableTypeDescriptor.getSupertype(Iterable.class).getType();
- return TypeDescriptor.of(iterableType.getActualTypeArguments()[0]);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
index 25d9c05..8247a58 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.transforms;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableLikeCoder;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -129,25 +128,12 @@
windowingStrategy = WindowingStrategy.globalDefault();
}
- return PCollection.<T>createPrimitiveOutputInternal(
+ return PCollection.createPrimitiveOutputInternal(
inputs.getPipeline(),
windowingStrategy,
- isBounded);
- }
-
- @Override
- protected Coder<?> getDefaultOutputCoder(PCollectionList<T> input)
- throws CannotProvideCoderException {
-
- // Take coder from first collection
- for (PCollection<T> pCollection : input.getAll()) {
- return pCollection.getCoder();
- }
-
- // No inputs
- throw new CannotProvideCoderException(
- this.getClass().getSimpleName() + " cannot provide a Coder for"
- + " empty " + PCollectionList.class.getSimpleName());
+ isBounded,
+ // Take coder from first collection. If there are none, will be left unspecified.
+ inputs.getAll().isEmpty() ? null : inputs.get(0).getCoder());
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
index 7516b25..3cb0d23 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
@@ -217,13 +217,11 @@
// merging windows as needed, using the windows assigned to the
// key/value input elements and the window merge operation of the
// window function associated with the input PCollection.
- return PCollection.createPrimitiveOutputInternal(input.getPipeline(),
- updateWindowingStrategy(input.getWindowingStrategy()), input.isBounded());
- }
-
- @Override
- protected Coder<KV<K, Iterable<V>>> getDefaultOutputCoder(PCollection<KV<K, V>> input) {
- return getOutputKvCoder(input.getCoder());
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ updateWindowingStrategy(input.getWindowingStrategy()),
+ input.isBounded(),
+ getOutputKvCoder(input.getCoder()));
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index b023363..a79b07b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -99,7 +99,7 @@
static class GroupIntoBatchesDoFn<K, InputT>
extends DoFn<KV<K, InputT>, KV<K, Iterable<InputT>>> {
- private static final Logger LOGGER = LoggerFactory.getLogger(GroupIntoBatchesDoFn.class);
+ private static final Logger LOG = LoggerFactory.getLogger(GroupIntoBatchesDoFn.class);
private static final String END_OF_WINDOW_ID = "endOFWindow";
private static final String BATCH_ID = "batch";
private static final String NUM_ELEMENTS_IN_BATCH_ID = "numElementsInBatch";
@@ -160,13 +160,13 @@
BoundedWindow window) {
Instant windowExpires = window.maxTimestamp().plus(allowedLateness);
- LOGGER.debug(
+ LOG.debug(
"*** SET TIMER *** to point in time {} for window {}",
windowExpires.toString(), window.toString());
timer.set(windowExpires);
key.write(c.element().getKey());
batch.add(c.element().getValue());
- LOGGER.debug("*** BATCH *** Add element for window {} ", window.toString());
+ LOG.debug("*** BATCH *** Add element for window {} ", window.toString());
// blind add is supported with combiningState
numElementsInBatch.add(1L);
Long num = numElementsInBatch.read();
@@ -175,7 +175,7 @@
batch.readLater();
}
if (num >= batchSize) {
- LOGGER.debug("*** END OF BATCH *** for window {}", window.toString());
+ LOG.debug("*** END OF BATCH *** for window {}", window.toString());
flushBatch(c, key, batch, numElementsInBatch);
}
}
@@ -188,7 +188,7 @@
@StateId(NUM_ELEMENTS_IN_BATCH_ID)
CombiningState<Long, long[], Long> numElementsInBatch,
BoundedWindow window) {
- LOGGER.debug(
+ LOG.debug(
"*** END OF WINDOW *** for timer timestamp {} in windows {}",
context.timestamp(), window.toString());
flushBatch(context, key, batch, numElementsInBatch);
@@ -205,7 +205,7 @@
c.output(KV.of(key.read(), values));
}
batch.clear();
- LOGGER.debug("*** BATCH *** clear");
+ LOG.debug("*** BATCH *** clear");
numElementsInBatch.clear();
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java
index f7028ec..f327df1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java
@@ -129,7 +129,7 @@
TimestampedValue.TimestampedValueCoder<T> inputTVCoder =
(TimestampedValue.TimestampedValueCoder<T>) inputCoder;
- return NullableCoder.of(inputTVCoder.<T>getValueCoder());
+ return NullableCoder.of(inputTVCoder.getValueCoder());
}
@Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
index 792a6d5..e1d6c11 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
@@ -18,38 +18,37 @@
package org.apache.beam.sdk.transforms;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.transforms.Contextful.Fn;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
/**
* {@code PTransform}s for mapping a simple function over the elements of a {@link PCollection}.
*/
public class MapElements<InputT, OutputT>
extends PTransform<PCollection<? extends InputT>, PCollection<OutputT>> {
- /**
- * Temporarily stores the argument of {@link #into(TypeDescriptor)} until combined with the
- * argument of {@link #via(SerializableFunction)} into the fully-specified {@link #fn}. Stays null
- * if constructed using {@link #via(SimpleFunction)} directly.
- */
+ @Nullable private final transient TypeDescriptor<InputT> inputType;
@Nullable private final transient TypeDescriptor<OutputT> outputType;
-
- /**
- * Non-null on a fully specified transform - is null only when constructed using {@link
- * #into(TypeDescriptor)}, until the fn is specified using {@link #via(SerializableFunction)}.
- */
- @Nullable private final SimpleFunction<InputT, OutputT> fn;
- private final DisplayData.ItemSpec<?> fnClassDisplayData;
+ @Nullable private final transient Object originalFnForDisplayData;
+ @Nullable private final Contextful<Fn<InputT, OutputT>> fn;
private MapElements(
- @Nullable SimpleFunction<InputT, OutputT> fn,
- @Nullable TypeDescriptor<OutputT> outputType,
- @Nullable Class<?> fnClass) {
+ @Nullable Contextful<Fn<InputT, OutputT>> fn,
+ @Nullable Object originalFnForDisplayData,
+ @Nullable TypeDescriptor<InputT> inputType,
+ TypeDescriptor<OutputT> outputType) {
this.fn = fn;
+ this.originalFnForDisplayData = originalFnForDisplayData;
+ this.inputType = inputType;
this.outputType = outputType;
- this.fnClassDisplayData = DisplayData.item("mapFn", fnClass).withLabel("Map Function");
}
/**
@@ -57,10 +56,11 @@
* takes an input {@code PCollection<InputT>} and returns a {@code PCollection<OutputT>}
* containing {@code fn.apply(v)} for every element {@code v} in the input.
*
- * <p>This overload is intended primarily for use in Java 7. In Java 8, the overload
- * {@link #via(SerializableFunction)} supports use of lambda for greater concision.
+ * <p>This overload is intended primarily for use in Java 7. In Java 8, the overload {@link
+ * #via(SerializableFunction)} supports use of lambda for greater concision.
*
* <p>Example of use in Java 7:
+ *
* <pre>{@code
* PCollection<String> words = ...;
* PCollection<Integer> wordsPerLine = words.apply(MapElements.via(
@@ -73,7 +73,8 @@
*/
public static <InputT, OutputT> MapElements<InputT, OutputT> via(
final SimpleFunction<InputT, OutputT> fn) {
- return new MapElements<>(fn, null, fn.getClass());
+ return new MapElements<>(
+ Contextful.fn(fn), fn, fn.getInputTypeDescriptor(), fn.getOutputTypeDescriptor());
}
/**
@@ -82,7 +83,7 @@
*/
public static <OutputT> MapElements<?, OutputT>
into(final TypeDescriptor<OutputT> outputType) {
- return new MapElements<>(null, outputType, null);
+ return new MapElements<>(null, null, null, outputType);
}
/**
@@ -104,10 +105,16 @@
*/
public <NewInputT> MapElements<NewInputT, OutputT> via(
SerializableFunction<NewInputT, OutputT> fn) {
+ return new MapElements<>(Contextful.fn(fn), fn, TypeDescriptors.inputOf(fn), outputType);
+ }
+
+ /**
+ * Like {@link #via(SerializableFunction)}, but supports access to context, such as side inputs.
+ */
+ @Experimental(Kind.CONTEXTFUL)
+ public <NewInputT> MapElements<NewInputT, OutputT> via(Contextful<Fn<NewInputT, OutputT>> fn) {
return new MapElements<>(
- SimpleFunction.fromSerializableFunctionWithOutputType(fn, outputType),
- null,
- fn.getClass());
+ fn, fn.getClosure(), TypeDescriptors.inputOf(fn.getClosure()), outputType);
}
@Override
@@ -118,8 +125,8 @@
ParDo.of(
new DoFn<InputT, OutputT>() {
@ProcessElement
- public void processElement(ProcessContext c) {
- c.output(fn.apply(c.element()));
+ public void processElement(ProcessContext c) throws Exception {
+ c.output(fn.getClosure().apply(c.element(), Fn.Context.wrapProcessContext(c)));
}
@Override
@@ -129,21 +136,29 @@
@Override
public TypeDescriptor<InputT> getInputTypeDescriptor() {
- return fn.getInputTypeDescriptor();
+ return inputType;
}
@Override
public TypeDescriptor<OutputT> getOutputTypeDescriptor() {
- return fn.getOutputTypeDescriptor();
+ checkState(
+ outputType != null,
+ "%s output type descriptor was null; "
+ + "this probably means that getOutputTypeDescriptor() was called after "
+ + "serialization/deserialization, but it is only available prior to "
+ + "serialization, for constructing a pipeline and inferring coders",
+ MapElements.class.getSimpleName());
+ return outputType;
}
- }));
+ }).withSideInputs(fn.getRequirements().getSideInputs()));
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- builder
- .include("mapFn", fn)
- .add(fnClassDisplayData);
+ builder.add(DisplayData.item("class", originalFnForDisplayData.getClass()));
+ if (originalFnForDisplayData instanceof HasDisplayData) {
+ builder.include("fn", (HasDisplayData) originalFnForDisplayData);
+ }
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
index 91851bc..384404a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
@@ -19,6 +19,7 @@
import java.io.Serializable;
import java.util.Comparator;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.transforms.Combine.BinaryCombineFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -158,11 +159,11 @@
}
public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder(T identity) {
- return new MaxFn<T>(identity, new Top.Largest<T>());
+ return new MaxFn<T>(identity, new Top.Natural<T>());
}
public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder() {
- return new MaxFn<T>(null, new Top.Largest<T>());
+ return new MaxFn<T>(null, new Top.Natural<T>());
}
/**
@@ -214,11 +215,11 @@
private static class MaxFn<T> extends BinaryCombineFn<T> {
- private final T identity;
+ @Nullable private final T identity;
private final Comparator<? super T> comparator;
private <ComparatorT extends Comparator<? super T> & Serializable> MaxFn(
- T identity, ComparatorT comparator) {
+ @Nullable T identity, ComparatorT comparator) {
this.identity = identity;
this.comparator = comparator;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
index 109f4e5..65b3e6e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
@@ -19,6 +19,7 @@
import java.io.Serializable;
import java.util.Comparator;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.transforms.Combine.BinaryCombineFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -158,11 +159,11 @@
}
public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder(T identity) {
- return new MinFn<T>(identity, new Top.Largest<T>());
+ return new MinFn<T>(identity, new Top.Natural<T>());
}
public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder() {
- return new MinFn<T>(null, new Top.Largest<T>());
+ return new MinFn<T>(null, new Top.Natural<T>());
}
/**
@@ -214,11 +215,11 @@
private static class MinFn<T> extends BinaryCombineFn<T> {
- private final T identity;
+ @Nullable private final T identity;
private final Comparator<? super T> comparator;
private <ComparatorT extends Comparator<? super T> & Serializable> MinFn(
- T identity, ComparatorT comparator) {
+ @Nullable T identity, ComparatorT comparator) {
this.identity = identity;
this.comparator = comparator;
}
@@ -236,8 +237,7 @@
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- builder.add(DisplayData.item("comparer", comparator.getClass())
- .withLabel("Record Comparer"));
+ builder.add(DisplayData.item("comparer", comparator.getClass()).withLabel("Record Comparer"));
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
index d5df944..139d82a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
@@ -22,6 +22,7 @@
import java.io.Serializable;
import java.util.Collections;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
@@ -176,8 +177,12 @@
public abstract class PTransform<InputT extends PInput, OutputT extends POutput>
implements Serializable /* See the note above */, HasDisplayData {
/**
- * Applies this {@code PTransform} on the given {@code InputT}, and returns its
- * {@code Output}.
+ * Override this method to specify how this {@code PTransform} should be expanded
+ * on the given {@code InputT}.
+ *
+ * <p>NOTE: This method should not be called directly. Instead apply the
+ * {@code PTransform} should be applied to the {@code InputT} using the {@code apply}
+ * method.
*
* <p>Composite transforms, which are defined in terms of other transforms,
* should return the output of one of the composed transforms. Non-composite
@@ -193,7 +198,7 @@
*
* <p>By default, does nothing.
*/
- public void validate(PipelineOptions options) {}
+ public void validate(@Nullable PipelineOptions options) {}
/**
* Returns all {@link PValue PValues} that are consumed as inputs to this {@link PTransform} that
@@ -223,13 +228,13 @@
* The base name of this {@code PTransform}, e.g., from defaults, or
* {@code null} if not yet assigned.
*/
- protected final transient String name;
+ @Nullable protected final transient String name;
protected PTransform() {
this.name = null;
}
- protected PTransform(String name) {
+ protected PTransform(@Nullable String name) {
this.name = name;
}
@@ -273,13 +278,16 @@
}
/**
- * Returns the default {@code Coder} to use for the output of this
- * single-output {@code PTransform}.
+ * Returns the default {@code Coder} to use for the output of this single-output {@code
+ * PTransform}.
*
* <p>By default, always throws
*
* @throws CannotProvideCoderException if no coder can be inferred
+ * @deprecated Instead, the PTransform should explicitly call {@link PCollection#setCoder} on the
+ * returned PCollection.
*/
+ @Deprecated
protected Coder<?> getDefaultOutputCoder() throws CannotProvideCoderException {
throw new CannotProvideCoderException("PTransform.getOutputCoder called.");
}
@@ -291,7 +299,10 @@
* <p>By default, always throws.
*
* @throws CannotProvideCoderException if none can be inferred.
+ * @deprecated Instead, the PTransform should explicitly call {@link PCollection#setCoder} on the
+ * returned PCollection.
*/
+ @Deprecated
protected Coder<?> getDefaultOutputCoder(@SuppressWarnings("unused") InputT input)
throws CannotProvideCoderException {
return getDefaultOutputCoder();
@@ -304,7 +315,10 @@
* <p>By default, always throws.
*
* @throws CannotProvideCoderException if none can be inferred.
+ * @deprecated Instead, the PTransform should explicitly call {@link PCollection#setCoder} on the
+ * returned PCollection.
*/
+ @Deprecated
public <T> Coder<T> getDefaultOutputCoder(
InputT input, @SuppressWarnings("unused") PCollection<T> output)
throws CannotProvideCoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index edf1419..2ad84fb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -20,7 +20,6 @@
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
@@ -33,6 +32,7 @@
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -46,10 +46,10 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.NameUtils;
-import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
@@ -455,6 +455,27 @@
}
}
+ private static void validateStateApplicableForInput(
+ DoFn<?, ?> fn,
+ PCollection<?> input) {
+ Coder<?> inputCoder = input.getCoder();
+ checkArgument(
+ inputCoder instanceof KvCoder,
+ "%s requires its input to use %s in order to use state and timers.",
+ ParDo.class.getSimpleName(),
+ KvCoder.class.getSimpleName());
+
+ KvCoder<?, ?> kvCoder = (KvCoder<?, ?>) inputCoder;
+ try {
+ kvCoder.getKeyCoder().verifyDeterministic();
+ } catch (Coder.NonDeterministicException exc) {
+ throw new IllegalArgumentException(
+ String.format(
+ "%s requires a deterministic key coder in order to use state and timers",
+ ParDo.class.getSimpleName()));
+ }
+ }
+
/**
* Try to provide coders for as many of the type arguments of given
* {@link DoFnSignature.StateDeclaration} as possible.
@@ -515,7 +536,8 @@
if (methodSignature.windowT() != null) {
checkArgument(
methodSignature.windowT().isSupertypeOf(actualWindowT),
- "%s expects window type %s, which is not a supertype of actual window type %s",
+ "%s unable to provide window -- expected window type from parameter (%s) is not a "
+ + "supertype of actual window type assigned by windowing (%s)",
methodSignature.targetMethod(),
methodSignature.windowT(),
actualWindowT);
@@ -566,7 +588,7 @@
DoFn<InputT, OutputT> fn,
List<PCollectionView<?>> sideInputs,
DisplayData.ItemSpec<? extends Class<?>> fnDisplayData) {
- this.fn = SerializableUtils.clone(fn);
+ this.fn = fn;
this.fnDisplayData = fnDisplayData;
this.sideInputs = sideInputs;
}
@@ -614,19 +636,21 @@
@Override
public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
- finishSpecifyingStateSpecs(fn, input.getPipeline().getCoderRegistry(), input.getCoder());
+ CoderRegistry registry = input.getPipeline().getCoderRegistry();
+ finishSpecifyingStateSpecs(fn, registry, input.getCoder());
TupleTag<OutputT> mainOutput = new TupleTag<>();
- return input.apply(withOutputTags(mainOutput, TupleTagList.empty())).get(mainOutput);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- protected Coder<OutputT> getDefaultOutputCoder(PCollection<? extends InputT> input)
- throws CannotProvideCoderException {
- return input.getPipeline().getCoderRegistry().getCoder(
- getFn().getOutputTypeDescriptor(),
- getFn().getInputTypeDescriptor(),
- ((PCollection<InputT>) input).getCoder());
+ PCollection<OutputT> res =
+ input.apply(withOutputTags(mainOutput, TupleTagList.empty())).get(mainOutput);
+ try {
+ res.setCoder(
+ registry.getCoder(
+ getFn().getOutputTypeDescriptor(),
+ getFn().getInputTypeDescriptor(),
+ ((PCollection<InputT>) input).getCoder()));
+ } catch (CannotProvideCoderException e) {
+ // Ignore and leave coder unset.
+ }
+ return res;
}
@Override
@@ -662,11 +686,7 @@
*/
@Override
public Map<TupleTag<?>, PValue> getAdditionalInputs() {
- ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder();
- for (PCollectionView<?> sideInput : sideInputs) {
- additionalInputs.put(sideInput.getTagInternal(), sideInput.getPCollection());
- }
- return additionalInputs.build();
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
}
@@ -696,7 +716,7 @@
this.sideInputs = sideInputs;
this.mainOutputTag = mainOutputTag;
this.additionalOutputTags = additionalOutputTags;
- this.fn = SerializableUtils.clone(fn);
+ this.fn = fn;
this.fnDisplayData = fnDisplayData;
}
@@ -739,13 +759,33 @@
validateWindowType(input, fn);
// Use coder registry to determine coders for all StateSpec defined in the fn signature.
- finishSpecifyingStateSpecs(fn, input.getPipeline().getCoderRegistry(), input.getCoder());
+ CoderRegistry registry = input.getPipeline().getCoderRegistry();
+ finishSpecifyingStateSpecs(fn, registry, input.getCoder());
+
+ DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
+ if (signature.usesState() || signature.usesTimers()) {
+ validateStateApplicableForInput(fn, input);
+ }
PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal(
input.getPipeline(),
TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
+ // TODO
+ Collections.<TupleTag<?>, Coder<?>>emptyMap(),
input.getWindowingStrategy(),
input.isBounded());
+ @SuppressWarnings("unchecked")
+ Coder<InputT> inputCoder = ((PCollection<InputT>) input).getCoder();
+ for (PCollection<?> out : outputs.getAll().values()) {
+ try {
+ out.setCoder(
+ (Coder)
+ registry.getCoder(
+ out.getTypeDescriptor(), getFn().getInputTypeDescriptor(), inputCoder));
+ } catch (CannotProvideCoderException e) {
+ // Ignore and let coder inference happen later.
+ }
+ }
// The fn will likely be an instance of an anonymous subclass
// such as DoFn<Integer, String> { }, thus will have a high-fidelity
@@ -756,24 +796,6 @@
}
@Override
- protected Coder<OutputT> getDefaultOutputCoder() {
- throw new RuntimeException(
- "internal error: shouldn't be calling this on a multi-output ParDo");
- }
-
- @Override
- public <T> Coder<T> getDefaultOutputCoder(
- PCollection<? extends InputT> input, PCollection<T> output)
- throws CannotProvideCoderException {
- @SuppressWarnings("unchecked")
- Coder<InputT> inputCoder = ((PCollection<InputT>) input).getCoder();
- return input.getPipeline().getCoderRegistry().getCoder(
- output.getTypeDescriptor(),
- getFn().getInputTypeDescriptor(),
- inputCoder);
- }
-
- @Override
protected String getKindString() {
return String.format("ParMultiDo(%s)", NameUtils.approximateSimpleName(getFn()));
}
@@ -807,11 +829,7 @@
*/
@Override
public Map<TupleTag<?>, PValue> getAdditionalInputs() {
- ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder();
- for (PCollectionView<?> sideInput : sideInputs) {
- additionalInputs.put(sideInput.getTagInternal(), sideInput.getPCollection());
- }
- return additionalInputs.build();
+ return PCollectionViews.toAdditionalInputs(sideInputs);
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java
new file mode 100644
index 0000000..f90e8f3
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/** Describes the run-time requirements of a {@link Contextful}, such as access to side inputs. */
+@Experimental(Kind.CONTEXTFUL)
+public final class Requirements implements Serializable {
+ private final Collection<PCollectionView<?>> sideInputs;
+
+ private Requirements(Collection<PCollectionView<?>> sideInputs) {
+ this.sideInputs = sideInputs;
+ }
+
+ /** The side inputs that this {@link Contextful} needs access to. */
+ public Collection<PCollectionView<?>> getSideInputs() {
+ return sideInputs;
+ }
+
+ /** Describes the need for access to the given side inputs. */
+ public static Requirements requiresSideInputs(Collection<PCollectionView<?>> sideInputs) {
+ return new Requirements(sideInputs);
+ }
+
+ /** Like {@link #requiresSideInputs(Collection)}. */
+ public static Requirements requiresSideInputs(PCollectionView<?>... sideInputs) {
+ return requiresSideInputs(Arrays.asList(sideInputs));
+ }
+
+ /** Describes an empty set of requirements. */
+ public static Requirements empty() {
+ return new Requirements(Collections.<PCollectionView<?>>emptyList());
+ }
+
+ /** Whether this is an empty set of requirements. */
+ public boolean isEmpty() {
+ return sideInputs.isEmpty();
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
index 3b7122c..68e4560 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.sdk.transforms;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
@@ -55,6 +57,15 @@
return new Reshuffle<K, V>();
}
+ /**
+ * Encapsulates the sequence "pair input with unique key, apply {@link
+ * Reshuffle#of}, drop the key" commonly used to break fusion.
+ */
+ @Experimental
+ public static <T> ViaRandomKey<T> viaRandomKey() {
+ return new ViaRandomKey<T>();
+ }
+
@Override
public PCollection<KV<K, V>> expand(PCollection<KV<K, V>> input) {
WindowingStrategy<?, ?> originalStrategy = input.getWindowingStrategy();
@@ -94,4 +105,40 @@
"RestoreOriginalTimestamps",
ReifyTimestamps.<K, V>extractFromValues());
}
+
+ /** Implementation of {@link #viaRandomKey()}. */
+ public static class ViaRandomKey<T> extends PTransform<PCollection<T>, PCollection<T>> {
+ private ViaRandomKey() {}
+
+ @Override
+ public PCollection<T> expand(PCollection<T> input) {
+ return input
+ .apply("Pair with random key", ParDo.of(new AssignShardFn<T>()))
+ .apply(Reshuffle.<Integer, T>of())
+ .apply(Values.<T>create());
+ }
+
+ private static class AssignShardFn<T> extends DoFn<T, KV<Integer, T>> {
+ private int shard;
+
+ @Setup
+ public void setup() {
+ shard = ThreadLocalRandom.current().nextInt();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) {
+ ++shard;
+ // Smear the shard into something more random-looking, to avoid issues
+ // with runners that don't properly hash the key being shuffled, but rely
+ // on it being random-looking. E.g. Spark takes the Java hashCode() of keys,
+ // which for Integer is a no-op and it is an issue:
+ // http://hydronitrogen.com/poor-hash-partitioning-of-timestamps-integers-and-longs-in-
+ // spark.html
+ // This hashing strategy is copied from com.google.common.collect.Hashing.smear().
+ int hashOfShard = 0x1b873593 * Integer.rotateLeft(shard * 0xcc9e2d51, 15);
+ context.output(KV.of(hashOfShard, context.element()));
+ }
+ }
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SerializableFunctions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SerializableFunctions.java
new file mode 100644
index 0000000..d057d81
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SerializableFunctions.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.transforms;
+
+/** Useful {@link SerializableFunction} overrides. */
+public class SerializableFunctions {
+ private static class Identity<T> implements SerializableFunction<T, T> {
+ @Override
+ public T apply(T input) {
+ return input;
+ }
+ }
+
+ private static class Constant<InT, OutT> implements SerializableFunction<InT, OutT> {
+ OutT value;
+
+ Constant(OutT value) {
+ this.value = value;
+ }
+
+ @Override
+ public OutT apply(InT input) {
+ return value;
+ }
+ }
+
+ public static <T> SerializableFunction<T, T> identity() {
+ return new Identity<>();
+ }
+
+ public static <InT, OutT> SerializableFunction<InT, OutT> constant(OutT value) {
+ return new Constant<>(value);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
index ccade4d..6b65416 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
@@ -151,6 +151,16 @@
public int identity() {
return 0;
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(this.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
private static class SumLongFn extends Combine.BinaryCombineLongFn {
@@ -164,6 +174,16 @@
public long identity() {
return 0;
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(this.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
private static class SumDoubleFn extends Combine.BinaryCombineDoubleFn {
@@ -177,5 +197,15 @@
public double identity() {
return 0;
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null && other.getClass().equals(this.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index 99ec49b..35d6703 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -29,6 +29,7 @@
import java.util.List;
import java.util.Objects;
import java.util.PriorityQueue;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.CoderRegistry;
@@ -144,7 +145,7 @@
* {@code KV}s and return the top values associated with each key.
*/
public static <T extends Comparable<T>> Combine.Globally<T, List<T>> smallest(int count) {
- return Combine.globally(new TopCombineFn<>(count, new Smallest<T>()));
+ return Combine.globally(new TopCombineFn<>(count, new Reversed<T>()));
}
/**
@@ -188,7 +189,7 @@
* {@code KV}s and return the top values associated with each key.
*/
public static <T extends Comparable<T>> Combine.Globally<T, List<T>> largest(int count) {
- return Combine.globally(new TopCombineFn<>(count, new Largest<T>()));
+ return Combine.globally(new TopCombineFn<>(count, new Natural<T>()));
}
/**
@@ -281,7 +282,7 @@
public static <K, V extends Comparable<V>>
PTransform<PCollection<KV<K, V>>, PCollection<KV<K, List<V>>>>
smallestPerKey(int count) {
- return Combine.perKey(new TopCombineFn<>(count, new Smallest<V>()));
+ return Combine.perKey(new TopCombineFn<>(count, new Reversed<V>()));
}
/**
@@ -327,13 +328,13 @@
public static <K, V extends Comparable<V>>
PerKey<K, V, List<V>>
largestPerKey(int count) {
- return Combine.perKey(new TopCombineFn<>(count, new Largest<V>()));
+ return Combine.perKey(new TopCombineFn<>(count, new Natural<V>()));
}
/**
- * A {@code Serializable} {@code Comparator} that that uses the compared elements' natural
- * ordering.
+ * @deprecated use {@link Natural} instead
*/
+ @Deprecated
public static class Largest<T extends Comparable<? super T>>
implements Comparator<T>, Serializable {
@Override
@@ -343,10 +344,34 @@
}
/**
+ * A {@code Serializable} {@code Comparator} that that uses the compared elements' natural
+ * ordering.
+ */
+ public static class Natural<T extends Comparable<? super T>>
+ implements Comparator<T>, Serializable {
+ @Override
+ public int compare(T a, T b) {
+ return a.compareTo(b);
+ }
+ }
+
+ /**
+ * @deprecated use {@link Reversed} instead
+ */
+ @Deprecated
+ public static class Smallest<T extends Comparable<? super T>>
+ implements Comparator<T>, Serializable {
+ @Override
+ public int compare(T a, T b) {
+ return b.compareTo(a);
+ }
+ }
+
+ /**
* {@code Serializable} {@code Comparator} that that uses the reverse of the compared elements'
* natural ordering.
*/
- public static class Smallest<T extends Comparable<? super T>>
+ public static class Reversed<T extends Comparable<? super T>>
implements Comparator<T>, Serializable {
@Override
public int compare(T a, T b) {
@@ -429,14 +454,14 @@
*
* <p>Only one of asList and asQueue may be non-null.
*/
- private PriorityQueue<T> asQueue;
+ @Nullable private PriorityQueue<T> asQueue;
/**
* A list in with largest first, the form of extractOutput().
*
* <p>Only one of asList and asQueue may be non-null.
*/
- private List<T> asList;
+ @Nullable private List<T> asList;
/** The user-provided Comparator. */
private final ComparatorT compareFn;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index bcbdb24..eaa7925 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -19,6 +19,7 @@
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.coders.Coder;
@@ -116,16 +117,17 @@
* {@code
* PCollection<Page> pages = ... // pages fit into memory
* PCollection<UrlVisit> urlVisits = ... // very large collection
- * final PCollectionView<Map<URL, Page>> = urlToPage
+ * final PCollectionView<Map<URL, Page>> urlToPageView = pages
* .apply(WithKeys.of( ... )) // extract the URL from the page
* .apply(View.<URL, Page>asMap());
*
* PCollection PageVisits = urlVisits
- * .apply(ParDo.withSideInputs(urlToPage)
+ * .apply(ParDo.withSideInputs(urlToPageView)
* .of(new DoFn<UrlVisit, PageVisit>() {
* {@literal @}Override
* void processElement(ProcessContext context) {
* UrlVisit urlVisit = context.element();
+ * Map<URL, Page> urlToPage = context.sideInput(urlToPageView);
* Page page = urlToPage.get(urlVisit.getUrl());
* c.output(new PageVisit(page, urlVisit.getVisitData()));
* }
@@ -173,7 +175,7 @@
* {@link PCollectionView} mapping each window to a {@link List} containing
* all of the elements in the window.
*
- * <p>The resulting list is required to fit in memory.
+ * <p>Unlike with {@link #asIterable}, the resulting list is required to fit in memory.
*/
public static <T> AsList<T> asList() {
return new AsList<>();
@@ -228,7 +230,7 @@
* <pre>
* {@code
* PCollection<KV<K, V>> input = ... // maybe more than one occurrence of a some keys
- * PCollectionView<Map<K, V>> output = input.apply(View.<K, V>asMultimap());
+ * PCollectionView<Map<K, Iterable<V>>> output = input.apply(View.<K, V>asMultimap());
* }</pre>
*
* <p>Currently, the resulting map is required to fit into memory.
@@ -256,8 +258,10 @@
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<T, List<T>>of(PCollectionViews.listView(
- input, input.getWindowingStrategy(), input.getCoder())));
+ PCollectionView<List<T>> view =
+ PCollectionViews.listView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<T, List<T>>of(view));
+ return view;
}
}
@@ -281,8 +285,10 @@
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<T, Iterable<T>>of(PCollectionViews.iterableView(
- input, input.getWindowingStrategy(), input.getCoder())));
+ PCollectionView<Iterable<T>> view =
+ PCollectionViews.iterableView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<T, Iterable<T>>of(view));
+ return view;
}
}
@@ -295,7 +301,7 @@
*/
@Internal
public static class AsSingleton<T> extends PTransform<PCollection<T>, PCollectionView<T>> {
- private final T defaultValue;
+ @Nullable private final T defaultValue;
private final boolean hasDefault;
private AsSingleton() {
@@ -348,8 +354,8 @@
private static class SingletonCombineFn<T> extends Combine.BinaryCombineFn<T> {
private final boolean hasDefault;
- private final Coder<T> valueCoder;
- private final byte[] defaultValue;
+ @Nullable private final Coder<T> valueCoder;
+ @Nullable private final byte[] defaultValue;
private SingletonCombineFn(boolean hasDefault, Coder<T> coder, T defaultValue) {
this.hasDefault = hasDefault;
@@ -422,11 +428,10 @@
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(
- PCollectionViews.multimapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder())));
+ PCollectionView<Map<K, Iterable<V>>> view =
+ PCollectionViews.multimapView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
+ return view;
}
}
@@ -458,11 +463,10 @@
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<KV<K, V>, Map<K, V>>of(
- PCollectionViews.mapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder())));
+ PCollectionView<Map<K, V>> view =
+ PCollectionViews.mapView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<KV<K, V>, Map<K, V>>of(view));
+ return view;
}
}
@@ -479,7 +483,7 @@
*/
@Internal
public static class CreatePCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private PCollectionView<ViewT> view;
private CreatePCollectionView(PCollectionView<ViewT> view) {
@@ -494,7 +498,7 @@
/**
* Return the {@link PCollectionView} that is returned by applying this {@link PTransform}.
*
- * <p>This should not be used to obtain the output of any given application of this
+ * @deprecated This should not be used to obtain the output of any given application of this
* {@link PTransform}. That should be obtained by inspecting the {@link Node}
* that contains this {@link CreatePCollectionView}, as this view may have been replaced within
* pipeline surgery.
@@ -505,8 +509,9 @@
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return view;
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder());
}
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java
new file mode 100644
index 0000000..75c2fe4
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java
@@ -0,0 +1,1066 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.sdk.transforms.Contextful.Fn.Context.wrapProcessContext;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.hash.Funnel;
+import com.google.common.hash.Funnels;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.hash.PrimitiveSink;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.BooleanCoder;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.DurationCoder;
+import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.MapCoder;
+import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.sdk.values.TypeDescriptors.TypeVariableExtractor;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.joda.time.ReadableDuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Given a "poll function" that produces a potentially growing set of outputs for an input, this
+ * transform simultaneously continuously watches the growth of output sets of all inputs, until a
+ * per-input termination condition is reached.
+ *
+ * <p>The output is returned as an unbounded {@link PCollection} of {@code KV<InputT, OutputT>},
+ * where each {@code OutputT} is associated with the {@code InputT} that produced it, and is
+ * assigned with the timestamp that the poll function returned when this output was detected for the
+ * first time.
+ *
+ * <p>Hypothetical usage example for watching new files in a collection of directories, where for
+ * each directory we assume that new files will not appear if the directory contains a file named
+ * ".complete":
+ *
+ * <pre>{@code
+ * PCollection<String> directories = ...; // E.g. Create.of(single directory)
+ * PCollection<KV<String, String>> matches = filepatterns.apply(Watch.<String, String>growthOf(
+ * new PollFn<String, String>() {
+ * public PollResult<String> apply(TimestampedValue<String> input) {
+ * String directory = input.getValue();
+ * List<TimestampedValue<String>> outputs = new ArrayList<>();
+ * ... List the directory and get creation times of all files ...
+ * boolean isComplete = ... does a file ".complete" exist in the directory ...
+ * return isComplete ? PollResult.complete(outputs) : PollResult.incomplete(outputs);
+ * }
+ * })
+ * // Poll each directory every 5 seconds
+ * .withPollInterval(Duration.standardSeconds(5))
+ * // Stop watching each directory 12 hours after it's seen even if it's incomplete
+ * .withTerminationPerInput(afterTotalOf(Duration.standardHours(12)));
+ * }</pre>
+ *
+ * <p>By default, the watermark for a particular input is computed from a poll result as "earliest
+ * timestamp of new elements in this poll result". It can also be set explicitly via {@link
+ * Growth.PollResult#withWatermark} if the {@link Growth.PollFn} can provide a more optimistic
+ * estimate.
+ *
+ * <p>Note: This transform works only in runners supporting Splittable DoFn: see <a
+ * href="https://beam.apache.org/documentation/runners/capability-matrix/">capability matrix</a>.
+ */
+@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+public class Watch {
+ private static final Logger LOG = LoggerFactory.getLogger(Watch.class);
+
+ /** Watches the growth of the given poll function. See class documentation for more details. */
+ public static <InputT, OutputT> Growth<InputT, OutputT> growthOf(
+ Contextful<Growth.PollFn<InputT, OutputT>> pollFn) {
+ return new AutoValue_Watch_Growth.Builder<InputT, OutputT>()
+ .setTerminationPerInput(Watch.Growth.<InputT>never())
+ .setPollFn(pollFn)
+ .build();
+ }
+
+ /** Watches the growth of the given poll function. See class documentation for more details. */
+ public static <InputT, OutputT> Growth<InputT, OutputT> growthOf(
+ Growth.PollFn<InputT, OutputT> pollFn, Requirements requirements) {
+ return growthOf(Contextful.of(pollFn, requirements));
+ }
+
+ /** Watches the growth of the given poll function. See class documentation for more details. */
+ public static <InputT, OutputT> Growth<InputT, OutputT> growthOf(
+ Growth.PollFn<InputT, OutputT> pollFn) {
+ return growthOf(pollFn, Requirements.empty());
+ }
+
+ /** Implementation of {@link #growthOf}. */
+ @AutoValue
+ public abstract static class Growth<InputT, OutputT>
+ extends PTransform<PCollection<InputT>, PCollection<KV<InputT, OutputT>>> {
+ /** The result of a single invocation of a {@link PollFn}. */
+ public static final class PollResult<OutputT> {
+ private final List<TimestampedValue<OutputT>> outputs;
+ // null means unspecified (infer automatically).
+ @Nullable private final Instant watermark;
+
+ private PollResult(List<TimestampedValue<OutputT>> outputs, @Nullable Instant watermark) {
+ this.outputs = outputs;
+ this.watermark = watermark;
+ }
+
+ List<TimestampedValue<OutputT>> getOutputs() {
+ return outputs;
+ }
+
+ @Nullable
+ Instant getWatermark() {
+ return watermark;
+ }
+
+ /**
+ * Sets the watermark - an approximate lower bound on timestamps of future new outputs from
+ * this {@link PollFn}.
+ */
+ public PollResult<OutputT> withWatermark(Instant watermark) {
+ checkNotNull(watermark, "watermark");
+ return new PollResult<>(outputs, watermark);
+ }
+
+ /**
+ * Constructs a {@link PollResult} with the given outputs and declares that there will be no
+ * new outputs for the current input. The {@link PollFn} will not be called again for this
+ * input.
+ */
+ public static <OutputT> PollResult<OutputT> complete(
+ List<TimestampedValue<OutputT>> outputs) {
+ return new PollResult<>(outputs, BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+
+ /** Like {@link #complete(List)}, but assigns the same timestamp to all new outputs. */
+ public static <OutputT> PollResult<OutputT> complete(
+ Instant timestamp, List<OutputT> outputs) {
+ return new PollResult<>(
+ addTimestamp(timestamp, outputs), BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+
+ /**
+ * Constructs a {@link PollResult} with the given outputs and declares that new outputs might
+ * appear for the current input. By default, {@link Watch} will estimate the watermark for
+ * future new outputs as equal to the earliest of the new outputs from this {@link
+ * PollResult}. To specify a more exact watermark, use {@link #withWatermark(Instant)}.
+ */
+ public static <OutputT> PollResult<OutputT> incomplete(
+ List<TimestampedValue<OutputT>> outputs) {
+ return new PollResult<>(outputs, null);
+ }
+
+ /** Like {@link #incomplete(List)}, but assigns the same timestamp to all new outputs. */
+ public static <OutputT> PollResult<OutputT> incomplete(
+ Instant timestamp, List<OutputT> outputs) {
+ return new PollResult<>(addTimestamp(timestamp, outputs), null);
+ }
+
+ private static <OutputT> List<TimestampedValue<OutputT>> addTimestamp(
+ Instant timestamp, List<OutputT> outputs) {
+ List<TimestampedValue<OutputT>> res = Lists.newArrayListWithExpectedSize(outputs.size());
+ for (OutputT output : outputs) {
+ res.add(TimestampedValue.of(output, timestamp));
+ }
+ return res;
+ }
+ }
+
+ /**
+ * A function that computes the current set of outputs for the given input, in the form of a
+ * {@link PollResult}.
+ */
+ public abstract static class PollFn<InputT, OutputT>
+ implements Contextful.Fn<InputT, PollResult<OutputT>> {}
+
+ /**
+ * A strategy for determining whether it is time to stop polling the current input regardless of
+ * whether its output is complete or not.
+ *
+ * <p>Some built-in termination conditions are {@link #never}, {@link #afterTotalOf} and {@link
+ * #afterTimeSinceNewOutput}. Conditions can be combined using {@link #eitherOf} and {@link
+ * #allOf}. Users can also develop custom termination conditions, for example, one might imagine
+ * a condition that terminates after a given time after the first output appears for the input
+ * (unlike {@link #afterTotalOf} which operates relative to when the input itself arrives).
+ *
+ * <p>A {@link TerminationCondition} is provided to {@link
+ * Growth#withTerminationPerInput(TerminationCondition)} and is used to maintain an independent
+ * state of the termination condition for every input, represented as {@code StateT} which must
+ * be immutable, non-null, and encodable via {@link #getStateCoder()}.
+ *
+ * <p>All functions take the wall-clock timestamp as {@link Instant} for convenience of
+ * unit-testing custom termination conditions.
+ */
+ public interface TerminationCondition<InputT, StateT> extends Serializable {
+ /** Used to encode the state of this {@link TerminationCondition}. */
+ Coder<StateT> getStateCoder();
+
+ /**
+ * Called by the {@link Watch} transform to create a new independent termination state for a
+ * newly arrived {@code InputT}.
+ */
+ StateT forNewInput(Instant now, @Nullable InputT input);
+
+ /**
+ * Called by the {@link Watch} transform to compute a new termination state, in case after
+ * calling the {@link PollFn} for the current input, the {@link PollResult} included a
+ * previously unseen {@code OutputT}.
+ */
+ StateT onSeenNewOutput(Instant now, StateT state);
+
+ /**
+ * Called by the {@link Watch} transform to determine whether the given termination state
+ * signals that {@link Watch} should stop calling {@link PollFn} for the current input,
+ * regardless of whether the last {@link PollResult} was complete or incomplete.
+ */
+ boolean canStopPolling(Instant now, StateT state);
+
+ /** Creates a human-readable representation of the given state of this condition. */
+ String toString(StateT state);
+ }
+
+ /**
+ * Returns a {@link TerminationCondition} that never holds (i.e., poll each input until its
+ * output is complete).
+ */
+ public static <InputT> Never<InputT> never() {
+ return new Never<>();
+ }
+
+ /**
+ * Wraps a given input-independent {@link TerminationCondition} as an equivalent condition
+ * with a given input type, passing {@code null} to the original condition as input.
+ */
+ public static <InputT, StateT> TerminationCondition<InputT, StateT> ignoreInput(
+ TerminationCondition<?, StateT> condition) {
+ return new IgnoreInput<>(condition);
+ }
+
+ /**
+ * Returns a {@link TerminationCondition} that holds after the given time has elapsed after the
+ * current input was seen.
+ */
+ public static <InputT> AfterTotalOf<InputT> afterTotalOf(ReadableDuration timeSinceInput) {
+ return afterTotalOf(SerializableFunctions.<InputT, ReadableDuration>constant(timeSinceInput));
+ }
+
+ /** Like {@link #afterTotalOf(ReadableDuration)}, but the duration is input-dependent. */
+ public static <InputT> AfterTotalOf<InputT> afterTotalOf(
+ SerializableFunction<InputT, ReadableDuration> timeSinceInput) {
+ return new AfterTotalOf<>(timeSinceInput);
+ }
+
+ /**
+ * Returns a {@link TerminationCondition} that holds after the given time has elapsed after the
+ * last time the {@link PollResult} for the current input contained a previously unseen output.
+ */
+ public static <InputT> AfterTimeSinceNewOutput<InputT> afterTimeSinceNewOutput(
+ ReadableDuration timeSinceNewOutput) {
+ return afterTimeSinceNewOutput(
+ SerializableFunctions.<InputT, ReadableDuration>constant(timeSinceNewOutput));
+ }
+
+ /**
+ * Like {@link #afterTimeSinceNewOutput(ReadableDuration)}, but the duration is input-dependent.
+ */
+ public static <InputT> AfterTimeSinceNewOutput<InputT> afterTimeSinceNewOutput(
+ SerializableFunction<InputT, ReadableDuration> timeSinceNewOutput) {
+ return new AfterTimeSinceNewOutput<>(timeSinceNewOutput);
+ }
+
+ /**
+ * Returns a {@link TerminationCondition} that holds when at least one of the given two
+ * conditions holds.
+ */
+ public static <InputT, FirstStateT, SecondStateT>
+ BinaryCombined<InputT, FirstStateT, SecondStateT> eitherOf(
+ TerminationCondition<InputT, FirstStateT> first,
+ TerminationCondition<InputT, SecondStateT> second) {
+ return new BinaryCombined<>(BinaryCombined.Operation.OR, first, second);
+ }
+
+ /**
+ * Returns a {@link TerminationCondition} that holds when both of the given two conditions hold.
+ */
+ public static <InputT, FirstStateT, SecondStateT>
+ BinaryCombined<InputT, FirstStateT, SecondStateT> allOf(
+ TerminationCondition<InputT, FirstStateT> first,
+ TerminationCondition<InputT, SecondStateT> second) {
+ return new BinaryCombined<>(BinaryCombined.Operation.AND, first, second);
+ }
+
+ // Uses Integer rather than Void for state, because termination state must be non-null.
+ static class Never<InputT> implements TerminationCondition<InputT, Integer> {
+ @Override
+ public Coder<Integer> getStateCoder() {
+ return VarIntCoder.of();
+ }
+
+ @Override
+ public Integer forNewInput(Instant now, InputT input) {
+ return 0;
+ }
+
+ @Override
+ public Integer onSeenNewOutput(Instant now, Integer state) {
+ return state;
+ }
+
+ @Override
+ public boolean canStopPolling(Instant now, Integer state) {
+ return false;
+ }
+
+ @Override
+ public String toString(Integer state) {
+ return "Never";
+ }
+ }
+
+ static class IgnoreInput<InputT, StateT> implements TerminationCondition<InputT, StateT> {
+ private final TerminationCondition<?, StateT> wrapped;
+
+ IgnoreInput(TerminationCondition<?, StateT> wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public Coder<StateT> getStateCoder() {
+ return wrapped.getStateCoder();
+ }
+
+ @Override
+ public StateT forNewInput(Instant now, InputT input) {
+ return wrapped.forNewInput(now, null);
+ }
+
+ @Override
+ public StateT onSeenNewOutput(Instant now, StateT state) {
+ return wrapped.onSeenNewOutput(now, state);
+ }
+
+ @Override
+ public boolean canStopPolling(Instant now, StateT state) {
+ return wrapped.canStopPolling(now, state);
+ }
+
+ @Override
+ public String toString(StateT state) {
+ return wrapped.toString(state);
+ }
+ }
+
+ static class AfterTotalOf<InputT>
+ implements TerminationCondition<
+ InputT, KV<Instant /* timeStarted */, ReadableDuration /* maxTimeSinceInput */>> {
+ private final SerializableFunction<InputT, ReadableDuration> maxTimeSinceInput;
+
+ private AfterTotalOf(SerializableFunction<InputT, ReadableDuration> maxTimeSinceInput) {
+ this.maxTimeSinceInput = maxTimeSinceInput;
+ }
+
+ @Override
+ public Coder<KV<Instant, ReadableDuration>> getStateCoder() {
+ return KvCoder.of(InstantCoder.of(), DurationCoder.of());
+ }
+
+ @Override
+ public KV<Instant, ReadableDuration> forNewInput(Instant now, InputT input) {
+ return KV.of(now, maxTimeSinceInput.apply(input));
+ }
+
+ @Override
+ public KV<Instant, ReadableDuration> onSeenNewOutput(
+ Instant now, KV<Instant, ReadableDuration> state) {
+ return state;
+ }
+
+ @Override
+ public boolean canStopPolling(Instant now, KV<Instant, ReadableDuration> state) {
+ return new Duration(state.getKey(), now).isLongerThan(state.getValue());
+ }
+
+ @Override
+ public String toString(KV<Instant, ReadableDuration> state) {
+ return "AfterTotalOf{"
+ + "timeStarted="
+ + state.getKey()
+ + ", maxTimeSinceInput="
+ + state.getValue()
+ + '}';
+ }
+ }
+
+ static class AfterTimeSinceNewOutput<InputT>
+ implements TerminationCondition<
+ InputT,
+ KV<Instant /* timeOfLastNewOutput */, ReadableDuration /* maxTimeSinceNewOutput */>> {
+ private final SerializableFunction<InputT, ReadableDuration> maxTimeSinceNewOutput;
+
+ private AfterTimeSinceNewOutput(
+ SerializableFunction<InputT, ReadableDuration> maxTimeSinceNewOutput) {
+ this.maxTimeSinceNewOutput = maxTimeSinceNewOutput;
+ }
+
+ @Override
+ public Coder<KV<Instant, ReadableDuration>> getStateCoder() {
+ return KvCoder.of(NullableCoder.of(InstantCoder.of()), DurationCoder.of());
+ }
+
+ @Override
+ public KV<Instant, ReadableDuration> forNewInput(Instant now, InputT input) {
+ return KV.of(null, maxTimeSinceNewOutput.apply(input));
+ }
+
+ @Override
+ public KV<Instant, ReadableDuration> onSeenNewOutput(
+ Instant now, KV<Instant, ReadableDuration> state) {
+ return KV.of(now, state.getValue());
+ }
+
+ @Override
+ public boolean canStopPolling(Instant now, KV<Instant, ReadableDuration> state) {
+ Instant timeOfLastNewOutput = state.getKey();
+ ReadableDuration maxTimeSinceNewOutput = state.getValue();
+ return timeOfLastNewOutput != null
+ && new Duration(timeOfLastNewOutput, now).isLongerThan(maxTimeSinceNewOutput);
+ }
+
+ @Override
+ public String toString(KV<Instant, ReadableDuration> state) {
+ return "AfterTimeSinceNewOutput{"
+ + "timeOfLastNewOutput="
+ + state.getKey()
+ + ", maxTimeSinceNewOutput="
+ + state.getValue()
+ + '}';
+ }
+ }
+
+ static class BinaryCombined<InputT, FirstStateT, SecondStateT>
+ implements TerminationCondition<InputT, KV<FirstStateT, SecondStateT>> {
+ private enum Operation {
+ OR,
+ AND
+ }
+
+ private final Operation operation;
+ private final TerminationCondition<InputT, FirstStateT> first;
+ private final TerminationCondition<InputT, SecondStateT> second;
+
+ public BinaryCombined(
+ Operation operation,
+ TerminationCondition<InputT, FirstStateT> first,
+ TerminationCondition<InputT, SecondStateT> second) {
+ this.operation = operation;
+ this.first = first;
+ this.second = second;
+ }
+
+ @Override
+ public Coder<KV<FirstStateT, SecondStateT>> getStateCoder() {
+ return KvCoder.of(first.getStateCoder(), second.getStateCoder());
+ }
+
+ @Override
+ public KV<FirstStateT, SecondStateT> forNewInput(Instant now, InputT input) {
+ return KV.of(first.forNewInput(now, input), second.forNewInput(now, input));
+ }
+
+ @Override
+ public KV<FirstStateT, SecondStateT> onSeenNewOutput(
+ Instant now, KV<FirstStateT, SecondStateT> state) {
+ return KV.of(
+ first.onSeenNewOutput(now, state.getKey()),
+ second.onSeenNewOutput(now, state.getValue()));
+ }
+
+ @Override
+ public boolean canStopPolling(Instant now, KV<FirstStateT, SecondStateT> state) {
+ switch (operation) {
+ case OR:
+ return first.canStopPolling(now, state.getKey())
+ || second.canStopPolling(now, state.getValue());
+ case AND:
+ return first.canStopPolling(now, state.getKey())
+ && second.canStopPolling(now, state.getValue());
+ default:
+ throw new UnsupportedOperationException("Unexpected operation " + operation);
+ }
+ }
+
+ @Override
+ public String toString(KV<FirstStateT, SecondStateT> state) {
+ return operation
+ + "{first="
+ + first.toString(state.getKey())
+ + ", second="
+ + second.toString(state.getValue())
+ + '}';
+ }
+ }
+
+ abstract Contextful<PollFn<InputT, OutputT>> getPollFn();
+
+ @Nullable
+ abstract Duration getPollInterval();
+
+ @Nullable
+ abstract TerminationCondition<InputT, ?> getTerminationPerInput();
+
+ @Nullable
+ abstract Coder<OutputT> getOutputCoder();
+
+ abstract Builder<InputT, OutputT> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<InputT, OutputT> {
+ abstract Builder<InputT, OutputT> setPollFn(Contextful<PollFn<InputT, OutputT>> pollFn);
+
+ abstract Builder<InputT, OutputT> setTerminationPerInput(
+ TerminationCondition<InputT, ?> terminationPerInput);
+
+ abstract Builder<InputT, OutputT> setPollInterval(Duration pollInterval);
+
+ abstract Builder<InputT, OutputT> setOutputCoder(Coder<OutputT> outputCoder);
+
+ abstract Growth<InputT, OutputT> build();
+ }
+
+ /** Specifies a {@link TerminationCondition} that will be independently used for every input. */
+ public Growth<InputT, OutputT> withTerminationPerInput(
+ TerminationCondition<InputT, ?> terminationPerInput) {
+ return toBuilder().setTerminationPerInput(terminationPerInput).build();
+ }
+
+ /**
+ * Specifies how long to wait after a call to {@link PollFn} before calling it again (if at all
+ * - according to {@link PollResult} and the {@link TerminationCondition}).
+ */
+ public Growth<InputT, OutputT> withPollInterval(Duration pollInterval) {
+ return toBuilder().setPollInterval(pollInterval).build();
+ }
+
+ /**
+ * Specifies a {@link Coder} to use for the outputs. If unspecified, it will be inferred from
+ * the output type of {@link PollFn} whenever possible.
+ *
+ * <p>The coder must be deterministic, because the transform will compare encoded outputs for
+ * deduplication between polling rounds.
+ */
+ public Growth<InputT, OutputT> withOutputCoder(Coder<OutputT> outputCoder) {
+ return toBuilder().setOutputCoder(outputCoder).build();
+ }
+
+ @Override
+ public PCollection<KV<InputT, OutputT>> expand(PCollection<InputT> input) {
+ checkNotNull(getPollInterval(), "pollInterval");
+ checkNotNull(getTerminationPerInput(), "terminationPerInput");
+
+ Coder<OutputT> outputCoder = getOutputCoder();
+ if (outputCoder == null) {
+ // If a coder was not specified explicitly, infer it from the OutputT type parameter
+ // of the PollFn.
+ TypeDescriptor<OutputT> outputT =
+ TypeDescriptors.extractFromTypeParameters(
+ getPollFn().getClosure(),
+ PollFn.class,
+ new TypeVariableExtractor<PollFn<InputT, OutputT>, OutputT>() {});
+ try {
+ outputCoder = input.getPipeline().getCoderRegistry().getCoder(outputT);
+ } catch (CannotProvideCoderException e) {
+ throw new RuntimeException(
+ "Unable to infer coder for OutputT. Specify it explicitly using withOutputCoder().");
+ }
+ }
+ try {
+ outputCoder.verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ throw new IllegalArgumentException(
+ "Output coder " + outputCoder + " must be deterministic");
+ }
+
+ return input
+ .apply(ParDo.of(new WatchGrowthFn<>(this, outputCoder))
+ .withSideInputs(getPollFn().getRequirements().getSideInputs()))
+ .setCoder(KvCoder.of(input.getCoder(), outputCoder));
+ }
+ }
+
+ private static class WatchGrowthFn<InputT, OutputT, TerminationStateT>
+ extends DoFn<InputT, KV<InputT, OutputT>> {
+ private final Watch.Growth<InputT, OutputT> spec;
+ private final Coder<OutputT> outputCoder;
+
+ private WatchGrowthFn(Growth<InputT, OutputT> spec, Coder<OutputT> outputCoder) {
+ this.spec = spec;
+ this.outputCoder = outputCoder;
+ }
+
+ @ProcessElement
+ public ProcessContinuation process(
+ ProcessContext c, final GrowthTracker<OutputT, TerminationStateT> tracker)
+ throws Exception {
+ if (!tracker.hasPending() && !tracker.currentRestriction().isOutputComplete) {
+ LOG.debug("{} - polling input", c.element());
+ Growth.PollResult<OutputT> res =
+ spec.getPollFn().getClosure().apply(c.element(), wrapProcessContext(c));
+ // TODO (https://issues.apache.org/jira/browse/BEAM-2680):
+ // Consider truncating the pending outputs if there are too many, to avoid blowing
+ // up the state. In that case, we'd rely on the next poll cycle to provide more outputs.
+ // All outputs would still have to be stored in state.completed, but it is more compact
+ // because it stores hashes and because it could potentially be garbage-collected.
+ int numPending = tracker.addNewAsPending(res);
+ if (numPending > 0) {
+ LOG.info(
+ "{} - polling returned {} results, of which {} were new. The output is {}.",
+ c.element(),
+ res.getOutputs().size(),
+ numPending,
+ BoundedWindow.TIMESTAMP_MAX_VALUE.equals(res.getWatermark())
+ ? "complete"
+ : "incomplete");
+ }
+ }
+ while (tracker.hasPending()) {
+ c.updateWatermark(tracker.getWatermark());
+
+ TimestampedValue<OutputT> nextPending = tracker.tryClaimNextPending();
+ if (nextPending == null) {
+ return stop();
+ }
+ c.outputWithTimestamp(
+ KV.of(c.element(), nextPending.getValue()), nextPending.getTimestamp());
+ }
+ Instant watermark = tracker.getWatermark();
+ if (watermark != null) {
+ // Null means the poll result did not provide a watermark and there were no new elements,
+ // so we have no information to update the watermark and should keep it as-is.
+ c.updateWatermark(watermark);
+ }
+ // No more pending outputs - future output will come from more polling,
+ // unless output is complete or termination condition is reached.
+ if (tracker.shouldPollMore()) {
+ return resume().withResumeDelay(spec.getPollInterval());
+ }
+ return stop();
+ }
+
+ private Growth.TerminationCondition<InputT, TerminationStateT> getTerminationCondition() {
+ return ((Growth.TerminationCondition<InputT, TerminationStateT>)
+ spec.getTerminationPerInput());
+ }
+
+ @GetInitialRestriction
+ public GrowthState<OutputT, TerminationStateT> getInitialRestriction(InputT element) {
+ return new GrowthState<>(getTerminationCondition().forNewInput(Instant.now(), element));
+ }
+
+ @NewTracker
+ public GrowthTracker<OutputT, TerminationStateT> newTracker(
+ GrowthState<OutputT, TerminationStateT> restriction) {
+ return new GrowthTracker<>(outputCoder, restriction, getTerminationCondition());
+ }
+
+ @GetRestrictionCoder
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Coder<GrowthState<OutputT, TerminationStateT>> getRestrictionCoder() {
+ return GrowthStateCoder.of(
+ outputCoder, (Coder) spec.getTerminationPerInput().getStateCoder());
+ }
+ }
+
+ @VisibleForTesting
+ static class GrowthState<OutputT, TerminationStateT> {
+ // Hashes and timestamps of outputs that have already been output and should be omitted
+ // from future polls. Timestamps are preserved to allow garbage-collecting this state
+ // in the future, e.g. dropping elements from "completed" and from addNewAsPending() if their
+ // timestamp is more than X behind the watermark.
+ // As of writing, we don't do this, but preserve the information for forward compatibility
+ // in case of pipeline update. TODO: do this.
+ private final Map<HashCode, Instant> completed;
+ // Outputs that are known to be present in a poll result, but have not yet been returned
+ // from a ProcessElement call, sorted by timestamp to help smooth watermark progress.
+ private final List<TimestampedValue<OutputT>> pending;
+ // If true, processing of this restriction should only output "pending". Otherwise, it should
+ // also continue polling.
+ private final boolean isOutputComplete;
+ // Can be null only if isOutputComplete is true.
+ @Nullable private final TerminationStateT terminationState;
+ // A lower bound on timestamps of future outputs from PollFn, excluding completed and pending.
+ @Nullable private final Instant pollWatermark;
+
+ GrowthState(TerminationStateT terminationState) {
+ this.completed = Collections.emptyMap();
+ this.pending = Collections.emptyList();
+ this.isOutputComplete = false;
+ this.terminationState = checkNotNull(terminationState);
+ this.pollWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ }
+
+ GrowthState(
+ Map<HashCode, Instant> completed,
+ List<TimestampedValue<OutputT>> pending,
+ boolean isOutputComplete,
+ @Nullable TerminationStateT terminationState,
+ @Nullable Instant pollWatermark) {
+ if (!isOutputComplete) {
+ checkNotNull(terminationState);
+ }
+ this.completed = Collections.unmodifiableMap(completed);
+ this.pending = Collections.unmodifiableList(pending);
+ this.isOutputComplete = isOutputComplete;
+ this.terminationState = terminationState;
+ this.pollWatermark = pollWatermark;
+ }
+
+ public String toString(Growth.TerminationCondition<?, TerminationStateT> terminationCondition) {
+ return "GrowthState{"
+ + "completed=<"
+ + completed.size()
+ + " elements>, pending=<"
+ + pending.size()
+ + " elements"
+ + (pending.isEmpty() ? "" : (", earliest " + pending.get(0)))
+ + ">, isOutputComplete="
+ + isOutputComplete
+ + ", terminationState="
+ + terminationCondition.toString(terminationState)
+ + ", pollWatermark="
+ + pollWatermark
+ + '}';
+ }
+ }
+
+ @VisibleForTesting
+ static class GrowthTracker<OutputT, TerminationStateT>
+ implements RestrictionTracker<GrowthState<OutputT, TerminationStateT>> {
+ private final Funnel<OutputT> coderFunnel;
+ private final Growth.TerminationCondition<?, TerminationStateT> terminationCondition;
+
+ // The restriction describing the entire work to be done by the current ProcessElement call.
+ // Changes only in checkpoint().
+ private GrowthState<OutputT, TerminationStateT> state;
+
+ // Mutable state changed by the ProcessElement call itself, and used to compute the primary
+ // and residual restrictions in checkpoint().
+
+ // Remaining pending outputs; initialized from state.pending (if non-empty) or in
+ // addNewAsPending(); drained via tryClaimNextPending().
+ private LinkedList<TimestampedValue<OutputT>> pending;
+ // Outputs that have been claimed in the current ProcessElement call. A prefix of "pending".
+ private List<TimestampedValue<OutputT>> claimed = Lists.newArrayList();
+ private boolean isOutputComplete;
+ @Nullable private TerminationStateT terminationState;
+ @Nullable private Instant pollWatermark;
+ private boolean shouldStop = false;
+
+ GrowthTracker(final Coder<OutputT> outputCoder, GrowthState<OutputT, TerminationStateT> state,
+ Growth.TerminationCondition<?, TerminationStateT> terminationCondition) {
+ this.coderFunnel =
+ new Funnel<OutputT>() {
+ @Override
+ public void funnel(OutputT from, PrimitiveSink into) {
+ try {
+ outputCoder.encode(from, Funnels.asOutputStream(into));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ this.terminationCondition = terminationCondition;
+ this.state = state;
+ this.isOutputComplete = state.isOutputComplete;
+ this.pollWatermark = state.pollWatermark;
+ this.terminationState = state.terminationState;
+ this.pending = Lists.newLinkedList(state.pending);
+ }
+
+ @Override
+ public synchronized GrowthState<OutputT, TerminationStateT> currentRestriction() {
+ return state;
+ }
+
+ @Override
+ public synchronized GrowthState<OutputT, TerminationStateT> checkpoint() {
+ // primary should contain exactly the work claimed in the current ProcessElement call - i.e.
+ // claimed outputs become pending, and it shouldn't poll again.
+ GrowthState<OutputT, TerminationStateT> primary =
+ new GrowthState<>(
+ state.completed /* completed */,
+ claimed /* pending */,
+ true /* isOutputComplete */,
+ null /* terminationState */,
+ BoundedWindow.TIMESTAMP_MAX_VALUE /* pollWatermark */);
+
+ // residual should contain exactly the work *not* claimed in the current ProcessElement call -
+ // unclaimed pending outputs plus future polling outputs.
+ Map<HashCode, Instant> newCompleted = Maps.newHashMap(state.completed);
+ for (TimestampedValue<OutputT> claimedOutput : claimed) {
+ newCompleted.put(hash128(claimedOutput.getValue()), claimedOutput.getTimestamp());
+ }
+ GrowthState<OutputT, TerminationStateT> residual =
+ new GrowthState<>(
+ newCompleted /* completed */,
+ pending /* pending */,
+ isOutputComplete /* isOutputComplete */,
+ terminationState,
+ pollWatermark);
+
+ // Morph ourselves into primary, except for "pending" - the current call has already claimed
+ // everything from it.
+ this.state = primary;
+ this.isOutputComplete = primary.isOutputComplete;
+ this.pollWatermark = primary.pollWatermark;
+ this.terminationState = null;
+ this.pending = Lists.newLinkedList();
+
+ this.shouldStop = true;
+ return residual;
+ }
+
+ private HashCode hash128(OutputT value) {
+ return Hashing.murmur3_128().hashObject(value, coderFunnel);
+ }
+
+ @Override
+ public synchronized void checkDone() throws IllegalStateException {
+ if (shouldStop) {
+ return;
+ }
+ checkState(!shouldPollMore(), "Polling is still allowed to continue");
+ checkState(pending.isEmpty(), "There are %s unclaimed pending outputs", pending.size());
+ }
+
+ @VisibleForTesting
+ synchronized boolean hasPending() {
+ return !pending.isEmpty();
+ }
+
+ @VisibleForTesting
+ @Nullable
+ synchronized TimestampedValue<OutputT> tryClaimNextPending() {
+ if (shouldStop) {
+ return null;
+ }
+ checkState(!pending.isEmpty(), "No more unclaimed pending outputs");
+ TimestampedValue<OutputT> value = pending.removeFirst();
+ claimed.add(value);
+ return value;
+ }
+
+ @VisibleForTesting
+ synchronized boolean shouldPollMore() {
+ return !isOutputComplete
+ && !terminationCondition.canStopPolling(Instant.now(), terminationState);
+ }
+
+ @VisibleForTesting
+ synchronized int addNewAsPending(Growth.PollResult<OutputT> pollResult) {
+ checkState(
+ state.pending.isEmpty(),
+ "Should have drained all old pending outputs before adding new, "
+ + "but there are %s old pending outputs",
+ state.pending.size());
+ List<TimestampedValue<OutputT>> newPending = Lists.newArrayList();
+ for (TimestampedValue<OutputT> output : pollResult.getOutputs()) {
+ OutputT value = output.getValue();
+ if (state.completed.containsKey(hash128(value))) {
+ continue;
+ }
+ // TODO (https://issues.apache.org/jira/browse/BEAM-2680):
+ // Consider adding only at most N pending elements and ignoring others,
+ // instead relying on future poll rounds to provide them, in order to avoid
+ // blowing up the state. Combined with garbage collection of GrowthState.completed,
+ // this would make the transform scalable to very large poll results.
+ newPending.add(TimestampedValue.of(value, output.getTimestamp()));
+ }
+ if (!newPending.isEmpty()) {
+ terminationState = terminationCondition.onSeenNewOutput(Instant.now(), terminationState);
+ }
+ this.pending =
+ Lists.newLinkedList(
+ Ordering.natural()
+ .onResultOf(
+ new Function<TimestampedValue<OutputT>, Instant>() {
+ @Override
+ public Instant apply(TimestampedValue<OutputT> output) {
+ return output.getTimestamp();
+ }
+ })
+ .sortedCopy(newPending));
+ // If poll result doesn't provide a watermark, assume that future new outputs may
+ // arrive with about the same timestamps as the current new outputs.
+ if (pollResult.getWatermark() != null) {
+ this.pollWatermark = pollResult.getWatermark();
+ } else if (!pending.isEmpty()) {
+ this.pollWatermark = pending.getFirst().getTimestamp();
+ }
+ if (BoundedWindow.TIMESTAMP_MAX_VALUE.equals(pollWatermark)) {
+ isOutputComplete = true;
+ }
+ return pending.size();
+ }
+
+ @VisibleForTesting
+ synchronized Instant getWatermark() {
+ // Future elements that can be claimed in this restriction come either from
+ // "pending" or from future polls, so the total watermark is
+ // min(watermark for future polling, earliest remaining pending element)
+ return Ordering.natural()
+ .nullsLast()
+ .min(pollWatermark, pending.isEmpty() ? null : pending.getFirst().getTimestamp());
+ }
+
+ @Override
+ public synchronized String toString() {
+ return "GrowthTracker{"
+ + "state="
+ + state.toString(terminationCondition)
+ + ", pending=<"
+ + pending.size()
+ + " elements"
+ + (pending.isEmpty() ? "" : (", earliest " + pending.get(0)))
+ + ">, claimed=<"
+ + claimed.size()
+ + " elements>, isOutputComplete="
+ + isOutputComplete
+ + ", terminationState="
+ + terminationState
+ + ", pollWatermark="
+ + pollWatermark
+ + ", shouldStop="
+ + shouldStop
+ + '}';
+ }
+ }
+
+ private static class HashCode128Coder extends AtomicCoder<HashCode> {
+ private static final HashCode128Coder INSTANCE = new HashCode128Coder();
+
+ public static HashCode128Coder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(HashCode value, OutputStream os) throws IOException {
+ checkArgument(
+ value.bits() == 128, "Expected a 128-bit hash code, but got %s bits", value.bits());
+ byte[] res = new byte[16];
+ value.writeBytesTo(res, 0, 16);
+ os.write(res);
+ }
+
+ @Override
+ public HashCode decode(InputStream is) throws IOException {
+ byte[] res = new byte[16];
+ int numRead = is.read(res, 0, 16);
+ checkArgument(numRead == 16, "Expected to read 16 bytes, but read %s", numRead);
+ return HashCode.fromBytes(res);
+ }
+ }
+
+ private static class GrowthStateCoder<OutputT, TerminationStateT>
+ extends StructuredCoder<GrowthState<OutputT, TerminationStateT>> {
+ public static <OutputT, TerminationStateT> GrowthStateCoder<OutputT, TerminationStateT> of(
+ Coder<OutputT> outputCoder, Coder<TerminationStateT> terminationStateCoder) {
+ return new GrowthStateCoder<>(outputCoder, terminationStateCoder);
+ }
+
+ private static final Coder<Boolean> BOOLEAN_CODER = BooleanCoder.of();
+ private static final Coder<Instant> INSTANT_CODER = NullableCoder.of(InstantCoder.of());
+ private static final Coder<HashCode> HASH_CODE_CODER = HashCode128Coder.of();
+
+ private final Coder<OutputT> outputCoder;
+ private final Coder<Map<HashCode, Instant>> completedCoder;
+ private final Coder<List<TimestampedValue<OutputT>>> pendingCoder;
+ private final Coder<TerminationStateT> terminationStateCoder;
+
+ private GrowthStateCoder(
+ Coder<OutputT> outputCoder, Coder<TerminationStateT> terminationStateCoder) {
+ this.outputCoder = outputCoder;
+ this.terminationStateCoder = terminationStateCoder;
+ this.completedCoder = MapCoder.of(HASH_CODE_CODER, INSTANT_CODER);
+ this.pendingCoder = ListCoder.of(TimestampedValue.TimestampedValueCoder.of(outputCoder));
+ }
+
+ @Override
+ public void encode(GrowthState<OutputT, TerminationStateT> value, OutputStream os)
+ throws IOException {
+ completedCoder.encode(value.completed, os);
+ pendingCoder.encode(value.pending, os);
+ BOOLEAN_CODER.encode(value.isOutputComplete, os);
+ terminationStateCoder.encode(value.terminationState, os);
+ INSTANT_CODER.encode(value.pollWatermark, os);
+ }
+
+ @Override
+ public GrowthState<OutputT, TerminationStateT> decode(InputStream is) throws IOException {
+ Map<HashCode, Instant> completed = completedCoder.decode(is);
+ List<TimestampedValue<OutputT>> pending = pendingCoder.decode(is);
+ boolean isOutputComplete = BOOLEAN_CODER.decode(is);
+ TerminationStateT terminationState = terminationStateCoder.decode(is);
+ Instant pollWatermark = INSTANT_CODER.decode(is);
+ return new GrowthState<>(
+ completed, pending, isOutputComplete, terminationState, pollWatermark);
+ }
+
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return Arrays.asList(outputCoder, terminationStateCoder);
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ outputCoder.verifyDeterministic();
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
index c66d1b1..23696e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
@@ -17,6 +17,10 @@
*/
package org.apache.beam.sdk.transforms;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
@@ -64,6 +68,8 @@
* be called on the result {@link PTransform}.
*/
public static <K, V> WithKeys<K, V> of(SerializableFunction<V, K> fn) {
+ checkNotNull(fn,
+ "WithKeys constructed with null function. Did you mean WithKeys.of((Void) null)?");
return new WithKeys<>(fn, null);
}
@@ -74,7 +80,7 @@
* given key.
*/
@SuppressWarnings("unchecked")
- public static <K, V> WithKeys<K, V> of(final K key) {
+ public static <K, V> WithKeys<K, V> of(@Nullable final K key) {
return new WithKeys<>(
new SerializableFunction<V, K>() {
@Override
@@ -82,14 +88,14 @@
return key;
}
},
- (Class<K>) (key == null ? null : key.getClass()));
+ (Class<K>) (key == null ? Void.class : key.getClass()));
}
/////////////////////////////////////////////////////////////////////////////
private SerializableFunction<V, K> fn;
- private transient Class<K> keyClass;
+ @CheckForNull private transient Class<K> keyClass;
private WithKeys(SerializableFunction<V, K> fn, Class<K> keyClass) {
this.fn = fn;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
index 3c4337b..1b4b48f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
@@ -710,14 +710,14 @@
*/
private static final FormattedItemValue NULL_VALUES = new FormattedItemValue(null);
- private final Object shortValue;
- private final Object longValue;
+ @Nullable private final Object shortValue;
+ @Nullable private final Object longValue;
- private FormattedItemValue(Object longValue) {
+ private FormattedItemValue(@Nullable Object longValue) {
this(longValue, null);
}
- private FormattedItemValue(Object longValue, Object shortValue) {
+ private FormattedItemValue(@Nullable Object longValue, @Nullable Object shortValue) {
this.longValue = longValue;
this.shortValue = shortValue;
}
@@ -735,8 +735,8 @@
private final Set<HasDisplayData> visitedComponents;
private final Map<Path, HasDisplayData> visitedPathMap;
- private Path latestPath;
- private Class<?> latestNs;
+ @Nullable private Path latestPath;
+ @Nullable private Class<?> latestNs;
private InternalBuilder() {
this.entries = Maps.newHashMap();
@@ -796,8 +796,9 @@
// Don't re-wrap exceptions recursively.
throw e;
} catch (Throwable e) {
- String msg = String.format("Error while populating display data for component: %s",
- namespace.getName());
+ String msg = String.format(
+ "Error while populating display data for component '%s': %s",
+ namespace.getName(), e.getMessage());
throw new PopulateDisplayDataException(msg, e);
}
@@ -882,12 +883,12 @@
return item(key, Type.STRING, null);
}
Type type = inferType(got);
- if (type == null) {
- throw new RuntimeException(String.format("Unknown value type: %s", got));
+ if (type != null) {
+ return item(key, type, got);
}
- return item(key, type, got);
}
- return item(key, Type.STRING, value.toString());
+ // General case: not null and type not inferable. Fall back to toString of the VP itself.
+ return item(key, Type.STRING, String.valueOf(value));
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/package-info.java
index 4af3327..e4fff40 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/package-info.java
@@ -23,4 +23,8 @@
*
* @see org.apache.beam.sdk.transforms.display.HasDisplayData
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms.display;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index 877bb07..1dad9f4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -28,6 +28,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.CustomCoder;
@@ -116,15 +117,13 @@
// against a given tag would not match anything.
final Boolean[] containsTag = new Boolean[schema.size()];
for (int unionTag = 0; unionTag < schema.size(); unionTag++) {
- final int unionTag0 = unionTag;
- updateUnionTag(tail, containsTag, unionTag, unionTag0);
+ updateUnionTag(tail, containsTag, unionTag);
}
}
}
private <T> void updateUnionTag(
- final Reiterator<RawUnionValue> tail, final Boolean[] containsTag,
- int unionTag, final int unionTag0) {
+ final Reiterator<RawUnionValue> tail, final Boolean[] containsTag, final int unionTag) {
@SuppressWarnings("unchecked")
final Iterable<T> head = (Iterable<T>) valueMap.get(unionTag);
valueMap.set(
@@ -134,7 +133,7 @@
public Iterator<T> iterator() {
return Iterators.concat(
head.iterator(),
- new UnionValueIterator<T>(unionTag0, tail.copy(), containsTag));
+ new UnionValueIterator<T>(unionTag, tail.copy(), containsTag));
}
});
}
@@ -197,7 +196,8 @@
* <p>If tag was not part of the original {@link CoGroupByKey},
* throws an IllegalArgumentException.
*/
- public <V> V getOnly(TupleTag<V> tag, V defaultValue) {
+ @Nullable
+ public <V> V getOnly(TupleTag<V> tag, @Nullable V defaultValue) {
return innerGetOnly(tag, defaultValue, true);
}
@@ -356,9 +356,10 @@
this.valueMap = valueMap;
}
+ @Nullable
private <V> V innerGetOnly(
TupleTag<V> tag,
- V defaultValue,
+ @Nullable V defaultValue,
boolean useDefault) {
int index = schema.getIndex(tag);
if (index < 0) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java
index 2e7dd01..a9d1873 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -207,24 +208,21 @@
*/
private final List<TaggedKeyedPCollection<K, ?>> keyedCollections;
- private Coder<K> keyCoder;
+ @Nullable private Coder<K> keyCoder;
private final CoGbkResultSchema schema;
private final Pipeline pipeline;
KeyedPCollectionTuple(Pipeline pipeline) {
- this(pipeline,
- new ArrayList<TaggedKeyedPCollection<K, ?>>(),
- TupleTagList.empty(),
- null);
+ this(pipeline, new ArrayList<TaggedKeyedPCollection<K, ?>>(), TupleTagList.empty(), null);
}
KeyedPCollectionTuple(
Pipeline pipeline,
List<TaggedKeyedPCollection<K, ?>> keyedCollections,
TupleTagList tupleTagList,
- Coder<K> keyCoder) {
+ @Nullable Coder<K> keyCoder) {
this.pipeline = pipeline;
this.keyedCollections = keyedCollections;
this.schema = new CoGbkResultSchema(tupleTagList);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/RawUnionValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/RawUnionValue.java
index 07bfe69..7ac1faf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/RawUnionValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/RawUnionValue.java
@@ -20,18 +20,20 @@
// TODO: Think about making this a complete dynamic union by adding
// a schema. Type would then be defined by the corresponding schema entry.
+import javax.annotation.Nullable;
+
/**
* This corresponds to an integer union tag and value. The mapping of
* union tag to type must come from elsewhere.
*/
public class RawUnionValue {
private final int unionTag;
- private final Object value;
+ @Nullable private final Object value;
/**
* Constructs a partial union from the given union tag and value.
*/
- public RawUnionValue(int unionTag, Object value) {
+ public RawUnionValue(int unionTag, @Nullable Object value) {
this.unionTag = unionTag;
this.value = value;
}
@@ -40,7 +42,7 @@
return unionTag;
}
- public Object getValue() {
+ @Nullable public Object getValue() {
return value;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/package-info.java
index f4b315e..7aab329 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/package-info.java
@@ -19,4 +19,8 @@
* Defines the {@link org.apache.beam.sdk.transforms.join.CoGroupByKey} transform
* for joining multiple PCollections.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms.join;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/package-info.java
index 892dee9..634786b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/package-info.java
@@ -40,4 +40,8 @@
* for their own application-specific logic.
*
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
index 5d5887a..8ce3348 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
@@ -89,6 +89,7 @@
public static final String PROCESS_CONTEXT_PARAMETER_METHOD = "processContext";
public static final String ON_TIMER_CONTEXT_PARAMETER_METHOD = "onTimerContext";
public static final String WINDOW_PARAMETER_METHOD = "window";
+ public static final String PIPELINE_OPTIONS_PARAMETER_METHOD = "pipelineOptions";
public static final String RESTRICTION_TRACKER_PARAMETER_METHOD = "restrictionTracker";
public static final String STATE_PARAMETER_METHOD = "state";
public static final String TIMER_PARAMETER_METHOD = "timer";
@@ -393,7 +394,8 @@
/** Whether the target method returns non-void. */
private final boolean targetHasReturn;
- protected FieldDescription delegateField;
+ /** Starts {@code null}, initialized by {@link #prepare(InstrumentedType)}. */
+ @Nullable protected FieldDescription delegateField;
private final TypeDescription doFnType;
@@ -626,6 +628,11 @@
getExtraContextFactoryMethodDescription(TIMER_PARAMETER_METHOD, String.class)),
TypeCasting.to(new TypeDescription.ForLoadedType(Timer.class)));
}
+
+ @Override
+ public StackManipulation dispatch(DoFnSignature.Parameter.PipelineOptionsParameter p) {
+ return simpleExtraContextParameter(PIPELINE_OPTIONS_PARAMETER_METHOD);
+ }
});
}
@@ -634,6 +641,17 @@
* {@link ProcessElement} method.
*/
private static final class ProcessElementDelegation extends DoFnMethodDelegation {
+ private static final MethodDescription PROCESS_CONTINUATION_STOP_METHOD;
+
+ static {
+ try {
+ PROCESS_CONTINUATION_STOP_METHOD =
+ new MethodDescription.ForLoadedMethod(DoFn.ProcessContinuation.class.getMethod("stop"));
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException("Failed to locate ProcessContinuation.stop()");
+ }
+ }
+
private final DoFnSignature.ProcessElementMethod signature;
/** Implementation of {@link MethodDelegation} for the {@link ProcessElement} method. */
@@ -667,6 +685,16 @@
}
return new StackManipulation.Compound(pushParameters);
}
+
+ @Override
+ protected StackManipulation afterDelegation(MethodDescription instrumentedMethod) {
+ if (TypeDescription.VOID.equals(targetMethod.getReturnType().asErasure())) {
+ return new StackManipulation.Compound(
+ MethodInvocation.invoke(PROCESS_CONTINUATION_STOP_METHOD), MethodReturn.REFERENCE);
+ } else {
+ return MethodReturn.of(targetMethod.getReturnType().asErasure());
+ }
+ }
}
private static class UserCodeMethodInvocation implements StackManipulation {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
index e031337..5e31f2e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.transforms.reflect;
+
import com.google.common.base.CharMatcher;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -61,13 +62,14 @@
@SuppressWarnings("unchecked")
Class<? extends DoFn<?, ?>> fnClass = (Class<? extends DoFn<?, ?>>) fn.getClass();
-
try {
- Constructor<?> constructor = constructorCache.get(fnClass).get(timerId);
- @SuppressWarnings("unchecked")
- OnTimerInvoker<InputT, OutputT> invoker =
+ OnTimerMethodSpecifier onTimerMethodSpecifier =
+ OnTimerMethodSpecifier.forClassAndTimerId(fnClass, timerId);
+ Constructor<?> constructor = constructorCache.get(onTimerMethodSpecifier);
+
+ OnTimerInvoker<InputT, OutputT> invoker =
(OnTimerInvoker<InputT, OutputT>) constructor.newInstance(fn);
- return invoker;
+ return invoker;
} catch (InstantiationException
| IllegalAccessException
| IllegalArgumentException
@@ -97,50 +99,31 @@
private static final String FN_DELEGATE_FIELD_NAME = "delegate";
/**
- * A cache of constructors of generated {@link OnTimerInvoker} classes, keyed by {@link DoFn}
- * class and then by {@link TimerId}.
+ * A cache of constructors of generated {@link OnTimerInvoker} classes,
+ * keyed by {@link OnTimerMethodSpecifier}.
*
* <p>Needed because generating an invoker class is expensive, and to avoid generating an
* excessive number of classes consuming PermGen memory in Java's that still have PermGen.
*/
- private final LoadingCache<Class<? extends DoFn<?, ?>>, LoadingCache<String, Constructor<?>>>
- constructorCache =
- CacheBuilder.newBuilder()
- .build(
- new CacheLoader<
- Class<? extends DoFn<?, ?>>, LoadingCache<String, Constructor<?>>>() {
- @Override
- public LoadingCache<String, Constructor<?>> load(
- final Class<? extends DoFn<?, ?>> fnClass) throws Exception {
- return CacheBuilder.newBuilder().build(new OnTimerConstructorLoader(fnClass));
- }
- });
+ private final LoadingCache<OnTimerMethodSpecifier, Constructor<?>> constructorCache =
+ CacheBuilder.newBuilder().build(
+ new CacheLoader<OnTimerMethodSpecifier, Constructor<?>>() {
+ @Override
+ public Constructor<?> load(final OnTimerMethodSpecifier onTimerMethodSpecifier)
+ throws Exception {
+ DoFnSignature signature =
+ DoFnSignatures.getSignature(onTimerMethodSpecifier.fnClass());
+ Class<? extends OnTimerInvoker<?, ?>> invokerClass =
+ generateOnTimerInvokerClass(signature, onTimerMethodSpecifier.timerId());
+ try {
+ return invokerClass.getConstructor(signature.fnClass());
+ } catch (IllegalArgumentException | NoSuchMethodException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
- /**
- * A cache loader fixed to a particular {@link DoFn} class that loads constructors for the
- * invokers for its {@link OnTimer @OnTimer} methods.
- */
- private static class OnTimerConstructorLoader extends CacheLoader<String, Constructor<?>> {
-
- private final DoFnSignature signature;
-
- public OnTimerConstructorLoader(Class<? extends DoFn<?, ?>> clazz) {
- this.signature = DoFnSignatures.getSignature(clazz);
- }
-
- @Override
- public Constructor<?> load(String timerId) throws Exception {
- Class<? extends OnTimerInvoker<?, ?>> invokerClass =
- generateOnTimerInvokerClass(signature, timerId);
- try {
- return invokerClass.getConstructor(signature.fnClass());
- } catch (IllegalArgumentException | NoSuchMethodException | SecurityException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
+ }
+ });
+ /**
* Generates a {@link OnTimerInvoker} class for the given {@link DoFnSignature} and {@link
* TimerId}.
*/
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index 6fd4052..ec2bf34 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -19,6 +19,7 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.Timer;
import org.apache.beam.sdk.transforms.DoFn;
@@ -53,8 +54,10 @@
* Invoke the {@link DoFn.ProcessElement} method on the bound {@link DoFn}.
*
* @param extra Factory for producing extra parameter objects (such as window), if necessary.
+ * @return The {@link DoFn.ProcessContinuation} returned by the underlying method, or {@link
+ * DoFn.ProcessContinuation#stop()} if it returns {@code void}.
*/
- void invokeProcessElement(ArgumentProvider<InputT, OutputT> extra);
+ DoFn.ProcessContinuation invokeProcessElement(ArgumentProvider<InputT, OutputT> extra);
/** Invoke the appropriate {@link DoFn.OnTimer} method on the bound {@link DoFn}. */
void invokeOnTimer(String timerId, ArgumentProvider<InputT, OutputT> arguments);
@@ -100,7 +103,12 @@
*/
BoundedWindow window();
- /** Provide a {@link DoFn.StartBundleContext} to use with the given {@link DoFn}. */
+ /** Provide {@link PipelineOptions}. */
+ PipelineOptions pipelineOptions();
+
+ /**
+ * Provide a {@link DoFn.StartBundleContext} to use with the given {@link DoFn}.
+ */
DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn);
/** Provide a {@link DoFn.FinishBundleContext} to use with the given {@link DoFn}. */
@@ -125,46 +133,81 @@
Timer timer(String timerId);
}
- /** For testing only, this {@link ArgumentProvider} returns {@code null} for all parameters. */
+ /**
+ * For testing only, this {@link ArgumentProvider} throws {@link UnsupportedOperationException}
+ * for all parameters.
+ */
class FakeArgumentProvider<InputT, OutputT> implements ArgumentProvider<InputT, OutputT> {
@Override
public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public BoundedWindow window() {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
+ }
+
+ @Override
+ public PipelineOptions pipelineOptions() {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
DoFn<InputT, OutputT> doFn) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public State state(String stateId) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
@Override
public Timer timer(String timerId) {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
public RestrictionTracker<?> restrictionTracker() {
- return null;
+ throw new UnsupportedOperationException(
+ String.format(
+ "Should never call non-overridden methods of %s",
+ FakeArgumentProvider.class.getSimpleName()));
}
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
index 33c5a6a..44b87a0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.transforms.reflect;
-import java.io.Serializable;
import org.apache.beam.sdk.transforms.DoFn;
/** Static utilities for working with {@link DoFnInvoker}. */
@@ -36,13 +35,5 @@
return ByteBuddyDoFnInvokerFactory.only().newByteBuddyInvoker(fn);
}
- /** TODO: remove this when Dataflow worker uses the DoFn overload. */
- @Deprecated
- @SuppressWarnings({"unchecked"})
- public static <InputT, OutputT> DoFnInvoker<InputT, OutputT> invokerFor(
- Serializable fn) {
- return invokerFor((DoFn) fn);
- }
-
private DoFnInvokers() {}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 0b4bf90..bfad69e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -27,11 +27,13 @@
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.Timer;
import org.apache.beam.sdk.state.TimerSpec;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.DoFn.StateId;
import org.apache.beam.sdk.transforms.DoFn.TimerId;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
@@ -193,6 +195,8 @@
return cases.dispatch((StateParameter) this);
} else if (this instanceof TimerParameter) {
return cases.dispatch((TimerParameter) this);
+ } else if (this instanceof PipelineOptionsParameter) {
+ return cases.dispatch((PipelineOptionsParameter) this);
} else {
throw new IllegalStateException(
String.format("Attempt to case match on unknown %s subclass %s",
@@ -212,6 +216,7 @@
ResultT dispatch(RestrictionTrackerParameter p);
ResultT dispatch(StateParameter p);
ResultT dispatch(TimerParameter p);
+ ResultT dispatch(PipelineOptionsParameter p);
/**
* A base class for a visitor with a default method for cases it is not interested in.
@@ -259,6 +264,11 @@
public ResultT dispatch(TimerParameter p) {
return dispatchDefault(p);
}
+
+ @Override
+ public ResultT dispatch(PipelineOptionsParameter p) {
+ return dispatchDefault(p);
+ }
}
}
@@ -287,6 +297,11 @@
return new AutoValue_DoFnSignature_Parameter_WindowParameter(windowT);
}
+ /** Returns a {@link PipelineOptionsParameter}. */
+ public static PipelineOptionsParameter pipelineOptions() {
+ return new AutoValue_DoFnSignature_Parameter_PipelineOptionsParameter();
+ }
+
/**
* Returns a {@link RestrictionTrackerParameter}.
*/
@@ -306,6 +321,14 @@
}
/**
+ * Descriptor for a {@link Parameter} of a subtype of {@link PipelineOptions}.
+ */
+ @AutoValue
+ public abstract static class PipelineOptionsParameter extends Parameter {
+ PipelineOptionsParameter() {}
+ }
+
+ /**
* Descriptor for a {@link Parameter} of type {@link DoFn.StartBundleContext}.
*
* <p>All such descriptors are equal.
@@ -314,6 +337,7 @@
public abstract static class StartBundleContextParameter extends Parameter {
StartBundleContextParameter() {}
}
+
/**
* Descriptor for a {@link Parameter} of type {@link DoFn.FinishBundleContext}.
*
@@ -410,16 +434,21 @@
@Nullable
public abstract TypeDescriptor<? extends BoundedWindow> windowT();
+ /** Whether this {@link DoFn} returns a {@link ProcessContinuation} or void. */
+ public abstract boolean hasReturnValue();
+
static ProcessElementMethod create(
Method targetMethod,
List<Parameter> extraParameters,
TypeDescriptor<?> trackerT,
- @Nullable TypeDescriptor<? extends BoundedWindow> windowT) {
+ @Nullable TypeDescriptor<? extends BoundedWindow> windowT,
+ boolean hasReturnValue) {
return new AutoValue_DoFnSignature_ProcessElementMethod(
targetMethod,
Collections.unmodifiableList(extraParameters),
trackerT,
- windowT);
+ windowT,
+ hasReturnValue);
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index bb191b1..c54c44f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.sdk.transforms.reflect;
+import static com.google.common.base.Preconditions.checkState;
+
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicates;
@@ -42,6 +44,7 @@
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.State;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.Timer;
@@ -78,19 +81,23 @@
ImmutableList.of(
Parameter.ProcessContextParameter.class,
Parameter.WindowParameter.class,
+ Parameter.PipelineOptionsParameter.class,
Parameter.TimerParameter.class,
Parameter.StateParameter.class);
private static final Collection<Class<? extends Parameter>>
ALLOWED_SPLITTABLE_PROCESS_ELEMENT_PARAMETERS =
ImmutableList.of(
- Parameter.ProcessContextParameter.class, Parameter.RestrictionTrackerParameter.class);
+ Parameter.PipelineOptionsParameter.class,
+ Parameter.ProcessContextParameter.class,
+ Parameter.RestrictionTrackerParameter.class);
private static final Collection<Class<? extends Parameter>>
ALLOWED_ON_TIMER_PARAMETERS =
ImmutableList.of(
Parameter.OnTimerContextParameter.class,
Parameter.WindowParameter.class,
+ Parameter.PipelineOptionsParameter.class,
Parameter.TimerParameter.class,
Parameter.StateParameter.class);
@@ -187,6 +194,15 @@
extraParameters, Predicates.instanceOf(WindowParameter.class));
}
+ /**
+ * Indicates whether a {@link Parameter.PipelineOptionsParameter} is
+ * known in this context.
+ */
+ public boolean hasPipelineOptionsParamter() {
+ return Iterables.any(
+ extraParameters, Predicates.instanceOf(Parameter.PipelineOptionsParameter.class));
+ }
+
/** The window type, if any, used by this method. */
@Nullable
public TypeDescriptor<? extends BoundedWindow> getWindowType() {
@@ -426,6 +442,8 @@
* <li>If the {@link DoFn} (or any of its supertypes) is annotated as {@link
* DoFn.BoundedPerElement} or {@link DoFn.UnboundedPerElement}, use that. Only one of
* these must be specified.
+ * <li>If {@link DoFn.ProcessElement} returns {@link DoFn.ProcessContinuation}, assume it is
+ * unbounded. Otherwise (if it returns {@code void}), assume it is bounded.
* <li>If {@link DoFn.ProcessElement} returns {@code void}, but the {@link DoFn} is annotated
* {@link DoFn.UnboundedPerElement}, this is an error.
* </ol>
@@ -451,7 +469,10 @@
}
if (processElement.isSplittable()) {
if (isBounded == null) {
- isBounded = PCollection.IsBounded.BOUNDED;
+ isBounded =
+ processElement.hasReturnValue()
+ ? PCollection.IsBounded.UNBOUNDED
+ : PCollection.IsBounded.BOUNDED;
}
} else {
errors.checkArgument(
@@ -460,6 +481,7 @@
+ ((isBounded == PCollection.IsBounded.BOUNDED)
? DoFn.BoundedPerElement.class.getSimpleName()
: DoFn.UnboundedPerElement.class.getSimpleName()));
+ checkState(!processElement.hasReturnValue(), "Should have been inferred splittable");
isBounded = PCollection.IsBounded.BOUNDED;
}
return isBounded;
@@ -696,8 +718,10 @@
TypeDescriptor<?> outputT,
FnAnalysisContext fnContext) {
errors.checkArgument(
- void.class.equals(m.getReturnType()),
- "Must return void");
+ void.class.equals(m.getReturnType())
+ || DoFn.ProcessContinuation.class.equals(m.getReturnType()),
+ "Must return void or %s",
+ DoFn.ProcessContinuation.class.getSimpleName());
MethodAnalysisContext methodContext = MethodAnalysisContext.create();
@@ -737,7 +761,11 @@
}
return DoFnSignature.ProcessElementMethod.create(
- m, methodContext.getExtraParameters(), trackerT, windowT);
+ m,
+ methodContext.getExtraParameters(),
+ trackerT,
+ windowT,
+ DoFn.ProcessContinuation.class.equals(m.getReturnType()));
}
private static void checkParameterOneOf(
@@ -789,6 +817,12 @@
"Multiple %s parameters",
BoundedWindow.class.getSimpleName());
return Parameter.boundedWindow((TypeDescriptor<? extends BoundedWindow>) paramT);
+ } else if (PipelineOptions.class.equals(rawType)) {
+ methodErrors.checkArgument(
+ !methodContext.hasPipelineOptionsParamter(),
+ "Multiple %s parameters",
+ PipelineOptions.class.getSimpleName());
+ return Parameter.pipelineOptions();
} else if (RestrictionTracker.class.isAssignableFrom(rawType)) {
methodErrors.checkArgument(
!methodContext.hasRestrictionTrackerParameter(),
@@ -1262,6 +1296,7 @@
return ImmutableMap.copyOf(declarations);
}
+ @Nullable
private static Method findAnnotatedMethod(
ErrorReporter errors, Class<? extends Annotation> anno, Class<?> fnClazz, boolean required) {
Collection<Method> matches = declaredMethodsWithAnnotation(anno, fnClazz, DoFn.class);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/OnTimerMethodSpecifier.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/OnTimerMethodSpecifier.java
new file mode 100644
index 0000000..edf7e3c
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/OnTimerMethodSpecifier.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.reflect;
+
+import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/**
+ * Used by {@link ByteBuddyOnTimerInvokerFactory} to Dynamically generate
+ * {@link OnTimerInvoker} instances for invoking a particular
+ * {@link DoFn.TimerId} on a particular {@link DoFn}.
+ */
+
+@AutoValue
+abstract class OnTimerMethodSpecifier {
+ public abstract Class<? extends DoFn<?, ?>> fnClass();
+ public abstract String timerId();
+ public static OnTimerMethodSpecifier
+ forClassAndTimerId(Class<? extends DoFn<?, ?>> fnClass, String timerId){
+ return new AutoValue_OnTimerMethodSpecifier(fnClass, timerId);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
index fe2f6b1..48b128c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
@@ -22,5 +22,8 @@
* and creating {@link org.apache.beam.sdk.transforms.reflect.DoFnSignature}'s and
* {@link org.apache.beam.sdk.transforms.reflect.DoFnInvoker}'s from them.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms.reflect;
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRange.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRange.java
deleted file mode 100644
index 104f5f2..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRange.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.transforms.splittabledofn;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.io.Serializable;
-
-/** A restriction represented by a range of integers [from, to). */
-public class OffsetRange
- implements Serializable, HasDefaultTracker<OffsetRange, OffsetRangeTracker> {
- private final long from;
- private final long to;
-
- public OffsetRange(long from, long to) {
- checkArgument(from <= to, "Malformed range [%s, %s)", from, to);
- this.from = from;
- this.to = to;
- }
-
- public long getFrom() {
- return from;
- }
-
- public long getTo() {
- return to;
- }
-
- @Override
- public OffsetRangeTracker newTracker() {
- return new OffsetRangeTracker(this);
- }
-
- @Override
- public String toString() {
- return "[" + from + ", " + to + ')';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- OffsetRange that = (OffsetRange) o;
-
- if (from != that.from) {
- return false;
- }
- return to == that.to;
- }
-
- @Override
- public int hashCode() {
- int result = (int) (from ^ (from >>> 32));
- result = 31 * result + (int) (to ^ (to >>> 32));
- return result;
- }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java
index 0271a0d..8ec2c6b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java
@@ -21,6 +21,9 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import com.google.common.base.MoreObjects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.transforms.DoFn;
/**
@@ -29,8 +32,8 @@
*/
public class OffsetRangeTracker implements RestrictionTracker<OffsetRange> {
private OffsetRange range;
- private Long lastClaimedOffset = null;
- private Long lastAttemptedOffset = null;
+ @Nullable private Long lastClaimedOffset = null;
+ @Nullable private Long lastAttemptedOffset = null;
public OffsetRangeTracker(OffsetRange range) {
this.range = checkNotNull(range);
@@ -99,4 +102,13 @@
lastAttemptedOffset + 1,
range.getTo());
}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("range", range)
+ .add("lastClaimedOffset", lastClaimedOffset)
+ .add("lastAttemptedOffset", lastAttemptedOffset)
+ .toString();
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
index 27ef68f..8cb0a6b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
@@ -31,10 +31,13 @@
RestrictionT currentRestriction();
/**
- * Signals that the current {@link DoFn.ProcessElement} call should terminate as soon as possible.
- * Modifies {@link #currentRestriction}. Returns a restriction representing the rest of the work:
- * the old value of {@link #currentRestriction} is equivalent to the new value and the return
- * value of this method combined. Must be called at most once on a given object.
+ * Signals that the current {@link DoFn.ProcessElement} call should terminate as soon as possible:
+ * after this method returns, the tracker MUST refuse all future claim calls, and {@link
+ * #checkDone} MUST succeed.
+ *
+ * <p>Modifies {@link #currentRestriction}. Returns a restriction representing the rest of the
+ * work: the old value of {@link #currentRestriction} is equivalent to the new value and the
+ * return value of this method combined. Must be called at most once on a given object.
*/
RestrictionT checkpoint();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
index 4523032..82538ea 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
@@ -19,4 +19,8 @@
* Defines utilities related to <a href="https://s.apache.org/splittable-do-fn">splittable</a>
* {@link org.apache.beam.sdk.transforms.DoFn}.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms.splittabledofn;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
index 74223b5..92fa3c5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
@@ -21,15 +21,19 @@
import org.joda.time.Instant;
/**
- * A {@code BoundedWindow} represents a finite grouping of elements, with an
- * upper bound (larger timestamps represent more recent data) on the timestamps
- * of elements that can be placed in the window. This finiteness means that for
- * every window, at some point in time, all data for that window will have
- * arrived and can be processed together.
+ * A {@link BoundedWindow} represents window information assigned to data elements.
*
- * <p>Windows must also implement {@link Object#equals} and
- * {@link Object#hashCode} such that windows that are logically equal will
- * be treated as equal by {@code equals()} and {@code hashCode()}.
+ * <p>It has one method {@link #maxTimestamp()} to define an upper bound (inclusive) for element
+ * timestamps. A {@link WindowFn} must assign an element only to windows where {@link
+ * #maxTimestamp()} is greater than or equal to the element timestamp. When the watermark passes the
+ * maximum timestamp, all data for a window is estimated to be received.
+ *
+ * <p>A window does not need to have a lower bound. Only the upper bound is mandatory because it
+ * governs management of triggering and discarding of the window.
+ *
+ * <p>Windows must also implement {@link Object#equals} and {@link Object#hashCode} such that
+ * windows that are logically equal will be treated as equal by {@code equals()} and {@code
+ * hashCode()}.
*/
public abstract class BoundedWindow {
// The min and max timestamps that won't overflow when they are converted to
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
index fada50a..989c431 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
@@ -145,6 +145,18 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same number of days, start date "
+ + "and time zone are compatible.",
+ DaysWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
@@ -245,6 +257,18 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same number of months, "
+ + "day of month, start date and time zone are compatible.",
+ MonthsWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
@@ -354,6 +378,18 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same number of years, month of year, "
+ + "day of month, start date and time zone are compatible.",
+ YearsWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
index 8683a60..8b16916 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
@@ -101,6 +101,17 @@
return this.equals(other);
}
+ @Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same size and offset are compatible.",
+ FixedWindows.class.getSimpleName()));
+ }
+ }
+
public Duration getSize() {
return size;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
index bc3438b..c68c497 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.transforms.windowing;
+import com.google.auto.value.AutoValue;
import java.util.Collection;
import java.util.Collections;
import org.apache.beam.sdk.coders.Coder;
@@ -44,16 +45,28 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is only compatible with %s.",
+ GlobalWindows.class.getSimpleName(), GlobalWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public Coder<GlobalWindow> windowCoder() {
return GlobalWindow.Coder.INSTANCE;
}
@Override
public WindowMappingFn<GlobalWindow> getDefaultWindowMappingFn() {
- return new GlobalWindowMappingFn();
+ return new AutoValue_GlobalWindows_GlobalWindowMappingFn();
}
- static class GlobalWindowMappingFn extends WindowMappingFn<GlobalWindow> {
+ @AutoValue
+ abstract static class GlobalWindowMappingFn extends WindowMappingFn<GlobalWindow> {
@Override
public GlobalWindow getSideInputWindow(BoundedWindow mainWindow) {
return GlobalWindow.INSTANCE;
@@ -66,6 +79,11 @@
}
@Override
+ public boolean assignsToOneWindow() {
+ return true;
+ }
+
+ @Override
public boolean equals(Object other) {
return other instanceof GlobalWindows;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java
new file mode 100644
index 0000000..20746af
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.windowing;
+
+/**
+ * Exception thrown by {@link WindowFn#verifyCompatibility(WindowFn)} if two compared
+ * WindowFns are not compatible, including the explanation of incompatibility.
+ */
+public class IncompatibleWindowException extends Exception {
+ private WindowFn<?, ?> givenWindowFn;
+ private String reason;
+
+ public IncompatibleWindowException(WindowFn<?, ?> windowFn, String reason) {
+ this.givenWindowFn = windowFn;
+ this.reason = reason;
+ }
+
+ @Override
+ public String getMessage() {
+ String windowFn = givenWindowFn.getClass().getSimpleName();
+ return String.format("The given WindowFn is %s. %s", windowFn, reason);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
index 49535a0..e58cd15 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
@@ -75,6 +75,17 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same originalWindowFn are compatible.",
+ InvalidWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public WindowMappingFn<W> getDefaultWindowMappingFn() {
throw new UnsupportedOperationException("InvalidWindows is not allowed in side inputs");
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
index 0a68021..0421868 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
@@ -21,6 +21,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
/**
@@ -61,9 +62,10 @@
}
private static class MergeCandidate {
- private IntervalWindow union;
+ @Nullable private IntervalWindow union;
private final List<IntervalWindow> parts;
public MergeCandidate() {
+ union = null;
parts = new ArrayList<>();
}
public MergeCandidate(IntervalWindow window) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PartitioningWindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PartitioningWindowFn.java
index 40ee68a..341ba27 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PartitioningWindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PartitioningWindowFn.java
@@ -58,4 +58,9 @@
public Instant getOutputTime(Instant inputTimestamp, W window) {
return inputTimestamp;
}
+
+ @Override
+ public final boolean assignsToOneWindow() {
+ return true;
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
index 7390728..9d6aecc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
@@ -81,6 +81,17 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is only compatible with %s.",
+ Sessions.class.getSimpleName(), Sessions.class.getSimpleName()));
+ }
+ }
+
+ @Override
public WindowMappingFn<IntervalWindow> getDefaultWindowMappingFn() {
throw new UnsupportedOperationException("Sessions is not allowed in side inputs");
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
index 650dc37..150b956 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
@@ -148,6 +148,22 @@
}
@Override
+ public boolean assignsToOneWindow() {
+ return !this.period.isShorterThan(this.size);
+ }
+
+ @Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "Only %s objects with the same size, period and offset are compatible.",
+ SlidingWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
index 519ab67..6985565 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
@@ -23,6 +23,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.transforms.GroupByKey;
@@ -71,9 +72,9 @@
@Experimental(Experimental.Kind.TRIGGER)
public abstract class Trigger implements Serializable {
- protected final List<Trigger> subTriggers;
+ @Nullable protected final List<Trigger> subTriggers;
- protected Trigger(List<Trigger> subTriggers) {
+ protected Trigger(@Nullable List<Trigger> subTriggers) {
this.subTriggers = subTriggers;
}
@@ -107,15 +108,16 @@
}
/**
- * Subclasses should override this to return the {@link #getContinuationTrigger} of this
- * {@link Trigger}. For convenience, this is provided the continuation trigger of each of the
+ * Subclasses should override this to return the {@link #getContinuationTrigger} of this {@link
+ * Trigger}. For convenience, this is provided the continuation trigger of each of the
* sub-triggers in the same order as {@link #subTriggers}.
*
- * @param continuationTriggers null if {@link #subTriggers} is null, otherwise contains the
- * result of {@link #getContinuationTrigger()} on each of the
- * subTriggers in the same order.
+ * @param continuationTriggers {@code null} if {@link #subTriggers} is {@code null}, otherwise
+ * contains the result of {@link #getContinuationTrigger()} on each of the subTriggers in the
+ * same order.
*/
- protected abstract Trigger getContinuationTrigger(List<Trigger> continuationTriggers);
+ @Nullable
+ protected abstract Trigger getContinuationTrigger(@Nullable List<Trigger> continuationTriggers);
/**
* <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
@@ -224,7 +226,7 @@
*/
@Internal
public abstract static class OnceTrigger extends Trigger {
- protected OnceTrigger(List<Trigger> subTriggers) {
+ protected OnceTrigger(@Nullable List<Trigger> subTriggers) {
super(subTriggers);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index dc4863b..3ec8136 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -19,10 +19,10 @@
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Ordering;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
@@ -141,6 +141,7 @@
*/
@AutoValue
public abstract class Window<T> extends PTransform<PCollection<T>, PCollection<T>> {
+
/**
* Specifies the conditions under which a final pane will be created when a window is permanently
* closed.
@@ -161,6 +162,24 @@
}
/**
+ * Specifies the conditions under which an on-time pane will be created when a window is closed.
+ */
+ public enum OnTimeBehavior {
+ /**
+ * Always fire the on-time pane. Even if there is no new data since the previous firing,
+ * an element will be produced.
+ *
+ * <p>This is the default behavior.
+ */
+ FIRE_ALWAYS,
+ /**
+ * Only fire the on-time pane if there is new data since the previous firing.
+ */
+ FIRE_IF_NON_EMPTY
+
+ }
+
+ /**
* Creates a {@code Window} {@code PTransform} that uses the given
* {@link WindowFn} to window the data.
*
@@ -193,6 +212,7 @@
@Nullable abstract AccumulationMode getAccumulationMode();
@Nullable abstract Duration getAllowedLateness();
@Nullable abstract ClosingBehavior getClosingBehavior();
+ @Nullable abstract OnTimeBehavior getOnTimeBehavior();
@Nullable abstract TimestampCombiner getTimestampCombiner();
abstract Builder<T> toBuilder();
@@ -204,6 +224,7 @@
abstract Builder<T> setAccumulationMode(AccumulationMode mode);
abstract Builder<T> setAllowedLateness(Duration allowedLateness);
abstract Builder<T> setClosingBehavior(ClosingBehavior closingBehavior);
+ abstract Builder<T> setOnTimeBehavior(OnTimeBehavior onTimeBehavior);
abstract Builder<T> setTimestampCombiner(TimestampCombiner timestampCombiner);
abstract Window<T> build();
@@ -297,6 +318,15 @@
}
/**
+ * <b><i>(Experimental)</i></b> Override the default {@link OnTimeBehavior}, to control
+ * whether to output an empty on-time pane.
+ */
+ @Experimental(Kind.TRIGGER)
+ public Window<T> withOnTimeBehavior(OnTimeBehavior behavior) {
+ return toBuilder().setOnTimeBehavior(behavior).build();
+ }
+
+ /**
* Get the output strategy of this {@link Window Window PTransform}. For internal use
* only.
*/
@@ -313,11 +343,15 @@
result = result.withMode(getAccumulationMode());
}
if (getAllowedLateness() != null) {
- result = result.withAllowedLateness(getAllowedLateness());
+ result = result.withAllowedLateness(Ordering.natural().max(getAllowedLateness(),
+ inputStrategy.getAllowedLateness()));
}
if (getClosingBehavior() != null) {
result = result.withClosingBehavior(getClosingBehavior());
}
+ if (getOnTimeBehavior() != null) {
+ result = result.withOnTimeBehavior(getOnTimeBehavior());
+ }
if (getTimestampCombiner() != null) {
result = result.withTimestampCombiner(getTimestampCombiner());
}
@@ -366,6 +400,7 @@
WindowingStrategy<?, ?> outputStrategy =
getOutputStrategyInternal(input.getWindowingStrategy());
+
if (getWindowFn() == null) {
// A new PCollection must be created in case input is reused in a different location as the
// two PCollections will, in general, have a different windowing strategy.
@@ -417,11 +452,6 @@
}
@Override
- protected Coder<?> getDefaultOutputCoder(PCollection<T> input) {
- return input.getCoder();
- }
-
- @Override
protected String getKindString() {
return "Window.Into()";
}
@@ -448,7 +478,7 @@
@Override
public PCollection<T> expand(PCollection<T> input) {
return PCollection.createPrimitiveOutputInternal(
- input.getPipeline(), updatedStrategy, input.isBounded());
+ input.getPipeline(), updatedStrategy, input.isBounded(), input.getCoder());
}
@Override
@@ -456,6 +486,7 @@
original.populateDisplayData(builder);
}
+ @Nullable
public WindowFn<T, ?> getWindowFn() {
return updatedStrategy.getWindowFn();
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
index 52ebd61..ffe85f3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
@@ -114,10 +114,31 @@
/**
* Returns whether this performs the same merging as the given
* {@code WindowFn}.
+ *
+ * @deprecated please override verifyCompatibility to throw a useful error message;
+ * we will remove isCompatible at version 3.0.0
*/
+ @Deprecated
public abstract boolean isCompatible(WindowFn<?, ?> other);
/**
+ * Throw {@link IncompatibleWindowException} if this WindowFn does not perform the same merging as
+ * the given ${@code WindowFn}.
+ *
+ * @throws IncompatibleWindowException if compared WindowFns are not compatible.
+ */
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is not compatible with %s",
+ this.getClass().getSimpleName(),
+ other.getClass().getSimpleName()));
+ }
+ }
+
+ /**
* Returns the {@link Coder} used for serializing the windows used
* by this windowFn.
*/
@@ -159,6 +180,17 @@
}
/**
+ * Returns true if this {@link WindowFn} always assigns an element to exactly one window.
+ *
+ * <p>If this varies per-element, or cannot be determined, conservatively return false.
+ *
+ * <p>By default, returns false.
+ */
+ public boolean assignsToOneWindow() {
+ return false;
+ }
+
+ /**
* Returns a {@link TypeDescriptor} capturing what is known statically about the window type of
* this {@link WindowFn} instance's most-derived class.
*
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/package-info.java
index 406e279..332a7b0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/package-info.java
@@ -47,4 +47,8 @@
* {@link org.apache.beam.sdk.transforms.windowing.AfterWatermark} for details on the
* watermark.
*/
+@DefaultAnnotation(NonNull.class)
package org.apache.beam.sdk.transforms.windowing;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
index 735190b..1266d75 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
@@ -834,6 +834,8 @@
.pruningPattern("org[.]apache[.]beam[.].*Test")
// Exposes Guava, but not intended for users
.pruningClassName("org.apache.beam.sdk.util.common.ReflectHelpers")
+ // test only
+ .pruningClassName("org.apache.beam.sdk.testing.InterceptingUrlClassLoader")
.pruningPrefix("java");
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
index 271bce0..2f9e0493 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
@@ -75,7 +75,7 @@
@Beta
final class ClassPath {
- private static final Logger logger = LoggerFactory.getLogger(ClassPath.class.getName());
+ private static final Logger LOG = LoggerFactory.getLogger(ClassPath.class.getName());
private static final Predicate<ClassInfo> IS_TOP_LEVEL =
new Predicate<ClassInfo>() {
@@ -374,7 +374,7 @@
return;
}
} catch (SecurityException e) {
- logger.warn("Cannot access " + file + ": " + e);
+ LOG.warn("Cannot access " + file + ": " + e);
return;
}
if (file.isDirectory()) {
@@ -429,7 +429,7 @@
url = getClassPathEntry(jarFile, path);
} catch (MalformedURLException e) {
// Ignore bad entry
- logger.warn("Invalid Class-Path entry: " + path);
+ LOG.warn("Invalid Class-Path entry: " + path);
continue;
}
if (url.getProtocol().equals("file")) {
@@ -509,7 +509,7 @@
throws IOException {
File[] files = directory.listFiles();
if (files == null) {
- logger.warn("Cannot read directory " + directory);
+ LOG.warn("Cannot read directory " + directory);
// IO error, just skip the directory
return;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
index da77829..cfd8fde 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
@@ -153,7 +153,7 @@
* {@link Coder}.
*/
public static <T> T clone(Coder<T> coder, T value) throws CoderException {
- return decodeFromByteArray(coder, encodeToByteArray(coder, value, Coder.Context.OUTER));
+ return decodeFromByteArray(coder, encodeToByteArray(coder, value));
}
/**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java
new file mode 100644
index 0000000..0800b21
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * Wrapper class holding the necessary information to serialize a {@link DoFn}.
+ *
+ * @param <InputT> the type of the (main) input elements of the {@link DoFn}
+ * @param <OutputT> the type of the (main) output elements of the {@link DoFn}
+ */
+public class DoFnInfo<InputT, OutputT> implements Serializable {
+ private final DoFn<InputT, OutputT> doFn;
+ private final WindowingStrategy<?, ?> windowingStrategy;
+ private final Iterable<PCollectionView<?>> sideInputViews;
+ private final Coder<InputT> inputCoder;
+ private final long mainOutput;
+ private final Map<Long, TupleTag<?>> outputMap;
+
+ /**
+ * Creates a {@link DoFnInfo} for the given {@link DoFn}.
+ */
+ public static <InputT, OutputT> DoFnInfo<InputT, OutputT> forFn(
+ DoFn<InputT, OutputT> doFn,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Iterable<PCollectionView<?>> sideInputViews,
+ Coder<InputT> inputCoder,
+ long mainOutput,
+ Map<Long, TupleTag<?>> outputMap) {
+ return new DoFnInfo<>(
+ doFn, windowingStrategy, sideInputViews, inputCoder, mainOutput, outputMap);
+ }
+
+ public DoFnInfo<InputT, OutputT> withFn(DoFn<InputT, OutputT> newFn) {
+ return DoFnInfo.forFn(newFn,
+ windowingStrategy,
+ sideInputViews,
+ inputCoder,
+ mainOutput,
+ outputMap);
+ }
+
+ private DoFnInfo(
+ DoFn<InputT, OutputT> doFn,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Iterable<PCollectionView<?>> sideInputViews,
+ Coder<InputT> inputCoder,
+ long mainOutput,
+ Map<Long, TupleTag<?>> outputMap) {
+ this.doFn = doFn;
+ this.windowingStrategy = windowingStrategy;
+ this.sideInputViews = sideInputViews;
+ this.inputCoder = inputCoder;
+ this.mainOutput = mainOutput;
+ this.outputMap = outputMap;
+ }
+
+ /** Returns the embedded function. */
+ public DoFn<InputT, OutputT> getDoFn() {
+ return doFn;
+ }
+
+ public WindowingStrategy<?, ?> getWindowingStrategy() {
+ return windowingStrategy;
+ }
+
+ public Iterable<PCollectionView<?>> getSideInputViews() {
+ return sideInputViews;
+ }
+
+ public Coder<InputT> getInputCoder() {
+ return inputCoder;
+ }
+
+ public long getMainOutput() {
+ return mainOutput;
+ }
+
+ public Map<Long, TupleTag<?>> getOutputMap() {
+ return outputMap;
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
index a61e3a6..54ac77c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
@@ -23,6 +23,7 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -84,6 +85,16 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ throw new UnsupportedOperationException(
+ String.format(
+ "%s.verifyCompatibility() should never be called."
+ + " It is a private implementation detail of sdk utilities."
+ + " This message indicates a bug in the Beam SDK.",
+ getClass().getCanonicalName()));
+ }
+
+ @Override
public Coder<BoundedWindow> windowCoder() {
// Safe because the prior WindowFn provides both the windows and the coder.
// The Coder is _not_ actually a coder for an arbitrary BoundedWindow.
@@ -100,9 +111,13 @@
getClass().getCanonicalName()));
}
- @Deprecated
@Override
public Instant getOutputTime(Instant inputTimestamp, BoundedWindow window) {
return inputTimestamp;
}
+
+ @Override
+ public boolean assignsToOneWindow() {
+ return true;
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MutationDetectors.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MutationDetectors.java
index 3b593bf..79b960a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MutationDetectors.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MutationDetectors.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.sdk.util;
-import java.util.Arrays;
-import java.util.Objects;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
@@ -30,12 +28,12 @@
private MutationDetectors() {}
/**
- * Creates a new {@code MutationDetector} for the provided {@code value} that uses the provided
- * {@link Coder} to perform deep copies and comparisons by serializing and deserializing values.
- *
- * <p>It is permissible for {@code value} to be {@code null}. Since {@code null} is immutable,
- * the mutation check will always succeed.
- */
+ * Creates a new {@code MutationDetector} for the provided {@code value} that uses the provided
+ * {@link Coder} to perform deep copies and comparisons by serializing and deserializing values.
+ *
+ * <p>It is permissible for {@code value} to be {@code null}. Since {@code null} is immutable,
+ * the mutation check will always succeed.
+ */
public static <T> MutationDetector forValueWithCoder(T value, Coder<T> coder)
throws CoderException {
if (value == null) {
@@ -59,7 +57,6 @@
* A {@link MutationDetector} for {@code null}, which is immutable.
*/
private static class NoopMutationDetector implements MutationDetector {
-
@Override
public void verifyUnmodified() { }
@@ -76,6 +73,7 @@
private static class CodedValueMutationDetector<T> implements MutationDetector {
private final Coder<T> coder;
+ private final T clonedOriginalValue;
/**
* A saved pointer to an in-memory value provided upon construction, which we will check for
@@ -97,11 +95,23 @@
private final T clonedOriginalObject;
/**
+ * The structural value from {@link #possiblyModifiedObject}. It will be used during every call
+ * to {@link #verifyUnmodified}, which could be called many times throughout the lifetime of
+ * this {@link CodedValueMutationDetector}.
+ */
+ private final Object originalStructuralValue;
+
+ /**
* Create a mutation detector for the provided {@code value}, using the provided {@link Coder}
* for cloning and checking serialized forms for equality.
*/
public CodedValueMutationDetector(T value, Coder<T> coder) throws CoderException {
this.coder = coder;
+ // We need to clone the original value before getting it's structural value.
+ // If the object is consistent with equals, the Structural value will be the
+ // exact same object reference making it impossible to detect changes.
+ clonedOriginalValue = CoderUtils.clone(coder, value);
+ this.originalStructuralValue = coder.structuralValue(clonedOriginalValue);
this.possiblyModifiedObject = value;
this.encodedOriginalObject = CoderUtils.encodeToByteArray(coder, value);
this.clonedOriginalObject = CoderUtils.decodeFromByteArray(coder, encodedOriginalObject);
@@ -117,49 +127,16 @@
}
private void verifyUnmodifiedThrowingCheckedExceptions() throws CoderException {
- // If either object believes they are equal, we trust that and short-circuit deeper checks.
- if (Objects.equals(possiblyModifiedObject, clonedOriginalObject)
- || Objects.equals(clonedOriginalObject, possiblyModifiedObject)) {
- return;
+ // Since there is no guarantee that cloning an object via the coder will
+ // return the exact same type as value, We are cloning the possiblyModifiedObject
+ // before getting it's structural value. This way we are guaranteed to compare the same
+ // types.
+ T possiblyModifiedClonedValue = CoderUtils.clone(coder, possiblyModifiedObject);
+ Object newStructuralValue = coder.structuralValue(possiblyModifiedClonedValue);
+ if (originalStructuralValue.equals(newStructuralValue)) {
+ return;
}
-
- // Since retainedObject is in general an instance of a subclass of T, when it is cloned to
- // clonedObject using a Coder<T>, the two will generally be equivalent viewed as a T, but in
- // general neither retainedObject.equals(clonedObject) nor clonedObject.equals(retainedObject)
- // will hold.
- //
- // For example, CoderUtils.clone(IterableCoder<Integer>, IterableSubclass<Integer>) will
- // produce an ArrayList<Integer> with the same contents as the IterableSubclass, but the
- // latter will quite reasonably not consider itself equivalent to an ArrayList (and vice
- // versa).
- //
- // To enable a reasonable comparison, we clone retainedObject again here, converting it to
- // the same sort of T that the Coder<T> output when it created clonedObject.
- T clonedPossiblyModifiedObject = CoderUtils.clone(coder, possiblyModifiedObject);
-
- // If deepEquals() then we trust the equals implementation.
- // This deliberately allows fields to escape this check.
- if (Objects.deepEquals(clonedPossiblyModifiedObject, clonedOriginalObject)) {
- return;
- }
-
- // If not deepEquals(), the class may just have a poor equals() implementation.
- // So we next try checking their serialized forms. We re-serialize instead of checking
- // encodedObject, because the Coder may treat it differently.
- //
- // For example, an unbounded Iterable will be encoded in an unbounded way, but decoded into an
- // ArrayList, which will then be re-encoded in a bounded format. So we really do need to
- // encode-decode-encode retainedObject.
- if (Arrays.equals(
- CoderUtils.encodeToByteArray(coder, clonedOriginalObject),
- CoderUtils.encodeToByteArray(coder, clonedPossiblyModifiedObject))) {
- return;
- }
-
- // If we got here, then they are not deepEquals() and do not have deepEquals() encodings.
- // Even if there is some conceptual sense in which the objects are equivalent, it has not
- // been adequately expressed in code.
- illegalMutation(clonedOriginalObject, clonedPossiblyModifiedObject);
+ illegalMutation(clonedOriginalObject, possiblyModifiedClonedValue);
}
private void illegalMutation(T previousValue, T newValue) throws CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
new file mode 100644
index 0000000..4951958f
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+
+/**
+ * A wrapper around {@link Throwable} that preserves the stack trace on serialization, unlike
+ * regular {@link Throwable}.
+ */
+public final class SerializableThrowable implements Serializable {
+ @Nullable private final Throwable throwable;
+ @Nullable private final StackTraceElement[] stackTrace;
+
+ public SerializableThrowable(@Nullable Throwable t) {
+ this.throwable = t;
+ this.stackTrace = (t == null) ? null : t.getStackTrace();
+ }
+
+ @Nullable
+ public Throwable getThrowable() {
+ return throwable;
+ }
+
+ private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
+ is.defaultReadObject();
+ if (throwable != null) {
+ throwable.setStackTrace(stackTrace);
+ }
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
index d4bfd0b..cf5a6f3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
@@ -24,12 +24,16 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.ObjectStreamClass;
import java.io.Serializable;
+import java.lang.reflect.Proxy;
import java.util.Arrays;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;
@@ -67,7 +71,7 @@
public static Object deserializeFromByteArray(byte[] encodedValue,
String description) {
try {
- try (ObjectInputStream ois = new ObjectInputStream(
+ try (ObjectInputStream ois = new ContextualObjectInputStream(
new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
return ois.readObject();
}
@@ -79,16 +83,31 @@
}
public static <T extends Serializable> T ensureSerializable(T value) {
- @SuppressWarnings("unchecked")
- T copy = (T) deserializeFromByteArray(serializeToByteArray(value),
- value.toString());
- return copy;
+ return clone(value);
}
public static <T extends Serializable> T clone(T value) {
+ final Thread thread = Thread.currentThread();
+ final ClassLoader tccl = thread.getContextClassLoader();
+ ClassLoader loader = tccl;
+ try {
+ if (tccl.loadClass(value.getClass().getName()) != value.getClass()) {
+ loader = value.getClass().getClassLoader();
+ }
+ } catch (final NoClassDefFoundError | ClassNotFoundException e) {
+ loader = value.getClass().getClassLoader();
+ }
+ if (loader == null) {
+ loader = tccl; // will likely fail but the best we can do
+ }
+ thread.setContextClassLoader(loader);
@SuppressWarnings("unchecked")
- T copy = (T) deserializeFromByteArray(serializeToByteArray(value),
- value.toString());
+ final T copy;
+ try {
+ copy = (T) deserializeFromByteArray(serializeToByteArray(value), value.toString());
+ } finally {
+ thread.setContextClassLoader(tccl);
+ }
return copy;
}
@@ -144,4 +163,40 @@
exn);
}
}
+
+ private static final class ContextualObjectInputStream extends ObjectInputStream {
+ private ContextualObjectInputStream(final InputStream in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ protected Class<?> resolveClass(final ObjectStreamClass classDesc)
+ throws IOException, ClassNotFoundException {
+ // note: staying aligned on JVM default but can need class filtering here to avoid 0day issue
+ final String n = classDesc.getName();
+ final ClassLoader classloader = ReflectHelpers.findClassLoader();
+ try {
+ return Class.forName(n, false, classloader);
+ } catch (final ClassNotFoundException e) {
+ return super.resolveClass(classDesc);
+ }
+ }
+
+ @Override
+ protected Class resolveProxyClass(final String[] interfaces)
+ throws IOException, ClassNotFoundException {
+ final ClassLoader classloader = ReflectHelpers.findClassLoader();
+
+ final Class[] cinterfaces = new Class[interfaces.length];
+ for (int i = 0; i < interfaces.length; i++) {
+ cinterfaces[i] = classloader.loadClass(interfaces[i]);
+ }
+
+ try {
+ return Proxy.getProxyClass(classloader, cinterfaces);
+ } catch (final IllegalArgumentException e) {
+ throw new ClassNotFoundException(null, e);
+ }
+ }
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
new file mode 100644
index 0000000..999f27a
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.values;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+
+/**
+ * {@link BeamRecord} is an immutable tuple-like type to represent one element in a
+ * {@link PCollection}. The fields are described with a {@link BeamRecordType}.
+ *
+ * <p>By default, {@link BeamRecordType} only contains the name for each field. It
+ * can be extended to support more sophisticated validation by overwriting
+ * {@link BeamRecordType#validateValueType(int, Object)}.
+ *
+ * <p>A Coder {@link BeamRecordCoder} is provided, which wraps the Coder for each data field.
+ */
+@Experimental
+public class BeamRecord implements Serializable {
+ //immutable list of field values.
+ private List<Object> dataValues;
+ private BeamRecordType dataType;
+
+ /**
+ * Creates a BeamRecord.
+ * @param dataType type of the record
+ * @param rawDataValues values of the record, record's size must match size of
+ * the {@code BeamRecordType}, or can be null, if it is null
+ * then every field is null.
+ */
+ public BeamRecord(BeamRecordType dataType, List<Object> rawDataValues) {
+ if (dataType.getFieldNames().size() != rawDataValues.size()) {
+ throw new IllegalArgumentException(
+ "Field count in BeamRecordType(" + dataType.getFieldNames().size()
+ + ") and rawDataValues(" + rawDataValues.size() + ") must match!");
+ }
+
+ this.dataType = dataType;
+ this.dataValues = new ArrayList<>(dataType.getFieldCount());
+
+ for (int idx = 0; idx < dataType.getFieldCount(); ++idx) {
+ dataValues.add(null);
+ }
+
+ for (int idx = 0; idx < dataType.getFieldCount(); ++idx) {
+ addField(idx, rawDataValues.get(idx));
+ }
+ }
+
+ /**
+ * see {@link #BeamRecord(BeamRecordType, List)}.
+ */
+ public BeamRecord(BeamRecordType dataType, Object... rawdataValues) {
+ this(dataType, Arrays.asList(rawdataValues));
+ }
+
+ private void addField(int index, Object fieldValue) {
+ dataType.validateValueType(index, fieldValue);
+ dataValues.set(index, fieldValue);
+ }
+
+ /**
+ * Get value by field name.
+ */
+ public Object getFieldValue(String fieldName) {
+ return getFieldValue(dataType.getFieldNames().indexOf(fieldName));
+ }
+
+ /**
+ * Get a {@link Byte} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Byte getByte(String fieldName) {
+ return (Byte) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Short} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Short getShort(String fieldName) {
+ return (Short) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Integer} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Integer getInteger(String fieldName) {
+ return (Integer) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Float} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Float getFloat(String fieldName) {
+ return (Float) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Double} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Double getDouble(String fieldName) {
+ return (Double) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Long} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Long getLong(String fieldName) {
+ return (Long) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link String} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public String getString(String fieldName) {
+ return (String) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Date} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Date getDate(String fieldName) {
+ return (Date) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link GregorianCalendar} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public GregorianCalendar getGregorianCalendar(String fieldName) {
+ return (GregorianCalendar) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link BigDecimal} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public BigDecimal getBigDecimal(String fieldName) {
+ return (BigDecimal) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get a {@link Boolean} value by field name, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Boolean getBoolean(String fieldName) {
+ return (Boolean) getFieldValue(fieldName);
+ }
+
+ /**
+ * Get value by field index.
+ */
+ public Object getFieldValue(int fieldIdx) {
+ return dataValues.get(fieldIdx);
+ }
+
+ /**
+ * Get a {@link Byte} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Byte getByte(int idx) {
+ return (Byte) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Short} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Short getShort(int idx) {
+ return (Short) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Integer} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Integer getInteger(int idx) {
+ return (Integer) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Float} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Float getFloat(int idx) {
+ return (Float) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Double} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Double getDouble(int idx) {
+ return (Double) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Long} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Long getLong(int idx) {
+ return (Long) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link String} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public String getString(int idx) {
+ return (String) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Date} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Date getDate(int idx) {
+ return (Date) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link GregorianCalendar} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public GregorianCalendar getGregorianCalendar(int idx) {
+ return (GregorianCalendar) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link BigDecimal} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public BigDecimal getBigDecimal(int idx) {
+ return (BigDecimal) getFieldValue(idx);
+ }
+
+ /**
+ * Get a {@link Boolean} value by field index, {@link ClassCastException} is thrown
+ * if type doesn't match.
+ */
+ public Boolean getBoolean(int idx) {
+ return (Boolean) getFieldValue(idx);
+ }
+
+ /**
+ * Return the size of data fields.
+ */
+ public int getFieldCount() {
+ return dataValues.size();
+ }
+
+ /**
+ * Return the list of data values.
+ */
+ public List<Object> getDataValues() {
+ return Collections.unmodifiableList(dataValues);
+ }
+
+ /**
+ * Return {@link BeamRecordType} which describes the fields.
+ */
+ public BeamRecordType getDataType() {
+ return dataType;
+ }
+
+ @Override
+ public String toString() {
+ return "BeamRecord [dataValues=" + dataValues + ", dataType=" + dataType + "]";
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ BeamRecord other = (BeamRecord) obj;
+ return toString().equals(other.toString());
+ }
+
+ @Override public int hashCode() {
+ return 31 * getDataType().hashCode() + getDataValues().hashCode();
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecordType.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecordType.java
new file mode 100644
index 0000000..620361c
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecordType.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.values;
+
+import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.coders.Coder;
+
+/**
+ * {@link BeamRecordType} describes the fields in {@link BeamRecord}, extra checking can be added
+ * by overwriting {@link BeamRecordType#validateValueType(int, Object)}.
+ */
+@Experimental
+public class BeamRecordType implements Serializable{
+ private List<String> fieldNames;
+ private List<Coder> fieldCoders;
+
+ /**
+ * Create a {@link BeamRecordType} with a name and Coder for each field.
+ */
+ public BeamRecordType(List<String> fieldNames, List<Coder> fieldCoders) {
+ if (fieldNames.size() != fieldCoders.size()) {
+ throw new IllegalStateException(
+ "the size of fieldNames and fieldCoders need to be the same.");
+ }
+ this.fieldNames = fieldNames;
+ this.fieldCoders = fieldCoders;
+ }
+
+ /**
+ * Validate input fieldValue for a field.
+ * @throws IllegalArgumentException throw exception when the validation fails.
+ */
+ public void validateValueType(int index, Object fieldValue)
+ throws IllegalArgumentException{
+ //do nothing by default.
+ }
+
+ /**
+ * Return the coder for {@link BeamRecord}, which wraps {@link #fieldCoders} for each field.
+ */
+ public BeamRecordCoder getRecordCoder(){
+ return BeamRecordCoder.of(this, fieldCoders);
+ }
+
+ /**
+ * Returns an immutable list of field names.
+ */
+ public List<String> getFieldNames(){
+ return ImmutableList.copyOf(fieldNames);
+ }
+
+ /**
+ * Return the name of field by index.
+ */
+ public String getFieldNameByIndex(int index){
+ return fieldNames.get(index);
+ }
+
+ /**
+ * Find the index of a given field.
+ */
+ public int findIndexOfField(String fieldName){
+ return fieldNames.indexOf(fieldName);
+ }
+
+ /**
+ * Return the count of fields.
+ */
+ public int getFieldCount(){
+ return fieldNames.size();
+ }
+
+ @Override
+ public String toString() {
+ return "BeamRecordType [fieldsName=" + fieldNames + "]";
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index f210fd8..e8bf9b8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -20,6 +20,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
+import java.util.Collections;
+import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Internal;
@@ -226,6 +228,11 @@
return super.getName();
}
+ @Override
+ public final Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, this);
+ }
+
/**
* Sets the name of this {@link PCollection}. Returns {@code this}.
*
@@ -314,6 +321,11 @@
private IsBounded isBounded;
+ /**
+ * A local {@link TupleTag} used in the expansion of this {@link PValueBase}.
+ */
+ private final TupleTag<?> tag = new TupleTag<>();
+
private PCollection(Pipeline p) {
super(p);
}
@@ -354,10 +366,15 @@
public static <T> PCollection<T> createPrimitiveOutputInternal(
Pipeline pipeline,
WindowingStrategy<?, ?> windowingStrategy,
- IsBounded isBounded) {
- return new PCollection<T>(pipeline)
+ IsBounded isBounded,
+ @Nullable Coder<T> coder) {
+ PCollection<T> res = new PCollection<T>(pipeline)
.setWindowingStrategyInternal(windowingStrategy)
.setIsBoundedInternal(isBounded);
+ if (coder != null) {
+ res.setCoder(coder);
+ }
+ return res;
}
private static class CoderOrFailure<T> {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index 793994f..9799d0e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -24,6 +24,7 @@
import java.util.Objects;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection.IsBounded;
@@ -201,6 +202,7 @@
public static PCollectionTuple ofPrimitiveOutputsInternal(
Pipeline pipeline,
TupleTagList outputTags,
+ Map<TupleTag<?>, Coder<?>> coders,
WindowingStrategy<?, ?> windowingStrategy,
IsBounded isBounded) {
Map<TupleTag<?>, PCollection<?>> pcollectionMap = new LinkedHashMap<>();
@@ -217,10 +219,10 @@
// erasure as the correct type. When a transform adds
// elements to `outputCollection` they will be of type T.
@SuppressWarnings("unchecked")
- TypeDescriptor<Object> token = (TypeDescriptor<Object>) outputTag.getTypeDescriptor();
- PCollection<Object> outputCollection = PCollection
- .createPrimitiveOutputInternal(pipeline, windowingStrategy, isBounded)
- .setTypeDescriptor(token);
+ PCollection outputCollection =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline, windowingStrategy, isBounded, coders.get(outputTag))
+ .setTypeDescriptor((TypeDescriptor) outputTag.getTypeDescriptor());
pcollectionMap.put(outputTag, outputCollection);
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
index 74887c7..f2a3097 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
@@ -21,6 +21,7 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import java.io.IOException;
@@ -38,6 +39,7 @@
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.transforms.Materialization;
import org.apache.beam.sdk.transforms.Materializations;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
@@ -139,6 +141,18 @@
}
/**
+ * Expands a list of {@link PCollectionView} into the form needed for
+ * {@link PTransform#getAdditionalInputs()}.
+ */
+ public static Map<TupleTag<?>, PValue> toAdditionalInputs(Iterable<PCollectionView<?>> views) {
+ ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder();
+ for (PCollectionView<?> view : views) {
+ additionalInputs.put(view.getTagInternal(), view.getPCollection());
+ }
+ return additionalInputs.build();
+ }
+
+ /**
* Implementation of conversion of singleton {@code Iterable<WindowedValue<T>>} to {@code T}.
*
* <p>For internal use only.
@@ -170,6 +184,14 @@
}
/**
+ * Returns if a default value was specified.
+ */
+ @Internal
+ public boolean hasDefault() {
+ return hasDefault;
+ }
+
+ /**
* Returns the default value that was specified.
*
* <p>For internal use only.
@@ -273,6 +295,16 @@
}
}));
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof ListViewFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return ListViewFn.class.hashCode();
+ }
}
/**
@@ -491,5 +523,10 @@
public String toString() {
return MoreObjects.toStringHelper(this).add("tag", tag).toString();
}
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, pCollection);
+ }
}
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
index 1089028..71f9465 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
@@ -37,8 +37,8 @@
/**
* {@inheritDoc}.
*
- * <p>A {@link PValue} always expands into itself. Calling {@link #expand()} on a PValue is almost
- * never appropriate.
+ * @deprecated A {@link PValue} always expands into itself. Calling {@link #expand()} on a PValue
+ * is almost never appropriate.
*/
@Deprecated
Map<TupleTag<?>, PValue> expand();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
index 6f638d7..f312eac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
@@ -19,8 +19,6 @@
import static com.google.common.base.Preconditions.checkState;
-import java.util.Collections;
-import java.util.Map;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.transforms.PTransform;
@@ -87,11 +85,6 @@
private String name;
/**
- * A local {@link TupleTag} used in the expansion of this {@link PValueBase}.
- */
- private TupleTag<?> tag = new TupleTag<>();
-
- /**
* Whether this {@link PValueBase} has been finalized, and its core
* properties, e.g., name, can no longer be changed.
*/
@@ -108,11 +101,6 @@
}
@Override
- public final Map<TupleTag<?>, PValue> expand() {
- return Collections.<TupleTag<?>, PValue>singletonMap(tag, this);
- }
-
- @Override
public void finishSpecifying(PInput input, PTransform<?, ?> transform) {
finishedSpecifying = true;
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ShardedKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ShardedKey.java
new file mode 100644
index 0000000..e56af13
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ShardedKey.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.values;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/** A key and a shard number. */
+public class ShardedKey<K> implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final K key;
+ private final int shardNumber;
+
+ public static <K> ShardedKey<K> of(K key, int shardNumber) {
+ return new ShardedKey<>(key, shardNumber);
+ }
+
+ private ShardedKey(K key, int shardNumber) {
+ this.key = key;
+ this.shardNumber = shardNumber;
+ }
+
+ public K getKey() {
+ return key;
+ }
+
+ public int getShardNumber() {
+ return shardNumber;
+ }
+
+ @Override
+ public String toString() {
+ return "key: " + key + " shard: " + shardNumber;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ShardedKey)) {
+ return false;
+ }
+ ShardedKey<K> other = (ShardedKey<K>) o;
+ return Objects.equals(key, other.key) && Objects.equals(shardNumber, other.shardNumber);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, shardNumber);
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java
index 14f2cb8..dd6a0fd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java
@@ -328,30 +328,64 @@
}
/**
- * Returns a new {@code TypeDescriptor} where type variables represented by
- * {@code typeParameter} are substituted by {@code typeDescriptor}. For example, it can be used to
- * construct {@code Map<K, V>} for any {@code K} and {@code V} type: <pre> {@code
- * static <K, V> TypeDescriptor<Map<K, V>> mapOf(
- * TypeDescriptor<K> keyType, TypeDescriptor<V> valueType) {
- * return new TypeDescriptor<Map<K, V>>() {}
- * .where(new TypeParameter<K>() {}, keyType)
- * .where(new TypeParameter<V>() {}, valueType);
- * }}</pre>
+ * Returns a new {@code TypeDescriptor} where the type variable represented by {@code
+ * typeParameter} are substituted by {@code typeDescriptor}. For example, it can be used to
+ * construct {@code Map<K, V>} for any {@code K} and {@code V} type:
+ *
+ * <pre>{@code
+ * static <K, V> TypeDescriptor<Map<K, V>> mapOf(
+ * TypeDescriptor<K> keyType, TypeDescriptor<V> valueType) {
+ * return new TypeDescriptor<Map<K, V>>() {}
+ * .where(new TypeParameter<K>() {}, keyType)
+ * .where(new TypeParameter<V>() {}, valueType);
+ * }
+ * }</pre>
*
* @param <X> The parameter type
* @param typeParameter the parameter type variable
* @param typeDescriptor the actual type to substitute
*/
@SuppressWarnings("unchecked")
- public <X> TypeDescriptor<T> where(TypeParameter<X> typeParameter,
- TypeDescriptor<X> typeDescriptor) {
- TypeResolver resolver =
- new TypeResolver()
- .where(
- typeParameter.typeVariable, typeDescriptor.getType());
+ public <X> TypeDescriptor<T> where(
+ TypeParameter<X> typeParameter, TypeDescriptor<X> typeDescriptor) {
+ return where(typeParameter.typeVariable, typeDescriptor.getType());
+ }
+
+ /**
+ * A more general form of {@link #where(TypeParameter, TypeDescriptor)} that returns a new {@code
+ * TypeDescriptor} by matching {@code formal} against {@code actual} to resolve type variables in
+ * the current {@link TypeDescriptor}.
+ */
+ @SuppressWarnings("unchecked")
+ public TypeDescriptor<T> where(Type formal, Type actual) {
+ TypeResolver resolver = new TypeResolver().where(formal, actual);
return (TypeDescriptor<T>) TypeDescriptor.of(resolver.resolveType(token.getType()));
}
+ /**
+ * Returns whether this {@link TypeDescriptor} has any unresolved type parameters, as opposed to
+ * being a concrete type.
+ *
+ * <p>For example:
+ * <pre>{@code
+ * TypeDescriptor.of(new ArrayList<String>() {}.getClass()).hasUnresolvedTypeParameters()
+ * => false, because the anonymous class is instantiated with a concrete type
+ *
+ * class TestUtils {
+ * <T> ArrayList<T> createTypeErasedList() {
+ * return new ArrayList<T>() {};
+ * }
+ * }
+ *
+ * TypeDescriptor.of(TestUtils.<String>createTypeErasedList().getClass())
+ * => true, because the type variable T got type-erased and the anonymous ArrayList class
+ * is instantiated with an unresolved type variable T.
+ * }</pre>
+ */
+ public boolean hasUnresolvedParameters() {
+ return hasUnresolvedParameters(getType());
+ }
+
@Override
public String toString() {
return token.toString();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptors.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptors.java
index a4626c9..e59f84b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptors.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptors.java
@@ -17,16 +17,20 @@
*/
package org.apache.beam.sdk.values;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.List;
import java.util.Set;
+import org.apache.beam.sdk.transforms.Contextful;
+import org.apache.beam.sdk.transforms.SerializableFunction;
/**
- * A utility class containing the Java primitives for
- * {@link TypeDescriptor} equivalents. Also, has methods
- * for classes that wrap Java primitives like {@link KV},
- * {@link Set}, {@link List}, and {@link Iterable}.
+ * A utility class for creating {@link TypeDescriptor} objects for different types, such as Java
+ * primitive types, containers and {@link KV KVs} of other {@link TypeDescriptor} objects, and
+ * extracting type variables of parameterized types (e.g. extracting the {@code OutputT} type
+ * variable of a {@code DoFn<InputT, OutputT>}).
*/
public class TypeDescriptors {
/**
@@ -286,4 +290,119 @@
return typeDescriptor;
}
+
+ /**
+ * A helper interface for use with {@link #extractFromTypeParameters(Object, Class,
+ * TypeVariableExtractor)}.
+ */
+ public interface TypeVariableExtractor<InputT, OutputT> {}
+
+ /**
+ * Extracts a type from the actual type parameters of a parameterized class, subject to Java type
+ * erasure. The type to extract is specified in a way that is safe w.r.t. changing the type
+ * signature of the parameterized class, as opposed to specifying the name or index of a type
+ * variable.
+ *
+ * <p>Example of use:
+ * <pre>{@code
+ * class Foo<BarT> {
+ * private SerializableFunction<BarT, String> fn;
+ *
+ * TypeDescriptor<BarT> inferBarTypeDescriptorFromFn() {
+ * return TypeDescriptors.extractFromTypeParameters(
+ * fn,
+ * SerializableFunction.class,
+ * // The actual type of "fn" is matched against the input type of the extractor,
+ * // and the obtained values of type variables of the superclass are substituted
+ * // into the output type of the extractor.
+ * new TypeVariableExtractor<SerializableFunction<BarT, String>, BarT>() {});
+ * }
+ * }
+ * }</pre>
+ *
+ * @param instance The object being analyzed
+ * @param supertype Parameterized superclass of interest
+ * @param extractor A class for specifying the type to extract from the supertype
+ *
+ * @return A {@link TypeDescriptor} for the actual value of the result type of the extractor,
+ * potentially containing unresolved type variables if the type was erased.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, V> TypeDescriptor<V> extractFromTypeParameters(
+ T instance, Class<? super T> supertype, TypeVariableExtractor<T, V> extractor) {
+ return extractFromTypeParameters(
+ (TypeDescriptor<T>) TypeDescriptor.of(instance.getClass()), supertype, extractor);
+ }
+
+ /**
+ * Like {@link #extractFromTypeParameters(Object, Class, TypeVariableExtractor)}, but takes a
+ * {@link TypeDescriptor} of the instance being analyzed rather than the instance itself.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, V> TypeDescriptor<V> extractFromTypeParameters(
+ TypeDescriptor<T> type, Class<? super T> supertype, TypeVariableExtractor<T, V> extractor) {
+ // Get the type signature of the extractor, e.g.
+ // TypeVariableExtractor<SerializableFunction<BarT, String>, BarT>
+ TypeDescriptor<TypeVariableExtractor<T, V>> extractorSupertype =
+ (TypeDescriptor<TypeVariableExtractor<T, V>>)
+ TypeDescriptor.of(extractor.getClass()).getSupertype(TypeVariableExtractor.class);
+
+ // Get the actual type argument, e.g. SerializableFunction<BarT, String>
+ Type inputT = ((ParameterizedType) extractorSupertype.getType()).getActualTypeArguments()[0];
+
+ // Get the actual supertype of the type being analyzed, hopefully with all type parameters
+ // resolved, e.g. SerializableFunction<Integer, String>
+ TypeDescriptor supertypeDescriptor = type.getSupertype(supertype);
+
+ // Substitute actual supertype into the extractor, e.g.
+ // TypeVariableExtractor<SerializableFunction<Integer, String>, Integer>
+ TypeDescriptor<TypeVariableExtractor<T, V>> extractorT =
+ extractorSupertype.where(inputT, supertypeDescriptor.getType());
+
+ // Get output of the extractor.
+ Type outputT = ((ParameterizedType) extractorT.getType()).getActualTypeArguments()[1];
+ return (TypeDescriptor<V>) TypeDescriptor.of(outputT);
+ }
+
+ /**
+ * Returns a type descriptor for the input of the given {@link SerializableFunction}, subject to
+ * Java type erasure: may contain unresolved type variables if the type was erased.
+ */
+ public static <InputT, OutputT> TypeDescriptor<InputT> inputOf(
+ SerializableFunction<InputT, OutputT> fn) {
+ return extractFromTypeParameters(
+ fn,
+ SerializableFunction.class,
+ new TypeVariableExtractor<SerializableFunction<InputT, OutputT>, InputT>() {});
+ }
+
+ /**
+ * Returns a type descriptor for the output of the given {@link SerializableFunction}, subject to
+ * Java type erasure: may contain unresolved type variables if the type was erased.
+ */
+ public static <InputT, OutputT> TypeDescriptor<OutputT> outputOf(
+ SerializableFunction<InputT, OutputT> fn) {
+ return extractFromTypeParameters(
+ fn,
+ SerializableFunction.class,
+ new TypeVariableExtractor<SerializableFunction<InputT, OutputT>, OutputT>() {});
+ }
+
+ /** Like {@link #inputOf(SerializableFunction)} but for {@link Contextful.Fn}. */
+ public static <InputT, OutputT> TypeDescriptor<InputT> inputOf(
+ Contextful.Fn<InputT, OutputT> fn) {
+ return TypeDescriptors.extractFromTypeParameters(
+ fn,
+ Contextful.Fn.class,
+ new TypeDescriptors.TypeVariableExtractor<Contextful.Fn<InputT, OutputT>, InputT>() {});
+ }
+
+ /** Like {@link #outputOf(SerializableFunction)} but for {@link Contextful.Fn}. */
+ public static <InputT, OutputT> TypeDescriptor<OutputT> outputOf(
+ Contextful.Fn<InputT, OutputT> fn) {
+ return TypeDescriptors.extractFromTypeParameters(
+ fn,
+ Contextful.Fn.class,
+ new TypeDescriptors.TypeVariableExtractor<Contextful.Fn<InputT, OutputT>, OutputT>() {});
+ }
}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
index 8a773e2..3b74e69 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
@@ -29,6 +29,7 @@
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.Trigger;
import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
+import org.apache.beam.sdk.transforms.windowing.Window.OnTimeBehavior;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.joda.time.Duration;
@@ -59,6 +60,7 @@
private final AccumulationMode mode;
private final Duration allowedLateness;
private final ClosingBehavior closingBehavior;
+ private final OnTimeBehavior onTimeBehavior;
private final TimestampCombiner timestampCombiner;
private final boolean triggerSpecified;
private final boolean modeSpecified;
@@ -71,7 +73,8 @@
AccumulationMode mode, boolean modeSpecified,
Duration allowedLateness, boolean allowedLatenessSpecified,
TimestampCombiner timestampCombiner, boolean timestampCombinerSpecified,
- ClosingBehavior closingBehavior) {
+ ClosingBehavior closingBehavior,
+ OnTimeBehavior onTimeBehavior) {
this.windowFn = windowFn;
this.trigger = trigger;
this.triggerSpecified = triggerSpecified;
@@ -80,6 +83,7 @@
this.allowedLateness = allowedLateness;
this.allowedLatenessSpecified = allowedLatenessSpecified;
this.closingBehavior = closingBehavior;
+ this.onTimeBehavior = onTimeBehavior;
this.timestampCombiner = timestampCombiner;
this.timestampCombinerSpecified = timestampCombinerSpecified;
}
@@ -98,7 +102,8 @@
AccumulationMode.DISCARDING_FIRED_PANES, false,
DEFAULT_ALLOWED_LATENESS, false,
TimestampCombiner.END_OF_WINDOW, false,
- ClosingBehavior.FIRE_IF_NON_EMPTY);
+ ClosingBehavior.FIRE_IF_NON_EMPTY,
+ OnTimeBehavior.FIRE_ALWAYS);
}
public WindowFn<T, W> getWindowFn() {
@@ -133,6 +138,10 @@
return closingBehavior;
}
+ public OnTimeBehavior getOnTimeBehavior() {
+ return onTimeBehavior;
+ }
+
public TimestampCombiner getTimestampCombiner() {
return timestampCombiner;
}
@@ -152,7 +161,8 @@
mode, modeSpecified,
allowedLateness, allowedLatenessSpecified,
timestampCombiner, timestampCombinerSpecified,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
/**
@@ -166,7 +176,8 @@
mode, true,
allowedLateness, allowedLatenessSpecified,
timestampCombiner, timestampCombinerSpecified,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
/**
@@ -183,7 +194,8 @@
mode, modeSpecified,
allowedLateness, allowedLatenessSpecified,
timestampCombiner, timestampCombinerSpecified,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
/**
@@ -197,7 +209,8 @@
mode, modeSpecified,
allowedLateness, true,
timestampCombiner, timestampCombinerSpecified,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
public WindowingStrategy<T, W> withClosingBehavior(ClosingBehavior closingBehavior) {
@@ -207,7 +220,19 @@
mode, modeSpecified,
allowedLateness, allowedLatenessSpecified,
timestampCombiner, timestampCombinerSpecified,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
+ }
+
+ public WindowingStrategy<T, W> withOnTimeBehavior(OnTimeBehavior onTimeBehavior) {
+ return new WindowingStrategy<T, W>(
+ windowFn,
+ trigger, triggerSpecified,
+ mode, modeSpecified,
+ allowedLateness, allowedLatenessSpecified,
+ timestampCombiner, timestampCombinerSpecified,
+ closingBehavior,
+ onTimeBehavior);
}
@Experimental(Experimental.Kind.OUTPUT_TIME)
@@ -219,7 +244,8 @@
mode, modeSpecified,
allowedLateness, allowedLatenessSpecified,
timestampCombiner, true,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
@Override
@@ -246,6 +272,7 @@
&& getMode().equals(other.getMode())
&& getAllowedLateness().equals(other.getAllowedLateness())
&& getClosingBehavior().equals(other.getClosingBehavior())
+ && getOnTimeBehavior().equals(other.getOnTimeBehavior())
&& getTrigger().equals(other.getTrigger())
&& getTimestampCombiner().equals(other.getTimestampCombiner())
&& getWindowFn().equals(other.getWindowFn());
@@ -278,6 +305,7 @@
mode, true,
allowedLateness, true,
timestampCombiner, true,
- closingBehavior);
+ closingBehavior,
+ onTimeBehavior);
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
index 2cc3f04..57fdd75 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
@@ -30,10 +30,9 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.io.GenerateSequence;
@@ -51,12 +50,13 @@
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.ValidatesRunner;
-import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.Max;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
@@ -394,39 +394,40 @@
}
@Test
- public void testReplacedNames() {
+ public void testReplaceWithExistingName() {
pipeline.enableAbandonedNodeEnforcement(false);
- final PCollection<String> originalInput = pipeline.apply(Create.of("foo", "bar", "baz"));
- class OriginalTransform extends PTransform<PCollection<String>, PCollection<Long>> {
+ final PCollection<Integer> originalInput = pipeline.apply(Create.of(1, 2, 3));
+ class OriginalTransform extends PTransform<PCollection<Integer>, PCollection<Integer>> {
@Override
- public PCollection<Long> expand(PCollection<String> input) {
- return input.apply("custom_name", Count.<String>globally());
+ public PCollection<Integer> expand(PCollection<Integer> input) {
+ return input.apply("custom_name", Sum.integersGlobally());
}
}
- class ReplacementTransform extends PTransform<PCollection<String>, PCollection<Long>> {
+ class ReplacementTransform extends PTransform<PCollection<Integer>, PCollection<Integer>> {
@Override
- public PCollection<Long> expand(PCollection<String> input) {
- return input.apply("custom_name", Count.<String>globally());
+ public PCollection<Integer> expand(PCollection<Integer> input) {
+ return input.apply("custom_name", Max.integersGlobally());
}
}
class ReplacementOverrideFactory
implements PTransformOverrideFactory<
- PCollection<String>, PCollection<Long>, OriginalTransform> {
- @Override
- public PTransformReplacement<PCollection<String>, PCollection<Long>> getReplacementTransform(
- AppliedPTransform<PCollection<String>, PCollection<Long>, OriginalTransform> transform) {
+ PCollection<Integer>, PCollection<Integer>, OriginalTransform> {
+
+ @Override public PTransformReplacement<PCollection<Integer>, PCollection<Integer>>
+ getReplacementTransform(
+ AppliedPTransform<PCollection<Integer>,
+ PCollection<Integer>, OriginalTransform> transform) {
return PTransformReplacement.of(originalInput, new ReplacementTransform());
}
@Override
public Map<PValue, ReplacementOutput> mapOutputs(
- Map<TupleTag<?>, PValue> outputs, PCollection<Long> newOutput) {
+ Map<TupleTag<?>, PValue> outputs, PCollection<Integer> newOutput) {
return Collections.<PValue, ReplacementOutput>singletonMap(
newOutput,
ReplacementOutput.of(
- TaggedPValue.ofExpandedValue(
- Iterables.getOnlyElement(outputs.values())),
- TaggedPValue.ofExpandedValue(newOutput)));
+ TaggedPValue.ofExpandedValue(Iterables.getOnlyElement(outputs.values())),
+ TaggedPValue.ofExpandedValue(newOutput)));
}
}
@@ -441,24 +442,26 @@
pipeline.replaceAll(
Collections.singletonList(
PTransformOverride.of(new OriginalMatcher(), new ReplacementOverrideFactory())));
- final Set<String> names = new HashSet<>();
+ final Map<String, Class<?>> nameToTransformClass = new HashMap<>();
pipeline.traverseTopologically(
new PipelineVisitor.Defaults() {
@Override
public void leaveCompositeTransform(Node node) {
if (!node.isRootNode()) {
- names.add(node.getFullName());
+ nameToTransformClass.put(node.getFullName(), node.getTransform().getClass());
}
}
@Override
public void visitPrimitiveTransform(Node node) {
- names.add(node.getFullName());
+ nameToTransformClass.put(node.getFullName(), node.getTransform().getClass());
}
});
- assertThat(names, hasItem("original_application/custom_name"));
- assertThat(names, not(hasItem("original_application/custom_name2")));
+ assertThat(nameToTransformClass.keySet(), hasItem("original_application/custom_name"));
+ assertThat(nameToTransformClass.keySet(), not(hasItem("original_application/custom_name2")));
+ Assert.assertEquals(nameToTransformClass.get("original_application/custom_name"),
+ Max.integersGlobally().getClass());
}
static class GenerateSequenceToCreateOverride
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/TestUtils.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/TestUtils.java
index 1224f10..5ccc1ac 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/TestUtils.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/TestUtils.java
@@ -17,15 +17,9 @@
*/
package org.apache.beam.sdk;
-import static org.junit.Assert.assertThat;
-
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
import org.apache.beam.sdk.values.KV;
-import org.hamcrest.CoreMatchers;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;
@@ -127,86 +121,4 @@
.appendText(")");
}
}
-
- ////////////////////////////////////////////////////////////////////////////
- // Utilities for testing CombineFns, ensuring they give correct results
- // across various permutations and shardings of the input.
-
- public static <InputT, AccumT, OutputT> void checkCombineFn(
- CombineFn<InputT, AccumT, OutputT> fn, List<InputT> input, final OutputT expected) {
- checkCombineFn(fn, input, CoreMatchers.is(expected));
- }
-
- public static <InputT, AccumT, OutputT> void checkCombineFn(
- CombineFn<InputT, AccumT, OutputT> fn, List<InputT> input, Matcher<? super OutputT> matcher) {
- checkCombineFnInternal(fn, input, matcher);
- Collections.shuffle(input);
- checkCombineFnInternal(fn, input, matcher);
- }
-
- private static <InputT, AccumT, OutputT> void checkCombineFnInternal(
- CombineFn<InputT, AccumT, OutputT> fn, List<InputT> input, Matcher<? super OutputT> matcher) {
- int size = input.size();
- checkCombineFnShards(fn, Collections.singletonList(input), matcher);
- checkCombineFnShards(fn, shardEvenly(input, 2), matcher);
- if (size > 4) {
- checkCombineFnShards(fn, shardEvenly(input, size / 2), matcher);
- checkCombineFnShards(
- fn, shardEvenly(input, (int) (size / Math.sqrt(size))), matcher);
- }
- checkCombineFnShards(fn, shardExponentially(input, 1.4), matcher);
- checkCombineFnShards(fn, shardExponentially(input, 2), matcher);
- checkCombineFnShards(fn, shardExponentially(input, Math.E), matcher);
- }
-
- public static <InputT, AccumT, OutputT> void checkCombineFnShards(
- CombineFn<InputT, AccumT, OutputT> fn,
- List<? extends Iterable<InputT>> shards,
- Matcher<? super OutputT> matcher) {
- checkCombineFnShardsInternal(fn, shards, matcher);
- Collections.shuffle(shards);
- checkCombineFnShardsInternal(fn, shards, matcher);
- }
-
- private static <InputT, AccumT, OutputT> void checkCombineFnShardsInternal(
- CombineFn<InputT, AccumT, OutputT> fn,
- Iterable<? extends Iterable<InputT>> shards,
- Matcher<? super OutputT> matcher) {
- List<AccumT> accumulators = new ArrayList<>();
- int maybeCompact = 0;
- for (Iterable<InputT> shard : shards) {
- AccumT accumulator = fn.createAccumulator();
- for (InputT elem : shard) {
- accumulator = fn.addInput(accumulator, elem);
- }
- if (maybeCompact++ % 2 == 0) {
- accumulator = fn.compact(accumulator);
- }
- accumulators.add(accumulator);
- }
- AccumT merged = fn.mergeAccumulators(accumulators);
- assertThat(fn.extractOutput(merged), matcher);
- }
-
- private static <T> List<List<T>> shardEvenly(List<T> input, int numShards) {
- List<List<T>> shards = new ArrayList<>(numShards);
- for (int i = 0; i < numShards; i++) {
- shards.add(input.subList(i * input.size() / numShards,
- (i + 1) * input.size() / numShards));
- }
- return shards;
- }
-
- private static <T> List<List<T>> shardExponentially(
- List<T> input, double base) {
- assert base > 1.0;
- List<List<T>> shards = new ArrayList<>();
- int end = input.size();
- while (end > 0) {
- int start = (int) (end / base);
- shards.add(input.subList(start, end));
- end = start;
- }
- return shards;
- }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
index 8942a9e..deecb96 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
@@ -26,10 +26,8 @@
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import com.google.common.io.ByteStreams;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
@@ -44,7 +42,7 @@
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
-import org.apache.avro.AvroTypeException;
+import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.avro.generic.GenericData;
@@ -60,6 +58,7 @@
import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.testing.CoderProperties;
+import org.apache.beam.sdk.testing.InterceptingUrlClassLoader;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -166,42 +165,16 @@
}
/**
- * A classloader that intercepts loading of Pojo and makes a new one.
- */
- private static class InterceptingUrlClassLoader extends ClassLoader {
-
- private InterceptingUrlClassLoader(ClassLoader parent) {
- super(parent);
- }
-
- @Override
- public Class<?> loadClass(String name) throws ClassNotFoundException {
- if (name.equals(AvroCoderTestPojo.class.getName())) {
- // Quite a hack?
- try {
- String classAsResource = name.replace('.', '/') + ".class";
- byte[] classBytes =
- ByteStreams.toByteArray(getParent().getResourceAsStream(classAsResource));
- return defineClass(name, classBytes, 0, classBytes.length);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- return getParent().loadClass(name);
- }
- }
- }
-
- /**
* Tests that {@link AvroCoder} works around issues in Avro where cache classes might be
* from the wrong ClassLoader, causing confusing "Cannot cast X to X" error messages.
*/
@Test
public void testTwoClassLoaders() throws Exception {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
ClassLoader loader1 =
- new InterceptingUrlClassLoader(Thread.currentThread().getContextClassLoader());
+ new InterceptingUrlClassLoader(contextClassLoader, AvroCoderTestPojo.class.getName());
ClassLoader loader2 =
- new InterceptingUrlClassLoader(Thread.currentThread().getContextClassLoader());
+ new InterceptingUrlClassLoader(contextClassLoader, AvroCoderTestPojo.class.getName());
Class<?> pojoClass1 = loader1.loadClass(AvroCoderTestPojo.class.getName());
Class<?> pojoClass2 = loader2.loadClass(AvroCoderTestPojo.class.getName());
@@ -502,7 +475,7 @@
try {
ReflectData.get().getSchema(SubclassHidingParent.class);
fail("Expected AvroTypeException");
- } catch (AvroTypeException e) {
+ } catch (AvroRuntimeException e) {
assertThat(e.getMessage(), containsString("mapField"));
assertThat(e.getMessage(), containsString("two fields named"));
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index d1113f7..b6430e5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -467,4 +467,73 @@
AutoRegistrationClassCoder.INSTANCE));
}
}
+
+ @Test
+ public void testCoderPrecedence() throws Exception {
+ CoderRegistry registry = CoderRegistry.createDefault();
+
+ // DefaultCoder precedes CoderProviderRegistrar
+ assertEquals(AvroCoder.of(MyValueA.class), registry.getCoder(MyValueA.class));
+
+ // CoderProviderRegistrar precedes SerializableCoder
+ assertEquals(MyValueBCoder.INSTANCE, registry.getCoder(MyValueB.class));
+
+ // fallbacks to SerializableCoder at last
+ assertEquals(SerializableCoder.of(MyValueC.class), registry.getCoder(MyValueC.class));
+ }
+
+ @DefaultCoder(AvroCoder.class)
+ private static class MyValueA implements Serializable {}
+
+ private static class MyValueB implements Serializable {}
+
+ private static class MyValueC implements Serializable {}
+
+ private static class MyValueACoder extends CustomCoder<MyValueA> {
+ private static final MyValueACoder INSTANCE = new MyValueACoder();
+
+ @Override
+ public void encode(MyValueA value, OutputStream outStream) throws CoderException, IOException {}
+
+ @Override
+ public MyValueA decode(InputStream inStream) throws CoderException, IOException {
+ return null;
+ }
+ }
+
+ /**
+ * A {@link CoderProviderRegistrar} to demonstrate default {@link Coder} registration.
+ */
+ @AutoService(CoderProviderRegistrar.class)
+ public static class MyValueACoderProviderRegistrar implements CoderProviderRegistrar {
+ @Override
+ public List<CoderProvider> getCoderProviders() {
+ return ImmutableList.of(
+ CoderProviders.forCoder(TypeDescriptor.of(MyValueA.class), MyValueACoder.INSTANCE));
+ }
+ }
+
+ private static class MyValueBCoder extends CustomCoder<MyValueB> {
+ private static final MyValueBCoder INSTANCE = new MyValueBCoder();
+
+ @Override
+ public void encode(MyValueB value, OutputStream outStream) throws CoderException, IOException {}
+
+ @Override
+ public MyValueB decode(InputStream inStream) throws CoderException, IOException {
+ return null;
+ }
+ }
+
+ /**
+ * A {@link CoderProviderRegistrar} to demonstrate default {@link Coder} registration.
+ */
+ @AutoService(CoderProviderRegistrar.class)
+ public static class MyValueBCoderProviderRegistrar implements CoderProviderRegistrar {
+ @Override
+ public List<CoderProvider> getCoderProviders() {
+ return ImmutableList.of(
+ CoderProviders.forCoder(TypeDescriptor.of(MyValueB.class), MyValueBCoder.INSTANCE));
+ }
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
index 1db7a2b..d50bc0a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
@@ -71,7 +71,7 @@
@RunWith(Parameterized.class)
public class CommonCoderTest {
private static final String STANDARD_CODERS_YAML_PATH =
- "/org/apache/beam/fn/v1/standard_coders.yaml";
+ "/org/apache/beam/model/fnexecution/v1/standard_coders.yaml";
private static final Map<String, Class<?>> coders = ImmutableMap.<String, Class<?>>builder()
.put("urn:beam:coders:bytes:0.1", ByteCoder.class)
@@ -130,7 +130,7 @@
private static List<OneCoderTestSpec> loadStandardCodersSuite() throws IOException {
InputStream stream = CommonCoderTest.class.getResourceAsStream(STANDARD_CODERS_YAML_PATH);
if (stream == null) {
- fail("null stream");
+ fail("Could not load standard coder specs as resource:" + STANDARD_CODERS_YAML_PATH);
}
// Would like to use the InputStream directly with Jackson, but Jackson does not seem to
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
index aa8d94c..274fef4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
@@ -89,7 +89,8 @@
private static class OldCustomSerializableCoder extends SerializableCoder<OldCustomRecord> {
// Extending SerializableCoder isn't trivial, but it can be done.
- @Deprecated // old form using a Class
+
+ // Old form using a Class.
@SuppressWarnings("unchecked")
public static <T extends Serializable> SerializableCoder<T> of(Class<T> recordType) {
checkArgument(OldCustomRecord.class.isAssignableFrom(recordType));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/VoidCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/VoidCoderTest.java
index e618dbb..4e0f1b7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/VoidCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/VoidCoderTest.java
@@ -18,6 +18,8 @@
package org.apache.beam.sdk.coders;
import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -30,11 +32,18 @@
*/
@RunWith(JUnit4.class)
public class VoidCoderTest {
-
private static final Coder<Void> TEST_CODER = VoidCoder.of();
@Test
public void testEncodedTypeDescriptor() throws Exception {
assertThat(TEST_CODER.getEncodedTypeDescriptor(), equalTo(TypeDescriptor.of(Void.class)));
}
+
+ @Test
+ public void testStructuralValueSharesSameObject() {
+ assertEquals(TEST_CODER.structuralValue(null), TEST_CODER.structuralValue(null));
+ // This is a minor performance optimization to not encode and compare empty byte
+ // arrays.
+ assertSame(TEST_CODER.structuralValue(null), TEST_CODER.structuralValue(null));
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
index d71f2f7..239c9f4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
@@ -19,6 +19,7 @@
import static com.google.common.base.MoreObjects.firstNonNull;
import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC;
+import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasItem;
@@ -28,18 +29,24 @@
import static org.junit.Assert.assertTrue;
import com.google.common.base.MoreObjects;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
@@ -47,6 +54,7 @@
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.reflect.Nullable;
@@ -56,8 +64,9 @@
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
-import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -65,13 +74,23 @@
import org.apache.beam.sdk.testing.UsesTestStream;
import org.apache.beam.sdk.testing.ValidatesRunner;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.Watch;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TimestampedValue;
import org.joda.time.Duration;
import org.joda.time.Instant;
@@ -83,20 +102,19 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for AvroIO Read and Write transforms.
- */
+/** Tests for AvroIO Read and Write transforms. */
@RunWith(JUnit4.class)
-public class AvroIOTest {
+public class AvroIOTest implements Serializable {
@Rule
- public TestPipeline p = TestPipeline.create();
+ public transient TestPipeline writePipeline = TestPipeline.create();
@Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
+ public transient TestPipeline readPipeline = TestPipeline.create();
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
+ @Rule public transient TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Rule public transient ExpectedException expectedException = ExpectedException.none();
@Test
public void testAvroIOGetName() {
@@ -108,11 +126,14 @@
static class GenericClass {
int intField;
String stringField;
+
public GenericClass() {}
- public GenericClass(int intValue, String stringValue) {
- this.intField = intValue;
- this.stringField = stringValue;
+
+ public GenericClass(int intField, String stringField) {
+ this.intField = intField;
+ this.stringField = stringField;
}
+
@Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
@@ -120,10 +141,12 @@
.add("stringField", stringField)
.toString();
}
+
@Override
public int hashCode() {
return Objects.hash(intField, stringField);
}
+
@Override
public boolean equals(Object other) {
if (other == null || !(other instanceof GenericClass)) {
@@ -134,78 +157,385 @@
}
}
+ private static class ParseGenericClass
+ implements SerializableFunction<GenericRecord, GenericClass> {
+ @Override
+ public GenericClass apply(GenericRecord input) {
+ return new GenericClass(
+ (int) input.get("intField"), input.get("stringField").toString());
+ }
+ }
+
+ private static final String SCHEMA_STRING =
+ "{\"namespace\": \"example.avro\",\n"
+ + " \"type\": \"record\",\n"
+ + " \"name\": \"AvroGeneratedUser\",\n"
+ + " \"fields\": [\n"
+ + " {\"name\": \"name\", \"type\": \"string\"},\n"
+ + " {\"name\": \"favorite_number\", \"type\": [\"int\", \"null\"]},\n"
+ + " {\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]}\n"
+ + " ]\n"
+ + "}";
+
+ private static final Schema SCHEMA = new Schema.Parser().parse(SCHEMA_STRING);
+
@Test
@Category(NeedsRunner.class)
- public void testAvroIOWriteAndReadASingleFile() throws Throwable {
- List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"),
- new GenericClass(5, "bar"));
+ public void testWriteThenReadJavaClass() throws Throwable {
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
File outputFile = tmpFolder.newFile("output.avro");
- p.apply(Create.of(values))
- .apply(AvroIO.write(GenericClass.class)
- .to(outputFile.getAbsolutePath())
- .withoutSharding());
- p.run();
+ writePipeline
+ .apply(Create.of(values))
+ .apply(
+ AvroIO.write(GenericClass.class)
+ .to(writePipeline.newProvider(outputFile.getAbsolutePath()))
+ .withoutSharding());
+ writePipeline.run();
- PCollection<GenericClass> input =
- p.apply(
+ PAssert.that(
+ readPipeline.apply(
+ "Read",
+ AvroIO.read(GenericClass.class)
+ .from(readPipeline.newProvider(outputFile.getAbsolutePath()))))
+ .containsInAnyOrder(values);
+
+ readPipeline.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteThenReadCustomType() throws Throwable {
+ List<Long> values = Arrays.asList(0L, 1L, 2L);
+ File outputFile = tmpFolder.newFile("output.avro");
+
+ writePipeline
+ .apply(Create.of(values))
+ .apply(
+ AvroIO.<Long, GenericClass>writeCustomType()
+ .to(writePipeline.newProvider(outputFile.getAbsolutePath()))
+ .withFormatFunction(new CreateGenericClass())
+ .withSchema(ReflectData.get().getSchema(GenericClass.class))
+ .withoutSharding());
+ writePipeline.run();
+
+ PAssert.that(
+ readPipeline
+ .apply(
+ "Read",
+ AvroIO.read(GenericClass.class)
+ .from(readPipeline.newProvider(outputFile.getAbsolutePath())))
+ .apply(
+ MapElements.via(
+ new SimpleFunction<GenericClass, Long>() {
+ @Override
+ public Long apply(GenericClass input) {
+ return (long) input.intField;
+ }
+ })))
+ .containsInAnyOrder(values);
+
+ readPipeline.run();
+ }
+
+ private <T extends GenericRecord> void testWriteThenReadGeneratedClass(
+ AvroIO.Write<T> writeTransform,
+ AvroIO.Read<T> readTransform
+ ) throws Exception {
+ File outputFile = tmpFolder.newFile("output.avro");
+
+ List<T> values =
+ ImmutableList.of(
+ (T) new AvroGeneratedUser("Bob", 256, null),
+ (T) new AvroGeneratedUser("Alice", 128, null),
+ (T) new AvroGeneratedUser("Ted", null, "white"));
+
+ writePipeline
+ .apply(Create.<T>of(values))
+ .apply(
+ writeTransform
+ .to(writePipeline.newProvider(outputFile.getAbsolutePath()))
+ .withoutSharding());
+ writePipeline.run();
+
+ PAssert.that(
+ readPipeline.apply(
+ "Read",
+ readTransform
+ .from(readPipeline.newProvider(outputFile.getAbsolutePath()))))
+ .containsInAnyOrder(values);
+
+ readPipeline.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteThenReadGeneratedClassWithClass() throws Throwable {
+ testWriteThenReadGeneratedClass(
+ AvroIO.write(AvroGeneratedUser.class), AvroIO.read(AvroGeneratedUser.class));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteThenReadGeneratedClassWithSchema() throws Throwable {
+ testWriteThenReadGeneratedClass(
+ AvroIO.writeGenericRecords(SCHEMA), AvroIO.readGenericRecords(SCHEMA));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteThenReadGeneratedClassWithSchemaString() throws Throwable {
+ testWriteThenReadGeneratedClass(
+ AvroIO.writeGenericRecords(SCHEMA.toString()),
+ AvroIO.readGenericRecords(SCHEMA.toString()));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteSingleFileThenReadUsingAllMethods() throws Throwable {
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
+ File outputFile = tmpFolder.newFile("output.avro");
+
+ writePipeline.apply(Create.of(values))
+ .apply(AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding());
+ writePipeline.run();
+
+ // Test the same data using all versions of read().
+ PCollection<String> path =
+ readPipeline.apply("Create path", Create.of(outputFile.getAbsolutePath()));
+ PAssert.that(
+ readPipeline.apply(
+ "Read", AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
+ .containsInAnyOrder(values);
+ PAssert.that(
+ readPipeline.apply(
+ "Read withHintMatchesManyFiles",
AvroIO.read(GenericClass.class)
- .from(outputFile.getAbsolutePath()));
+ .from(outputFile.getAbsolutePath())
+ .withHintMatchesManyFiles()))
+ .containsInAnyOrder(values);
+ PAssert.that(
+ path.apply(
+ "ReadAll", AvroIO.readAll(GenericClass.class).withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(values);
+ PAssert.that(
+ readPipeline.apply(
+ "Parse",
+ AvroIO.parseGenericRecords(new ParseGenericClass())
+ .from(outputFile.getAbsolutePath())
+ .withCoder(AvroCoder.of(GenericClass.class))))
+ .containsInAnyOrder(values);
+ PAssert.that(
+ readPipeline.apply(
+ "Parse withHintMatchesManyFiles",
+ AvroIO.parseGenericRecords(new ParseGenericClass())
+ .from(outputFile.getAbsolutePath())
+ .withCoder(AvroCoder.of(GenericClass.class))
+ .withHintMatchesManyFiles()))
+ .containsInAnyOrder(values);
+ PAssert.that(
+ path.apply(
+ "ParseAll",
+ AvroIO.parseAllGenericRecords(new ParseGenericClass())
+ .withCoder(AvroCoder.of(GenericClass.class))
+ .withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(values);
- PAssert.that(input).containsInAnyOrder(values);
- p.run();
+ readPipeline.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteThenReadMultipleFilepatterns() throws Throwable {
+ List<GenericClass> firstValues = Lists.newArrayList();
+ List<GenericClass> secondValues = Lists.newArrayList();
+ for (int i = 0; i < 10; ++i) {
+ firstValues.add(new GenericClass(i, "a" + i));
+ secondValues.add(new GenericClass(i, "b" + i));
+ }
+ writePipeline
+ .apply("Create first", Create.of(firstValues))
+ .apply(
+ "Write first",
+ AvroIO.write(GenericClass.class)
+ .to(tmpFolder.getRoot().getAbsolutePath() + "/first")
+ .withNumShards(2));
+ writePipeline
+ .apply("Create second", Create.of(secondValues))
+ .apply(
+ "Write second",
+ AvroIO.write(GenericClass.class)
+ .to(tmpFolder.getRoot().getAbsolutePath() + "/second")
+ .withNumShards(3));
+ writePipeline.run();
+
+ // Test readAll() and parseAllGenericRecords().
+ PCollection<String> paths =
+ readPipeline.apply(
+ "Create paths",
+ Create.of(
+ tmpFolder.getRoot().getAbsolutePath() + "/first*",
+ tmpFolder.getRoot().getAbsolutePath() + "/second*"));
+ PAssert.that(
+ paths.apply(
+ "Read all", AvroIO.readAll(GenericClass.class).withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(Iterables.concat(firstValues, secondValues));
+ PAssert.that(
+ paths.apply(
+ "Parse all",
+ AvroIO.parseAllGenericRecords(new ParseGenericClass())
+ .withCoder(AvroCoder.of(GenericClass.class))
+ .withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(Iterables.concat(firstValues, secondValues));
+
+ readPipeline.run();
+ }
+
+ private static class CreateGenericClass extends SimpleFunction<Long, GenericClass> {
+ @Override
+ public GenericClass apply(Long i) {
+ return new GenericClass(i.intValue(), "value" + i);
+ }
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testContinuouslyWriteAndReadMultipleFilepatterns() throws Throwable {
+ SimpleFunction<Long, GenericClass> mapFn = new CreateGenericClass();
+ List<GenericClass> firstValues = Lists.newArrayList();
+ List<GenericClass> secondValues = Lists.newArrayList();
+ for (int i = 0; i < 7; ++i) {
+ (i < 3 ? firstValues : secondValues).add(mapFn.apply((long) i));
+ }
+ // Configure windowing of the input so that it fires every time a new element is generated,
+ // so that files are written continuously.
+ Window<Long> window = Window.<Long>into(FixedWindows.of(Duration.millis(100)))
+ .withAllowedLateness(Duration.ZERO)
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))
+ .discardingFiredPanes();
+ readPipeline.apply(
+ "Sequence first",
+ GenerateSequence.from(0).to(3).withRate(1, Duration.millis(300)))
+ .apply("Window first", window)
+ .apply("Map first", MapElements.via(mapFn))
+ .apply(
+ "Write first",
+ AvroIO.write(GenericClass.class)
+ .to(tmpFolder.getRoot().getAbsolutePath() + "/first")
+ .withNumShards(2).withWindowedWrites());
+ readPipeline.apply(
+ "Sequence second",
+ GenerateSequence.from(3).to(7).withRate(1, Duration.millis(300)))
+ .apply("Window second", window)
+ .apply("Map second", MapElements.via(mapFn))
+ .apply(
+ "Write second",
+ AvroIO.write(GenericClass.class)
+ .to(tmpFolder.getRoot().getAbsolutePath() + "/second")
+ .withNumShards(3).withWindowedWrites());
+
+ // Test read(), readAll(), parse(), and parseAllGenericRecords() with watchForNewFiles().
+ PAssert.that(
+ readPipeline.apply(
+ "Read",
+ AvroIO.read(GenericClass.class)
+ .from(tmpFolder.getRoot().getAbsolutePath() + "/first*")
+ .watchForNewFiles(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3)))))
+ .containsInAnyOrder(firstValues);
+ PAssert.that(
+ readPipeline.apply(
+ "Parse",
+ AvroIO.parseGenericRecords(new ParseGenericClass())
+ .from(tmpFolder.getRoot().getAbsolutePath() + "/first*")
+ .watchForNewFiles(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3)))))
+ .containsInAnyOrder(firstValues);
+
+ PCollection<String> paths =
+ readPipeline.apply(
+ "Create paths",
+ Create.of(
+ tmpFolder.getRoot().getAbsolutePath() + "/first*",
+ tmpFolder.getRoot().getAbsolutePath() + "/second*"));
+ PAssert.that(
+ paths.apply(
+ "Read all",
+ AvroIO.readAll(GenericClass.class)
+ .watchForNewFiles(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3)))
+ .withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(Iterables.concat(firstValues, secondValues));
+ PAssert.that(
+ paths.apply(
+ "Parse all",
+ AvroIO.parseAllGenericRecords(new ParseGenericClass())
+ .withCoder(AvroCoder.of(GenericClass.class))
+ .watchForNewFiles(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3)))
+ .withDesiredBundleSizeBytes(10)))
+ .containsInAnyOrder(Iterables.concat(firstValues, secondValues));
+ readPipeline.run();
}
@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
- public void testAvroIOCompressedWriteAndReadASingleFile() throws Throwable {
- List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"),
- new GenericClass(5, "bar"));
+ public void testCompressedWriteAndReadASingleFile() throws Throwable {
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
File outputFile = tmpFolder.newFile("output.avro");
- p.apply(Create.of(values))
- .apply(AvroIO.write(GenericClass.class)
- .to(outputFile.getAbsolutePath())
- .withoutSharding()
- .withCodec(CodecFactory.deflateCodec(9)));
- p.run();
+ writePipeline.apply(Create.of(values))
+ .apply(
+ AvroIO.write(GenericClass.class)
+ .to(outputFile.getAbsolutePath())
+ .withoutSharding()
+ .withCodec(CodecFactory.deflateCodec(9)));
+ writePipeline.run();
- PCollection<GenericClass> input = p
- .apply(AvroIO.read(GenericClass.class)
- .from(outputFile.getAbsolutePath()));
+ PAssert.that(
+ readPipeline.apply(AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
+ .containsInAnyOrder(values);
+ readPipeline.run();
- PAssert.that(input).containsInAnyOrder(values);
- p.run();
- DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile),
- new GenericDatumReader());
- assertEquals("deflate", dataFileStream.getMetaString("avro.codec"));
+ try (DataFileStream dataFileStream =
+ new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) {
+ assertEquals("deflate", dataFileStream.getMetaString("avro.codec"));
+ }
}
@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
- public void testAvroIONullCodecWriteAndReadASingleFile() throws Throwable {
- List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"),
- new GenericClass(5, "bar"));
+ public void testWriteThenReadASingleFileWithNullCodec() throws Throwable {
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
File outputFile = tmpFolder.newFile("output.avro");
- p.apply(Create.of(values))
- .apply(AvroIO.write(GenericClass.class)
- .to(outputFile.getAbsolutePath())
- .withoutSharding()
- .withCodec(CodecFactory.nullCodec()));
- p.run();
+ writePipeline.apply(Create.of(values))
+ .apply(
+ AvroIO.write(GenericClass.class)
+ .to(outputFile.getAbsolutePath())
+ .withoutSharding()
+ .withCodec(CodecFactory.nullCodec()));
+ writePipeline.run();
- PCollection<GenericClass> input = p
- .apply(AvroIO.read(GenericClass.class)
- .from(outputFile.getAbsolutePath()));
+ PAssert.that(
+ readPipeline.apply(AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
+ .containsInAnyOrder(values);
+ readPipeline.run();
- PAssert.that(input).containsInAnyOrder(values);
- p.run();
- DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile),
- new GenericDatumReader());
- assertEquals("null", dataFileStream.getMetaString("avro.codec"));
+ try (DataFileStream dataFileStream =
+ new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) {
+ assertEquals("null", dataFileStream.getMetaString("avro.codec"));
+ }
}
@DefaultCoder(AvroCoder.class)
@@ -213,12 +543,15 @@
int intField;
String stringField;
@Nullable String nullableField;
+
public GenericClassV2() {}
+
public GenericClassV2(int intValue, String stringValue, String nullableValue) {
this.intField = intValue;
this.stringField = stringValue;
this.nullableField = nullableValue;
}
+
@Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
@@ -227,10 +560,12 @@
.add("nullableField", nullableField)
.toString();
}
+
@Override
public int hashCode() {
return Objects.hash(intField, stringField, nullableField);
}
+
@Override
public boolean equals(Object other) {
if (other == null || !(other instanceof GenericClassV2)) {
@@ -244,105 +579,115 @@
}
/**
- * Tests that {@code AvroIO} can read an upgraded version of an old class, as long as the
- * schema resolution process succeeds. This test covers the case when a new, {@code @Nullable}
- * field has been added.
+ * Tests that {@code AvroIO} can read an upgraded version of an old class, as long as the schema
+ * resolution process succeeds. This test covers the case when a new, {@code @Nullable} field has
+ * been added.
*
* <p>For more information, see http://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution
*/
@Test
@Category(NeedsRunner.class)
- public void testAvroIOWriteAndReadSchemaUpgrade() throws Throwable {
- List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"),
- new GenericClass(5, "bar"));
+ public void testWriteThenReadSchemaUpgrade() throws Throwable {
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
File outputFile = tmpFolder.newFile("output.avro");
- p.apply(Create.of(values))
- .apply(AvroIO.write(GenericClass.class)
- .to(outputFile.getAbsolutePath())
- .withoutSharding());
- p.run();
+ writePipeline.apply(Create.of(values))
+ .apply(AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding());
+ writePipeline.run();
- List<GenericClassV2> expected = ImmutableList.of(new GenericClassV2(3, "hi", null),
- new GenericClassV2(5, "bar", null));
+ List<GenericClassV2> expected =
+ ImmutableList.of(new GenericClassV2(3, "hi", null), new GenericClassV2(5, "bar", null));
- PCollection<GenericClassV2> input =
- p.apply(
- AvroIO.read(GenericClassV2.class)
- .from(outputFile.getAbsolutePath()));
-
- PAssert.that(input).containsInAnyOrder(expected);
- p.run();
+ PAssert.that(
+ readPipeline.apply(
+ AvroIO.read(GenericClassV2.class).from(outputFile.getAbsolutePath())))
+ .containsInAnyOrder(expected);
+ readPipeline.run();
}
private static class WindowedFilenamePolicy extends FilenamePolicy {
- final String outputFilePrefix;
+ final ResourceId outputFilePrefix;
- WindowedFilenamePolicy(String outputFilePrefix) {
+ WindowedFilenamePolicy(ResourceId outputFilePrefix) {
this.outputFilePrefix = outputFilePrefix;
}
@Override
public ResourceId windowedFilename(
- ResourceId outputDirectory, WindowedContext input, String extension) {
- String filename = String.format(
- "%s-%s-%s-of-%s-pane-%s%s%s",
- outputFilePrefix,
- input.getWindow(),
- input.getShardNumber(),
- input.getNumShards() - 1,
- input.getPaneInfo().getIndex(),
- input.getPaneInfo().isLast() ? "-final" : "",
- extension);
- return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ String filenamePrefix =
+ outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), "");
+
+ String filename =
+ String.format(
+ "%s-%s-%s-of-%s-pane-%s%s%s",
+ filenamePrefix,
+ window,
+ shardNumber,
+ numShards - 1,
+ paneInfo.getIndex(),
+ paneInfo.isLast() ? "-final" : "",
+ outputFileHints.getSuggestedFilenameSuffix());
+ return outputFilePrefix
+ .getCurrentDirectory()
+ .resolve(filename, RESOLVE_FILE);
}
@Override
public ResourceId unwindowedFilename(
- ResourceId outputDirectory, Context input, String extension) {
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
throw new UnsupportedOperationException("Expecting windowed outputs only");
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
- builder.add(DisplayData.item("fileNamePrefix", outputFilePrefix)
- .withLabel("File Name Prefix"));
+ builder.add(
+ DisplayData.item("fileNamePrefix", outputFilePrefix.toString())
+ .withLabel("File Name Prefix"));
}
}
- @Rule
- public TestPipeline windowedAvroWritePipeline = TestPipeline.create();
+ @Rule public transient TestPipeline windowedAvroWritePipeline = TestPipeline.create();
@Test
@Category({ValidatesRunner.class, UsesTestStream.class})
- public void testWindowedAvroIOWrite() throws Throwable {
+ public void testWriteWindowed() throws Throwable {
Path baseDir = Files.createTempDirectory(tmpFolder.getRoot().toPath(), "testwrite");
String baseFilename = baseDir.resolve("prefix").toString();
Instant base = new Instant(0);
ArrayList<GenericClass> allElements = new ArrayList<>();
ArrayList<TimestampedValue<GenericClass>> firstWindowElements = new ArrayList<>();
- ArrayList<Instant> firstWindowTimestamps = Lists.newArrayList(
- base.plus(Duration.standardSeconds(0)), base.plus(Duration.standardSeconds(10)),
- base.plus(Duration.standardSeconds(20)), base.plus(Duration.standardSeconds(30)));
+ ArrayList<Instant> firstWindowTimestamps =
+ Lists.newArrayList(
+ base.plus(Duration.standardSeconds(0)), base.plus(Duration.standardSeconds(10)),
+ base.plus(Duration.standardSeconds(20)), base.plus(Duration.standardSeconds(30)));
Random random = new Random();
for (int i = 0; i < 100; ++i) {
GenericClass item = new GenericClass(i, String.valueOf(i));
allElements.add(item);
- firstWindowElements.add(TimestampedValue.of(item,
- firstWindowTimestamps.get(random.nextInt(firstWindowTimestamps.size()))));
+ firstWindowElements.add(
+ TimestampedValue.of(
+ item, firstWindowTimestamps.get(random.nextInt(firstWindowTimestamps.size()))));
}
ArrayList<TimestampedValue<GenericClass>> secondWindowElements = new ArrayList<>();
- ArrayList<Instant> secondWindowTimestamps = Lists.newArrayList(
- base.plus(Duration.standardSeconds(60)), base.plus(Duration.standardSeconds(70)),
- base.plus(Duration.standardSeconds(80)), base.plus(Duration.standardSeconds(90)));
+ ArrayList<Instant> secondWindowTimestamps =
+ Lists.newArrayList(
+ base.plus(Duration.standardSeconds(60)), base.plus(Duration.standardSeconds(70)),
+ base.plus(Duration.standardSeconds(80)), base.plus(Duration.standardSeconds(90)));
for (int i = 100; i < 200; ++i) {
GenericClass item = new GenericClass(i, String.valueOf(i));
allElements.add(new GenericClass(i, String.valueOf(i)));
- secondWindowElements.add(TimestampedValue.of(item,
- secondWindowTimestamps.get(random.nextInt(secondWindowTimestamps.size()))));
+ secondWindowElements.add(
+ TimestampedValue.of(
+ item, secondWindowTimestamps.get(random.nextInt(secondWindowTimestamps.size()))));
}
TimestampedValue<GenericClass>[] firstWindowArray =
@@ -350,24 +695,30 @@
TimestampedValue<GenericClass>[] secondWindowArray =
secondWindowElements.toArray(new TimestampedValue[100]);
- TestStream<GenericClass> values = TestStream.create(AvroCoder.of(GenericClass.class))
- .advanceWatermarkTo(new Instant(0))
- .addElements(firstWindowArray[0],
- Arrays.copyOfRange(firstWindowArray, 1, firstWindowArray.length))
- .advanceWatermarkTo(new Instant(0).plus(Duration.standardMinutes(1)))
- .addElements(secondWindowArray[0],
- Arrays.copyOfRange(secondWindowArray, 1, secondWindowArray.length))
- .advanceWatermarkToInfinity();
+ TestStream<GenericClass> values =
+ TestStream.create(AvroCoder.of(GenericClass.class))
+ .advanceWatermarkTo(new Instant(0))
+ .addElements(
+ firstWindowArray[0],
+ Arrays.copyOfRange(firstWindowArray, 1, firstWindowArray.length))
+ .advanceWatermarkTo(new Instant(0).plus(Duration.standardMinutes(1)))
+ .addElements(
+ secondWindowArray[0],
+ Arrays.copyOfRange(secondWindowArray, 1, secondWindowArray.length))
+ .advanceWatermarkToInfinity();
- FilenamePolicy policy = new WindowedFilenamePolicy(baseFilename);
+ FilenamePolicy policy =
+ new WindowedFilenamePolicy(FileBasedSink.convertToFileResourceIfPossible(baseFilename));
windowedAvroWritePipeline
.apply(values)
.apply(Window.<GenericClass>into(FixedWindows.of(Duration.standardMinutes(1))))
- .apply(AvroIO.write(GenericClass.class)
- .to(baseFilename)
- .withFilenamePolicy(policy)
- .withWindowedWrites()
- .withNumShards(2));
+ .apply(
+ AvroIO.write(GenericClass.class)
+ .to(policy)
+ .withTempDirectory(
+ StaticValueProvider.of(FileSystems.matchNewResource(baseDir.toString(), true)))
+ .withWindowedWrites()
+ .withNumShards(2));
windowedAvroWritePipeline.run();
// Validate that the data written matches the expected elements in the expected order
@@ -375,11 +726,17 @@
for (int shard = 0; shard < 2; shard++) {
for (int window = 0; window < 2; window++) {
Instant windowStart = new Instant(0).plus(Duration.standardMinutes(window));
- IntervalWindow intervalWindow = new IntervalWindow(
- windowStart, Duration.standardMinutes(1));
+ IntervalWindow intervalWindow =
+ new IntervalWindow(windowStart, Duration.standardMinutes(1));
expectedFiles.add(
- new File(baseFilename + "-" + intervalWindow.toString() + "-" + shard
- + "-of-1" + "-pane-0-final"));
+ new File(
+ baseFilename
+ + "-"
+ + intervalWindow.toString()
+ + "-"
+ + shard
+ + "-of-1"
+ + "-pane-0-final"));
}
}
@@ -387,9 +744,10 @@
for (File outputFile : expectedFiles) {
assertTrue("Expected output file " + outputFile.getAbsolutePath(), outputFile.exists());
try (DataFileReader<GenericClass> reader =
- new DataFileReader<>(outputFile,
- new ReflectDatumReader<GenericClass>(
- ReflectData.get().getSchema(GenericClass.class)))) {
+ new DataFileReader<>(
+ outputFile,
+ new ReflectDatumReader<GenericClass>(
+ ReflectData.get().getSchema(GenericClass.class)))) {
Iterators.addAll(actualElements, reader);
}
outputFile.delete();
@@ -397,71 +755,241 @@
assertThat(actualElements, containsInAnyOrder(allElements.toArray()));
}
+ private static final String SCHEMA_TEMPLATE_STRING =
+ "{\"namespace\": \"example.avro\",\n"
+ + " \"type\": \"record\",\n"
+ + " \"name\": \"TestTemplateSchema$$\",\n"
+ + " \"fields\": [\n"
+ + " {\"name\": \"$$full\", \"type\": \"string\"},\n"
+ + " {\"name\": \"$$suffix\", \"type\": [\"string\", \"null\"]}\n"
+ + " ]\n"
+ + "}";
+
+ private static String schemaFromPrefix(String prefix) {
+ return SCHEMA_TEMPLATE_STRING.replace("$$", prefix);
+ }
+
+ private static GenericRecord createRecord(String record, String prefix, Schema schema) {
+ GenericRecord genericRecord = new GenericData.Record(schema);
+ genericRecord.put(prefix + "full", record);
+ genericRecord.put(prefix + "suffix", record.substring(1));
+ return genericRecord;
+ }
+
+ private static class TestDynamicDestinations
+ extends DynamicAvroDestinations<String, String, GenericRecord> {
+ ResourceId baseDir;
+ PCollectionView<Map<String, String>> schemaView;
+
+ TestDynamicDestinations(ResourceId baseDir, PCollectionView<Map<String, String>> schemaView) {
+ this.baseDir = baseDir;
+ this.schemaView = schemaView;
+ }
+
+ @Override
+ public Schema getSchema(String destination) {
+ // Return a per-destination schema.
+ String schema = sideInput(schemaView).get(destination);
+ return new Schema.Parser().parse(schema);
+ }
+
+ @Override
+ public List<PCollectionView<?>> getSideInputs() {
+ return ImmutableList.<PCollectionView<?>>of(schemaView);
+ }
+
+ @Override
+ public GenericRecord formatRecord(String record) {
+ String prefix = record.substring(0, 1);
+ return createRecord(record, prefix, getSchema(prefix));
+ }
+
+ @Override
+ public String getDestination(String element) {
+ // Destination is based on first character of string.
+ return element.substring(0, 1);
+ }
+
+ @Override
+ public String getDefaultDestination() {
+ return "";
+ }
+
+ @Override
+ public FilenamePolicy getFilenamePolicy(String destination) {
+ return DefaultFilenamePolicy.fromStandardParameters(
+ StaticValueProvider.of(
+ baseDir.resolve("file_" + destination + ".txt", RESOLVE_FILE)),
+ null,
+ null,
+ false);
+ }
+ }
+
+ private enum Sharding {
+ RUNNER_DETERMINED,
+ WITHOUT_SHARDING,
+ FIXED_3_SHARDS
+ }
+
+ private void testDynamicDestinationsWithSharding(Sharding sharding) throws Exception {
+ ResourceId baseDir =
+ FileSystems.matchNewResource(
+ Files.createTempDirectory(tmpFolder.getRoot().toPath(), "testDynamicDestinations")
+ .toString(),
+ true);
+
+ List<String> elements = Lists.newArrayList("aaaa", "aaab", "baaa", "baab", "caaa", "caab");
+ Multimap<String, GenericRecord> expectedElements = ArrayListMultimap.create();
+ Map<String, String> schemaMap = Maps.newHashMap();
+ for (String element : elements) {
+ String prefix = element.substring(0, 1);
+ String jsonSchema = schemaFromPrefix(prefix);
+ schemaMap.put(prefix, jsonSchema);
+ expectedElements.put(
+ prefix, createRecord(element, prefix, new Schema.Parser().parse(jsonSchema)));
+ }
+ PCollectionView<Map<String, String>> schemaView =
+ writePipeline
+ .apply("createSchemaView", Create.of(schemaMap))
+ .apply(View.<String, String>asMap());
+
+ PCollection<String> input =
+ writePipeline.apply("createInput", Create.of(elements).withCoder(StringUtf8Coder.of()));
+ AvroIO.TypedWrite<String, String, GenericRecord> write =
+ AvroIO.<String>writeCustomTypeToGenericRecords()
+ .to(new TestDynamicDestinations(baseDir, schemaView))
+ .withTempDirectory(baseDir);
+
+ switch (sharding) {
+ case RUNNER_DETERMINED:
+ break;
+ case WITHOUT_SHARDING:
+ write = write.withoutSharding();
+ break;
+ case FIXED_3_SHARDS:
+ write = write.withNumShards(3);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown sharding " + sharding);
+ }
+
+ input.apply(write);
+ writePipeline.run();
+
+ // Validate that the data written matches the expected elements in the expected order.
+
+ for (String prefix : expectedElements.keySet()) {
+ String shardPattern;
+ switch (sharding) {
+ case RUNNER_DETERMINED:
+ shardPattern = "*";
+ break;
+ case WITHOUT_SHARDING:
+ shardPattern = "00000-of-00001";
+ break;
+ case FIXED_3_SHARDS:
+ shardPattern = "*-of-00003";
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown sharding " + sharding);
+ }
+ String expectedFilepattern =
+ baseDir.resolve("file_" + prefix + ".txt-" + shardPattern, RESOLVE_FILE).toString();
+
+ PCollection<GenericRecord> records =
+ readPipeline.apply(
+ "read_" + prefix,
+ AvroIO.readGenericRecords(schemaFromPrefix(prefix)).from(expectedFilepattern));
+ PAssert.that(records).containsInAnyOrder(expectedElements.get(prefix));
+ }
+ readPipeline.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsRunnerDeterminedSharding() throws Exception {
+ testDynamicDestinationsWithSharding(Sharding.RUNNER_DETERMINED);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsWithoutSharding() throws Exception {
+ testDynamicDestinationsWithSharding(Sharding.WITHOUT_SHARDING);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsWithNumShards() throws Exception {
+ testDynamicDestinationsWithSharding(Sharding.FIXED_3_SHARDS);
+ }
+
@Test
public void testWriteWithDefaultCodec() throws Exception {
- AvroIO.Write<String> write = AvroIO.write(String.class)
- .to("/tmp/foo/baz");
- assertEquals(CodecFactory.deflateCodec(6).toString(), write.getCodec().toString());
+ AvroIO.Write<String> write = AvroIO.write(String.class).to("/tmp/foo/baz");
+ assertEquals(CodecFactory.deflateCodec(6).toString(), write.inner.getCodec().toString());
}
@Test
public void testWriteWithCustomCodec() throws Exception {
- AvroIO.Write<String> write = AvroIO.write(String.class)
- .to("/tmp/foo/baz")
- .withCodec(CodecFactory.snappyCodec());
- assertEquals(SNAPPY_CODEC, write.getCodec().toString());
+ AvroIO.Write<String> write =
+ AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.snappyCodec());
+ assertEquals(SNAPPY_CODEC, write.inner.getCodec().toString());
}
@Test
@SuppressWarnings("unchecked")
public void testWriteWithSerDeCustomDeflateCodec() throws Exception {
- AvroIO.Write<String> write = AvroIO.write(String.class)
- .to("/tmp/foo/baz")
- .withCodec(CodecFactory.deflateCodec(9));
+ AvroIO.Write<String> write =
+ AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.deflateCodec(9));
assertEquals(
CodecFactory.deflateCodec(9).toString(),
- SerializableUtils.clone(write.getCodec()).getCodec().toString());
+ SerializableUtils.clone(write.inner.getCodec()).getCodec().toString());
}
@Test
@SuppressWarnings("unchecked")
public void testWriteWithSerDeCustomXZCodec() throws Exception {
- AvroIO.Write<String> write = AvroIO.write(String.class)
- .to("/tmp/foo/baz")
- .withCodec(CodecFactory.xzCodec(9));
+ AvroIO.Write<String> write =
+ AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.xzCodec(9));
assertEquals(
CodecFactory.xzCodec(9).toString(),
- SerializableUtils.clone(write.getCodec()).getCodec().toString());
+ SerializableUtils.clone(write.inner.getCodec()).getCodec().toString());
}
@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
public void testMetadata() throws Exception {
- List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"),
- new GenericClass(5, "bar"));
+ List<GenericClass> values =
+ ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
File outputFile = tmpFolder.newFile("output.avro");
- p.apply(Create.of(values))
- .apply(AvroIO.write(GenericClass.class)
- .to(outputFile.getAbsolutePath())
- .withoutSharding()
- .withMetadata(ImmutableMap.<String, Object>of(
- "stringKey", "stringValue",
- "longKey", 100L,
- "bytesKey", "bytesValue".getBytes())));
- p.run();
+ writePipeline.apply(Create.of(values))
+ .apply(
+ AvroIO.write(GenericClass.class)
+ .to(outputFile.getAbsolutePath())
+ .withoutSharding()
+ .withMetadata(
+ ImmutableMap.<String, Object>of(
+ "stringKey",
+ "stringValue",
+ "longKey",
+ 100L,
+ "bytesKey",
+ "bytesValue".getBytes())));
+ writePipeline.run();
- DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile),
- new GenericDatumReader());
- assertEquals("stringValue", dataFileStream.getMetaString("stringKey"));
- assertEquals(100L, dataFileStream.getMetaLong("longKey"));
- assertArrayEquals("bytesValue".getBytes(), dataFileStream.getMeta("bytesKey"));
+ try (DataFileStream dataFileStream =
+ new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) {
+ assertEquals("stringValue", dataFileStream.getMetaString("stringKey"));
+ assertEquals(100L, dataFileStream.getMetaLong("longKey"));
+ assertArrayEquals("bytesValue".getBytes(), dataFileStream.getMeta("bytesKey"));
+ }
}
-
@SuppressWarnings("deprecation") // using AvroCoder#createDatumReader for tests.
private void runTestWrite(String[] expectedElements, int numShards) throws IOException {
File baseOutputFile = new File(tmpFolder.getRoot(), "prefix");
@@ -475,11 +1003,13 @@
System.out.println("no sharding");
write = write.withoutSharding();
}
- p.apply(Create.of(ImmutableList.copyOf(expectedElements))).apply(write);
- p.run();
+ writePipeline.apply(Create.of(ImmutableList.copyOf(expectedElements))).apply(write);
+ writePipeline.run();
String shardNameTemplate =
- firstNonNull(write.getShardTemplate(), DefaultFilenamePolicy.DEFAULT_SHARD_TEMPLATE);
+ firstNonNull(
+ write.inner.getShardTemplate(),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
assertTestOutputs(expectedElements, numShards, outputFilePrefix, shardNameTemplate);
}
@@ -493,15 +1023,22 @@
expectedFiles.add(
new File(
DefaultFilenamePolicy.constructName(
- outputFilePrefix, shardNameTemplate, "" /* no suffix */, i, numShards)));
+ FileBasedSink.convertToFileResourceIfPossible(outputFilePrefix),
+ shardNameTemplate,
+ "" /* no suffix */,
+ i,
+ numShards,
+ null,
+ null)
+ .toString()));
}
List<String> actualElements = new ArrayList<>();
for (File outputFile : expectedFiles) {
assertTrue("Expected output file " + outputFile.getName(), outputFile.exists());
try (DataFileReader<String> reader =
- new DataFileReader<>(outputFile,
- new ReflectDatumReader(ReflectData.get().getSchema(String.class)))) {
+ new DataFileReader<>(
+ outputFile, new ReflectDatumReader(ReflectData.get().getSchema(String.class)))) {
Iterators.addAll(actualElements, reader);
}
}
@@ -543,37 +1080,35 @@
AvroIO.readGenericRecords(Schema.create(Schema.Type.STRING)).from("/foo.*");
Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read);
- assertThat("AvroIO.Read should include the file pattern in its primitive transform",
- displayData, hasItem(hasDisplayItem("filePattern")));
+ assertThat(
+ "AvroIO.Read should include the file pattern in its primitive transform",
+ displayData,
+ hasItem(hasDisplayItem("filePattern")));
}
@Test
public void testWriteDisplayData() {
- AvroIO.Write<GenericClass> write = AvroIO.write(GenericClass.class)
- .to("/foo")
- .withShardNameTemplate("-SS-of-NN-")
- .withSuffix("bar")
- .withNumShards(100)
- .withCodec(CodecFactory.snappyCodec());
+ AvroIO.Write<GenericClass> write =
+ AvroIO.write(GenericClass.class)
+ .to("/foo")
+ .withShardNameTemplate("-SS-of-NN-")
+ .withSuffix("bar")
+ .withNumShards(100)
+ .withCodec(CodecFactory.snappyCodec());
DisplayData displayData = DisplayData.from(write);
assertThat(displayData, hasDisplayItem("filePrefix", "/foo"));
assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-"));
assertThat(displayData, hasDisplayItem("fileSuffix", "bar"));
- assertThat(displayData, hasDisplayItem("schema", GenericClass.class));
+ assertThat(
+ displayData,
+ hasDisplayItem(
+ "schema",
+ "{\"type\":\"record\",\"name\":\"GenericClass\",\"namespace\":\"org.apache.beam.sdk.io"
+ + ".AvroIOTest$\",\"fields\":[{\"name\":\"intField\",\"type\":\"int\"},"
+ + "{\"name\":\"stringField\",\"type\":\"string\"}]}"));
assertThat(displayData, hasDisplayItem("numShards", 100));
assertThat(displayData, hasDisplayItem("codec", CodecFactory.snappyCodec().toString()));
}
-
- @Test
- public void testWindowedWriteRequiresFilenamePolicy() {
- PCollection<String> emptyInput = p.apply(Create.empty(StringUtf8Coder.of()));
- AvroIO.Write write = AvroIO.write(String.class).to("/tmp/some/file").withWindowedWrites();
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage(
- "When using windowed writes, a filename policy must be set via withFilenamePolicy()");
- emptyInput.apply(write);
- }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTransformTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTransformTest.java
deleted file mode 100644
index b4f7a79..0000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTransformTest.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io;
-
-import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import javax.annotation.Nullable;
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.testing.NeedsRunner;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollection;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Suite;
-
-/**
- * A test suite for {@link AvroIO.Write} and {@link AvroIO.Read} transforms.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- AvroIOTransformTest.AvroIOReadTransformTest.class,
- AvroIOTransformTest.AvroIOWriteTransformTest.class
-})
-public class AvroIOTransformTest {
-
- // TODO: Stop requiring local files
-
- @Rule
- public final transient TestPipeline pipeline = TestPipeline.create();
-
- @Rule
- public final TemporaryFolder tmpFolder = new TemporaryFolder();
-
- private static final Schema.Parser parser = new Schema.Parser();
-
- private static final String SCHEMA_STRING =
- "{\"namespace\": \"example.avro\",\n"
- + " \"type\": \"record\",\n"
- + " \"name\": \"AvroGeneratedUser\",\n"
- + " \"fields\": [\n"
- + " {\"name\": \"name\", \"type\": \"string\"},\n"
- + " {\"name\": \"favorite_number\", \"type\": [\"int\", \"null\"]},\n"
- + " {\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]}\n"
- + " ]\n"
- + "}";
-
- private static final Schema SCHEMA = parser.parse(SCHEMA_STRING);
-
- private static AvroGeneratedUser[] generateAvroObjects() {
- final AvroGeneratedUser user1 = new AvroGeneratedUser();
- user1.setName("Bob");
- user1.setFavoriteNumber(256);
-
- final AvroGeneratedUser user2 = new AvroGeneratedUser();
- user2.setName("Alice");
- user2.setFavoriteNumber(128);
-
- final AvroGeneratedUser user3 = new AvroGeneratedUser();
- user3.setName("Ted");
- user3.setFavoriteColor("white");
-
- return new AvroGeneratedUser[] { user1, user2, user3 };
- }
-
- /**
- * Tests for AvroIO Read transforms, using classes generated from {@code user.avsc}.
- */
- @RunWith(Parameterized.class)
- public static class AvroIOReadTransformTest extends AvroIOTransformTest {
-
- private static GenericRecord[] generateAvroGenericRecords() {
- final GenericRecord user1 = new GenericData.Record(SCHEMA);
- user1.put("name", "Bob");
- user1.put("favorite_number", 256);
-
- final GenericRecord user2 = new GenericData.Record(SCHEMA);
- user2.put("name", "Alice");
- user2.put("favorite_number", 128);
-
- final GenericRecord user3 = new GenericData.Record(SCHEMA);
- user3.put("name", "Ted");
- user3.put("favorite_color", "white");
-
- return new GenericRecord[] { user1, user2, user3 };
- }
-
- private void generateAvroFile(final AvroGeneratedUser[] elements,
- final File avroFile) throws IOException {
- final DatumWriter<AvroGeneratedUser> userDatumWriter =
- new SpecificDatumWriter<>(AvroGeneratedUser.class);
- try (DataFileWriter<AvroGeneratedUser> dataFileWriter =
- new DataFileWriter<>(userDatumWriter)) {
- dataFileWriter.create(elements[0].getSchema(), avroFile);
- for (final AvroGeneratedUser user : elements) {
- dataFileWriter.append(user);
- }
- }
- }
-
- private <T> void runTestRead(@Nullable final String applyName,
- final AvroIO.Read<T> readBuilder,
- final String expectedName,
- final T[] expectedOutput) throws Exception {
-
- final File avroFile = tmpFolder.newFile("file.avro");
- generateAvroFile(generateAvroObjects(), avroFile);
- final AvroIO.Read<T> read = readBuilder.from(avroFile.getPath());
- final PCollection<T> output =
- applyName == null ? pipeline.apply(read) : pipeline.apply(applyName, read);
-
- PAssert.that(output).containsInAnyOrder(expectedOutput);
-
- pipeline.run();
-
- assertEquals(expectedName, output.getName());
- }
-
- @Parameterized.Parameters(name = "{2}_with_{4}")
- public static Iterable<Object[]> data() throws IOException {
-
- final String generatedClass = "GeneratedClass";
- final String fromSchema = "SchemaObject";
- final String fromSchemaString = "SchemaString";
-
- return
- ImmutableList.<Object[]>builder()
- .add(
-
- // test read using generated class
- new Object[] {
- null,
- AvroIO.read(AvroGeneratedUser.class),
- "AvroIO.Read/Read.out",
- generateAvroObjects(),
- generatedClass
- },
- new Object[] {
- "MyRead",
- AvroIO.read(AvroGeneratedUser.class),
- "MyRead/Read.out",
- generateAvroObjects(),
- generatedClass
- },
-
- // test read using schema object
- new Object[] {
- null,
- AvroIO.readGenericRecords(SCHEMA),
- "AvroIO.Read/Read.out",
- generateAvroGenericRecords(),
- fromSchema
- },
- new Object[] {
- "MyRead",
- AvroIO.readGenericRecords(SCHEMA),
- "MyRead/Read.out",
- generateAvroGenericRecords(),
- fromSchema
- },
-
- // test read using schema string
- new Object[] {
- null,
- AvroIO.readGenericRecords(SCHEMA_STRING),
- "AvroIO.Read/Read.out",
- generateAvroGenericRecords(),
- fromSchemaString
- },
- new Object[] {
- "MyRead",
- AvroIO.readGenericRecords(SCHEMA_STRING),
- "MyRead/Read.out",
- generateAvroGenericRecords(),
- fromSchemaString
- })
- .build();
- }
-
- @SuppressWarnings("DefaultAnnotationParam")
- @Parameterized.Parameter(0)
- public String transformName;
-
- @Parameterized.Parameter(1)
- public AvroIO.Read readTransform;
-
- @Parameterized.Parameter(2)
- public String expectedReadTransformName;
-
- @Parameterized.Parameter(3)
- public Object[] expectedOutput;
-
- @Parameterized.Parameter(4)
- public String testAlias;
-
- @Test
- @Category(NeedsRunner.class)
- public void testRead() throws Exception {
- runTestRead(transformName, readTransform, expectedReadTransformName, expectedOutput);
- }
- }
-
- /**
- * Tests for AvroIO Write transforms, using classes generated from {@code user.avsc}.
- */
- @RunWith(Parameterized.class)
- public static class AvroIOWriteTransformTest extends AvroIOTransformTest {
-
- private static final String WRITE_TRANSFORM_NAME = "AvroIO.Write";
-
- private List<AvroGeneratedUser> readAvroFile(final File avroFile) throws IOException {
- final DatumReader<AvroGeneratedUser> userDatumReader =
- new SpecificDatumReader<>(AvroGeneratedUser.class);
- final List<AvroGeneratedUser> users = new ArrayList<>();
- try (DataFileReader<AvroGeneratedUser> dataFileReader =
- new DataFileReader<>(avroFile, userDatumReader)) {
- while (dataFileReader.hasNext()) {
- users.add(dataFileReader.next());
- }
- }
- return users;
- }
-
- @Parameterized.Parameters(name = "{0}_with_{1}")
- public static Iterable<Object[]> data() throws IOException {
-
- final String generatedClass = "GeneratedClass";
- final String fromSchema = "SchemaObject";
- final String fromSchemaString = "SchemaString";
-
- return
- ImmutableList.<Object[]>builder()
- .add(
- new Object[] {
- AvroIO.write(AvroGeneratedUser.class),
- generatedClass
- },
- new Object[] {
- AvroIO.writeGenericRecords(SCHEMA),
- fromSchema
- },
-
- new Object[] {
- AvroIO.writeGenericRecords(SCHEMA_STRING),
- fromSchemaString
- })
- .build();
- }
-
- @SuppressWarnings("DefaultAnnotationParam")
- @Parameterized.Parameter(0)
- public AvroIO.Write writeTransform;
-
- @Parameterized.Parameter(1)
- public String testAlias;
-
- private <T> void runTestWrite(final AvroIO.Write<T> writeBuilder)
- throws Exception {
-
- final File avroFile = tmpFolder.newFile("file.avro");
- final AvroGeneratedUser[] users = generateAvroObjects();
- final AvroIO.Write<T> write = writeBuilder.to(avroFile.getPath());
-
- @SuppressWarnings("unchecked") final
- PCollection<T> input =
- pipeline.apply(Create.of(Arrays.asList((T[]) users))
- .withCoder((Coder<T>) AvroCoder.of(AvroGeneratedUser.class)));
- input.apply(write.withoutSharding());
-
- pipeline.run();
-
- assertEquals(WRITE_TRANSFORM_NAME, write.getName());
- assertThat(readAvroFile(avroFile), containsInAnyOrder(users));
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWrite() throws Exception {
- runTestWrite(writeTransform);
- }
-
- // TODO: for Write only, test withSuffix, withNumShards,
- // withShardNameTemplate and withoutSharding.
- }
-}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java
index d6facba..714e029 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java
@@ -21,7 +21,6 @@
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -60,6 +59,7 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.SerializableUtils;
import org.hamcrest.Matchers;
@@ -408,11 +408,6 @@
source = AvroSource.from(filename).withSchema(schemaString);
records = SourceTestUtils.readFromSource(source, null);
assertEqualsWithGeneric(expected, records);
-
- // Create a source with no schema
- source = AvroSource.from(filename);
- records = SourceTestUtils.readFromSource(source, null);
- assertEqualsWithGeneric(expected, records);
}
@Test
@@ -439,39 +434,39 @@
String filename = generateTestFile("tmp.avro", birds, SyncBehavior.SYNC_DEFAULT, 0,
AvroCoder.of(Bird.class), DataFileConstants.NULL_CODEC);
Metadata fileMetadata = FileSystems.matchSingleFileSpec(filename);
- String schemaA = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString();
- String schemaB = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString();
- assertNotSame(schemaA, schemaB);
-
- AvroSource<GenericRecord> sourceA = AvroSource.from(filename).withSchema(schemaA);
- AvroSource<GenericRecord> sourceB = AvroSource.from(filename).withSchema(schemaB);
- assertSame(sourceA.getSchema(), sourceB.getSchema());
+ String schema = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString();
+ // Add "" to the schema to make sure it is not interned.
+ AvroSource<GenericRecord> sourceA = AvroSource.from(filename).withSchema("" + schema);
+ AvroSource<GenericRecord> sourceB = AvroSource.from(filename).withSchema("" + schema);
+ assertSame(sourceA.getReaderSchemaString(), sourceB.getReaderSchemaString());
// Ensure that deserialization still goes through interning
AvroSource<GenericRecord> sourceC = SerializableUtils.clone(sourceB);
- assertSame(sourceA.getSchema(), sourceC.getSchema());
+ assertSame(sourceA.getReaderSchemaString(), sourceC.getReaderSchemaString());
}
@Test
- public void testSchemaIsInterned() throws Exception {
- List<Bird> birds = createRandomRecords(100);
- String filename = generateTestFile("tmp.avro", birds, SyncBehavior.SYNC_DEFAULT, 0,
+ public void testParseFn() throws Exception {
+ List<Bird> expected = createRandomRecords(100);
+ String filename = generateTestFile("tmp.avro", expected, SyncBehavior.SYNC_DEFAULT, 0,
AvroCoder.of(Bird.class), DataFileConstants.NULL_CODEC);
- Metadata fileMetadata = FileSystems.matchSingleFileSpec(filename);
- String schemaA = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString();
- String schemaB = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString();
- assertNotSame(schemaA, schemaB);
- AvroSource<GenericRecord> sourceA = (AvroSource<GenericRecord>) AvroSource.from(filename)
- .withSchema(schemaA).createForSubrangeOfFile(fileMetadata, 0L, 0L);
- AvroSource<GenericRecord> sourceB = (AvroSource<GenericRecord>) AvroSource.from(filename)
- .withSchema(schemaB).createForSubrangeOfFile(fileMetadata, 0L, 0L);
- assertSame(sourceA.getReadSchema(), sourceA.getFileSchema());
- assertSame(sourceA.getReadSchema(), sourceB.getReadSchema());
- assertSame(sourceA.getReadSchema(), sourceB.getFileSchema());
-
- // Schemas are transient and not serialized thus we don't need to worry about interning
- // after deserialization.
+ AvroSource<Bird> source =
+ AvroSource.from(filename)
+ .withParseFn(
+ new SerializableFunction<GenericRecord, Bird>() {
+ @Override
+ public Bird apply(GenericRecord input) {
+ return new Bird(
+ (long) input.get("number"),
+ input.get("species").toString(),
+ input.get("quality").toString(),
+ (long) input.get("quantity"));
+ }
+ },
+ AvroCoder.of(Bird.class));
+ List<Bird> actual = SourceTestUtils.readFromSource(source, null);
+ assertThat(actual, containsInAnyOrder(expected.toArray()));
}
private void assertEqualsWithGeneric(List<Bird> expected, List<GenericRecord> actual) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
index 3fff319..f932d43 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
@@ -19,7 +19,6 @@
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFor;
-import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertEquals;
@@ -29,6 +28,7 @@
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import com.google.common.collect.HashMultiset;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
@@ -60,24 +60,16 @@
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.testing.NeedsRunner;
-import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-import org.hamcrest.Matchers;
import org.joda.time.Instant;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.internal.matchers.ThrowableMessageMatcher;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
@@ -90,9 +82,6 @@
public class CompressedSourceTest {
@Rule
- public TestPipeline p = TestPipeline.create();
-
- @Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@Rule
@@ -102,7 +91,6 @@
* Test reading nonempty input with gzip.
*/
@Test
- @Category(NeedsRunner.class)
public void testReadGzip() throws Exception {
byte[] input = generateInput(5000);
runReadTest(input, CompressionMode.GZIP);
@@ -174,7 +162,6 @@
* Test reading nonempty input with bzip2.
*/
@Test
- @Category(NeedsRunner.class)
public void testReadBzip2() throws Exception {
byte[] input = generateInput(5000);
runReadTest(input, CompressionMode.BZIP2);
@@ -184,7 +171,6 @@
* Test reading nonempty input with zip.
*/
@Test
- @Category(NeedsRunner.class)
public void testReadZip() throws Exception {
byte[] input = generateInput(5000);
runReadTest(input, CompressionMode.ZIP);
@@ -194,7 +180,6 @@
* Test reading nonempty input with deflate.
*/
@Test
- @Category(NeedsRunner.class)
public void testReadDeflate() throws Exception {
byte[] input = generateInput(5000);
runReadTest(input, CompressionMode.DEFLATE);
@@ -204,7 +189,6 @@
* Test reading empty input with gzip.
*/
@Test
- @Category(NeedsRunner.class)
public void testEmptyReadGzip() throws Exception {
byte[] input = generateInput(0);
runReadTest(input, CompressionMode.GZIP);
@@ -232,7 +216,6 @@
* to be the concatenation of those individual files.
*/
@Test
- @Category(NeedsRunner.class)
public void testReadConcatenatedGzip() throws IOException {
byte[] header = "a,b,c\n".getBytes(StandardCharsets.UTF_8);
byte[] body = "1,2,3\n4,5,6\n7,8,9\n".getBytes(StandardCharsets.UTF_8);
@@ -246,17 +229,46 @@
CompressedSource<Byte> source =
CompressedSource.from(new ByteSource(tmpFile.getAbsolutePath(), 1))
.withDecompression(CompressionMode.GZIP);
- PCollection<Byte> output = p.apply(Read.from(source));
+ List<Byte> actual = SourceTestUtils.readFromSource(source, PipelineOptionsFactory.create());
+ assertEquals(Bytes.asList(expected), actual);
+ }
- PAssert.that(output).containsInAnyOrder(Bytes.asList(expected));
- p.run();
+ /**
+ * Test a bzip2 file containing multiple streams is correctly decompressed.
+ *
+ * <p>A bzip2 file may contain multiple streams and should decompress as the concatenation of
+ * those streams.
+ */
+ @Test
+ public void testReadMultiStreamBzip2() throws IOException {
+ CompressionMode mode = CompressionMode.BZIP2;
+ byte[] input1 = generateInput(5, 587973);
+ byte[] input2 = generateInput(5, 387374);
+
+ ByteArrayOutputStream stream1 = new ByteArrayOutputStream();
+ try (OutputStream os = getOutputStreamForMode(mode, stream1)) {
+ os.write(input1);
+ }
+
+ ByteArrayOutputStream stream2 = new ByteArrayOutputStream();
+ try (OutputStream os = getOutputStreamForMode(mode, stream2)) {
+ os.write(input2);
+ }
+
+ File tmpFile = tmpFolder.newFile();
+ try (OutputStream os = new FileOutputStream(tmpFile)) {
+ os.write(stream1.toByteArray());
+ os.write(stream2.toByteArray());
+ }
+
+ byte[] output = Bytes.concat(input1, input2);
+ verifyReadContents(output, tmpFile, mode);
}
/**
* Test reading empty input with bzip2.
*/
@Test
- @Category(NeedsRunner.class)
public void testCompressedReadBzip2() throws Exception {
byte[] input = generateInput(0);
runReadTest(input, CompressionMode.BZIP2);
@@ -266,7 +278,6 @@
* Test reading according to filepattern when the file is bzipped.
*/
@Test
- @Category(NeedsRunner.class)
public void testCompressedAccordingToFilepatternGzip() throws Exception {
byte[] input = generateInput(100);
File tmpFile = tmpFolder.newFile("test.gz");
@@ -278,7 +289,6 @@
* Test reading according to filepattern when the file is gzipped.
*/
@Test
- @Category(NeedsRunner.class)
public void testCompressedAccordingToFilepatternBzip2() throws Exception {
byte[] input = generateInput(100);
File tmpFile = tmpFolder.newFile("test.bz2");
@@ -290,7 +300,6 @@
* Test reading multiple files with different compression.
*/
@Test
- @Category(NeedsRunner.class)
public void testHeterogeneousCompression() throws Exception {
String baseName = "test-input";
@@ -300,32 +309,30 @@
// Every sort of compression
File uncompressedFile = tmpFolder.newFile(baseName + ".bin");
- generated = generateInput(1000);
+ generated = generateInput(1000, 1);
Files.write(generated, uncompressedFile);
expected.addAll(Bytes.asList(generated));
File gzipFile = tmpFolder.newFile(baseName + ".gz");
- generated = generateInput(1000);
+ generated = generateInput(1000, 2);
writeFile(gzipFile, generated, CompressionMode.GZIP);
expected.addAll(Bytes.asList(generated));
File bzip2File = tmpFolder.newFile(baseName + ".bz2");
- generated = generateInput(1000);
- writeFile(bzip2File, generateInput(1000), CompressionMode.BZIP2);
+ generated = generateInput(1000, 3);
+ writeFile(bzip2File, generated, CompressionMode.BZIP2);
expected.addAll(Bytes.asList(generated));
String filePattern = new File(tmpFolder.getRoot().toString(), baseName + ".*").toString();
CompressedSource<Byte> source =
CompressedSource.from(new ByteSource(filePattern, 1));
- PCollection<Byte> output = p.apply(Read.from(source));
-
- PAssert.that(output).containsInAnyOrder(expected);
- p.run();
+ List<Byte> actual = SourceTestUtils.readFromSource(source, PipelineOptionsFactory.create());
+ assertEquals(HashMultiset.create(actual), HashMultiset.create(expected));
}
@Test
- public void testUncompressedFileIsSplittable() throws Exception {
+ public void testUncompressedFileWithAutoIsSplittable() throws Exception {
String baseName = "test-input";
File uncompressedFile = tmpFolder.newFile(baseName + ".bin");
@@ -337,6 +344,21 @@
SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
}
+
+ @Test
+ public void testUncompressedFileWithUncompressedIsSplittable() throws Exception {
+ String baseName = "test-input";
+
+ File uncompressedFile = tmpFolder.newFile(baseName + ".bin");
+ Files.write(generateInput(10), uncompressedFile);
+
+ CompressedSource<Byte> source =
+ CompressedSource.from(new ByteSource(uncompressedFile.getPath(), 1))
+ .withDecompression(CompressionMode.UNCOMPRESSED);
+ assertTrue(source.isSplittable());
+ SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
+ }
+
@Test
public void testGzipFileIsNotSplittable() throws Exception {
String baseName = "test-input";
@@ -366,7 +388,6 @@
* this due to properties of services that we read from.
*/
@Test
- @Category(NeedsRunner.class)
public void testFalseGzipStream() throws Exception {
byte[] input = generateInput(1000);
File tmpFile = tmpFolder.newFile("test.gz");
@@ -379,15 +400,11 @@
* we fail.
*/
@Test
- @Category(NeedsRunner.class)
public void testFalseBzip2Stream() throws Exception {
byte[] input = generateInput(1000);
File tmpFile = tmpFolder.newFile("test.bz2");
Files.write(input, tmpFile);
- thrown.expectCause(Matchers.allOf(
- instanceOf(IOException.class),
- ThrowableMessageMatcher.hasMessage(
- containsString("Stream is not in the BZip2 format"))));
+ thrown.expectMessage("Stream is not in the BZip2 format");
verifyReadContents(input, tmpFile, CompressionMode.BZIP2);
}
@@ -396,7 +413,6 @@
* the gzip header is two bytes.
*/
@Test
- @Category(NeedsRunner.class)
public void testEmptyReadGzipUncompressed() throws Exception {
byte[] input = generateInput(0);
File tmpFile = tmpFolder.newFile("test.gz");
@@ -409,7 +425,6 @@
* the gzip header is two bytes.
*/
@Test
- @Category(NeedsRunner.class)
public void testOneByteReadGzipUncompressed() throws Exception {
byte[] input = generateInput(1);
File tmpFile = tmpFolder.newFile("test.gz");
@@ -421,15 +436,14 @@
* Test reading multiple files.
*/
@Test
- @Category(NeedsRunner.class)
public void testCompressedReadMultipleFiles() throws Exception {
- int numFiles = 10;
+ int numFiles = 3;
String baseName = "test_input-";
String filePattern = new File(tmpFolder.getRoot().toString(), baseName + "*").toString();
List<Byte> expected = new ArrayList<>();
for (int i = 0; i < numFiles; i++) {
- byte[] generated = generateInput(1000);
+ byte[] generated = generateInput(100);
File tmpFile = tmpFolder.newFile(baseName + i);
writeFile(tmpFile, generated, CompressionMode.GZIP);
expected.addAll(Bytes.asList(generated));
@@ -438,10 +452,8 @@
CompressedSource<Byte> source =
CompressedSource.from(new ByteSource(filePattern, 1))
.withDecompression(CompressionMode.GZIP);
- PCollection<Byte> output = p.apply(Read.from(source));
-
- PAssert.that(output).containsInAnyOrder(expected);
- p.run();
+ List<Byte> actual = SourceTestUtils.readFromSource(source, PipelineOptionsFactory.create());
+ assertEquals(HashMultiset.create(expected), HashMultiset.create(actual));
}
@Test
@@ -470,7 +482,16 @@
*/
private byte[] generateInput(int size) {
// Arbitrary but fixed seed
- Random random = new Random(285930);
+ return generateInput(size, 285930);
+ }
+
+
+ /**
+ * Generate byte array of given size.
+ */
+ private byte[] generateInput(int size, int seed) {
+ // Arbitrary but fixed seed
+ Random random = new Random(seed);
byte[] buff = new byte[size];
random.nextBytes(buff);
return buff;
@@ -550,20 +571,23 @@
}
private void verifyReadContents(byte[] expected, File inputFile,
- @Nullable DecompressingChannelFactory decompressionFactory) {
+ @Nullable DecompressingChannelFactory decompressionFactory) throws IOException {
CompressedSource<Byte> source =
CompressedSource.from(new ByteSource(inputFile.toPath().toString(), 1));
if (decompressionFactory != null) {
source = source.withDecompression(decompressionFactory);
}
- PCollection<KV<Long, Byte>> output = p.apply(Read.from(source))
- .apply(ParDo.of(new ExtractIndexFromTimestamp()));
- ArrayList<KV<Long, Byte>> expectedOutput = new ArrayList<>();
+ List<KV<Long, Byte>> actualOutput = Lists.newArrayList();
+ try (BoundedReader<Byte> reader = source.createReader(PipelineOptionsFactory.create())) {
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ actualOutput.add(KV.of(reader.getCurrentTimestamp().getMillis(), reader.getCurrent()));
+ }
+ }
+ List<KV<Long, Byte>> expectedOutput = Lists.newArrayList();
for (int i = 0; i < expected.length; i++) {
expectedOutput.add(KV.of((long) i, expected[i]));
}
- PAssert.that(output).containsInAnyOrder(expectedOutput);
- p.run();
+ assertEquals(expectedOutput, actualOutput);
}
/**
@@ -596,7 +620,7 @@
}
@Override
- public Coder<Byte> getDefaultOutputCoder() {
+ public Coder<Byte> getOutputCoder() {
return SerializableCoder.of(Byte.class);
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
index c895da8..9dc6d33 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
@@ -17,9 +17,9 @@
*/
package org.apache.beam.sdk.io;
-import static org.apache.beam.sdk.io.DefaultFilenamePolicy.constructName;
import static org.junit.Assert.assertEquals;
+import org.apache.beam.sdk.io.fs.ResourceId;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -29,27 +29,109 @@
*/
@RunWith(JUnit4.class)
public class DefaultFilenamePolicyTest {
+
+ private static String constructName(
+ String baseFilename,
+ String shardTemplate,
+ String suffix,
+ int shardNum,
+ int numShards,
+ String paneStr,
+ String windowStr) {
+ ResourceId constructed =
+ DefaultFilenamePolicy.constructName(
+ FileSystems.matchNewResource(baseFilename, false),
+ shardTemplate,
+ suffix,
+ shardNum,
+ numShards,
+ paneStr,
+ windowStr);
+ return constructed.toString();
+ }
+
@Test
public void testConstructName() {
- assertEquals("output-001-of-123.txt",
- constructName("output", "-SSS-of-NNN", ".txt", 1, 123));
+ assertEquals(
+ "/path/to/output-001-of-123.txt",
+ constructName("/path/to/output", "-SSS-of-NNN", ".txt", 1, 123, null, null));
- assertEquals("out.txt/part-00042",
- constructName("out.txt", "/part-SSSSS", "", 42, 100));
+ assertEquals(
+ "/path/to/out.txt/part-00042",
+ constructName("/path/to/out.txt", "/part-SSSSS", "", 42, 100, null, null));
- assertEquals("out.txt",
- constructName("ou", "t.t", "xt", 1, 1));
+ assertEquals("/path/to/out.txt", constructName("/path/to/ou", "t.t", "xt", 1, 1, null, null));
- assertEquals("out0102shard.txt",
- constructName("out", "SSNNshard", ".txt", 1, 2));
+ assertEquals(
+ "/path/to/out0102shard.txt",
+ constructName("/path/to/out", "SSNNshard", ".txt", 1, 2, null, null));
- assertEquals("out-2/1.part-1-of-2.txt",
- constructName("out", "-N/S.part-S-of-N", ".txt", 1, 2));
+ assertEquals(
+ "/path/to/out-2/1.part-1-of-2.txt",
+ constructName("/path/to/out", "-N/S.part-S-of-N", ".txt", 1, 2, null, null));
}
@Test
public void testConstructNameWithLargeShardCount() {
- assertEquals("out-100-of-5000.txt",
- constructName("out", "-SS-of-NN", ".txt", 100, 5000));
+ assertEquals(
+ "/out-100-of-5000.txt", constructName("/out", "-SS-of-NN", ".txt", 100, 5000, null, null));
}
+
+ @Test
+ public void testConstructWindowedName() {
+ assertEquals(
+ "/path/to/output-001-of-123.txt",
+ constructName("/path/to/output", "-SSS-of-NNN", ".txt", 1, 123, null, null));
+
+ assertEquals(
+ "/path/to/output-001-of-123-PPP-W.txt",
+ constructName("/path/to/output", "-SSS-of-NNN-PPP-W", ".txt", 1, 123, null, null));
+
+ assertEquals(
+ "/path/to/out" + ".txt/part-00042-myPaneStr-myWindowStr",
+ constructName(
+ "/path/to/out.txt", "/part-SSSSS-P-W", "", 42, 100, "myPaneStr", "myWindowStr"));
+
+ assertEquals(
+ "/path/to/out.txt",
+ constructName("/path/to/ou", "t.t", "xt", 1, 1, "myPaneStr2", "anotherWindowStr"));
+
+ assertEquals(
+ "/path/to/out0102shard-oneMoreWindowStr-anotherPaneStr.txt",
+ constructName(
+ "/path/to/out", "SSNNshard-W-P", ".txt", 1, 2, "anotherPaneStr", "oneMoreWindowStr"));
+
+ assertEquals(
+ "/out-2/1.part-1-of-2-slidingWindow1-myPaneStr3-windowslidingWindow1-"
+ + "panemyPaneStr3.txt",
+ constructName(
+ "/out",
+ "-N/S.part-S-of-N-W-P-windowW-paneP",
+ ".txt",
+ 1,
+ 2,
+ "myPaneStr3",
+ "slidingWindow1"));
+
+ // test first/last pane
+ assertEquals(
+ "/out.txt/part-00042-myWindowStr-pane-11-true-false",
+ constructName(
+ "/out.txt", "/part-SSSSS-W-P", "", 42, 100, "pane-11-true-false", "myWindowStr"));
+
+ assertEquals(
+ "/path/to/out.txt",
+ constructName("/path/to/ou", "t.t", "xt", 1, 1, "pane", "anotherWindowStr"));
+
+ assertEquals(
+ "/out0102shard-oneMoreWindowStr-pane--1-false-false-pane--1-false-false.txt",
+ constructName(
+ "/out", "SSNNshard-W-P-P", ".txt", 1, 2, "pane--1-false-false", "oneMoreWindowStr"));
+
+ assertEquals(
+ "/path/to/out-2/1.part-1-of-2-sWindow1-winsWindow1-ppaneL.txt",
+ constructName(
+ "/path/to/out", "-N/S.part-S-of-N-W-winW-pP", ".txt", 1, 2, "paneL", "sWindow1"));
+ }
+
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
index 6615a2e..a7644b6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
@@ -39,7 +39,7 @@
}
@Override
- public String getFilenameSuffix() {
+ public String getSuggestedFilenameSuffix() {
return ".drunk";
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
index caad759..0a96b7e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
@@ -48,8 +48,6 @@
import org.apache.beam.sdk.io.FileBasedSink.CompressionType;
import org.apache.beam.sdk.io.FileBasedSink.FileResult;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
-import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.Context;
-import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
import org.apache.beam.sdk.io.FileBasedSink.WriteOperation;
import org.apache.beam.sdk.io.FileBasedSink.Writer;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
@@ -62,9 +60,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for {@link FileBasedSink}.
- */
+/** Tests for {@link FileBasedSink}. */
@RunWith(JUnit4.class)
public class FileBasedSinkTest {
@Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
@@ -87,14 +83,14 @@
}
/**
- * Writer opens the correct file, writes the header, footer, and elements in the correct
- * order, and returns the correct filename.
+ * Writer opens the correct file, writes the header, footer, and elements in the correct order,
+ * and returns the correct filename.
*/
@Test
public void testWriter() throws Exception {
String testUid = "testId";
- ResourceId expectedTempFile = getBaseTempDirectory()
- .resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
+ ResourceId expectedTempFile =
+ getBaseTempDirectory().resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
List<String> values = Arrays.asList("sympathetic vulture", "boresome hummingbird");
List<String> expected = new ArrayList<>();
expected.add(SimpleSink.SimpleWriter.HEADER);
@@ -103,7 +99,7 @@
SimpleSink.SimpleWriter writer =
buildWriteOperationWithTempDir(getBaseTempDirectory()).createWriter();
- writer.openUnwindowed(testUid, -1);
+ writer.openUnwindowed(testUid, -1, null);
for (String value : values) {
writer.write(value);
}
@@ -114,9 +110,7 @@
assertFileContains(expected, expectedTempFile);
}
- /**
- * Assert that a file contains the lines provided, in the same order as expected.
- */
+ /** Assert that a file contains the lines provided, in the same order as expected. */
private void assertFileContains(List<String> expected, ResourceId file) throws Exception {
try (BufferedReader reader = new BufferedReader(new FileReader(file.toString()))) {
List<String> actual = new ArrayList<>();
@@ -140,9 +134,7 @@
}
}
- /**
- * Removes temporary files when temporary and output directories differ.
- */
+ /** Removes temporary files when temporary and output directories differ. */
@Test
public void testRemoveWithTempFilename() throws Exception {
testRemoveTemporaryFiles(3, getBaseTempDirectory());
@@ -198,23 +190,27 @@
throws Exception {
int numFiles = temporaryFiles.size();
- List<FileResult> fileResults = new ArrayList<>();
+ List<FileResult<Void>> fileResults = new ArrayList<>();
// Create temporary output bundles and output File objects.
for (int i = 0; i < numFiles; i++) {
fileResults.add(
- new FileResult(
+ new FileResult<Void>(
LocalResources.fromFile(temporaryFiles.get(i), false),
WriteFiles.UNKNOWN_SHARDNUM,
null,
+ null,
null));
}
- writeOp.finalize(fileResults);
+ writeOp.removeTemporaryFiles(writeOp.finalize(fileResults).keySet());
- ResourceId outputDirectory = writeOp.getSink().getBaseOutputDirectoryProvider().get();
for (int i = 0; i < numFiles; i++) {
- ResourceId outputFilename = writeOp.getSink().getFilenamePolicy()
- .unwindowedFilename(outputDirectory, new Context(i, numFiles), "");
+ ResourceId outputFilename =
+ writeOp
+ .getSink()
+ .getDynamicDestinations()
+ .getFilenamePolicy(null)
+ .unwindowedFilename(i, numFiles, CompressionType.UNCOMPRESSED);
assertTrue(new File(outputFilename.toString()).exists());
assertFalse(temporaryFiles.get(i).exists());
}
@@ -228,20 +224,19 @@
* Create n temporary and output files and verify that removeTemporaryFiles only removes temporary
* files.
*/
- private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory)
- throws Exception {
+ private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory) throws Exception {
String prefix = "file";
- SimpleSink sink =
- new SimpleSink(getBaseOutputDirectory(), prefix, "", "");
+ SimpleSink<Void> sink =
+ SimpleSink.makeSimpleSink(
+ getBaseOutputDirectory(), prefix, "", "", Compression.UNCOMPRESSED);
- WriteOperation<String> writeOp =
- new SimpleSink.SimpleWriteOperation(sink, tempDirectory);
+ WriteOperation<Void, String> writeOp =
+ new SimpleSink.SimpleWriteOperation<>(sink, tempDirectory);
List<File> temporaryFiles = new ArrayList<>();
List<File> outputFiles = new ArrayList<>();
for (int i = 0; i < numFiles; i++) {
- ResourceId tempResource =
- WriteOperation.buildTemporaryFilename(tempDirectory, prefix + i);
+ ResourceId tempResource = WriteOperation.buildTemporaryFilename(tempDirectory, prefix + i);
File tmpFile = new File(tempResource.toString());
tmpFile.getParentFile().mkdirs();
assertTrue("not able to create new temp file", tmpFile.createNewFile());
@@ -259,12 +254,9 @@
for (int i = 0; i < numFiles; i++) {
File temporaryFile = temporaryFiles.get(i);
assertThat(
- String.format("temp file %s exists", temporaryFile),
- temporaryFile.exists(), is(false));
+ String.format("temp file %s exists", temporaryFile), temporaryFile.exists(), is(false));
File outputFile = outputFiles.get(i);
- assertThat(
- String.format("output file %s exists", outputFile),
- outputFile.exists(), is(true));
+ assertThat(String.format("output file %s exists", outputFile), outputFile.exists(), is(true));
}
}
@@ -272,12 +264,10 @@
@Test
public void testCopyToOutputFiles() throws Exception {
SimpleSink.SimpleWriteOperation writeOp = buildWriteOperation();
- ResourceId outputDirectory = writeOp.getSink().getBaseOutputDirectoryProvider().get();
-
List<String> inputFilenames = Arrays.asList("input-1", "input-2", "input-3");
List<String> inputContents = Arrays.asList("1", "2", "3");
- List<String> expectedOutputFilenames = Arrays.asList(
- "file-00-of-03.test", "file-01-of-03.test", "file-02-of-03.test");
+ List<String> expectedOutputFilenames =
+ Arrays.asList("file-00-of-03.test", "file-01-of-03.test", "file-02-of-03.test");
Map<ResourceId, ResourceId> inputFilePaths = new HashMap<>();
List<ResourceId> expectedOutputPaths = new ArrayList<>();
@@ -292,9 +282,13 @@
File inputTmpFile = tmpFolder.newFile(inputFilenames.get(i));
List<String> lines = Collections.singletonList(inputContents.get(i));
writeFile(lines, inputTmpFile);
- inputFilePaths.put(LocalResources.fromFile(inputTmpFile, false),
- writeOp.getSink().getFilenamePolicy()
- .unwindowedFilename(outputDirectory, new Context(i, inputFilenames.size()), ""));
+ inputFilePaths.put(
+ LocalResources.fromFile(inputTmpFile, false),
+ writeOp
+ .getSink()
+ .getDynamicDestinations()
+ .getFilenamePolicy(null)
+ .unwindowedFilename(i, inputFilenames.size(), CompressionType.UNCOMPRESSED));
}
// Copy input files to output files.
@@ -311,35 +305,34 @@
ResourceId outputDirectory, FilenamePolicy policy, int numFiles) {
List<ResourceId> filenames = new ArrayList<>();
for (int i = 0; i < numFiles; i++) {
- filenames.add(policy.unwindowedFilename(outputDirectory, new Context(i, numFiles), ""));
+ filenames.add(policy.unwindowedFilename(i, numFiles, CompressionType.UNCOMPRESSED));
}
return filenames;
}
- /**
- * Output filenames are generated correctly when an extension is supplied.
- */
-
+ /** Output filenames are generated correctly when an extension is supplied. */
@Test
public void testGenerateOutputFilenames() {
List<ResourceId> expected;
List<ResourceId> actual;
ResourceId root = getBaseOutputDirectory();
- SimpleSink sink = new SimpleSink(root, "file", ".SSSSS.of.NNNNN", ".test");
- FilenamePolicy policy = sink.getFilenamePolicy();
+ SimpleSink<Void> sink =
+ SimpleSink.makeSimpleSink(
+ root, "file", ".SSSSS.of.NNNNN", ".test", Compression.UNCOMPRESSED);
+ FilenamePolicy policy = sink.getDynamicDestinations().getFilenamePolicy(null);
- expected = Arrays.asList(
- root.resolve("file.00000.of.00003.test", StandardResolveOptions.RESOLVE_FILE),
- root.resolve("file.00001.of.00003.test", StandardResolveOptions.RESOLVE_FILE),
- root.resolve("file.00002.of.00003.test", StandardResolveOptions.RESOLVE_FILE)
- );
+ expected =
+ Arrays.asList(
+ root.resolve("file.00000.of.00003.test", StandardResolveOptions.RESOLVE_FILE),
+ root.resolve("file.00001.of.00003.test", StandardResolveOptions.RESOLVE_FILE),
+ root.resolve("file.00002.of.00003.test", StandardResolveOptions.RESOLVE_FILE));
actual = generateDestinationFilenames(root, policy, 3);
assertEquals(expected, actual);
- expected = Collections.singletonList(
- root.resolve("file.00000.of.00001.test", StandardResolveOptions.RESOLVE_FILE)
- );
+ expected =
+ Collections.singletonList(
+ root.resolve("file.00000.of.00001.test", StandardResolveOptions.RESOLVE_FILE));
actual = generateDestinationFilenames(root, policy, 1);
assertEquals(expected, actual);
@@ -352,8 +345,9 @@
@Test
public void testCollidingOutputFilenames() throws IOException {
ResourceId root = getBaseOutputDirectory();
- SimpleSink sink = new SimpleSink(root, "file", "-NN", "test");
- SimpleSink.SimpleWriteOperation writeOp = new SimpleSink.SimpleWriteOperation(sink);
+ SimpleSink<Void> sink =
+ SimpleSink.makeSimpleSink(root, "file", "-NN", "test", Compression.UNCOMPRESSED);
+ SimpleSink.SimpleWriteOperation<Void> writeOp = new SimpleSink.SimpleWriteOperation<>(sink);
ResourceId temp1 = root.resolve("temp1", StandardResolveOptions.RESOLVE_FILE);
ResourceId temp2 = root.resolve("temp2", StandardResolveOptions.RESOLVE_FILE);
@@ -361,11 +355,11 @@
ResourceId output = root.resolve("file-03.test", StandardResolveOptions.RESOLVE_FILE);
// More than one shard does.
try {
- Iterable<FileResult> results =
+ Iterable<FileResult<Void>> results =
Lists.newArrayList(
- new FileResult(temp1, 1, null, null),
- new FileResult(temp2, 1, null, null),
- new FileResult(temp3, 1, null, null));
+ new FileResult<Void>(temp1, 1, null, null, null),
+ new FileResult<Void>(temp2, 1, null, null, null),
+ new FileResult<Void>(temp3, 1, null, null, null));
writeOp.buildOutputFilenames(results);
fail("Should have failed.");
} catch (IllegalStateException exn) {
@@ -379,20 +373,22 @@
List<ResourceId> expected;
List<ResourceId> actual;
ResourceId root = getBaseOutputDirectory();
- SimpleSink sink = new SimpleSink(root, "file", "-SSSSS-of-NNNNN", "");
- FilenamePolicy policy = sink.getFilenamePolicy();
+ SimpleSink<Void> sink =
+ SimpleSink.makeSimpleSink(
+ root, "file", "-SSSSS-of-NNNNN", "", Compression.UNCOMPRESSED);
+ FilenamePolicy policy = sink.getDynamicDestinations().getFilenamePolicy(null);
- expected = Arrays.asList(
- root.resolve("file-00000-of-00003", StandardResolveOptions.RESOLVE_FILE),
- root.resolve("file-00001-of-00003", StandardResolveOptions.RESOLVE_FILE),
- root.resolve("file-00002-of-00003", StandardResolveOptions.RESOLVE_FILE)
- );
+ expected =
+ Arrays.asList(
+ root.resolve("file-00000-of-00003", StandardResolveOptions.RESOLVE_FILE),
+ root.resolve("file-00001-of-00003", StandardResolveOptions.RESOLVE_FILE),
+ root.resolve("file-00002-of-00003", StandardResolveOptions.RESOLVE_FILE));
actual = generateDestinationFilenames(root, policy, 3);
assertEquals(expected, actual);
- expected = Collections.singletonList(
- root.resolve("file-00000-of-00001", StandardResolveOptions.RESOLVE_FILE)
- );
+ expected =
+ Collections.singletonList(
+ root.resolve("file-00000-of-00001", StandardResolveOptions.RESOLVE_FILE));
actual = generateDestinationFilenames(root, policy, 1);
assertEquals(expected, actual);
@@ -401,11 +397,11 @@
assertEquals(expected, actual);
}
- /** {@link CompressionType#BZIP2} correctly writes BZip2 data. */
+ /** {@link Compression#BZIP2} correctly writes BZip2 data. */
@Test
- public void testCompressionTypeBZIP2() throws FileNotFoundException, IOException {
+ public void testCompressionBZIP2() throws FileNotFoundException, IOException {
final File file =
- writeValuesWithWritableByteChannelFactory(CompressionType.BZIP2, "abc", "123");
+ writeValuesWithCompression(Compression.BZIP2, "abc", "123");
// Read Bzip2ed data back in using Apache commons API (de facto standard).
assertReadValues(
new BufferedReader(
@@ -416,10 +412,10 @@
"123");
}
- /** {@link CompressionType#GZIP} correctly writes Gzipped data. */
+ /** {@link Compression#GZIP} correctly writes Gzipped data. */
@Test
- public void testCompressionTypeGZIP() throws FileNotFoundException, IOException {
- final File file = writeValuesWithWritableByteChannelFactory(CompressionType.GZIP, "abc", "123");
+ public void testCompressionGZIP() throws FileNotFoundException, IOException {
+ final File file = writeValuesWithCompression(Compression.GZIP, "abc", "123");
// Read Gzipped data back in using standard API.
assertReadValues(
new BufferedReader(
@@ -429,11 +425,11 @@
"123");
}
- /** {@link CompressionType#DEFLATE} correctly writes deflate data. */
+ /** {@link Compression#DEFLATE} correctly writes deflate data. */
@Test
- public void testCompressionTypeDEFLATE() throws FileNotFoundException, IOException {
+ public void testCompressionDEFLATE() throws FileNotFoundException, IOException {
final File file =
- writeValuesWithWritableByteChannelFactory(CompressionType.DEFLATE, "abc", "123");
+ writeValuesWithCompression(Compression.DEFLATE, "abc", "123");
// Read Gzipped data back in using standard API.
assertReadValues(
new BufferedReader(
@@ -444,11 +440,11 @@
"123");
}
- /** {@link CompressionType#UNCOMPRESSED} correctly writes uncompressed data. */
+ /** {@link Compression#UNCOMPRESSED} correctly writes uncompressed data. */
@Test
- public void testCompressionTypeUNCOMPRESSED() throws FileNotFoundException, IOException {
+ public void testCompressionUNCOMPRESSED() throws FileNotFoundException, IOException {
final File file =
- writeValuesWithWritableByteChannelFactory(CompressionType.UNCOMPRESSED, "abc", "123");
+ writeValuesWithCompression(Compression.UNCOMPRESSED, "abc", "123");
// Read uncompressed data back in using standard API.
assertReadValues(
new BufferedReader(
@@ -465,12 +461,11 @@
}
}
- private File writeValuesWithWritableByteChannelFactory(final WritableByteChannelFactory factory,
- String... values)
- throws IOException {
+ private File writeValuesWithCompression(
+ Compression compression, String... values) throws IOException {
final File file = tmpFolder.newFile("test.gz");
final WritableByteChannel channel =
- factory.create(Channels.newChannel(new FileOutputStream(file)));
+ compression.writeCompressed(Channels.newChannel(new FileOutputStream(file)));
for (String value : values) {
channel.write(ByteBuffer.wrap((value + "\n").getBytes(StandardCharsets.UTF_8)));
}
@@ -486,10 +481,11 @@
public void testFileBasedWriterWithWritableByteChannelFactory() throws Exception {
final String testUid = "testId";
ResourceId root = getBaseOutputDirectory();
- WriteOperation<String> writeOp =
- new SimpleSink(root, "file", "-SS-of-NN", "txt", new DrunkWritableByteChannelFactory())
+ WriteOperation<Void, String> writeOp =
+ SimpleSink.makeSimpleSink(
+ root, "file", "-SS-of-NN", "txt", new DrunkWritableByteChannelFactory())
.createWriteOperation();
- final Writer<String> writer = writeOp.createWriter();
+ final Writer<Void, String> writer = writeOp.createWriter();
final ResourceId expectedFile =
writeOp.tempDirectory.get().resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
@@ -503,7 +499,7 @@
expected.add("footer");
expected.add("footer");
- writer.openUnwindowed(testUid, -1);
+ writer.openUnwindowed(testUid, -1, null);
writer.write("a");
writer.write("b");
final FileResult result = writer.close();
@@ -513,20 +509,20 @@
}
/** Build a SimpleSink with default options. */
- private SimpleSink buildSink() {
- return new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", ".test");
+ private SimpleSink<Void> buildSink() {
+ return SimpleSink.makeSimpleSink(
+ getBaseOutputDirectory(), "file", "-SS-of-NN", ".test", Compression.UNCOMPRESSED);
}
- /**
- * Build a SimpleWriteOperation with default options and the given temporary directory.
- */
- private SimpleSink.SimpleWriteOperation buildWriteOperationWithTempDir(ResourceId tempDirectory) {
- SimpleSink sink = buildSink();
- return new SimpleSink.SimpleWriteOperation(sink, tempDirectory);
+ /** Build a SimpleWriteOperation with default options and the given temporary directory. */
+ private SimpleSink.SimpleWriteOperation<Void> buildWriteOperationWithTempDir(
+ ResourceId tempDirectory) {
+ SimpleSink<Void> sink = buildSink();
+ return new SimpleSink.SimpleWriteOperation<>(sink, tempDirectory);
}
/** Build a write operation with the default options for it and its parent sink. */
- private SimpleSink.SimpleWriteOperation buildWriteOperation() {
+ private SimpleSink.SimpleWriteOperation<Void> buildWriteOperation() {
return buildSink().createWriteOperation();
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSourceTest.java
index c15e667..8ed61e8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSourceTest.java
@@ -47,6 +47,7 @@
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.FileBasedSource.FileBasedReader;
import org.apache.beam.sdk.io.Source.Reader;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -94,6 +95,15 @@
}
public TestFileBasedSource(
+ String fileOrPattern,
+ EmptyMatchTreatment emptyMatchTreatment,
+ long minBundleSize,
+ String splitHeader) {
+ super(StaticValueProvider.of(fileOrPattern), emptyMatchTreatment, minBundleSize);
+ this.splitHeader = splitHeader;
+ }
+
+ public TestFileBasedSource(
Metadata fileOrPattern,
long minBundleSize,
long startOffset,
@@ -104,10 +114,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
@@ -371,6 +378,47 @@
}
@Test
+ public void testEmptyFilepatternTreatmentDefaultDisallow() throws IOException {
+ PipelineOptions options = PipelineOptionsFactory.create();
+ TestFileBasedSource source =
+ new TestFileBasedSource(new File(tempFolder.getRoot(), "doesNotExist").getPath(), 64, null);
+ thrown.expect(FileNotFoundException.class);
+ readFromSource(source, options);
+ }
+
+ @Test
+ public void testEmptyFilepatternTreatmentAllow() throws IOException {
+ PipelineOptions options = PipelineOptionsFactory.create();
+ TestFileBasedSource source =
+ new TestFileBasedSource(
+ new File(tempFolder.getRoot(), "doesNotExist").getPath(),
+ EmptyMatchTreatment.ALLOW,
+ 64,
+ null);
+ TestFileBasedSource sourceWithWildcard =
+ new TestFileBasedSource(
+ new File(tempFolder.getRoot(), "doesNotExist*").getPath(),
+ EmptyMatchTreatment.ALLOW_IF_WILDCARD,
+ 64,
+ null);
+ assertEquals(0, readFromSource(source, options).size());
+ assertEquals(0, readFromSource(sourceWithWildcard, options).size());
+ }
+
+ @Test
+ public void testEmptyFilepatternTreatmentAllowIfWildcard() throws IOException {
+ PipelineOptions options = PipelineOptionsFactory.create();
+ TestFileBasedSource source =
+ new TestFileBasedSource(
+ new File(tempFolder.getRoot(), "doesNotExist").getPath(),
+ EmptyMatchTreatment.ALLOW_IF_WILDCARD,
+ 64,
+ null);
+ thrown.expect(FileNotFoundException.class);
+ readFromSource(source, options);
+ }
+
+ @Test
public void testCloseUnstartedFilePatternReader() throws IOException {
PipelineOptions options = PipelineOptionsFactory.create();
List<String> data1 = createStringDataset(3, 50);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java
new file mode 100644
index 0000000..7065bff
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import static org.hamcrest.Matchers.isA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.GZIPOutputStream;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesSplittableParDo;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Watch;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link FileIO}. */
+@RunWith(JUnit4.class)
+public class FileIOTest implements Serializable {
+ @Rule public transient TestPipeline p = TestPipeline.create();
+
+ @Rule public transient TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchAndMatchAll() throws IOException {
+ Path firstPath = tmpFolder.newFile("first").toPath();
+ Path secondPath = tmpFolder.newFile("second").toPath();
+ int firstSize = 37;
+ int secondSize = 42;
+ Files.write(firstPath, new byte[firstSize]);
+ Files.write(secondPath, new byte[secondSize]);
+
+ PAssert.that(
+ p.apply(
+ "Match existing",
+ FileIO.match().filepattern(tmpFolder.getRoot().getAbsolutePath() + "/*")))
+ .containsInAnyOrder(metadata(firstPath, firstSize), metadata(secondPath, secondSize));
+ PAssert.that(
+ p.apply(
+ "Match existing with provider",
+ FileIO.match()
+ .filepattern(p.newProvider(tmpFolder.getRoot().getAbsolutePath() + "/*"))))
+ .containsInAnyOrder(metadata(firstPath, firstSize), metadata(secondPath, secondSize));
+ PAssert.that(
+ p.apply("Create existing", Create.of(tmpFolder.getRoot().getAbsolutePath() + "/*"))
+ .apply("MatchAll existing", FileIO.matchAll()))
+ .containsInAnyOrder(metadata(firstPath, firstSize), metadata(secondPath, secondSize));
+
+ PAssert.that(
+ p.apply(
+ "Match non-existing ALLOW",
+ FileIO.match()
+ .filepattern(tmpFolder.getRoot().getAbsolutePath() + "/blah")
+ .withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW)))
+ .containsInAnyOrder();
+ PAssert.that(
+ p.apply(
+ "Create non-existing",
+ Create.of(tmpFolder.getRoot().getAbsolutePath() + "/blah"))
+ .apply(
+ "MatchAll non-existing ALLOW",
+ FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW)))
+ .containsInAnyOrder();
+
+ PAssert.that(
+ p.apply(
+ "Match non-existing ALLOW_IF_WILDCARD",
+ FileIO.match()
+ .filepattern(tmpFolder.getRoot().getAbsolutePath() + "/blah*")
+ .withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW_IF_WILDCARD)))
+ .containsInAnyOrder();
+ PAssert.that(
+ p.apply(
+ "Create non-existing wildcard + explicit",
+ Create.of(tmpFolder.getRoot().getAbsolutePath() + "/blah*"))
+ .apply(
+ "MatchAll non-existing ALLOW_IF_WILDCARD",
+ FileIO.matchAll()
+ .withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW_IF_WILDCARD)))
+ .containsInAnyOrder();
+ PAssert.that(
+ p.apply(
+ "Create non-existing wildcard + default",
+ Create.of(tmpFolder.getRoot().getAbsolutePath() + "/blah*"))
+ .apply("MatchAll non-existing default", FileIO.matchAll()))
+ .containsInAnyOrder();
+
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchDisallowEmptyDefault() throws IOException {
+ p.apply("Match", FileIO.match().filepattern(tmpFolder.getRoot().getAbsolutePath() + "/*"));
+
+ thrown.expectCause(isA(FileNotFoundException.class));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchDisallowEmptyExplicit() throws IOException {
+ p.apply(
+ FileIO.match()
+ .filepattern(tmpFolder.getRoot().getAbsolutePath() + "/*")
+ .withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+
+ thrown.expectCause(isA(FileNotFoundException.class));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchDisallowEmptyNonWildcard() throws IOException {
+ p.apply(
+ FileIO.match()
+ .filepattern(tmpFolder.getRoot().getAbsolutePath() + "/blah")
+ .withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW_IF_WILDCARD));
+
+ thrown.expectCause(isA(FileNotFoundException.class));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchAllDisallowEmptyExplicit() throws IOException {
+ p.apply(Create.of(tmpFolder.getRoot().getAbsolutePath() + "/*"))
+ .apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+ thrown.expectCause(isA(FileNotFoundException.class));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchAllDisallowEmptyNonWildcard() throws IOException {
+ p.apply(Create.of(tmpFolder.getRoot().getAbsolutePath() + "/blah"))
+ .apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW_IF_WILDCARD));
+ thrown.expectCause(isA(FileNotFoundException.class));
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testMatchWatchForNewFiles() throws IOException, InterruptedException {
+ final Path basePath = tmpFolder.getRoot().toPath().resolve("watch");
+ basePath.toFile().mkdir();
+ PCollection<MatchResult.Metadata> matchMetadata =
+ p.apply(
+ FileIO.match()
+ .filepattern(basePath.resolve("*").toString())
+ .continuously(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3))));
+ PCollection<MatchResult.Metadata> matchAllMetadata =
+ p.apply(Create.of(basePath.resolve("*").toString()))
+ .apply(
+ FileIO.matchAll()
+ .continuously(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3))));
+
+ Thread writer =
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(1000);
+ Files.write(basePath.resolve("first"), new byte[42]);
+ Thread.sleep(300);
+ Files.write(basePath.resolve("second"), new byte[37]);
+ Thread.sleep(300);
+ Files.write(basePath.resolve("third"), new byte[99]);
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ writer.start();
+
+ List<MatchResult.Metadata> expected =
+ Arrays.asList(
+ metadata(basePath.resolve("first"), 42),
+ metadata(basePath.resolve("second"), 37),
+ metadata(basePath.resolve("third"), 99));
+ PAssert.that(matchMetadata).containsInAnyOrder(expected);
+ PAssert.that(matchAllMetadata).containsInAnyOrder(expected);
+ p.run();
+
+ writer.join();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testRead() throws IOException {
+ final String path = tmpFolder.newFile("file").getAbsolutePath();
+ final String pathGZ = tmpFolder.newFile("file.gz").getAbsolutePath();
+ Files.write(new File(path).toPath(), "Hello world".getBytes());
+ try (Writer writer =
+ new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(pathGZ)))) {
+ writer.write("Hello world");
+ }
+
+ PCollection<MatchResult.Metadata> matches = p.apply("Match", FileIO.match().filepattern(path));
+ PCollection<FileIO.ReadableFile> decompressedAuto =
+ matches.apply("Read AUTO", FileIO.readMatches().withCompression(Compression.AUTO));
+ PCollection<FileIO.ReadableFile> decompressedDefault =
+ matches.apply("Read default", FileIO.readMatches());
+ PCollection<FileIO.ReadableFile> decompressedUncompressed =
+ matches.apply(
+ "Read UNCOMPRESSED", FileIO.readMatches().withCompression(Compression.UNCOMPRESSED));
+ for (PCollection<FileIO.ReadableFile> c :
+ Arrays.asList(decompressedAuto, decompressedDefault, decompressedUncompressed)) {
+ PAssert.thatSingleton(c)
+ .satisfies(
+ new SerializableFunction<FileIO.ReadableFile, Void>() {
+ @Override
+ public Void apply(FileIO.ReadableFile input) {
+ assertEquals(path, input.getMetadata().resourceId().toString());
+ assertEquals("Hello world".length(), input.getMetadata().sizeBytes());
+ assertEquals(Compression.UNCOMPRESSED, input.getCompression());
+ assertTrue(input.getMetadata().isReadSeekEfficient());
+ try {
+ assertEquals("Hello world", input.readFullyAsUTF8String());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+ });
+ }
+
+ PCollection<MatchResult.Metadata> matchesGZ =
+ p.apply("Match GZ", FileIO.match().filepattern(pathGZ));
+ PCollection<FileIO.ReadableFile> compressionAuto =
+ matchesGZ.apply("Read GZ AUTO", FileIO.readMatches().withCompression(Compression.AUTO));
+ PCollection<FileIO.ReadableFile> compressionDefault =
+ matchesGZ.apply("Read GZ default", FileIO.readMatches());
+ PCollection<FileIO.ReadableFile> compressionGzip =
+ matchesGZ.apply("Read GZ GZIP", FileIO.readMatches().withCompression(Compression.GZIP));
+ for (PCollection<FileIO.ReadableFile> c :
+ Arrays.asList(compressionAuto, compressionDefault, compressionGzip)) {
+ PAssert.thatSingleton(c)
+ .satisfies(
+ new SerializableFunction<FileIO.ReadableFile, Void>() {
+ @Override
+ public Void apply(FileIO.ReadableFile input) {
+ assertEquals(pathGZ, input.getMetadata().resourceId().toString());
+ assertFalse(input.getMetadata().sizeBytes() == "Hello world".length());
+ assertEquals(Compression.GZIP, input.getCompression());
+ assertFalse(input.getMetadata().isReadSeekEfficient());
+ try {
+ assertEquals("Hello world", input.readFullyAsUTF8String());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+ });
+ }
+
+ p.run();
+ }
+
+ private static MatchResult.Metadata metadata(Path path, int size) {
+ return MatchResult.Metadata.builder()
+ .setResourceId(FileSystems.matchNewResource(path.toString(), false /* isDirectory */))
+ .setIsReadSeekEfficient(true)
+ .setSizeBytes(size)
+ .build();
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
index a75c54d..3e393bf 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
@@ -83,18 +83,6 @@
}
@Test
- public void testDeleteThrowsNoSuchFileException() throws Exception {
- Path existingPath = temporaryFolder.newFile().toPath();
- Path nonExistentPath = existingPath.resolveSibling("non-existent");
-
- createFileWithContent(existingPath, "content1");
-
- thrown.expect(NoSuchFileException.class);
- FileSystems.delete(
- toResourceIds(ImmutableList.of(existingPath, nonExistentPath), false /* isDirectory */));
- }
-
- @Test
public void testDeleteIgnoreMissingFiles() throws Exception {
Path existingPath = temporaryFolder.newFile().toPath();
Path nonExistentPath = existingPath.resolveSibling("non-existent");
@@ -102,8 +90,7 @@
createFileWithContent(existingPath, "content1");
FileSystems.delete(
- toResourceIds(ImmutableList.of(existingPath, nonExistentPath), false /* isDirectory */),
- MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
+ toResourceIds(ImmutableList.of(existingPath, nonExistentPath), false /* isDirectory */));
}
@Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java
index 048908f..aaaeb83 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java
@@ -45,7 +45,9 @@
import org.apache.beam.sdk.io.fs.MatchResult;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
+import org.apache.beam.sdk.testing.RestoreSystemProperties;
import org.apache.beam.sdk.util.MimeTypes;
+import org.apache.commons.lang3.SystemUtils;
import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;
@@ -61,6 +63,7 @@
public class LocalFileSystemTest {
@Rule public ExpectedException thrown = ExpectedException.none();
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ @Rule public RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
private LocalFileSystem localFileSystem = new LocalFileSystem();
@Test
@@ -242,6 +245,52 @@
}
@Test
+ public void testMatchInDirectory() throws Exception {
+ List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString());
+ temporaryFolder.newFile("aa");
+ temporaryFolder.newFile("ab");
+
+ String expectedFile = expected.get(0);
+ int slashIndex = expectedFile.lastIndexOf('/');
+ if (SystemUtils.IS_OS_WINDOWS) {
+ slashIndex = expectedFile.lastIndexOf('\\');
+ }
+ String directory = expectedFile.substring(0, slashIndex);
+ String relative = expectedFile.substring(slashIndex + 1);
+ System.setProperty("user.dir", directory);
+ List<MatchResult> results = localFileSystem.match(ImmutableList.of(relative));
+ assertThat(
+ toFilenames(results),
+ containsInAnyOrder(expected.toArray(new String[expected.size()])));
+ }
+
+ @Test
+ public void testMatchWithFileSlashPrefix() throws Exception {
+ List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString());
+ temporaryFolder.newFile("aa");
+ temporaryFolder.newFile("ab");
+
+ String file = "file:/" + temporaryFolder.getRoot().toPath().resolve("a").toString();
+ List<MatchResult> results = localFileSystem.match(ImmutableList.of(file));
+ assertThat(
+ toFilenames(results),
+ containsInAnyOrder(expected.toArray(new String[expected.size()])));
+ }
+
+ @Test
+ public void testMatchWithFileThreeSlashesPrefix() throws Exception {
+ List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString());
+ temporaryFolder.newFile("aa");
+ temporaryFolder.newFile("ab");
+
+ String file = "file:///" + temporaryFolder.getRoot().toPath().resolve("a").toString();
+ List<MatchResult> results = localFileSystem.match(ImmutableList.of(file));
+ assertThat(
+ toFilenames(results),
+ containsInAnyOrder(expected.toArray(new String[expected.size()])));
+ }
+
+ @Test
public void testMatchMultipleWithoutSubdirectoryExpansion() throws Exception {
File unmatchedSubDir = temporaryFolder.newFolder("aaa");
File unmatchedSubDirFile = File.createTempFile("sub-dir-file", "", unmatchedSubDir);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/OffsetBasedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/OffsetBasedSourceTest.java
index 25168a3..81b2cc4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/OffsetBasedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/OffsetBasedSourceTest.java
@@ -62,10 +62,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<Integer> getDefaultOutputCoder() {
+ public Coder<Integer> getOutputCoder() {
return BigEndianIntegerCoder.of();
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java
index 74acf18..fc5575c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java
@@ -168,10 +168,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
}
@@ -204,10 +201,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
index c97313d..b59876f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
@@ -19,33 +19,76 @@
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.util.MimeTypes;
/**
- * A simple {@link FileBasedSink} that writes {@link String} values as lines with
- * header and footer.
+ * A simple {@link FileBasedSink} that writes {@link String} values as lines with header and footer.
*/
-class SimpleSink extends FileBasedSink<String> {
- public SimpleSink(ResourceId baseOutputDirectory, String prefix, String template, String suffix) {
- this(baseOutputDirectory, prefix, template, suffix, CompressionType.UNCOMPRESSED);
+class SimpleSink<DestinationT> extends FileBasedSink<String, DestinationT, String> {
+ public SimpleSink(
+ ResourceId tempDirectory,
+ DynamicDestinations<String, DestinationT, String> dynamicDestinations,
+ WritableByteChannelFactory writableByteChannelFactory) {
+ super(StaticValueProvider.of(tempDirectory), dynamicDestinations, writableByteChannelFactory);
}
- public SimpleSink(ResourceId baseOutputDirectory, String prefix, String template, String suffix,
- WritableByteChannelFactory writableByteChannelFactory) {
- super(
- StaticValueProvider.of(baseOutputDirectory),
- new DefaultFilenamePolicy(StaticValueProvider.of(prefix), template, suffix),
- writableByteChannelFactory);
+ public SimpleSink(
+ ResourceId tempDirectory,
+ DynamicDestinations<String, DestinationT, String> dynamicDestinations,
+ Compression compression) {
+ super(StaticValueProvider.of(tempDirectory), dynamicDestinations, compression);
+ }
+
+ public static SimpleSink<Void> makeSimpleSink(
+ ResourceId tempDirectory, FilenamePolicy filenamePolicy) {
+ return new SimpleSink<>(
+ tempDirectory,
+ DynamicFileDestinations.<String>constant(filenamePolicy),
+ Compression.UNCOMPRESSED);
+ }
+
+ public static SimpleSink<Void> makeSimpleSink(
+ ResourceId baseDirectory,
+ String prefix,
+ String shardTemplate,
+ String suffix,
+ WritableByteChannelFactory writableByteChannelFactory) {
+ DynamicDestinations<String, Void, String> dynamicDestinations =
+ DynamicFileDestinations.constant(
+ DefaultFilenamePolicy.fromParams(
+ new Params()
+ .withBaseFilename(
+ baseDirectory.resolve(prefix, StandardResolveOptions.RESOLVE_FILE))
+ .withShardTemplate(shardTemplate)
+ .withSuffix(suffix)));
+ return new SimpleSink<>(baseDirectory, dynamicDestinations, writableByteChannelFactory);
+ }
+
+ public static SimpleSink<Void> makeSimpleSink(
+ ResourceId baseDirectory,
+ String prefix,
+ String shardTemplate,
+ String suffix,
+ Compression compression) {
+ return makeSimpleSink(
+ baseDirectory,
+ prefix,
+ shardTemplate,
+ suffix,
+ FileBasedSink.CompressionType.fromCanonical(compression));
}
@Override
- public SimpleWriteOperation createWriteOperation() {
- return new SimpleWriteOperation(this);
+ public SimpleWriteOperation<DestinationT> createWriteOperation() {
+ return new SimpleWriteOperation<>(this);
}
- static final class SimpleWriteOperation extends WriteOperation<String> {
+ static final class SimpleWriteOperation<DestinationT>
+ extends WriteOperation<DestinationT, String> {
public SimpleWriteOperation(SimpleSink sink, ResourceId tempOutputDirectory) {
super(sink, tempOutputDirectory);
}
@@ -55,12 +98,12 @@
}
@Override
- public SimpleWriter createWriter() throws Exception {
- return new SimpleWriter(this);
+ public SimpleWriter<DestinationT> createWriter() throws Exception {
+ return new SimpleWriter<>(this);
}
}
- static final class SimpleWriter extends Writer<String> {
+ static final class SimpleWriter<DestinationT> extends Writer<DestinationT, String> {
static final String HEADER = "header";
static final String FOOTER = "footer";
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
index d564d3b..6e5e4da 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
@@ -17,11 +17,10 @@
*/
package org.apache.beam.sdk.io;
-import static org.apache.beam.sdk.io.TFRecordIO.CompressionType;
-import static org.apache.beam.sdk.io.TFRecordIO.CompressionType.AUTO;
-import static org.apache.beam.sdk.io.TFRecordIO.CompressionType.GZIP;
-import static org.apache.beam.sdk.io.TFRecordIO.CompressionType.NONE;
-import static org.apache.beam.sdk.io.TFRecordIO.CompressionType.ZLIB;
+import static org.apache.beam.sdk.io.Compression.AUTO;
+import static org.apache.beam.sdk.io.Compression.DEFLATE;
+import static org.apache.beam.sdk.io.Compression.GZIP;
+import static org.apache.beam.sdk.io.Compression.UNCOMPRESSED;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.Matchers.isIn;
import static org.junit.Assert.assertEquals;
@@ -144,7 +143,7 @@
public void testReadDisplayData() {
TFRecordIO.Read read = TFRecordIO.read()
.from("foo.*")
- .withCompressionType(GZIP)
+ .withCompression(GZIP)
.withoutValidation();
DisplayData displayData = DisplayData.from(read);
@@ -161,7 +160,7 @@
.withSuffix("bar")
.withShardNameTemplate("-SS-of-NN-")
.withNumShards(100)
- .withCompressionType(GZIP);
+ .withCompression(GZIP);
DisplayData displayData = DisplayData.from(write);
@@ -265,25 +264,25 @@
@Test
@Category(NeedsRunner.class)
public void runTestRoundTrip() throws IOException {
- runTestRoundTrip(LARGE, 10, ".tfrecords", NONE, NONE);
+ runTestRoundTrip(LARGE, 10, ".tfrecords", UNCOMPRESSED, UNCOMPRESSED);
}
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripWithEmptyData() throws IOException {
- runTestRoundTrip(EMPTY, 10, ".tfrecords", NONE, NONE);
+ runTestRoundTrip(EMPTY, 10, ".tfrecords", UNCOMPRESSED, UNCOMPRESSED);
}
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripWithOneShards() throws IOException {
- runTestRoundTrip(LARGE, 1, ".tfrecords", NONE, NONE);
+ runTestRoundTrip(LARGE, 1, ".tfrecords", UNCOMPRESSED, UNCOMPRESSED);
}
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripWithSuffix() throws IOException {
- runTestRoundTrip(LARGE, 10, ".suffix", NONE, NONE);
+ runTestRoundTrip(LARGE, 10, ".suffix", UNCOMPRESSED, UNCOMPRESSED);
}
@Test
@@ -295,13 +294,13 @@
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripZlib() throws IOException {
- runTestRoundTrip(LARGE, 10, ".tfrecords", ZLIB, ZLIB);
+ runTestRoundTrip(LARGE, 10, ".tfrecords", DEFLATE, DEFLATE);
}
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripUncompressedFilesWithAuto() throws IOException {
- runTestRoundTrip(LARGE, 10, ".tfrecords", NONE, AUTO);
+ runTestRoundTrip(LARGE, 10, ".tfrecords", UNCOMPRESSED, AUTO);
}
@Test
@@ -313,14 +312,14 @@
@Test
@Category(NeedsRunner.class)
public void runTestRoundTripZlibFilesWithAuto() throws IOException {
- runTestRoundTrip(LARGE, 10, ".tfrecords", ZLIB, AUTO);
+ runTestRoundTrip(LARGE, 10, ".tfrecords", DEFLATE, AUTO);
}
private void runTestRoundTrip(Iterable<String> elems,
int numShards,
String suffix,
- CompressionType writeCompressionType,
- CompressionType readCompressionType) throws IOException {
+ Compression writeCompression,
+ Compression readCompression) throws IOException {
String outputName = "file";
Path baseDir = Files.createTempDirectory(tempFolder, "test-rt");
String baseFilename = baseDir.resolve(outputName).toString();
@@ -328,14 +327,14 @@
TFRecordIO.Write write = TFRecordIO.write().to(baseFilename)
.withNumShards(numShards)
.withSuffix(suffix)
- .withCompressionType(writeCompressionType);
+ .withCompression(writeCompression);
p.apply(Create.of(elems).withCoder(StringUtf8Coder.of()))
.apply(ParDo.of(new StringToByteArray()))
.apply(write);
p.run();
TFRecordIO.Read read = TFRecordIO.read().from(baseFilename + "*")
- .withCompressionType(readCompressionType);
+ .withCompression(readCompression);
PCollection<String> output = p2.apply(read).apply(ParDo.of(new ByteArrayToString()));
PAssert.that(output).containsInAnyOrder(elems);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
new file mode 100644
index 0000000..e4fca47
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
@@ -0,0 +1,860 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.beam.sdk.TestUtils.LINES_ARRAY;
+import static org.apache.beam.sdk.TestUtils.NO_LINES_ARRAY;
+import static org.apache.beam.sdk.io.Compression.AUTO;
+import static org.apache.beam.sdk.io.Compression.BZIP2;
+import static org.apache.beam.sdk.io.Compression.DEFLATE;
+import static org.apache.beam.sdk.io.Compression.GZIP;
+import static org.apache.beam.sdk.io.Compression.UNCOMPRESSED;
+import static org.apache.beam.sdk.io.Compression.ZIP;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesSplittableParDo;
+import org.apache.beam.sdk.testing.ValidatesRunner;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ToString;
+import org.apache.beam.sdk.transforms.Watch;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.joda.time.Duration;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.Parameterized;
+
+/** Tests for {@link TextIO.Read}. */
+@RunWith(Enclosed.class)
+public class TextIOReadTest {
+ private static final int LINES_NUMBER_FOR_LARGE = 1000;
+ private static final List<String> EMPTY = Collections.emptyList();
+ private static final List<String> TINY =
+ Arrays.asList("Irritable eagle", "Optimistic jay", "Fanciful hawk");
+
+ private static final List<String> LARGE = makeLines(LINES_NUMBER_FOR_LARGE);
+
+ private static File writeToFile(
+ List<String> lines, TemporaryFolder folder, String fileName, Compression compression)
+ throws IOException {
+ File file = folder.getRoot().toPath().resolve(fileName).toFile();
+ OutputStream output = new FileOutputStream(file);
+ switch (compression) {
+ case UNCOMPRESSED:
+ break;
+ case GZIP:
+ output = new GZIPOutputStream(output);
+ break;
+ case BZIP2:
+ output = new BZip2CompressorOutputStream(output);
+ break;
+ case ZIP:
+ ZipOutputStream zipOutput = new ZipOutputStream(output);
+ zipOutput.putNextEntry(new ZipEntry("entry"));
+ output = zipOutput;
+ break;
+ case DEFLATE:
+ output = new DeflateCompressorOutputStream(output);
+ break;
+ default:
+ throw new UnsupportedOperationException(compression.toString());
+ }
+ writeToStreamAndClose(lines, output);
+ return file;
+ }
+
+ /**
+ * Helper that writes the given lines (adding a newline in between) to a stream, then closes the
+ * stream.
+ */
+ private static void writeToStreamAndClose(List<String> lines, OutputStream outputStream) {
+ try (PrintStream writer = new PrintStream(outputStream)) {
+ for (String line : lines) {
+ writer.println(line);
+ }
+ }
+ }
+
+ /** Helper to make an array of compressible strings. Returns ["word"i] for i in range(0,n). */
+ private static List<String> makeLines(int n) {
+ List<String> ret = new ArrayList<>();
+ for (int i = 0; i < n; ++i) {
+ ret.add("word" + i);
+ }
+ return ret;
+ }
+
+ /**
+ * Helper method that runs a variety of ways to read a single file using TextIO and checks that
+ * they all match the given expected output.
+ *
+ * <p>The transforms being verified are:
+ * <ul>
+ * <li>TextIO.read().from(filename).withCompression(compressionType)
+ * <li>TextIO.read().from(filename).withCompression(compressionType)
+ * .withHintMatchesManyFiles()
+ * <li>TextIO.readAll().withCompression(compressionType)
+ * </ul>
+ */
+ private static void assertReadingCompressedFileMatchesExpected(
+ File file, Compression compression, List<String> expected, Pipeline p) {
+
+ TextIO.Read read = TextIO.read().from(file.getPath()).withCompression(compression);
+
+ PAssert.that(p.apply("Read_" + file + "_" + compression.toString(), read))
+ .containsInAnyOrder(expected);
+
+ PAssert.that(
+ p.apply(
+ "Read_" + file + "_" + compression.toString() + "_many",
+ read.withHintMatchesManyFiles()))
+ .containsInAnyOrder(expected);
+
+ TextIO.ReadAll readAll =
+ TextIO.readAll().withCompression(compression);
+ PAssert.that(
+ p.apply("Create_" + file, Create.of(file.getPath()))
+ .apply("Read_" + compression.toString(), readAll))
+ .containsInAnyOrder(expected);
+ }
+
+ /**
+ * Create a zip file with the given lines.
+ *
+ * @param expected A list of expected lines, populated in the zip file.
+ * @param folder A temporary folder used to create files.
+ * @param filename Optionally zip file name (can be null).
+ * @param fieldsEntries Fields to write in zip entries.
+ * @return The zip filename.
+ * @throws Exception In case of a failure during zip file creation.
+ */
+ private static File createZipFile(
+ List<String> expected, TemporaryFolder folder, String filename, String[]... fieldsEntries)
+ throws Exception {
+ File tmpFile = folder.getRoot().toPath().resolve(filename).toFile();
+
+ ZipOutputStream out = new ZipOutputStream(new FileOutputStream(tmpFile));
+ PrintStream writer = new PrintStream(out, true /* auto-flush on write */);
+
+ int index = 0;
+ for (String[] entry : fieldsEntries) {
+ out.putNextEntry(new ZipEntry(Integer.toString(index)));
+ for (String field : entry) {
+ writer.println(field);
+ expected.add(field);
+ }
+ out.closeEntry();
+ index++;
+ }
+
+ writer.close();
+ out.close();
+
+ return tmpFile;
+ }
+
+ private static TextSource prepareSource(
+ TemporaryFolder temporaryFolder, byte[] data, byte[] delimiter) throws IOException {
+ Path path = temporaryFolder.newFile().toPath();
+ Files.write(path, data);
+ return new TextSource(
+ ValueProvider.StaticValueProvider.of(path.toString()),
+ EmptyMatchTreatment.DISALLOW,
+ delimiter);
+ }
+
+ private static String getFileSuffix(Compression compression) {
+ switch (compression) {
+ case UNCOMPRESSED:
+ return ".txt";
+ case GZIP:
+ return ".gz";
+ case BZIP2:
+ return ".bz2";
+ case ZIP:
+ return ".zip";
+ case DEFLATE:
+ return ".deflate";
+ default:
+ return "";
+ }
+ }
+
+ /** Tests for reading from different size of files with various Compression. */
+ @RunWith(Parameterized.class)
+ public static class CompressedReadTest {
+ @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+ @Rule public TestPipeline p = TestPipeline.create();
+
+ @Parameterized.Parameters(name = "{index}: {1}")
+ public static Iterable<Object[]> data() {
+ return ImmutableList.<Object[]>builder()
+ .add(new Object[] {EMPTY, UNCOMPRESSED})
+ .add(new Object[] {EMPTY, GZIP})
+ .add(new Object[] {EMPTY, BZIP2})
+ .add(new Object[] {EMPTY, ZIP})
+ .add(new Object[] {EMPTY, DEFLATE})
+ .add(new Object[] {TINY, UNCOMPRESSED})
+ .add(new Object[] {TINY, GZIP})
+ .add(new Object[] {TINY, BZIP2})
+ .add(new Object[] {TINY, ZIP})
+ .add(new Object[] {TINY, DEFLATE})
+ .add(new Object[] {LARGE, UNCOMPRESSED})
+ .add(new Object[] {LARGE, GZIP})
+ .add(new Object[] {LARGE, BZIP2})
+ .add(new Object[] {LARGE, ZIP})
+ .add(new Object[] {LARGE, DEFLATE})
+ .build();
+ }
+
+ @Parameterized.Parameter(0)
+ public List<String> lines;
+
+ @Parameterized.Parameter(1)
+ public Compression compression;
+
+ /** Tests reading from a small, compressed file with no extension. */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testCompressedReadWithoutExtension() throws Exception {
+ String fileName = lines.size() + "_" + compression + "_no_extension";
+ File fileWithNoExtension = writeToFile(lines, tempFolder, fileName, compression);
+ assertReadingCompressedFileMatchesExpected(fileWithNoExtension, compression, lines, p);
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testCompressedReadWithExtension() throws Exception {
+ String fileName =
+ lines.size() + "_" + compression + "_no_extension" + getFileSuffix(compression);
+ File fileWithExtension = writeToFile(lines, tempFolder, fileName, compression);
+
+ // Sanity check that we're properly testing compression.
+ if (lines.size() == LINES_NUMBER_FOR_LARGE && !compression.equals(UNCOMPRESSED)) {
+ File uncompressedFile = writeToFile(lines, tempFolder, "large.txt", UNCOMPRESSED);
+ assertThat(uncompressedFile.length(), greaterThan(fileWithExtension.length()));
+ }
+
+ assertReadingCompressedFileMatchesExpected(fileWithExtension, compression, lines, p);
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadWithAuto() throws Exception {
+ // Files with non-compressed extensions should work in AUTO and UNCOMPRESSED modes.
+ String fileName =
+ lines.size() + "_" + compression + "_no_extension" + getFileSuffix(compression);
+ File fileWithExtension = writeToFile(lines, tempFolder, fileName, compression);
+ assertReadingCompressedFileMatchesExpected(fileWithExtension, AUTO, lines, p);
+ p.run();
+ }
+ }
+
+ /** Tests for reading files with various delimiters. */
+ @RunWith(Parameterized.class)
+ public static class ReadWithDelimiterTest {
+ private static final ImmutableList<String> EXPECTED = ImmutableList.of("asdf", "hjkl", "xyz");
+ @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+ @Parameterized.Parameters(name = "{index}: {0}")
+ public static Iterable<Object[]> data() {
+ return ImmutableList.<Object[]>builder()
+ .add(new Object[] {"\n\n\n", ImmutableList.of("", "", "")})
+ .add(new Object[] {"asdf\nhjkl\nxyz\n", EXPECTED})
+ .add(new Object[] {"asdf\rhjkl\rxyz\r", EXPECTED})
+ .add(new Object[] {"asdf\r\nhjkl\r\nxyz\r\n", EXPECTED})
+ .add(new Object[] {"asdf\rhjkl\r\nxyz\n", EXPECTED})
+ .add(new Object[] {"asdf\nhjkl\nxyz", EXPECTED})
+ .add(new Object[] {"asdf\rhjkl\rxyz", EXPECTED})
+ .add(new Object[] {"asdf\r\nhjkl\r\nxyz", EXPECTED})
+ .add(new Object[] {"asdf\rhjkl\r\nxyz", EXPECTED})
+ .build();
+ }
+
+ @Parameterized.Parameter(0)
+ public String line;
+
+ @Parameterized.Parameter(1)
+ public ImmutableList<String> expected;
+
+ @Test
+ public void testReadLinesWithDelimiter() throws Exception {
+ runTestReadWithData(line.getBytes(UTF_8), expected);
+ }
+
+ @Test
+ public void testSplittingSource() throws Exception {
+ TextSource source = prepareSource(line.getBytes(UTF_8));
+ SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
+ }
+
+ private TextSource prepareSource(byte[] data) throws IOException {
+ return TextIOReadTest.prepareSource(tempFolder, data, null);
+ }
+
+ private void runTestReadWithData(byte[] data, List<String> expectedResults) throws Exception {
+ TextSource source = prepareSource(data);
+ List<String> actual = SourceTestUtils.readFromSource(source, PipelineOptionsFactory.create());
+ assertThat(
+ actual, containsInAnyOrder(new ArrayList<>(expectedResults).toArray(new String[0])));
+ }
+ }
+
+ /** Tests for some basic operations in {@link TextIO.Read}. */
+ @RunWith(JUnit4.class)
+ public static class BasicIOTest {
+ @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+ @Rule public TestPipeline p = TestPipeline.create();
+
+ private void runTestRead(String[] expected) throws Exception {
+ File tmpFile = tempFolder.newFile();
+ String filename = tmpFile.getPath();
+
+ try (PrintStream writer = new PrintStream(new FileOutputStream(tmpFile))) {
+ for (String elem : expected) {
+ byte[] encodedElem = CoderUtils.encodeToByteArray(StringUtf8Coder.of(), elem);
+ String line = new String(encodedElem);
+ writer.println(line);
+ }
+ }
+
+ TextIO.Read read = TextIO.read().from(filename);
+ PCollection<String> output = p.apply(read);
+
+ PAssert.that(output).containsInAnyOrder(expected);
+ p.run();
+ }
+
+ @Test
+ public void testDelimiterSelfOverlaps(){
+ assertFalse(TextIO.Read.isSelfOverlapping(new byte[]{'a', 'b', 'c'}));
+ assertFalse(TextIO.Read.isSelfOverlapping(new byte[]{'c', 'a', 'b', 'd', 'a', 'b'}));
+ assertFalse(TextIO.Read.isSelfOverlapping(new byte[]{'a', 'b', 'c', 'a', 'b', 'd'}));
+ assertTrue(TextIO.Read.isSelfOverlapping(new byte[]{'a', 'b', 'a'}));
+ assertTrue(TextIO.Read.isSelfOverlapping(new byte[]{'a', 'b', 'c', 'a', 'b'}));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadStringsWithCustomDelimiter() throws Exception {
+ final String[] inputStrings =
+ new String[] {
+ // incomplete delimiter
+ "To be, or not to be: that |is the question: ",
+ // incomplete delimiter
+ "To be, or not to be: that *is the question: ",
+ // complete delimiter
+ "Whether 'tis nobler in the mind to suffer |*",
+ // truncated delimiter
+ "The slings and arrows of outrageous fortune,|"
+ };
+
+ File tmpFile = tempFolder.newFile("tmpfile.txt");
+ String filename = tmpFile.getPath();
+
+ try (FileWriter writer = new FileWriter(tmpFile)) {
+ writer.write(Joiner.on("").join(inputStrings));
+ }
+
+ PAssert.that(p.apply(TextIO.read().from(filename).withDelimiter(new byte[] {'|', '*'})))
+ .containsInAnyOrder(
+ "To be, or not to be: that |is the question: To be, or not to be: "
+ + "that *is the question: Whether 'tis nobler in the mind to suffer ",
+ "The slings and arrows of outrageous fortune,|");
+ p.run();
+ }
+
+ @Test
+ public void testSplittingSourceWithCustomDelimiter() throws Exception {
+ List<String> testCases = Lists.newArrayList();
+ String infix = "first|*second|*|*third";
+ String[] affixes = new String[] {"", "|", "*", "|*"};
+ for (String prefix : affixes) {
+ for (String suffix : affixes) {
+ testCases.add(prefix + infix + suffix);
+ }
+ }
+ for (String testCase : testCases) {
+ SourceTestUtils.assertSplitAtFractionExhaustive(
+ TextIOReadTest.prepareSource(
+ tempFolder, testCase.getBytes(UTF_8), new byte[] {'|', '*'}),
+ PipelineOptionsFactory.create());
+ }
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadStrings() throws Exception {
+ runTestRead(LINES_ARRAY);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadEmptyStrings() throws Exception {
+ runTestRead(NO_LINES_ARRAY);
+ }
+
+ @Test
+ public void testReadNamed() throws Exception {
+ File emptyFile = tempFolder.newFile();
+ p.enableAbandonedNodeEnforcement(false);
+
+ assertEquals("TextIO.Read/Read.out", p.apply(TextIO.read().from("somefile")).getName());
+ assertEquals(
+ "MyRead/Read.out", p.apply("MyRead", TextIO.read().from(emptyFile.getPath())).getName());
+ }
+
+ @Test
+ public void testReadDisplayData() {
+ TextIO.Read read = TextIO.read().from("foo.*").withCompression(BZIP2);
+
+ DisplayData displayData = DisplayData.from(read);
+
+ assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
+ assertThat(displayData, hasDisplayItem("compressionType", BZIP2.toString()));
+ }
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testPrimitiveReadDisplayData() {
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
+
+ TextIO.Read read = TextIO.read().from("foobar");
+
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read);
+ assertThat(
+ "TextIO.Read should include the file prefix in its primitive display data",
+ displayData,
+ hasItem(hasDisplayItem(hasValue(startsWith("foobar")))));
+ }
+
+ /** Options for testing. */
+ public interface RuntimeTestOptions extends PipelineOptions {
+ ValueProvider<String> getInput();
+ void setInput(ValueProvider<String> value);
+ }
+
+ @Test
+ public void testRuntimeOptionsNotCalledInApply() throws Exception {
+ p.enableAbandonedNodeEnforcement(false);
+
+ RuntimeTestOptions options =
+ PipelineOptionsFactory.as(RuntimeTestOptions.class);
+
+ p.apply(TextIO.read().from(options.getInput()));
+ }
+
+ @Test
+ public void testCompressionIsSet() throws Exception {
+ TextIO.Read read = TextIO.read().from("/tmp/test");
+ assertEquals(AUTO, read.getCompression());
+ read = TextIO.read().from("/tmp/test").withCompression(GZIP);
+ assertEquals(GZIP, read.getCompression());
+ }
+
+ /**
+ * Tests reading from a small, uncompressed file with .gz extension. This must work in
+ * GZIP modes. This is needed because some network file systems / HTTP clients will
+ * transparently decompress gzipped content.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSmallCompressedGzipReadActuallyUncompressed() throws Exception {
+ File smallGzNotCompressed =
+ writeToFile(TINY, tempFolder, "tiny_uncompressed.gz", UNCOMPRESSED);
+ // Should work with GZIP compression set.
+ assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, GZIP, TINY, p);
+ p.run();
+ }
+
+ /**
+ * Tests reading from a small, uncompressed file with .gz extension. This must work in
+ * AUTO modes. This is needed because some network file systems / HTTP clients will
+ * transparently decompress gzipped content.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSmallCompressedAutoReadActuallyUncompressed() throws Exception {
+ File smallGzNotCompressed =
+ writeToFile(TINY, tempFolder, "tiny_uncompressed.gz", UNCOMPRESSED);
+ // Should also work with AUTO mode set.
+ assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, AUTO, TINY, p);
+ p.run();
+ }
+
+ /**
+ * Tests a zip file with no entries. This is a corner case not tested elsewhere as the default
+ * test zip files have a single entry.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testZipCompressedReadWithNoEntries() throws Exception {
+ File file = createZipFile(new ArrayList<String>(), tempFolder, "empty zip file");
+ assertReadingCompressedFileMatchesExpected(file, ZIP, EMPTY, p);
+ p.run();
+ }
+
+ /**
+ * Tests a zip file with multiple entries. This is a corner case not tested elsewhere as the
+ * default test zip files have a single entry.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testZipCompressedReadWithMultiEntriesFile() throws Exception {
+ String[] entry0 = new String[] {"first", "second", "three"};
+ String[] entry1 = new String[] {"four", "five", "six"};
+ String[] entry2 = new String[] {"seven", "eight", "nine"};
+
+ List<String> expected = new ArrayList<>();
+
+ File file =
+ createZipFile(expected, tempFolder, "multiple entries", entry0, entry1, entry2);
+ assertReadingCompressedFileMatchesExpected(file, ZIP, expected, p);
+ p.run();
+ }
+
+ /**
+ * Read a ZIP compressed file containing data, multiple empty entries, and then more data. We
+ * expect just the data back.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testZipCompressedReadWithComplexEmptyAndPresentEntries() throws Exception {
+ File file =
+ createZipFile(
+ new ArrayList<String>(),
+ tempFolder,
+ "complex empty and present entries",
+ new String[] {"cat"},
+ new String[] {},
+ new String[] {},
+ new String[] {"dog"});
+
+ assertReadingCompressedFileMatchesExpected(
+ file, ZIP, Arrays.asList("cat", "dog"), p);
+ p.run();
+ }
+
+ @Test
+ public void testTextIOGetName() {
+ assertEquals("TextIO.Read", TextIO.read().from("somefile").getName());
+ assertEquals("TextIO.Read", TextIO.read().from("somefile").toString());
+ }
+
+ private TextSource prepareSource(byte[] data) throws IOException {
+ return TextIOReadTest.prepareSource(tempFolder, data, null);
+ }
+
+ @Test
+ public void testProgressEmptyFile() throws IOException {
+ try (BoundedSource.BoundedReader<String> reader =
+ prepareSource(new byte[0]).createReader(PipelineOptionsFactory.create())) {
+ // Check preconditions before starting.
+ assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // Assert empty
+ assertFalse(reader.start());
+
+ // Check postconditions after finishing
+ assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(0, reader.getSplitPointsRemaining());
+ }
+ }
+
+ @Test
+ public void testProgressTextFile() throws IOException {
+ String file = "line1\nline2\nline3";
+ try (BoundedSource.BoundedReader<String> reader =
+ prepareSource(file.getBytes()).createReader(PipelineOptionsFactory.create())) {
+ // Check preconditions before starting
+ assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // Line 1
+ assertTrue(reader.start());
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // Line 2
+ assertTrue(reader.advance());
+ assertEquals(1, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // Line 3
+ assertTrue(reader.advance());
+ assertEquals(2, reader.getSplitPointsConsumed());
+ assertEquals(1, reader.getSplitPointsRemaining());
+
+ // Check postconditions after finishing
+ assertFalse(reader.advance());
+ assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(3, reader.getSplitPointsConsumed());
+ assertEquals(0, reader.getSplitPointsRemaining());
+ }
+ }
+
+ @Test
+ public void testProgressAfterSplitting() throws IOException {
+ String file = "line1\nline2\nline3";
+ BoundedSource<String> source = prepareSource(file.getBytes());
+ BoundedSource<String> remainder;
+
+ // Create the remainder, verifying properties pre- and post-splitting.
+ try (BoundedSource.BoundedReader<String> readerOrig =
+ source.createReader(PipelineOptionsFactory.create())) {
+ // Preconditions.
+ assertEquals(0.0, readerOrig.getFractionConsumed(), 1e-6);
+ assertEquals(0, readerOrig.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, readerOrig.getSplitPointsRemaining());
+
+ // First record, before splitting.
+ assertTrue(readerOrig.start());
+ assertEquals(0, readerOrig.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, readerOrig.getSplitPointsRemaining());
+
+ // Split. 0.1 is in line1, so should now be able to detect last record.
+ remainder = readerOrig.splitAtFraction(0.1);
+ System.err.println(readerOrig.getCurrentSource());
+ assertNotNull(remainder);
+
+ // First record, after splitting.
+ assertEquals(0, readerOrig.getSplitPointsConsumed());
+ assertEquals(1, readerOrig.getSplitPointsRemaining());
+
+ // Finish and postconditions.
+ assertFalse(readerOrig.advance());
+ assertEquals(1.0, readerOrig.getFractionConsumed(), 1e-6);
+ assertEquals(1, readerOrig.getSplitPointsConsumed());
+ assertEquals(0, readerOrig.getSplitPointsRemaining());
+ }
+
+ // Check the properties of the remainder.
+ try (BoundedSource.BoundedReader<String> reader =
+ remainder.createReader(PipelineOptionsFactory.create())) {
+ // Preconditions.
+ assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // First record should be line 2.
+ assertTrue(reader.start());
+ assertEquals(0, reader.getSplitPointsConsumed());
+ assertEquals(
+ BoundedSource.BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
+
+ // Second record is line 3
+ assertTrue(reader.advance());
+ assertEquals(1, reader.getSplitPointsConsumed());
+ assertEquals(1, reader.getSplitPointsRemaining());
+
+ // Check postconditions after finishing
+ assertFalse(reader.advance());
+ assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
+ assertEquals(2, reader.getSplitPointsConsumed());
+ assertEquals(0, reader.getSplitPointsRemaining());
+ }
+ }
+
+ @Test
+ public void testInitialSplitAutoModeTxt() throws Exception {
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ long desiredBundleSize = 1000;
+ File largeTxt = writeToFile(LARGE, tempFolder, "large.txt", UNCOMPRESSED);
+
+ // Sanity check: file is at least 2 bundles long.
+ assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize));
+
+ FileBasedSource<String> source = TextIO.read().from(largeTxt.getPath()).getSource();
+ List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options);
+
+ // At least 2 splits and they are equal to reading the whole file.
+ assertThat(splits, hasSize(greaterThan(1)));
+ SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
+ }
+
+ @Test
+ public void testInitialSplitAutoModeGz() throws Exception {
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ long desiredBundleSize = 1000;
+ File largeGz = writeToFile(LARGE, tempFolder, "large.gz", GZIP);
+ // Sanity check: file is at least 2 bundles long.
+ assertThat(largeGz.length(), greaterThan(2 * desiredBundleSize));
+
+ FileBasedSource<String> source = TextIO.read().from(largeGz.getPath()).getSource();
+ List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options);
+
+ // Exactly 1 split, even in AUTO mode, since it is a gzip file.
+ assertThat(splits, hasSize(equalTo(1)));
+ SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
+ }
+
+ @Test
+ public void testInitialSplitGzipModeTxt() throws Exception {
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ long desiredBundleSize = 1000;
+ File largeTxt = writeToFile(LARGE, tempFolder, "large.txt", UNCOMPRESSED);
+ // Sanity check: file is at least 2 bundles long.
+ assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize));
+
+ FileBasedSource<String> source =
+ TextIO.read().from(largeTxt.getPath()).withCompression(GZIP).getSource();
+ List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options);
+
+ // Exactly 1 split, even though splittable text file, since using GZIP mode.
+ assertThat(splits, hasSize(equalTo(1)));
+ SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadAll() throws IOException {
+ Path tempFolderPath = tempFolder.getRoot().toPath();
+ writeToFile(TINY, tempFolder, "readAllTiny1.zip", ZIP);
+ writeToFile(TINY, tempFolder, "readAllTiny2.txt", UNCOMPRESSED);
+ writeToFile(LARGE, tempFolder, "readAllLarge1.zip", ZIP);
+ writeToFile(LARGE, tempFolder, "readAllLarge2.txt", UNCOMPRESSED);
+ PCollection<String> lines =
+ p.apply(
+ Create.of(
+ tempFolderPath.resolve("readAllTiny*").toString(),
+ tempFolderPath.resolve("readAllLarge*").toString()))
+ .apply(TextIO.readAll().withCompression(AUTO));
+ PAssert.that(lines).containsInAnyOrder(Iterables.concat(TINY, TINY, LARGE, LARGE));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReadFiles() throws IOException {
+ Path tempFolderPath = tempFolder.getRoot().toPath();
+ writeToFile(TINY, tempFolder, "readAllTiny1.zip", ZIP);
+ writeToFile(TINY, tempFolder, "readAllTiny2.txt", UNCOMPRESSED);
+ writeToFile(LARGE, tempFolder, "readAllLarge1.zip", ZIP);
+ writeToFile(LARGE, tempFolder, "readAllLarge2.txt", UNCOMPRESSED);
+ PCollection<String> lines =
+ p.apply(
+ Create.of(
+ tempFolderPath.resolve("readAllTiny*").toString(),
+ tempFolderPath.resolve("readAllLarge*").toString()))
+ .apply(FileIO.matchAll())
+ .apply(FileIO.readMatches().withCompression(AUTO))
+ .apply(TextIO.readFiles().withDesiredBundleSizeBytes(10));
+ PAssert.that(lines).containsInAnyOrder(Iterables.concat(TINY, TINY, LARGE, LARGE));
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testReadWatchForNewFiles() throws IOException, InterruptedException {
+ final Path basePath = tempFolder.getRoot().toPath().resolve("readWatch");
+ basePath.toFile().mkdir();
+
+ p.apply(GenerateSequence.from(0).to(10).withRate(1, Duration.millis(100)))
+ .apply(
+ Window.<Long>into(FixedWindows.of(Duration.millis(150)))
+ .withAllowedLateness(Duration.ZERO)
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))
+ .discardingFiredPanes())
+ .apply(ToString.elements())
+ .apply(
+ TextIO.write()
+ .to(basePath.resolve("data").toString())
+ .withNumShards(1)
+ .withWindowedWrites());
+
+ PCollection<String> lines =
+ p.apply(
+ TextIO.read()
+ .from(basePath.resolve("*").toString())
+ .watchForNewFiles(
+ Duration.millis(100),
+ Watch.Growth.<String>afterTimeSinceNewOutput(Duration.standardSeconds(3))));
+
+ PAssert.that(lines).containsInAnyOrder("0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+ p.run();
+ }
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index 0d8fbbd..4fe1c56 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -17,1101 +17,15 @@
*/
package org.apache.beam.sdk.io;
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static org.apache.beam.sdk.TestUtils.LINES2_ARRAY;
-import static org.apache.beam.sdk.TestUtils.LINES_ARRAY;
-import static org.apache.beam.sdk.TestUtils.NO_LINES_ARRAY;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.AUTO;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.BZIP2;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.DEFLATE;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.GZIP;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.UNCOMPRESSED;
-import static org.apache.beam.sdk.io.TextIO.CompressionType.ZIP;
-import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
-import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.startsWith;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
-import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
-import org.apache.beam.sdk.io.TextIO.CompressionType;
-import org.apache.beam.sdk.io.fs.MatchResult;
-import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.options.ValueProvider;
-import org.apache.beam.sdk.testing.NeedsRunner;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.ValidatesRunner;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
-import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.runners.Suite;
-/**
- * Tests for {@link TextIO} {@link TextIO.Read} and {@link TextIO.Write} transforms.
- */
-// TODO: Change the tests to use ValidatesRunner instead of NeedsRunner
-@RunWith(JUnit4.class)
-@SuppressWarnings("unchecked")
+/** Tests for {@link TextIO} transforms. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ TextIOReadTest.class,
+ TextIOWriteTest.class
+})
public class TextIOTest {
- private static final String MY_HEADER = "myHeader";
- private static final String MY_FOOTER = "myFooter";
- private static final String[] EMPTY = new String[] {};
- private static final String[] TINY =
- new String[] {"Irritable eagle", "Optimistic jay", "Fanciful hawk"};
- private static final String[] LARGE = makeLines(1000);
-
- private static Path tempFolder;
- private static File emptyTxt;
- private static File tinyTxt;
- private static File largeTxt;
- private static File emptyGz;
- private static File tinyGz;
- private static File largeGz;
- private static File emptyBzip2;
- private static File tinyBzip2;
- private static File largeBzip2;
- private static File emptyZip;
- private static File tinyZip;
- private static File largeZip;
- private static File emptyDeflate;
- private static File tinyDeflate;
- private static File largeDeflate;
-
- @Rule
- public TestPipeline p = TestPipeline.create();
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- private static File writeToFile(String[] lines, String filename, CompressionType compression)
- throws IOException {
- File file = tempFolder.resolve(filename).toFile();
- OutputStream output = new FileOutputStream(file);
- switch (compression) {
- case UNCOMPRESSED:
- break;
- case GZIP:
- output = new GZIPOutputStream(output);
- break;
- case BZIP2:
- output = new BZip2CompressorOutputStream(output);
- break;
- case ZIP:
- ZipOutputStream zipOutput = new ZipOutputStream(output);
- zipOutput.putNextEntry(new ZipEntry("entry"));
- output = zipOutput;
- break;
- case DEFLATE:
- output = new DeflateCompressorOutputStream(output);
- break;
- default:
- throw new UnsupportedOperationException(compression.toString());
- }
- writeToStreamAndClose(lines, output);
- return file;
- }
-
- @BeforeClass
- public static void setupClass() throws IOException {
- tempFolder = Files.createTempDirectory("TextIOTest");
- // empty files
- emptyTxt = writeToFile(EMPTY, "empty.txt", CompressionType.UNCOMPRESSED);
- emptyGz = writeToFile(EMPTY, "empty.gz", GZIP);
- emptyBzip2 = writeToFile(EMPTY, "empty.bz2", BZIP2);
- emptyZip = writeToFile(EMPTY, "empty.zip", ZIP);
- emptyDeflate = writeToFile(EMPTY, "empty.deflate", DEFLATE);
- // tiny files
- tinyTxt = writeToFile(TINY, "tiny.txt", CompressionType.UNCOMPRESSED);
- tinyGz = writeToFile(TINY, "tiny.gz", GZIP);
- tinyBzip2 = writeToFile(TINY, "tiny.bz2", BZIP2);
- tinyZip = writeToFile(TINY, "tiny.zip", ZIP);
- tinyDeflate = writeToFile(TINY, "tiny.deflate", DEFLATE);
- // large files
- largeTxt = writeToFile(LARGE, "large.txt", CompressionType.UNCOMPRESSED);
- largeGz = writeToFile(LARGE, "large.gz", GZIP);
- largeBzip2 = writeToFile(LARGE, "large.bz2", BZIP2);
- largeZip = writeToFile(LARGE, "large.zip", ZIP);
- largeDeflate = writeToFile(LARGE, "large.deflate", DEFLATE);
- }
-
- @AfterClass
- public static void teardownClass() throws IOException {
- Files.walkFileTree(tempFolder, new SimpleFileVisitor<Path>() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- Files.delete(file);
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
- Files.delete(dir);
- return FileVisitResult.CONTINUE;
- }
- });
- }
-
- private <T> void runTestRead(String[] expected) throws Exception {
- File tmpFile = Files.createTempFile(tempFolder, "file", "txt").toFile();
- String filename = tmpFile.getPath();
-
- try (PrintStream writer = new PrintStream(new FileOutputStream(tmpFile))) {
- for (String elem : expected) {
- byte[] encodedElem = CoderUtils.encodeToByteArray(StringUtf8Coder.of(), elem);
- String line = new String(encodedElem);
- writer.println(line);
- }
- }
-
- TextIO.Read read = TextIO.read().from(filename);
-
- PCollection<String> output = p.apply(read);
-
- PAssert.that(output).containsInAnyOrder(expected);
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReadStrings() throws Exception {
- runTestRead(LINES_ARRAY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReadEmptyStrings() throws Exception {
- runTestRead(NO_LINES_ARRAY);
- }
-
- @Test
- public void testReadNamed() throws Exception {
- p.enableAbandonedNodeEnforcement(false);
-
- assertEquals(
- "TextIO.Read/Read.out",
- p.apply(TextIO.read().from("somefile")).getName());
- assertEquals(
- "MyRead/Read.out",
- p.apply("MyRead", TextIO.read().from(emptyTxt.getPath())).getName());
- }
-
- @Test
- public void testReadDisplayData() {
- TextIO.Read read = TextIO.read()
- .from("foo.*")
- .withCompressionType(BZIP2);
-
- DisplayData displayData = DisplayData.from(read);
-
- assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
- assertThat(displayData, hasDisplayItem("compressionType", BZIP2.toString()));
- }
-
- @Test
- @Category(ValidatesRunner.class)
- public void testPrimitiveReadDisplayData() {
- DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
-
- TextIO.Read read = TextIO.read()
- .from("foobar");
-
- Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read);
- assertThat("TextIO.Read should include the file prefix in its primitive display data",
- displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar")))));
- }
-
- private void runTestWrite(String[] elems) throws Exception {
- runTestWrite(elems, null, null, 1);
- }
-
- private void runTestWrite(String[] elems, int numShards) throws Exception {
- runTestWrite(elems, null, null, numShards);
- }
-
- private void runTestWrite(String[] elems, String header, String footer)
- throws Exception {
- runTestWrite(elems, header, footer, 1);
- }
-
- private void runTestWrite(
- String[] elems, String header, String footer, int numShards) throws Exception {
- String outputName = "file.txt";
- Path baseDir = Files.createTempDirectory(tempFolder, "testwrite");
- String baseFilename = baseDir.resolve(outputName).toString();
-
- PCollection<String> input =
- p.apply(Create.of(Arrays.asList(elems)).withCoder(StringUtf8Coder.of()));
-
- TextIO.Write write =
- TextIO.write().to(baseFilename)
- .withHeader(header)
- .withFooter(footer);
-
- if (numShards == 1) {
- write = write.withoutSharding();
- } else if (numShards > 0) {
- write = write.withNumShards(numShards).withShardNameTemplate(ShardNameTemplate.INDEX_OF_MAX);
- }
-
- input.apply(write);
-
- p.run();
-
- assertOutputFiles(elems, header, footer, numShards, baseDir, outputName,
- firstNonNull(write.getShardTemplate(), DefaultFilenamePolicy.DEFAULT_SHARD_TEMPLATE));
- }
-
- public static void assertOutputFiles(
- String[] elems,
- final String header,
- final String footer,
- int numShards,
- Path rootLocation,
- String outputName,
- String shardNameTemplate)
- throws Exception {
- List<File> expectedFiles = new ArrayList<>();
- if (numShards == 0) {
- String pattern = rootLocation.toAbsolutePath().resolve(outputName + "*").toString();
- List<MatchResult> matches = FileSystems.match(Collections.singletonList(pattern));
- for (Metadata expectedFile : Iterables.getOnlyElement(matches).metadata()) {
- expectedFiles.add(new File(expectedFile.resourceId().toString()));
- }
- } else {
- for (int i = 0; i < numShards; i++) {
- expectedFiles.add(
- new File(
- rootLocation.toString(),
- DefaultFilenamePolicy.constructName(
- outputName, shardNameTemplate, "", i, numShards)));
- }
- }
-
- List<List<String>> actual = new ArrayList<>();
-
- for (File tmpFile : expectedFiles) {
- try (BufferedReader reader = new BufferedReader(new FileReader(tmpFile))) {
- List<String> currentFile = new ArrayList<>();
- for (;;) {
- String line = reader.readLine();
- if (line == null) {
- break;
- }
- currentFile.add(line);
- }
- actual.add(currentFile);
- }
- }
-
- List<String> expectedElements = new ArrayList<>(elems.length);
- for (String elem : elems) {
- byte[] encodedElem = CoderUtils.encodeToByteArray(StringUtf8Coder.of(), elem);
- String line = new String(encodedElem);
- expectedElements.add(line);
- }
-
- List<String> actualElements =
- Lists.newArrayList(
- Iterables.concat(
- FluentIterable
- .from(actual)
- .transform(removeHeaderAndFooter(header, footer))
- .toList()));
-
- assertThat(actualElements, containsInAnyOrder(expectedElements.toArray()));
-
- assertTrue(Iterables.all(actual, haveProperHeaderAndFooter(header, footer)));
- }
-
- private static Function<List<String>, List<String>> removeHeaderAndFooter(final String header,
- final String footer) {
- return new Function<List<String>, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(List<String> lines) {
- ArrayList<String> newLines = Lists.newArrayList(lines);
- if (header != null) {
- newLines.remove(0);
- }
- if (footer != null) {
- int last = newLines.size() - 1;
- newLines.remove(last);
- }
- return newLines;
- }
- };
- }
-
- private static Predicate<List<String>> haveProperHeaderAndFooter(final String header,
- final String footer) {
- return new Predicate<List<String>>() {
- @Override
- public boolean apply(List<String> fileLines) {
- int last = fileLines.size() - 1;
- return (header == null || fileLines.get(0).equals(header))
- && (footer == null || fileLines.get(last).equals(footer));
- }
- };
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteStrings() throws Exception {
- runTestWrite(LINES_ARRAY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteEmptyStringsNoSharding() throws Exception {
- runTestWrite(NO_LINES_ARRAY, 0);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteEmptyStrings() throws Exception {
- runTestWrite(NO_LINES_ARRAY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testShardedWrite() throws Exception {
- runTestWrite(LINES_ARRAY, 5);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteWithHeader() throws Exception {
- runTestWrite(LINES_ARRAY, MY_HEADER, null);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteWithFooter() throws Exception {
- runTestWrite(LINES_ARRAY, null, MY_FOOTER);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteWithHeaderAndFooter() throws Exception {
- runTestWrite(LINES_ARRAY, MY_HEADER, MY_FOOTER);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testWriteWithWritableByteChannelFactory() throws Exception {
- Coder<String> coder = StringUtf8Coder.of();
- String outputName = "file.txt";
- Path baseDir = Files.createTempDirectory(tempFolder, "testwrite");
-
- PCollection<String> input = p.apply(Create.of(Arrays.asList(LINES2_ARRAY)).withCoder(coder));
-
- final WritableByteChannelFactory writableByteChannelFactory =
- new DrunkWritableByteChannelFactory();
- TextIO.Write write = TextIO.write().to(baseDir.resolve(outputName).toString())
- .withoutSharding().withWritableByteChannelFactory(writableByteChannelFactory);
- DisplayData displayData = DisplayData.from(write);
- assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "DRUNK"));
-
- input.apply(write);
-
- p.run();
-
- final List<String> drunkElems = new ArrayList<>(LINES2_ARRAY.length * 2 + 2);
- for (String elem : LINES2_ARRAY) {
- drunkElems.add(elem);
- drunkElems.add(elem);
- }
- assertOutputFiles(drunkElems.toArray(new String[0]), null, null, 1, baseDir,
- outputName + writableByteChannelFactory.getFilenameSuffix(), write.getShardTemplate());
- }
-
- @Test
- public void testWriteDisplayData() {
- TextIO.Write write = TextIO.write()
- .to("/foo")
- .withSuffix("bar")
- .withShardNameTemplate("-SS-of-NN-")
- .withNumShards(100)
- .withFooter("myFooter")
- .withHeader("myHeader");
-
- DisplayData displayData = DisplayData.from(write);
-
- assertThat(displayData, hasDisplayItem("filePrefix", "/foo"));
- assertThat(displayData, hasDisplayItem("fileSuffix", "bar"));
- assertThat(displayData, hasDisplayItem("fileHeader", "myHeader"));
- assertThat(displayData, hasDisplayItem("fileFooter", "myFooter"));
- assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-"));
- assertThat(displayData, hasDisplayItem("numShards", 100));
- assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "UNCOMPRESSED"));
- }
-
- @Test
- public void testWriteDisplayDataValidateThenHeader() {
- TextIO.Write write = TextIO.write()
- .to("foo")
- .withHeader("myHeader");
-
- DisplayData displayData = DisplayData.from(write);
-
- assertThat(displayData, hasDisplayItem("fileHeader", "myHeader"));
- }
-
- @Test
- public void testWriteDisplayDataValidateThenFooter() {
- TextIO.Write write = TextIO.write()
- .to("foo")
- .withFooter("myFooter");
-
- DisplayData displayData = DisplayData.from(write);
-
- assertThat(displayData, hasDisplayItem("fileFooter", "myFooter"));
- }
-
- /** Options for testing. */
- public interface RuntimeTestOptions extends PipelineOptions {
- ValueProvider<String> getInput();
- void setInput(ValueProvider<String> value);
-
- ValueProvider<String> getOutput();
- void setOutput(ValueProvider<String> value);
- }
-
- @Test
- public void testRuntimeOptionsNotCalledInApply() throws Exception {
- p.enableAbandonedNodeEnforcement(false);
-
- RuntimeTestOptions options = PipelineOptionsFactory.as(RuntimeTestOptions.class);
-
- p
- .apply(TextIO.read().from(options.getInput()))
- .apply(TextIO.write().to(options.getOutput()));
- }
-
- @Test
- public void testCompressionTypeIsSet() throws Exception {
- TextIO.Read read = TextIO.read().from("/tmp/test");
- assertEquals(AUTO, read.getCompressionType());
- read = TextIO.read().from("/tmp/test").withCompressionType(GZIP);
- assertEquals(GZIP, read.getCompressionType());
- }
-
- /**
- * Helper that writes the given lines (adding a newline in between) to a stream, then closes the
- * stream.
- */
- private static void writeToStreamAndClose(String[] lines, OutputStream outputStream) {
- try (PrintStream writer = new PrintStream(outputStream)) {
- for (String line : lines) {
- writer.println(line);
- }
- }
- }
-
- /**
- * Helper method that runs TextIO.read().from(filename).withCompressionType(compressionType)
- * and asserts that the results match the given expected output.
- */
- private void assertReadingCompressedFileMatchesExpected(
- File file, CompressionType compressionType, String[] expected) {
-
- TextIO.Read read =
- TextIO.read().from(file.getPath()).withCompressionType(compressionType);
- PCollection<String> output = p.apply("Read_" + file + "_" + compressionType.toString(), read);
-
- PAssert.that(output).containsInAnyOrder(expected);
- p.run();
- }
-
- /**
- * Helper to make an array of compressible strings. Returns ["word"i] for i in range(0,n).
- */
- private static String[] makeLines(int n) {
- String[] ret = new String[n];
- for (int i = 0; i < n; ++i) {
- ret[i] = "word" + i;
- }
- return ret;
- }
-
- /**
- * Tests reading from a small, gzipped file with no .gz extension but GZIP compression set.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testSmallCompressedGzipReadNoExtension() throws Exception {
- File smallGzNoExtension = writeToFile(TINY, "tiny_gz_no_extension", GZIP);
- assertReadingCompressedFileMatchesExpected(smallGzNoExtension, GZIP, TINY);
- }
-
- /**
- * Tests reading from a small, uncompressed file with .gz extension. This must work in AUTO or
- * GZIP modes. This is needed because some network file systems / HTTP clients will transparently
- * decompress gzipped content.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testSmallCompressedGzipReadActuallyUncompressed() throws Exception {
- File smallGzNotCompressed =
- writeToFile(TINY, "tiny_uncompressed.gz", CompressionType.UNCOMPRESSED);
- // Should work with GZIP compression set.
- assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, GZIP, TINY);
- // Should also work with AUTO mode set.
- assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, AUTO, TINY);
- }
-
- /**
- * Tests reading from a small, bzip2ed file with no .bz2 extension but BZIP2 compression set.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testSmallCompressedBzip2ReadNoExtension() throws Exception {
- File smallBz2NoExtension = writeToFile(TINY, "tiny_bz2_no_extension", BZIP2);
- assertReadingCompressedFileMatchesExpected(smallBz2NoExtension, BZIP2, TINY);
- }
-
- /**
- * Create a zip file with the given lines.
- *
- * @param expected A list of expected lines, populated in the zip file.
- * @param filename Optionally zip file name (can be null).
- * @param fieldsEntries Fields to write in zip entries.
- * @return The zip filename.
- * @throws Exception In case of a failure during zip file creation.
- */
- private String createZipFile(List<String> expected, String filename, String[]... fieldsEntries)
- throws Exception {
- File tmpFile = tempFolder.resolve(filename).toFile();
- String tmpFileName = tmpFile.getPath();
-
- ZipOutputStream out = new ZipOutputStream(new FileOutputStream(tmpFile));
- PrintStream writer = new PrintStream(out, true /* auto-flush on write */);
-
- int index = 0;
- for (String[] entry : fieldsEntries) {
- out.putNextEntry(new ZipEntry(Integer.toString(index)));
- for (String field : entry) {
- writer.println(field);
- expected.add(field);
- }
- out.closeEntry();
- index++;
- }
-
- writer.close();
- out.close();
-
- return tmpFileName;
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testTxtRead() throws Exception {
- // Files with non-compressed extensions should work in AUTO and UNCOMPRESSED modes.
- for (CompressionType type : new CompressionType[]{AUTO, UNCOMPRESSED}) {
- assertReadingCompressedFileMatchesExpected(emptyTxt, type, EMPTY);
- assertReadingCompressedFileMatchesExpected(tinyTxt, type, TINY);
- assertReadingCompressedFileMatchesExpected(largeTxt, type, LARGE);
- }
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testGzipCompressedRead() throws Exception {
- // Files with the right extensions should work in AUTO and GZIP modes.
- for (CompressionType type : new CompressionType[]{AUTO, GZIP}) {
- assertReadingCompressedFileMatchesExpected(emptyGz, type, EMPTY);
- assertReadingCompressedFileMatchesExpected(tinyGz, type, TINY);
- assertReadingCompressedFileMatchesExpected(largeGz, type, LARGE);
- }
-
- // Sanity check that we're properly testing compression.
- assertThat(largeTxt.length(), greaterThan(largeGz.length()));
-
- // GZIP files with non-gz extension should work in GZIP mode.
- File gzFile = writeToFile(TINY, "tiny_gz_no_extension", GZIP);
- assertReadingCompressedFileMatchesExpected(gzFile, GZIP, TINY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testBzip2CompressedRead() throws Exception {
- // Files with the right extensions should work in AUTO and BZIP2 modes.
- for (CompressionType type : new CompressionType[]{AUTO, BZIP2}) {
- assertReadingCompressedFileMatchesExpected(emptyBzip2, type, EMPTY);
- assertReadingCompressedFileMatchesExpected(tinyBzip2, type, TINY);
- assertReadingCompressedFileMatchesExpected(largeBzip2, type, LARGE);
- }
-
- // Sanity check that we're properly testing compression.
- assertThat(largeTxt.length(), greaterThan(largeBzip2.length()));
-
- // BZ2 files with non-bz2 extension should work in BZIP2 mode.
- File bz2File = writeToFile(TINY, "tiny_bz2_no_extension", BZIP2);
- assertReadingCompressedFileMatchesExpected(bz2File, BZIP2, TINY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testZipCompressedRead() throws Exception {
- // Files with the right extensions should work in AUTO and ZIP modes.
- for (CompressionType type : new CompressionType[]{AUTO, ZIP}) {
- assertReadingCompressedFileMatchesExpected(emptyZip, type, EMPTY);
- assertReadingCompressedFileMatchesExpected(tinyZip, type, TINY);
- assertReadingCompressedFileMatchesExpected(largeZip, type, LARGE);
- }
-
- // Sanity check that we're properly testing compression.
- assertThat(largeTxt.length(), greaterThan(largeZip.length()));
-
- // Zip files with non-zip extension should work in ZIP mode.
- File zipFile = writeToFile(TINY, "tiny_zip_no_extension", ZIP);
- assertReadingCompressedFileMatchesExpected(zipFile, ZIP, TINY);
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testDeflateCompressedRead() throws Exception {
- // Files with the right extensions should work in AUTO and ZIP modes.
- for (CompressionType type : new CompressionType[]{AUTO, DEFLATE}) {
- assertReadingCompressedFileMatchesExpected(emptyDeflate, type, EMPTY);
- assertReadingCompressedFileMatchesExpected(tinyDeflate, type, TINY);
- assertReadingCompressedFileMatchesExpected(largeDeflate, type, LARGE);
- }
-
- // Sanity check that we're properly testing compression.
- assertThat(largeTxt.length(), greaterThan(largeDeflate.length()));
-
- // Deflate files with non-deflate extension should work in DEFLATE mode.
- File deflateFile = writeToFile(TINY, "tiny_deflate_no_extension", DEFLATE);
- assertReadingCompressedFileMatchesExpected(deflateFile, DEFLATE, TINY);
- }
-
- /**
- * Tests a zip file with no entries. This is a corner case not tested elsewhere as the default
- * test zip files have a single entry.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testZipCompressedReadWithNoEntries() throws Exception {
- String filename = createZipFile(new ArrayList<String>(), "empty zip file");
- assertReadingCompressedFileMatchesExpected(new File(filename), CompressionType.ZIP, EMPTY);
- }
-
- /**
- * Tests a zip file with multiple entries. This is a corner case not tested elsewhere as the
- * default test zip files have a single entry.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testZipCompressedReadWithMultiEntriesFile() throws Exception {
- String[] entry0 = new String[]{"first", "second", "three"};
- String[] entry1 = new String[]{"four", "five", "six"};
- String[] entry2 = new String[]{"seven", "eight", "nine"};
-
- List<String> expected = new ArrayList<>();
-
- String filename = createZipFile(expected, "multiple entries", entry0, entry1, entry2);
- assertReadingCompressedFileMatchesExpected(
- new File(filename), CompressionType.ZIP, expected.toArray(new String[]{}));
- }
-
- /**
- * Read a ZIP compressed file containing data, multiple empty entries, and then more data. We
- * expect just the data back.
- */
- @Test
- @Category(NeedsRunner.class)
- public void testZipCompressedReadWithComplexEmptyAndPresentEntries() throws Exception {
- String filename = createZipFile(
- new ArrayList<String>(),
- "complex empty and present entries",
- new String[]{"cat"},
- new String[]{},
- new String[]{},
- new String[]{"dog"});
-
- assertReadingCompressedFileMatchesExpected(
- new File(filename), CompressionType.ZIP, new String[] {"cat", "dog"});
- }
-
- @Test
- public void testTextIOGetName() {
- assertEquals("TextIO.Read", TextIO.read().from("somefile").getName());
- assertEquals("TextIO.Write", TextIO.write().to("somefile").getName());
- assertEquals("TextIO.Read", TextIO.read().from("somefile").toString());
- }
-
- @Test
- public void testProgressEmptyFile() throws IOException {
- try (BoundedReader<String> reader =
- prepareSource(new byte[0]).createReader(PipelineOptionsFactory.create())) {
- // Check preconditions before starting.
- assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // Assert empty
- assertFalse(reader.start());
-
- // Check postconditions after finishing
- assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(0, reader.getSplitPointsRemaining());
- }
- }
-
- @Test
- public void testProgressTextFile() throws IOException {
- String file = "line1\nline2\nline3";
- try (BoundedReader<String> reader =
- prepareSource(file.getBytes()).createReader(PipelineOptionsFactory.create())) {
- // Check preconditions before starting
- assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // Line 1
- assertTrue(reader.start());
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // Line 2
- assertTrue(reader.advance());
- assertEquals(1, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // Line 3
- assertTrue(reader.advance());
- assertEquals(2, reader.getSplitPointsConsumed());
- assertEquals(1, reader.getSplitPointsRemaining());
-
- // Check postconditions after finishing
- assertFalse(reader.advance());
- assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(3, reader.getSplitPointsConsumed());
- assertEquals(0, reader.getSplitPointsRemaining());
- }
- }
-
- @Test
- public void testProgressAfterSplitting() throws IOException {
- String file = "line1\nline2\nline3";
- BoundedSource<String> source = prepareSource(file.getBytes());
- BoundedSource<String> remainder;
-
- // Create the remainder, verifying properties pre- and post-splitting.
- try (BoundedReader<String> readerOrig = source.createReader(PipelineOptionsFactory.create())) {
- // Preconditions.
- assertEquals(0.0, readerOrig.getFractionConsumed(), 1e-6);
- assertEquals(0, readerOrig.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, readerOrig.getSplitPointsRemaining());
-
- // First record, before splitting.
- assertTrue(readerOrig.start());
- assertEquals(0, readerOrig.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, readerOrig.getSplitPointsRemaining());
-
- // Split. 0.1 is in line1, so should now be able to detect last record.
- remainder = readerOrig.splitAtFraction(0.1);
- System.err.println(readerOrig.getCurrentSource());
- assertNotNull(remainder);
-
- // First record, after splitting.
- assertEquals(0, readerOrig.getSplitPointsConsumed());
- assertEquals(1, readerOrig.getSplitPointsRemaining());
-
- // Finish and postconditions.
- assertFalse(readerOrig.advance());
- assertEquals(1.0, readerOrig.getFractionConsumed(), 1e-6);
- assertEquals(1, readerOrig.getSplitPointsConsumed());
- assertEquals(0, readerOrig.getSplitPointsRemaining());
- }
-
- // Check the properties of the remainder.
- try (BoundedReader<String> reader = remainder.createReader(PipelineOptionsFactory.create())) {
- // Preconditions.
- assertEquals(0.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // First record should be line 2.
- assertTrue(reader.start());
- assertEquals(0, reader.getSplitPointsConsumed());
- assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining());
-
- // Second record is line 3
- assertTrue(reader.advance());
- assertEquals(1, reader.getSplitPointsConsumed());
- assertEquals(1, reader.getSplitPointsRemaining());
-
- // Check postconditions after finishing
- assertFalse(reader.advance());
- assertEquals(1.0, reader.getFractionConsumed(), 1e-6);
- assertEquals(2, reader.getSplitPointsConsumed());
- assertEquals(0, reader.getSplitPointsRemaining());
- }
- }
-
- @Test
- public void testReadEmptyLines() throws Exception {
- runTestReadWithData("\n\n\n".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("", "", ""));
- }
-
- @Test
- public void testReadFileWithLineFeedDelimiter() throws Exception {
- runTestReadWithData("asdf\nhjkl\nxyz\n".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithCarriageReturnDelimiter() throws Exception {
- runTestReadWithData("asdf\rhjkl\rxyz\r".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithCarriageReturnAndLineFeedDelimiter() throws Exception {
- runTestReadWithData("asdf\r\nhjkl\r\nxyz\r\n".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithMixedDelimiters() throws Exception {
- runTestReadWithData("asdf\rhjkl\r\nxyz\n".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithLineFeedDelimiterAndNonEmptyBytesAtEnd() throws Exception {
- runTestReadWithData("asdf\nhjkl\nxyz".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithCarriageReturnDelimiterAndNonEmptyBytesAtEnd() throws Exception {
- runTestReadWithData("asdf\rhjkl\rxyz".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithCarriageReturnAndLineFeedDelimiterAndNonEmptyBytesAtEnd()
- throws Exception {
- runTestReadWithData("asdf\r\nhjkl\r\nxyz".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- @Test
- public void testReadFileWithMixedDelimitersAndNonEmptyBytesAtEnd() throws Exception {
- runTestReadWithData("asdf\rhjkl\r\nxyz".getBytes(StandardCharsets.UTF_8),
- ImmutableList.of("asdf", "hjkl", "xyz"));
- }
-
- private void runTestReadWithData(byte[] data, List<String> expectedResults) throws Exception {
- TextSource source = prepareSource(data);
- List<String> actual = SourceTestUtils.readFromSource(source, PipelineOptionsFactory.create());
- assertThat(actual, containsInAnyOrder(new ArrayList<>(expectedResults).toArray(new String[0])));
- }
-
- @Test
- public void testSplittingSourceWithEmptyLines() throws Exception {
- TextSource source = prepareSource("\n\n\n".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithLineFeedDelimiter() throws Exception {
- TextSource source = prepareSource("asdf\nhjkl\nxyz\n".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithCarriageReturnDelimiter() throws Exception {
- TextSource source = prepareSource("asdf\rhjkl\rxyz\r".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithCarriageReturnAndLineFeedDelimiter() throws Exception {
- TextSource source = prepareSource(
- "asdf\r\nhjkl\r\nxyz\r\n".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithMixedDelimiters() throws Exception {
- TextSource source = prepareSource(
- "asdf\rhjkl\r\nxyz\n".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithLineFeedDelimiterAndNonEmptyBytesAtEnd() throws Exception {
- TextSource source = prepareSource("asdf\nhjkl\nxyz".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithCarriageReturnDelimiterAndNonEmptyBytesAtEnd()
- throws Exception {
- TextSource source = prepareSource("asdf\rhjkl\rxyz".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithCarriageReturnAndLineFeedDelimiterAndNonEmptyBytesAtEnd()
- throws Exception {
- TextSource source = prepareSource(
- "asdf\r\nhjkl\r\nxyz".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSplittingSourceWithMixedDelimitersAndNonEmptyBytesAtEnd() throws Exception {
- TextSource source = prepareSource("asdf\rhjkl\r\nxyz".getBytes(StandardCharsets.UTF_8));
- SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create());
- }
-
- private TextSource prepareSource(byte[] data) throws IOException {
- Path path = Files.createTempFile(tempFolder, "tempfile", "ext");
- Files.write(path, data);
- return new TextSource(ValueProvider.StaticValueProvider.of(path.toString()));
- }
-
- @Test
- public void testInitialSplitAutoModeTxt() throws Exception {
- PipelineOptions options = TestPipeline.testingPipelineOptions();
- long desiredBundleSize = 1000;
-
- // Sanity check: file is at least 2 bundles long.
- assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize));
-
- FileBasedSource<String> source = TextIO.read().from(largeTxt.getPath()).getSource();
- List<? extends FileBasedSource<String>> splits =
- source.split(desiredBundleSize, options);
-
- // At least 2 splits and they are equal to reading the whole file.
- assertThat(splits, hasSize(greaterThan(1)));
- SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
- }
-
- @Test
- public void testInitialSplitAutoModeGz() throws Exception {
- long desiredBundleSize = 1000;
- PipelineOptions options = TestPipeline.testingPipelineOptions();
-
- // Sanity check: file is at least 2 bundles long.
- assertThat(largeGz.length(), greaterThan(2 * desiredBundleSize));
-
- FileBasedSource<String> source = TextIO.read().from(largeGz.getPath()).getSource();
- List<? extends FileBasedSource<String>> splits =
- source.split(desiredBundleSize, options);
-
- // Exactly 1 split, even in AUTO mode, since it is a gzip file.
- assertThat(splits, hasSize(equalTo(1)));
- SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
- }
-
- @Test
- public void testInitialSplitGzipModeTxt() throws Exception {
- PipelineOptions options = TestPipeline.testingPipelineOptions();
- long desiredBundleSize = 1000;
-
- // Sanity check: file is at least 2 bundles long.
- assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize));
-
- FileBasedSource<String> source =
- TextIO.read().from(largeTxt.getPath()).withCompressionType(GZIP).getSource();
- List<? extends FileBasedSource<String>> splits =
- source.split(desiredBundleSize, options);
-
- // Exactly 1 split, even though splittable text file, since using GZIP mode.
- assertThat(splits, hasSize(equalTo(1)));
- SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
- }
-
- @Test
- public void testInitialSplitGzipModeGz() throws Exception {
- PipelineOptions options = TestPipeline.testingPipelineOptions();
- long desiredBundleSize = 1000;
-
- // Sanity check: file is at least 2 bundles long.
- assertThat(largeGz.length(), greaterThan(2 * desiredBundleSize));
-
- FileBasedSource<String> source =
- TextIO.read().from(largeGz.getPath()).withCompressionType(GZIP).getSource();
- List<? extends FileBasedSource<String>> splits =
- source.split(desiredBundleSize, options);
-
- // Exactly 1 split using .gz extension and using GZIP mode.
- assertThat(splits, hasSize(equalTo(1)));
- SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
- }
-
- @Test
- public void testWindowedWriteRequiresFilenamePolicy() {
- PCollection<String> emptyInput = p.apply(Create.empty(StringUtf8Coder.of()));
- TextIO.Write write = TextIO.write().to("/tmp/some/file").withWindowedWrites();
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage(
- "When using windowed writes, a filename policy must be set via withFilenamePolicy()");
- emptyInput.apply(write);
- }
+ // Empty.
}
-
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java
new file mode 100644
index 0000000..0f40067
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java
@@ -0,0 +1,643 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static org.apache.beam.sdk.TestUtils.LINES2_ARRAY;
+import static org.apache.beam.sdk.TestUtils.LINES_ARRAY;
+import static org.apache.beam.sdk.TestUtils.NO_LINES_ARRAY;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.DefaultCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
+import org.apache.beam.sdk.io.fs.MatchResult;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.io.fs.ResolveOptions;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+/** Tests for {@link TextIO.Write}. */
+public class TextIOWriteTest {
+ private static final String MY_HEADER = "myHeader";
+ private static final String MY_FOOTER = "myFooter";
+
+ private static Path tempFolder;
+
+ @Rule public TestPipeline p = TestPipeline.create();
+
+ @Rule public ExpectedException expectedException = ExpectedException.none();
+
+ @BeforeClass
+ public static void setupClass() throws IOException {
+ tempFolder = Files.createTempDirectory("TextIOTest");
+ }
+
+ @AfterClass
+ public static void teardownClass() throws IOException {
+ Files.walkFileTree(
+ tempFolder,
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ static class TestDynamicDestinations
+ extends FileBasedSink.DynamicDestinations<String, String, String> {
+ ResourceId baseDir;
+
+ TestDynamicDestinations(ResourceId baseDir) {
+ this.baseDir = baseDir;
+ }
+
+ @Override
+ public String formatRecord(String record) {
+ return record;
+ }
+
+ @Override
+ public String getDestination(String element) {
+ // Destination is based on first character of string.
+ return element.substring(0, 1);
+ }
+
+ @Override
+ public String getDefaultDestination() {
+ return "";
+ }
+
+ @Nullable
+ @Override
+ public Coder<String> getDestinationCoder() {
+ return StringUtf8Coder.of();
+ }
+
+ @Override
+ public FileBasedSink.FilenamePolicy getFilenamePolicy(String destination) {
+ return DefaultFilenamePolicy.fromStandardParameters(
+ ValueProvider.StaticValueProvider.of(
+ baseDir.resolve(
+ "file_" + destination + ".txt",
+ ResolveOptions.StandardResolveOptions.RESOLVE_FILE)),
+ null,
+ null,
+ false);
+ }
+ }
+
+ class StartsWith implements Predicate<String> {
+ String prefix;
+
+ StartsWith(String prefix) {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public boolean apply(@Nullable String input) {
+ return input.startsWith(prefix);
+ }
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinations() throws Exception {
+ ResourceId baseDir =
+ FileSystems.matchNewResource(
+ Files.createTempDirectory(tempFolder, "testDynamicDestinations").toString(), true);
+
+ List<String> elements = Lists.newArrayList("aaaa", "aaab", "baaa", "baab", "caaa", "caab");
+ PCollection<String> input = p.apply(Create.of(elements).withCoder(StringUtf8Coder.of()));
+ input.apply(TextIO.write().to(new TestDynamicDestinations(baseDir)).withTempDirectory(baseDir));
+ p.run();
+
+ assertOutputFiles(
+ Iterables.toArray(Iterables.filter(elements, new StartsWith("a")), String.class),
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_a.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ assertOutputFiles(
+ Iterables.toArray(Iterables.filter(elements, new StartsWith("b")), String.class),
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_b.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ assertOutputFiles(
+ Iterables.toArray(Iterables.filter(elements, new StartsWith("c")), String.class),
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_c.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ }
+
+ @DefaultCoder(AvroCoder.class)
+ private static class UserWriteType {
+ String destination;
+ String metadata;
+
+ UserWriteType() {
+ this.destination = "";
+ this.metadata = "";
+ }
+
+ UserWriteType(String destination, String metadata) {
+ this.destination = destination;
+ this.metadata = metadata;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("destination: %s metadata : %s", destination, metadata);
+ }
+ }
+
+ private static class SerializeUserWrite implements SerializableFunction<UserWriteType, String> {
+ @Override
+ public String apply(UserWriteType input) {
+ return input.toString();
+ }
+ }
+
+ private static class UserWriteDestination
+ implements SerializableFunction<UserWriteType, DefaultFilenamePolicy.Params> {
+ private ResourceId baseDir;
+
+ UserWriteDestination(ResourceId baseDir) {
+ this.baseDir = baseDir;
+ }
+
+ @Override
+ public DefaultFilenamePolicy.Params apply(UserWriteType input) {
+ return new DefaultFilenamePolicy.Params()
+ .withBaseFilename(
+ baseDir.resolve(
+ "file_" + input.destination.substring(0, 1) + ".txt",
+ ResolveOptions.StandardResolveOptions.RESOLVE_FILE));
+ }
+ }
+
+ private static class ExtractWriteDestination implements Function<UserWriteType, String> {
+ @Override
+ public String apply(@Nullable UserWriteType input) {
+ return input.destination;
+ }
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDefaultFilenamePolicy() throws Exception {
+ ResourceId baseDir =
+ FileSystems.matchNewResource(
+ Files.createTempDirectory(tempFolder, "testDynamicDestinations").toString(), true);
+
+ List<UserWriteType> elements =
+ Lists.newArrayList(
+ new UserWriteType("aaaa", "first"),
+ new UserWriteType("aaab", "second"),
+ new UserWriteType("baaa", "third"),
+ new UserWriteType("baab", "fourth"),
+ new UserWriteType("caaa", "fifth"),
+ new UserWriteType("caab", "sixth"));
+ PCollection<UserWriteType> input = p.apply(Create.of(elements));
+ input.apply(
+ TextIO.<UserWriteType>writeCustomType()
+ .to(
+ new UserWriteDestination(baseDir),
+ new DefaultFilenamePolicy.Params()
+ .withBaseFilename(
+ baseDir.resolve(
+ "empty", ResolveOptions.StandardResolveOptions.RESOLVE_FILE)))
+ .withFormatFunction(new SerializeUserWrite())
+ .withTempDirectory(FileSystems.matchNewResource(baseDir.toString(), true)));
+ p.run();
+
+ String[] aElements =
+ Iterables.toArray(
+ Iterables.transform(
+ Iterables.filter(
+ elements,
+ Predicates.compose(new StartsWith("a"), new ExtractWriteDestination())),
+ Functions.toStringFunction()),
+ String.class);
+ String[] bElements =
+ Iterables.toArray(
+ Iterables.transform(
+ Iterables.filter(
+ elements,
+ Predicates.compose(new StartsWith("b"), new ExtractWriteDestination())),
+ Functions.toStringFunction()),
+ String.class);
+ String[] cElements =
+ Iterables.toArray(
+ Iterables.transform(
+ Iterables.filter(
+ elements,
+ Predicates.compose(new StartsWith("c"), new ExtractWriteDestination())),
+ Functions.toStringFunction()),
+ String.class);
+ assertOutputFiles(
+ aElements,
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_a.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ assertOutputFiles(
+ bElements,
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_b.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ assertOutputFiles(
+ cElements,
+ null,
+ null,
+ 0,
+ baseDir.resolve("file_c.txt", ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE);
+ }
+
+ private void runTestWrite(String[] elems) throws Exception {
+ runTestWrite(elems, null, null, 1);
+ }
+
+ private void runTestWrite(String[] elems, int numShards) throws Exception {
+ runTestWrite(elems, null, null, numShards);
+ }
+
+ private void runTestWrite(String[] elems, String header, String footer) throws Exception {
+ runTestWrite(elems, header, footer, 1);
+ }
+
+ private static class MatchesFilesystem implements SerializableFunction<Iterable<String>, Void> {
+ private final ResourceId baseFilename;
+
+ MatchesFilesystem(ResourceId baseFilename) {
+ this.baseFilename = baseFilename;
+ }
+
+ @Override
+ public Void apply(Iterable<String> values) {
+ try {
+ String pattern = baseFilename.toString() + "*";
+ List<String> matches = Lists.newArrayList();
+ for (Metadata match :Iterables.getOnlyElement(
+ FileSystems.match(Collections.singletonList(pattern))).metadata()) {
+ matches.add(match.resourceId().toString());
+ }
+ assertThat(values, containsInAnyOrder(Iterables.toArray(matches, String.class)));
+ } catch (Exception e) {
+ fail("Exception caught " + e);
+ }
+ return null;
+ }
+ }
+
+ private void runTestWrite(String[] elems, String header, String footer, int numShards)
+ throws Exception {
+ String outputName = "file.txt";
+ Path baseDir = Files.createTempDirectory(tempFolder, "testwrite");
+ ResourceId baseFilename =
+ FileBasedSink.convertToFileResourceIfPossible(baseDir.resolve(outputName).toString());
+
+ PCollection<String> input =
+ p.apply("CreateInput", Create.of(Arrays.asList(elems)).withCoder(StringUtf8Coder.of()));
+
+ TextIO.TypedWrite<String, Void> write =
+ TextIO.write().to(baseFilename).withHeader(header).withFooter(footer).withOutputFilenames();
+
+ if (numShards == 1) {
+ write = write.withoutSharding();
+ } else if (numShards > 0) {
+ write = write.withNumShards(numShards).withShardNameTemplate(ShardNameTemplate.INDEX_OF_MAX);
+ }
+
+ WriteFilesResult<Void> result = input.apply(write);
+ PAssert.that(result.getPerDestinationOutputFilenames()
+ .apply("GetFilenames", Values.<String>create()))
+ .satisfies(new MatchesFilesystem(baseFilename));
+ p.run();
+
+ assertOutputFiles(
+ elems,
+ header,
+ footer,
+ numShards,
+ baseFilename,
+ firstNonNull(
+ write.getShardTemplate(),
+ DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE));
+ }
+
+ private static void assertOutputFiles(
+ String[] elems,
+ final String header,
+ final String footer,
+ int numShards,
+ ResourceId outputPrefix,
+ String shardNameTemplate)
+ throws Exception {
+ List<File> expectedFiles = new ArrayList<>();
+ if (numShards == 0) {
+ String pattern = outputPrefix.toString() + "*";
+ List<MatchResult> matches = FileSystems.match(Collections.singletonList(pattern));
+ for (Metadata expectedFile : Iterables.getOnlyElement(matches).metadata()) {
+ expectedFiles.add(new File(expectedFile.resourceId().toString()));
+ }
+ } else {
+ for (int i = 0; i < numShards; i++) {
+ expectedFiles.add(
+ new File(
+ DefaultFilenamePolicy.constructName(
+ outputPrefix, shardNameTemplate, "", i, numShards, null, null)
+ .toString()));
+ }
+ }
+
+ List<List<String>> actual = new ArrayList<>();
+
+ for (File tmpFile : expectedFiles) {
+ try (BufferedReader reader = new BufferedReader(new FileReader(tmpFile))) {
+ List<String> currentFile = new ArrayList<>();
+ while (true) {
+ String line = reader.readLine();
+ if (line == null) {
+ break;
+ }
+ currentFile.add(line);
+ }
+ actual.add(currentFile);
+ }
+ }
+
+ List<String> expectedElements = new ArrayList<>(elems.length);
+ for (String elem : elems) {
+ byte[] encodedElem = CoderUtils.encodeToByteArray(StringUtf8Coder.of(), elem);
+ String line = new String(encodedElem);
+ expectedElements.add(line);
+ }
+
+ List<String> actualElements =
+ Lists.newArrayList(
+ Iterables.concat(
+ FluentIterable.from(actual)
+ .transform(removeHeaderAndFooter(header, footer))
+ .toList()));
+
+ assertThat(actualElements, containsInAnyOrder(expectedElements.toArray()));
+
+ assertTrue(Iterables.all(actual, haveProperHeaderAndFooter(header, footer)));
+ }
+
+ private static Function<List<String>, List<String>> removeHeaderAndFooter(
+ final String header, final String footer) {
+ return new Function<List<String>, List<String>>() {
+ @Nullable
+ @Override
+ public List<String> apply(List<String> lines) {
+ ArrayList<String> newLines = Lists.newArrayList(lines);
+ if (header != null) {
+ newLines.remove(0);
+ }
+ if (footer != null) {
+ int last = newLines.size() - 1;
+ newLines.remove(last);
+ }
+ return newLines;
+ }
+ };
+ }
+
+ private static Predicate<List<String>> haveProperHeaderAndFooter(
+ final String header, final String footer) {
+ return new Predicate<List<String>>() {
+ @Override
+ public boolean apply(List<String> fileLines) {
+ int last = fileLines.size() - 1;
+ return (header == null || fileLines.get(0).equals(header))
+ && (footer == null || fileLines.get(last).equals(footer));
+ }
+ };
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteStrings() throws Exception {
+ runTestWrite(LINES_ARRAY);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteEmptyStringsNoSharding() throws Exception {
+ runTestWrite(NO_LINES_ARRAY, 0);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteEmptyStrings() throws Exception {
+ runTestWrite(NO_LINES_ARRAY);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testShardedWrite() throws Exception {
+ runTestWrite(LINES_ARRAY, 5);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteWithHeader() throws Exception {
+ runTestWrite(LINES_ARRAY, MY_HEADER, null);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteWithFooter() throws Exception {
+ runTestWrite(LINES_ARRAY, null, MY_FOOTER);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteWithHeaderAndFooter() throws Exception {
+ runTestWrite(LINES_ARRAY, MY_HEADER, MY_FOOTER);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testWriteWithWritableByteChannelFactory() throws Exception {
+ Coder<String> coder = StringUtf8Coder.of();
+ String outputName = "file.txt";
+ ResourceId baseDir =
+ FileSystems.matchNewResource(
+ Files.createTempDirectory(tempFolder, "testwrite").toString(), true);
+
+ PCollection<String> input = p.apply(Create.of(Arrays.asList(LINES2_ARRAY)).withCoder(coder));
+
+ final WritableByteChannelFactory writableByteChannelFactory =
+ new DrunkWritableByteChannelFactory();
+ TextIO.Write write =
+ TextIO.write()
+ .to(
+ baseDir
+ .resolve(outputName, ResolveOptions.StandardResolveOptions.RESOLVE_FILE)
+ .toString())
+ .withoutSharding()
+ .withWritableByteChannelFactory(writableByteChannelFactory);
+ DisplayData displayData = DisplayData.from(write);
+ assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "DRUNK"));
+
+ input.apply(write);
+
+ p.run();
+
+ final List<String> drunkElems = new ArrayList<>(LINES2_ARRAY.length * 2 + 2);
+ for (String elem : LINES2_ARRAY) {
+ drunkElems.add(elem);
+ drunkElems.add(elem);
+ }
+ assertOutputFiles(
+ drunkElems.toArray(new String[0]),
+ null,
+ null,
+ 1,
+ baseDir.resolve(
+ outputName + writableByteChannelFactory.getSuggestedFilenameSuffix(),
+ ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
+ write.inner.getShardTemplate());
+ }
+
+ @Test
+ public void testWriteDisplayData() {
+ TextIO.Write write =
+ TextIO.write()
+ .to("/foo")
+ .withSuffix("bar")
+ .withShardNameTemplate("-SS-of-NN-")
+ .withNumShards(100)
+ .withFooter("myFooter")
+ .withHeader("myHeader");
+
+ DisplayData displayData = DisplayData.from(write);
+
+ assertThat(displayData, hasDisplayItem("filePrefix", "/foo"));
+ assertThat(displayData, hasDisplayItem("fileSuffix", "bar"));
+ assertThat(displayData, hasDisplayItem("fileHeader", "myHeader"));
+ assertThat(displayData, hasDisplayItem("fileFooter", "myFooter"));
+ assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-"));
+ assertThat(displayData, hasDisplayItem("numShards", 100));
+ assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "UNCOMPRESSED"));
+ }
+
+ @Test
+ public void testWriteDisplayDataValidateThenHeader() {
+ TextIO.Write write = TextIO.write().to("foo").withHeader("myHeader");
+
+ DisplayData displayData = DisplayData.from(write);
+
+ assertThat(displayData, hasDisplayItem("fileHeader", "myHeader"));
+ }
+
+ @Test
+ public void testWriteDisplayDataValidateThenFooter() {
+ TextIO.Write write = TextIO.write().to("foo").withFooter("myFooter");
+
+ DisplayData displayData = DisplayData.from(write);
+
+ assertThat(displayData, hasDisplayItem("fileFooter", "myFooter"));
+ }
+
+ @Test
+ public void testGetName() {
+ assertEquals("TextIO.Write", TextIO.write().to("somefile").getName());
+ }
+
+ /** Options for testing. */
+ public interface RuntimeTestOptions extends PipelineOptions {
+ ValueProvider<String> getOutput();
+ void setOutput(ValueProvider<String> value);
+ }
+
+ @Test
+ public void testRuntimeOptionsNotCalledInApply() throws Exception {
+ p.enableAbandonedNodeEnforcement(false);
+
+ RuntimeTestOptions options = PipelineOptionsFactory.as(RuntimeTestOptions.class);
+
+ p.apply(Create.of("")).apply(TextIO.write().to(options.getOutput()));
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
index a5dacd1..e0f7b39 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.io;
+import static com.google.common.base.MoreObjects.firstNonNull;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFor;
import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -26,6 +27,7 @@
import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
@@ -34,13 +36,21 @@
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
+import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
+import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints;
import org.apache.beam.sdk.io.SimpleSink.SimpleWriter;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
@@ -61,13 +71,21 @@
import org.apache.beam.sdk.transforms.Top;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Sessions;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.commons.compress.utils.Sets;
import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -76,17 +94,12 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for the WriteFiles PTransform.
- */
+/** Tests for the WriteFiles PTransform. */
@RunWith(JUnit4.class)
public class WriteFilesTest {
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
- @Rule
- public final TestPipeline p = TestPipeline.create();
- @Rule
- public ExpectedException thrown = ExpectedException.none();
+ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+ @Rule public final TestPipeline p = TestPipeline.create();
+ @Rule public ExpectedException thrown = ExpectedException.none();
@SuppressWarnings("unchecked") // covariant cast
private static final PTransform<PCollection<String>, PCollection<String>> IDENTITY_MAP =
@@ -101,12 +114,12 @@
private static final PTransform<PCollection<String>, PCollectionView<Integer>>
SHARDING_TRANSFORM =
- new PTransform<PCollection<String>, PCollectionView<Integer>>() {
- @Override
- public PCollectionView<Integer> expand(PCollection<String> input) {
- return null;
- }
- };
+ new PTransform<PCollection<String>, PCollectionView<Integer>>() {
+ @Override
+ public PCollectionView<Integer> expand(PCollection<String> input) {
+ return null;
+ }
+ };
private static class WindowAndReshuffle<T> extends PTransform<PCollection<T>, PCollection<T>> {
private final Window<T> window;
@@ -148,30 +161,33 @@
}
private String getBaseOutputFilename() {
- return getBaseOutputDirectory()
- .resolve("file", StandardResolveOptions.RESOLVE_FILE).toString();
+ return getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE).toString();
}
- /**
- * Test a WriteFiles transform with a PCollection of elements.
- */
+ /** Test a WriteFiles transform with a PCollection of elements. */
@Test
@Category(NeedsRunner.class)
public void testWrite() throws IOException {
- List<String> inputs = Arrays.asList("Critical canary", "Apprehensive eagle",
- "Intimidating pigeon", "Pedantic gull", "Frisky finch");
- runWrite(inputs, IDENTITY_MAP, getBaseOutputFilename());
+ List<String> inputs =
+ Arrays.asList(
+ "Critical canary",
+ "Apprehensive eagle",
+ "Intimidating pigeon",
+ "Pedantic gull",
+ "Frisky finch");
+ runWrite(inputs, IDENTITY_MAP, getBaseOutputFilename(), WriteFiles.to(makeSimpleSink()));
}
- /**
- * Test that WriteFiles with an empty input still produces one shard.
- */
+ /** Test that WriteFiles with an empty input still produces one shard. */
@Test
@Category(NeedsRunner.class)
public void testEmptyWrite() throws IOException {
- runWrite(Collections.<String>emptyList(), IDENTITY_MAP, getBaseOutputFilename());
- checkFileContents(getBaseOutputFilename(), Collections.<String>emptyList(),
- Optional.of(1));
+ runWrite(
+ Collections.<String>emptyList(),
+ IDENTITY_MAP,
+ getBaseOutputFilename(),
+ WriteFiles.to(makeSimpleSink()));
+ checkFileContents(getBaseOutputFilename(), Collections.<String>emptyList(), Optional.of(1));
}
/**
@@ -185,16 +201,20 @@
Arrays.asList("one", "two", "three", "four", "five", "six"),
IDENTITY_MAP,
getBaseOutputFilename(),
- Optional.of(1));
+ WriteFiles.to(makeSimpleSink()));
}
private ResourceId getBaseOutputDirectory() {
return LocalResources.fromFile(tmpFolder.getRoot(), true)
.resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY);
-
}
- private SimpleSink makeSimpleSink() {
- return new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", "simple");
+
+ private SimpleSink<Void> makeSimpleSink() {
+ FilenamePolicy filenamePolicy =
+ new PerWindowFiles(
+ getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE),
+ "simple");
+ return SimpleSink.makeSimpleSink(getBaseOutputDirectory(), filenamePolicy);
}
@Test
@@ -213,8 +233,8 @@
timestamps.add(i + 1);
}
- SimpleSink sink = makeSimpleSink();
- WriteFiles<String> write = WriteFiles.to(sink).withSharding(new LargestInt());
+ SimpleSink<Void> sink = makeSimpleSink();
+ WriteFiles<String, ?, String> write = WriteFiles.to(sink).withSharding(new LargestInt());
p.apply(Create.timestamped(inputs, timestamps).withCoder(StringUtf8Coder.of()))
.apply(IDENTITY_MAP)
.apply(write);
@@ -235,53 +255,72 @@
Arrays.asList("one", "two", "three", "four", "five", "six"),
IDENTITY_MAP,
getBaseOutputFilename(),
- Optional.of(20));
+ WriteFiles.to(makeSimpleSink()).withNumShards(20));
}
- /**
- * Test a WriteFiles transform with an empty PCollection.
- */
+ /** Test a WriteFiles transform with an empty PCollection. */
@Test
@Category(NeedsRunner.class)
public void testWriteWithEmptyPCollection() throws IOException {
List<String> inputs = new ArrayList<>();
- runWrite(inputs, IDENTITY_MAP, getBaseOutputFilename());
+ runWrite(inputs, IDENTITY_MAP, getBaseOutputFilename(), WriteFiles.to(makeSimpleSink()));
}
- /**
- * Test a WriteFiles with a windowed PCollection.
- */
+ /** Test a WriteFiles with a windowed PCollection. */
@Test
@Category(NeedsRunner.class)
public void testWriteWindowed() throws IOException {
- List<String> inputs = Arrays.asList("Critical canary", "Apprehensive eagle",
- "Intimidating pigeon", "Pedantic gull", "Frisky finch");
+ List<String> inputs =
+ Arrays.asList(
+ "Critical canary",
+ "Apprehensive eagle",
+ "Intimidating pigeon",
+ "Pedantic gull",
+ "Frisky finch");
runWrite(
- inputs, new WindowAndReshuffle<>(Window.<String>into(FixedWindows.of(Duration.millis(2)))),
- getBaseOutputFilename());
+ inputs,
+ new WindowAndReshuffle<>(Window.<String>into(FixedWindows.of(Duration.millis(2)))),
+ getBaseOutputFilename(),
+ WriteFiles.to(makeSimpleSink()));
}
- /**
- * Test a WriteFiles with sessions.
- */
+ /** Test a WriteFiles with sessions. */
@Test
@Category(NeedsRunner.class)
public void testWriteWithSessions() throws IOException {
- List<String> inputs = Arrays.asList("Critical canary", "Apprehensive eagle",
- "Intimidating pigeon", "Pedantic gull", "Frisky finch");
+ List<String> inputs =
+ Arrays.asList(
+ "Critical canary",
+ "Apprehensive eagle",
+ "Intimidating pigeon",
+ "Pedantic gull",
+ "Frisky finch");
runWrite(
inputs,
- new WindowAndReshuffle<>(
- Window.<String>into(Sessions.withGapDuration(Duration.millis(1)))),
- getBaseOutputFilename());
+ new WindowAndReshuffle<>(Window.<String>into(Sessions.withGapDuration(Duration.millis(1)))),
+ getBaseOutputFilename(),
+ WriteFiles.to(makeSimpleSink()));
}
@Test
+ @Category(NeedsRunner.class)
+ public void testWriteSpilling() throws IOException {
+ List<String> inputs = Lists.newArrayList();
+ for (int i = 0; i < 100; ++i) {
+ inputs.add("mambo_number_" + i);
+ }
+ runWrite(
+ inputs,
+ Window.<String>into(FixedWindows.of(Duration.millis(2))),
+ getBaseOutputFilename(),
+ WriteFiles.to(makeSimpleSink()).withMaxNumWritersPerBundle(2).withWindowedWrites());
+ }
+
public void testBuildWrite() {
- SimpleSink sink = makeSimpleSink();
- WriteFiles<String> write = WriteFiles.to(sink).withNumShards(3);
- assertThat((SimpleSink) write.getSink(), is(sink));
+ SimpleSink<Void> sink = makeSimpleSink();
+ WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(3);
+ assertThat((SimpleSink<Void>) write.getSink(), is(sink));
PTransform<PCollection<String>, PCollectionView<Integer>> originalSharding =
write.getSharding();
@@ -290,25 +329,36 @@
assertThat(write.getNumShards().get(), equalTo(3));
assertThat(write.getSharding(), equalTo(originalSharding));
- WriteFiles<String> write2 = write.withSharding(SHARDING_TRANSFORM);
- assertThat((SimpleSink) write2.getSink(), is(sink));
+ WriteFiles<String, ?, ?> write2 = write.withSharding(SHARDING_TRANSFORM);
+ assertThat((SimpleSink<Void>) write2.getSink(), is(sink));
assertThat(write2.getSharding(), equalTo(SHARDING_TRANSFORM));
// original unchanged
- WriteFiles<String> writeUnsharded = write2.withRunnerDeterminedSharding();
+ WriteFiles<String, ?, ?> writeUnsharded = write2.withRunnerDeterminedSharding();
assertThat(writeUnsharded.getSharding(), nullValue());
assertThat(write.getSharding(), equalTo(originalSharding));
}
@Test
public void testDisplayData() {
- SimpleSink sink = new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", "") {
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- builder.add(DisplayData.item("foo", "bar"));
- }
- };
- WriteFiles<String> write = WriteFiles.to(sink);
+ DynamicDestinations<String, Void, String> dynamicDestinations =
+ DynamicFileDestinations.constant(
+ DefaultFilenamePolicy.fromParams(
+ new Params()
+ .withBaseFilename(
+ getBaseOutputDirectory()
+ .resolve("file", StandardResolveOptions.RESOLVE_FILE))
+ .withShardTemplate("-SS-of-NN")));
+ SimpleSink<Void> sink =
+ new SimpleSink<Void>(
+ getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("foo", "bar"));
+ }
+ };
+ WriteFiles<String, ?, String> write = WriteFiles.to(sink);
+
DisplayData displayData = DisplayData.from(write);
assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
@@ -316,29 +366,179 @@
}
@Test
+ @Category(NeedsRunner.class)
+ public void testUnboundedNeedsWindowed() {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage(
+ "Must use windowed writes when applying WriteFiles to an unbounded PCollection");
+
+ SimpleSink<Void> sink = makeSimpleSink();
+ p.apply(Create.of("foo")).setIsBoundedInternal(IsBounded.UNBOUNDED).apply(WriteFiles.to(sink));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testUnboundedNeedsSharding() {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage(
+ "When applying WriteFiles to an unbounded PCollection, "
+ + "must specify number of output shards explicitly");
+
+ SimpleSink<Void> sink = makeSimpleSink();
+ p.apply(Create.of("foo"))
+ .setIsBoundedInternal(IsBounded.UNBOUNDED)
+ .apply(WriteFiles.to(sink).withWindowedWrites());
+ p.run();
+ }
+
+ // Test DynamicDestinations class. Expects user values to be string-encoded integers.
+ // Stores the integer mod 5 as the destination, and uses that in the file prefix.
+ static class TestDestinations extends DynamicDestinations<String, Integer, String> {
+ private ResourceId baseOutputDirectory;
+
+ TestDestinations(ResourceId baseOutputDirectory) {
+ this.baseOutputDirectory = baseOutputDirectory;
+ }
+
+ @Override
+ public String formatRecord(String record) {
+ return "record_" + record;
+ }
+
+ @Override
+ public Integer getDestination(String element) {
+ return Integer.valueOf(element) % 5;
+ }
+
+ @Override
+ public Integer getDefaultDestination() {
+ return 0;
+ }
+
+ @Override
+ public FilenamePolicy getFilenamePolicy(Integer destination) {
+ return new PerWindowFiles(
+ baseOutputDirectory.resolve("file_" + destination, StandardResolveOptions.RESOLVE_FILE),
+ "simple");
+ }
+
+ @Override
+ public void populateDisplayData(Builder builder) {
+ super.populateDisplayData(builder);
+ }
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsBounded() throws Exception {
+ testDynamicDestinationsHelper(true, false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsUnbounded() throws Exception {
+ testDynamicDestinationsHelper(false, false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testDynamicDestinationsFillEmptyShards() throws Exception {
+ testDynamicDestinationsHelper(true, true);
+ }
+
+ private void testDynamicDestinationsHelper(boolean bounded, boolean emptyShards)
+ throws IOException {
+ TestDestinations dynamicDestinations = new TestDestinations(getBaseOutputDirectory());
+ SimpleSink<Integer> sink =
+ new SimpleSink<>(
+ getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED);
+
+ // Flag to validate that the pipeline options are passed to the Sink.
+ WriteOptions options = TestPipeline.testingPipelineOptions().as(WriteOptions.class);
+ options.setTestFlag("test_value");
+ Pipeline p = TestPipeline.create(options);
+
+ final int numInputs = 100;
+ List<String> inputs = Lists.newArrayList();
+ for (int i = 0; i < numInputs; ++i) {
+ inputs.add(Integer.toString(i));
+ }
+ // Prepare timestamps for the elements.
+ List<Long> timestamps = new ArrayList<>();
+ for (long i = 0; i < inputs.size(); i++) {
+ timestamps.add(i + 1);
+ }
+ // If emptyShards==true make numShards larger than the number of elements per destination.
+ // This will force every destination to generate some empty shards.
+ int numShards = emptyShards ? 2 * numInputs / 5 : 2;
+ WriteFiles<String, Integer, String> writeFiles = WriteFiles.to(sink).withNumShards(numShards);
+
+ PCollection<String> input = p.apply(Create.timestamped(inputs, timestamps));
+ if (!bounded) {
+ input.setIsBoundedInternal(IsBounded.UNBOUNDED);
+ input = input.apply(Window.<String>into(FixedWindows.of(Duration.standardDays(1))));
+ input.apply(writeFiles.withWindowedWrites());
+ } else {
+ input.apply(writeFiles);
+ }
+ p.run();
+
+ for (int i = 0; i < 5; ++i) {
+ ResourceId base =
+ getBaseOutputDirectory().resolve("file_" + i, StandardResolveOptions.RESOLVE_FILE);
+ List<String> expected = Lists.newArrayList();
+ for (int j = i; j < numInputs; j += 5) {
+ expected.add("record_" + j);
+ }
+ checkFileContents(base.toString(), expected, Optional.of(numShards));
+ }
+ }
+
+ @Test
public void testShardedDisplayData() {
- SimpleSink sink = new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", "") {
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- builder.add(DisplayData.item("foo", "bar"));
- }
- };
- WriteFiles<String> write = WriteFiles.to(sink).withNumShards(1);
+ DynamicDestinations<String, Void, String> dynamicDestinations =
+ DynamicFileDestinations.constant(
+ DefaultFilenamePolicy.fromParams(
+ new Params()
+ .withBaseFilename(
+ getBaseOutputDirectory()
+ .resolve("file", StandardResolveOptions.RESOLVE_FILE))
+ .withShardTemplate("-SS-of-NN")));
+ SimpleSink<Void> sink =
+ new SimpleSink<Void>(
+ getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("foo", "bar"));
+ }
+ };
+ WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(1);
DisplayData displayData = DisplayData.from(write);
assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
assertThat(displayData, includesDisplayDataFor("sink", sink));
- assertThat(displayData, hasDisplayItem("numShards", "1"));
+ assertThat(displayData, hasDisplayItem("numShards", 1));
}
@Test
public void testCustomShardStrategyDisplayData() {
- SimpleSink sink = new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", "") {
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- builder.add(DisplayData.item("foo", "bar"));
- }
- };
- WriteFiles<String> write =
+ DynamicDestinations<String, Void, String> dynamicDestinations =
+ DynamicFileDestinations.constant(
+ DefaultFilenamePolicy.fromParams(
+ new Params()
+ .withBaseFilename(
+ getBaseOutputDirectory()
+ .resolve("file", StandardResolveOptions.RESOLVE_FILE))
+ .withShardTemplate("-SS-of-NN")));
+ SimpleSink<Void> sink =
+ new SimpleSink<Void>(
+ getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("foo", "bar"));
+ }
+ };
+ WriteFiles<String, ?, String> write =
WriteFiles.to(sink)
.withSharding(
new PTransform<PCollection<String>, PCollectionView<Integer>>() {
@@ -364,22 +564,85 @@
* PCollection are written to the sink.
*/
private void runWrite(
- List<String> inputs, PTransform<PCollection<String>, PCollection<String>> transform,
- String baseName) throws IOException {
- runShardedWrite(inputs, transform, baseName, Optional.<Integer>absent());
+ List<String> inputs,
+ PTransform<PCollection<String>, PCollection<String>> transform,
+ String baseName,
+ WriteFiles<String, ?, String> write)
+ throws IOException {
+ runShardedWrite(inputs, transform, baseName, write);
+ }
+
+ private static class PerWindowFiles extends FilenamePolicy {
+ private static final DateTimeFormatter FORMATTER = ISODateTimeFormat.hourMinuteSecondMillis();
+ private final ResourceId baseFilename;
+ private final String suffix;
+
+ public PerWindowFiles(ResourceId baseFilename, String suffix) {
+ this.baseFilename = baseFilename;
+ this.suffix = suffix;
+ }
+
+ public String filenamePrefixForWindow(IntervalWindow window) {
+ String prefix =
+ baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), "");
+ return String.format(
+ "%s%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end()));
+ }
+
+ @Override
+ public ResourceId windowedFilename(
+ int shardNumber,
+ int numShards,
+ BoundedWindow window,
+ PaneInfo paneInfo,
+ OutputFileHints outputFileHints) {
+ DecimalFormat df = new DecimalFormat("0000");
+ IntervalWindow intervalWindow = (IntervalWindow) window;
+ String filename =
+ String.format(
+ "%s-%s-of-%s%s%s",
+ filenamePrefixForWindow(intervalWindow),
+ df.format(shardNumber),
+ df.format(numShards),
+ outputFileHints.getSuggestedFilenameSuffix(),
+ suffix);
+ return baseFilename
+ .getCurrentDirectory()
+ .resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ }
+
+ @Override
+ public ResourceId unwindowedFilename(
+ int shardNumber, int numShards, OutputFileHints outputFileHints) {
+ DecimalFormat df = new DecimalFormat("0000");
+ String prefix =
+ baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), "");
+ String filename =
+ String.format(
+ "%s-%s-of-%s%s%s",
+ prefix,
+ df.format(shardNumber),
+ df.format(numShards),
+ outputFileHints.getSuggestedFilenameSuffix(),
+ suffix);
+ return baseFilename
+ .getCurrentDirectory()
+ .resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+ }
}
/**
* Performs a WriteFiles transform with the desired number of shards. Verifies the WriteFiles
* transform calls the appropriate methods on a test sink in the correct order, as well as
- * verifies that the elements of a PCollection are written to the sink. If numConfiguredShards
- * is not null, also verifies that the output number of shards is correct.
+ * verifies that the elements of a PCollection are written to the sink. If numConfiguredShards is
+ * not null, also verifies that the output number of shards is correct.
*/
private void runShardedWrite(
List<String> inputs,
PTransform<PCollection<String>, PCollection<String>> transform,
String baseName,
- Optional<Integer> numConfiguredShards) throws IOException {
+ WriteFiles<String, ?, String> write)
+ throws IOException {
// Flag to validate that the pipeline options are passed to the Sink
WriteOptions options = TestPipeline.testingPipelineOptions().as(WriteOptions.class);
options.setTestFlag("test_value");
@@ -390,22 +653,21 @@
for (long i = 0; i < inputs.size(); i++) {
timestamps.add(i + 1);
}
-
- SimpleSink sink = makeSimpleSink();
- WriteFiles<String> write = WriteFiles.to(sink);
- if (numConfiguredShards.isPresent()) {
- write = write.withNumShards(numConfiguredShards.get());
- }
p.apply(Create.timestamped(inputs, timestamps).withCoder(StringUtf8Coder.of()))
.apply(transform)
.apply(write);
p.run();
- checkFileContents(baseName, inputs, numConfiguredShards);
+ Optional<Integer> numShards =
+ (write.getNumShards() != null)
+ ? Optional.of(write.getNumShards().get())
+ : Optional.<Integer>absent();
+ checkFileContents(baseName, inputs, numShards);
}
- static void checkFileContents(String baseName, List<String> inputs,
- Optional<Integer> numExpectedShards) throws IOException {
+ static void checkFileContents(
+ String baseName, List<String> inputs, Optional<Integer> numExpectedShards)
+ throws IOException {
List<File> outputFiles = Lists.newArrayList();
final String pattern = baseName + "*";
List<Metadata> metadata =
@@ -415,6 +677,22 @@
}
if (numExpectedShards.isPresent()) {
assertEquals(numExpectedShards.get().intValue(), outputFiles.size());
+ Pattern shardPattern = Pattern.compile("\\d{4}-of-\\d{4}");
+
+ Set<String> expectedShards = Sets.newHashSet();
+ DecimalFormat df = new DecimalFormat("0000");
+ for (int i = 0; i < numExpectedShards.get(); i++) {
+ expectedShards.add(
+ String.format("%s-of-%s", df.format(i), df.format(numExpectedShards.get())));
+ }
+
+ Set<String> outputShards = Sets.newHashSet();
+ for (File file : outputFiles) {
+ Matcher matcher = shardPattern.matcher(file.getName());
+ assertTrue(matcher.find());
+ assertTrue(outputShards.add(matcher.group()));
+ }
+ assertEquals(expectedShards, outputShards);
}
List<String> actual = Lists.newArrayList();
@@ -434,12 +712,11 @@
assertThat(actual, containsInAnyOrder(inputs.toArray()));
}
- /**
- * Options for test, exposed for PipelineOptionsFactory.
- */
+ /** Options for test, exposed for PipelineOptionsFactory. */
public interface WriteOptions extends TestPipelineOptions {
@Description("Test flag and value")
String getTestFlag();
+
void setTestFlag(String value);
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
index 8deaf44..0523d75 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
@@ -38,6 +38,7 @@
private static final ByteKey NEW_MIDDLE_KEY = ByteKey.of(0x24);
private static final ByteKey BEFORE_END_KEY = ByteKey.of(0x33);
private static final ByteKey END_KEY = ByteKey.of(0x34);
+ private static final ByteKey KEY_LARGER_THAN_END = ByteKey.of(0x35);
private static final double INITIAL_RANGE_SIZE = 0x34 - 0x12;
private static final ByteKeyRange INITIAL_RANGE = ByteKeyRange.of(INITIAL_START_KEY, END_KEY);
private static final double NEW_RANGE_SIZE = 0x34 - 0x14;
@@ -98,6 +99,28 @@
assertEquals(1 - 1 / INITIAL_RANGE_SIZE, tracker.getFractionConsumed(), delta);
}
+ @Test
+ public void testGetFractionConsumedAfterDone() {
+ ByteKeyRangeTracker tracker = ByteKeyRangeTracker.of(INITIAL_RANGE);
+ double delta = 0.00001;
+
+ assertTrue(tracker.tryReturnRecordAt(true, INITIAL_START_KEY));
+ tracker.markDone();
+
+ assertEquals(1.0, tracker.getFractionConsumed(), delta);
+ }
+
+ @Test
+ public void testGetFractionConsumedAfterOutOfRangeClaim() {
+ ByteKeyRangeTracker tracker = ByteKeyRangeTracker.of(INITIAL_RANGE);
+ double delta = 0.00001;
+
+ assertTrue(tracker.tryReturnRecordAt(true, INITIAL_START_KEY));
+ assertTrue(tracker.tryReturnRecordAt(false, KEY_LARGER_THAN_END));
+
+ assertEquals(1.0, tracker.getFractionConsumed(), delta);
+ }
+
/** Tests for {@link ByteKeyRangeTracker#getFractionConsumed()} with updated start key. */
@Test
public void testGetFractionConsumedUpdateStartKey() {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
index 5031952..030a759 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
@@ -96,7 +96,7 @@
if (result1 instanceof GaugeResult) {
return (((GaugeResult) result1).value()) == (((GaugeResult) result2).value());
} else {
- return result1.equals(result2);
+ return Objects.equals(result1, result2);
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
index bc768f8..bdcf892 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
@@ -48,6 +48,7 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
/**
@@ -71,6 +72,9 @@
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
+ @Rule
+ public final transient ExpectedException thrown = ExpectedException.none();
+
@After
public void tearDown() {
MetricsEnvironment.setCurrentContainer(null);
@@ -95,6 +99,30 @@
}
@Test
+ public void testCounterWithEmptyName() {
+ thrown.expect(IllegalArgumentException.class);
+ Metrics.counter(NS, "");
+ }
+
+ @Test
+ public void testCounterWithEmptyNamespace() {
+ thrown.expect(IllegalArgumentException.class);
+ Metrics.counter("", NAME);
+ }
+
+ @Test
+ public void testDistributionWithEmptyName() {
+ thrown.expect(IllegalArgumentException.class);
+ Metrics.distribution(NS, "");
+ }
+
+ @Test
+ public void testDistributionWithEmptyNamespace() {
+ thrown.expect(IllegalArgumentException.class);
+ Metrics.distribution("", NAME);
+ }
+
+ @Test
public void testDistributionToCell() {
MetricsContainer mockContainer = Mockito.mock(MetricsContainer.class);
Distribution mockDistribution = Mockito.mock(Distribution.class);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
index d40b5fc..f8de74a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
@@ -1629,9 +1629,47 @@
containsString("The pipeline runner that will be used to execute the pipeline."));
}
+ interface PipelineOptionsInheritedInvalid extends Invalid1,
+ InvalidPipelineOptions2, PipelineOptions {
+ String getFoo();
+ void setFoo(String value);
+ }
+
+ interface InvalidPipelineOptions1 {
+ String getBar();
+ void setBar(String value);
+ }
+
+ interface Invalid1 extends InvalidPipelineOptions1 {
+ String getBar();
+ void setBar(String value);
+ }
+
+ interface InvalidPipelineOptions2 {
+ String getBar();
+ void setBar(String value);
+ }
+
+ @Test
+ public void testAllFromPipelineOptions() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "All inherited interfaces of [org.apache.beam.sdk.options.PipelineOptionsFactoryTest"
+ + "$PipelineOptionsInheritedInvalid] should inherit from the PipelineOptions interface. "
+ + "The following inherited interfaces do not:\n"
+ + " - org.apache.beam.sdk.options.PipelineOptionsFactoryTest"
+ + "$InvalidPipelineOptions1\n"
+ + " - org.apache.beam.sdk.options.PipelineOptionsFactoryTest"
+ + "$InvalidPipelineOptions2");
+
+ PipelineOptionsInheritedInvalid options = PipelineOptionsFactory.as(
+ PipelineOptionsInheritedInvalid.class);
+ }
+
private String emptyStringErrorMessage() {
return emptyStringErrorMessage(null);
}
+
private String emptyStringErrorMessage(String type) {
String msg = "Empty argument value is only allowed for String, String Array, "
+ "Collections of Strings or any of these types in a parameterized ValueProvider";
@@ -1736,4 +1774,5 @@
jsonGenerator.writeString(jacksonIncompatible.value);
}
}
+
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
index 5e3211f..7f80c0c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
@@ -19,6 +19,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -36,6 +37,8 @@
/** Unit tests for {@link PipelineOptions}. */
@RunWith(JUnit4.class)
public class PipelineOptionsTest {
+ private static final String DEFAULT_USER_AGENT_NAME = "Apache_Beam_SDK_for_Java";
+
@Rule public ExpectedException expectedException = ExpectedException.none();
/** Interfaces used for testing that {@link PipelineOptions#as(Class)} functions. */
@@ -106,4 +109,12 @@
}
}
}
+
+ @Test
+ public void testUserAgentFactory() {
+ PipelineOptions options = PipelineOptionsFactory.create();
+ String userAgent = options.getUserAgent();
+ assertNotNull(userAgent);
+ assertTrue(userAgent.contains(DEFAULT_USER_AGENT_NAME));
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsValidatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsValidatorTest.java
index 120d5ed..f8cd00f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsValidatorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsValidatorTest.java
@@ -60,6 +60,18 @@
}
@Test
+ public void testWhenRequiredOptionIsSetAndClearedCli() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Missing required value for "
+ + "[--object, \"Fake Description\"].");
+
+ Required required = PipelineOptionsFactory.fromArgs(new String[]{"--object=blah"})
+ .as(Required.class);
+ required.setObject(null);
+ PipelineOptionsValidator.validateCli(Required.class, required);
+ }
+
+ @Test
public void testWhenRequiredOptionIsNeverSet() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Missing required value for "
@@ -70,6 +82,17 @@
PipelineOptionsValidator.validate(Required.class, required);
}
+
+ @Test
+ public void testWhenRequiredOptionIsNeverSetCli() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Missing required value for "
+ + "[--object, \"Fake Description\"].");
+
+ Required required = PipelineOptionsFactory.fromArgs(new String[]{}).as(Required.class);
+ PipelineOptionsValidator.validateCli(Required.class, required);
+ }
+
@Test
public void testWhenRequiredOptionIsNeverSetOnSuperInterface() {
expectedException.expect(IllegalArgumentException.class);
@@ -81,6 +104,16 @@
PipelineOptionsValidator.validate(Required.class, options);
}
+ @Test
+ public void testWhenRequiredOptionIsNeverSetOnSuperInterfaceCli() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Missing required value for "
+ + "[--object, \"Fake Description\"].");
+
+ PipelineOptions options = PipelineOptionsFactory.fromArgs(new String[]{}).create();
+ PipelineOptionsValidator.validateCli(Required.class, options);
+ }
+
/** A test interface that overrides the parent's method. */
public interface SubClassValidation extends Required {
@Override
@@ -100,6 +133,17 @@
PipelineOptionsValidator.validate(Required.class, required);
}
+ @Test
+ public void testValidationOnOverriddenMethodsCli() throws Exception {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Missing required value for "
+ + "[--object, \"Fake Description\"].");
+
+ SubClassValidation required = PipelineOptionsFactory.fromArgs(new String[]{})
+ .as(SubClassValidation.class);
+ PipelineOptionsValidator.validateCli(Required.class, required);
+ }
+
/** A test interface with a required group. */
public interface GroupRequired extends PipelineOptions {
@Validation.Required(groups = {"ham"})
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
index 2c43f57..fe8a0f9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
@@ -24,7 +24,6 @@
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -44,6 +43,7 @@
import com.google.common.collect.Maps;
import com.google.common.testing.EqualsTester;
import java.io.IOException;
+import java.io.NotSerializableException;
import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
@@ -54,13 +54,13 @@
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.hamcrest.Matchers;
import org.joda.time.Instant;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.internal.matchers.ThrowableMessageMatcher;
import org.junit.rules.ExpectedException;
import org.junit.rules.ExternalResource;
import org.junit.rules.TestRule;
@@ -795,7 +795,7 @@
expectedException.expectMessage(
ProxyInvocationHandler.PipelineOptionsDisplayData.class.getName());
- expectedException.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!!")));
+ expectedException.expectMessage("oh noes!!");
p.run();
}
@@ -1019,4 +1019,21 @@
DisplayData data = DisplayData.from(options);
assertThat(data, not(hasDisplayItem("value")));
}
+
+ private static class CapturesOptions implements Serializable {
+ PipelineOptions options = PipelineOptionsFactory.create();
+ }
+
+ @Test
+ public void testOptionsAreNotSerializable() {
+ expectedException.expectCause(Matchers.<Throwable>instanceOf(NotSerializableException.class));
+ SerializableUtils.clone(new CapturesOptions());
+ }
+
+ @Test
+ public void testGetOptionNameFromMethod() throws NoSuchMethodException {
+ ProxyInvocationHandler handler = new ProxyInvocationHandler(Maps.<String, Object>newHashMap());
+ handler.as(BaseOptions.class);
+ assertEquals("foo", handler.getOptionName(BaseOptions.class.getMethod("getFoo")));
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
new file mode 100644
index 0000000..565bbac
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.options;
+
+import static org.apache.beam.sdk.options.SdkHarnessOptions.LogLevel.WARN;
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import org.apache.beam.sdk.options.SdkHarnessOptions.SdkHarnessLogLevelOverrides;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SdkHarnessOptions}. */
+@RunWith(JUnit4.class)
+public class SdkHarnessOptionsTest {
+ private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+ ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+ @Rule public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testSdkHarnessLogLevelOverrideWithInvalidLogLevel() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Unsupported log level");
+ SdkHarnessLogLevelOverrides.from(ImmutableMap.of("Name", "FakeLevel"));
+ }
+
+ @Test
+ public void testSdkHarnessLogLevelOverrideForClass() throws Exception {
+ assertEquals("{\"org.junit.Test\":\"WARN\"}",
+ MAPPER.writeValueAsString(
+ new SdkHarnessLogLevelOverrides().addOverrideForClass(Test.class, WARN)));
+ }
+
+ @Test
+ public void testSdkHarnessLogLevelOverrideForPackage() throws Exception {
+ assertEquals("{\"org.junit\":\"WARN\"}",
+ MAPPER.writeValueAsString(
+ new SdkHarnessLogLevelOverrides().addOverrideForPackage(
+ Test.class.getPackage(), WARN)));
+ }
+
+ @Test
+ public void testSdkHarnessLogLevelOverrideForName() throws Exception {
+ assertEquals("{\"A\":\"WARN\"}",
+ MAPPER.writeValueAsString(
+ new SdkHarnessLogLevelOverrides().addOverrideForName("A", WARN)));
+ }
+
+ @Test
+ public void testSerializationAndDeserializationOf() throws Exception {
+ String testValue = "{\"A\":\"WARN\"}";
+ assertEquals(testValue,
+ MAPPER.writeValueAsString(
+ MAPPER.readValue(testValue, SdkHarnessLogLevelOverrides.class)));
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
index e596cc1..51a92e3 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
@@ -23,8 +23,8 @@
import static org.junit.Assert.assertTrue;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
@@ -88,7 +88,7 @@
ValueProvider<String> provider = StaticValueProvider.of("foo");
assertEquals("foo", provider.get());
assertTrue(provider.isAccessible());
- assertEquals("StaticValueProvider{value=foo}", provider.toString());
+ assertEquals("foo", provider.toString());
}
@Test
@@ -97,8 +97,9 @@
ValueProvider<String> provider = options.getFoo();
assertFalse(provider.isAccessible());
- expectedException.expect(RuntimeException.class);
- expectedException.expectMessage("Not called from a runtime context");
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Value only available at runtime");
+ expectedException.expectMessage("foo");
provider.get();
}
@@ -108,7 +109,7 @@
ValueProvider<String> provider = options.getFoo();
assertEquals("foo", ((RuntimeValueProvider) provider).propertyName());
assertEquals(
- "RuntimeValueProvider{propertyName=foo, default=null, value=null}",
+ "RuntimeValueProvider{propertyName=foo, default=null}",
provider.toString());
}
@@ -193,16 +194,16 @@
StaticValueProvider<String> provider = options.getBar();
}
+
@Test
public void testSerializeDeserializeNoArg() throws Exception {
TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
assertFalse(submitOptions.getFoo().isAccessible());
- String serializedOptions = MAPPER.writeValueAsString(submitOptions);
- String runnerString = ValueProviders.updateSerializedOptions(
- serializedOptions, ImmutableMap.of("foo", "quux"));
- TestOptions runtime = MAPPER.readValue(runnerString, PipelineOptions.class)
- .as(TestOptions.class);
+ ObjectNode root = MAPPER.valueToTree(submitOptions);
+ ((ObjectNode) root.get("options")).put("foo", "quux");
+ TestOptions runtime =
+ MAPPER.convertValue(root, PipelineOptions.class).as(TestOptions.class);
ValueProvider<String> vp = runtime.getFoo();
assertTrue(vp.isAccessible());
@@ -213,14 +214,13 @@
@Test
public void testSerializeDeserializeWithArg() throws Exception {
TestOptions submitOptions = PipelineOptionsFactory.fromArgs("--foo=baz").as(TestOptions.class);
- assertEquals("baz", submitOptions.getFoo().get());
assertTrue(submitOptions.getFoo().isAccessible());
- String serializedOptions = MAPPER.writeValueAsString(submitOptions);
+ assertEquals("baz", submitOptions.getFoo().get());
- String runnerString = ValueProviders.updateSerializedOptions(
- serializedOptions, ImmutableMap.of("foo", "quux"));
- TestOptions runtime = MAPPER.readValue(runnerString, PipelineOptions.class)
- .as(TestOptions.class);
+ ObjectNode root = MAPPER.valueToTree(submitOptions);
+ ((ObjectNode) root.get("options")).put("foo", "quux");
+ TestOptions runtime =
+ MAPPER.convertValue(root, PipelineOptions.class).as(TestOptions.class);
ValueProvider<String> vp = runtime.getFoo();
assertTrue(vp.isAccessible());
@@ -239,9 +239,7 @@
});
assertTrue(nvp.isAccessible());
assertEquals("foobar", nvp.get());
- assertEquals(
- "NestedValueProvider{value=StaticValueProvider{value=foo}}",
- nvp.toString());
+ assertEquals("foobar", nvp.toString());
}
@Test
@@ -266,7 +264,7 @@
assertEquals("bar", ((NestedValueProvider) doubleNvp).propertyName());
assertFalse(nvp.isAccessible());
expectedException.expect(RuntimeException.class);
- expectedException.expectMessage("Not called from a runtime context");
+ expectedException.expectMessage("Value only available at runtime");
nvp.get();
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 125e159..12fe633 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -19,6 +19,7 @@
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
@@ -32,6 +33,9 @@
import java.util.Set;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.Pipeline.PipelineVisitor.Defaults;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.io.Read;
@@ -79,7 +83,7 @@
@Before
public void setup() {
- hierarchy = new TransformHierarchy(pipeline);
+ hierarchy = new TransformHierarchy();
}
@Test
@@ -107,7 +111,7 @@
public void emptyCompositeSucceeds() {
PCollection<Long> created =
PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1));
hierarchy.setOutput(created);
hierarchy.popNode();
@@ -136,7 +140,7 @@
public void producingOwnAndOthersOutputsFails() {
PCollection<Long> created =
PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1));
hierarchy.setOutput(created);
hierarchy.popNode();
@@ -144,8 +148,11 @@
final PCollectionList<Long> appended =
pcList.and(
- PCollection.<Long>createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
+ PCollection.createPrimitiveOutputInternal(
+ pipeline,
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ VarLongCoder.of())
.setName("prim"));
hierarchy.pushNode(
"AddPc",
@@ -168,7 +175,7 @@
public void producingOwnOutputWithCompositeFails() {
final PCollection<Long> comp =
PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
PTransform<PBegin, PCollection<Long>> root =
new PTransform<PBegin, PCollection<Long>>() {
@Override
@@ -324,7 +331,7 @@
PCollection<Long> created =
PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
SingleOutput<Long, Long> pardo =
ParDo.of(
@@ -337,7 +344,7 @@
PCollection<Long> mapped =
PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
TransformHierarchy.Node compositeNode = hierarchy.pushNode("Create", begin, create);
hierarchy.finishSpecifyingInput();
@@ -492,4 +499,197 @@
assertThat(visitedPrimitiveNodes, containsInAnyOrder(upstreamNode, replacementParNode));
assertThat(visitedValues, Matchers.<PValue>containsInAnyOrder(upstream, output));
}
+
+ @Test
+ public void visitIsTopologicallyOrdered() {
+ PCollection<String> one =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of());
+ final PCollection<Integer> two =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, VarIntCoder.of());
+ final PDone done = PDone.in(pipeline);
+ final TupleTag<String> oneTag = new TupleTag<String>() {};
+ final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
+ final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
+
+ PTransform<PCollection<String>, PDone> multiConsumer =
+ new PTransform<PCollection<String>, PDone>() {
+ @Override
+ public PDone expand(PCollection<String> input) {
+ return done;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
+ }
+ };
+ hierarchy.pushNode("consumes_both", one, multiConsumer);
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final PTransform<PBegin, PCollectionTuple> producer =
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return oneAndTwo;
+ }
+ };
+ hierarchy.pushNode(
+ "encloses_producer",
+ PBegin.in(pipeline),
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return input.apply(producer);
+ }
+ });
+ hierarchy.pushNode(
+ "creates_one_and_two",
+ PBegin.in(pipeline), producer);
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+
+ hierarchy.pushNode("second_copy_of_consumes_both", one, multiConsumer);
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final Set<Node> visitedNodes = new HashSet<>();
+ final Set<Node> exitedNodes = new HashSet<>();
+ final Set<PValue> visitedValues = new HashSet<>();
+ hierarchy.visit(
+ new PipelineVisitor.Defaults() {
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(Node node) {
+ for (PValue input : node.getInputs().values()) {
+ assertThat(visitedValues, hasItem(input));
+ }
+ assertThat(
+ "Nodes should not be visited more than once", visitedNodes, not(hasItem(node)));
+ if (!node.isRootNode()) {
+ assertThat(
+ "Nodes should always be visited after their enclosing nodes",
+ visitedNodes,
+ hasItem(node.getEnclosingNode()));
+ }
+ visitedNodes.add(node);
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ assertThat(visitedNodes, hasItem(node));
+ if (!node.isRootNode()) {
+ assertThat(
+ "Nodes should always be left before their enclosing nodes are left",
+ exitedNodes,
+ not(hasItem(node.getEnclosingNode())));
+ }
+ assertThat(exitedNodes, not(hasItem(node)));
+ exitedNodes.add(node);
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ assertThat(visitedNodes, hasItem(node.getEnclosingNode()));
+ assertThat(exitedNodes, not(hasItem(node.getEnclosingNode())));
+ assertThat(
+ "Nodes should not be visited more than once", visitedNodes, not(hasItem(node)));
+ for (PValue input : node.getInputs().values()) {
+ assertThat(visitedValues, hasItem(input));
+ }
+ visitedNodes.add(node);
+ }
+
+ @Override
+ public void visitValue(PValue value, Node producer) {
+ assertThat(visitedNodes, hasItem(producer));
+ assertThat(visitedValues, not(hasItem(value)));
+ visitedValues.add(value);
+ }
+ });
+ assertThat("Should have visited all the nodes", visitedNodes.size(), equalTo(5));
+ assertThat("Should have left all of the visited composites", exitedNodes.size(), equalTo(2));
+ }
+
+ @Test
+ public void visitDoesNotVisitSkippedNodes() {
+ PCollection<String> one =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline,
+ WindowingStrategy.globalDefault(),
+ IsBounded.BOUNDED,
+ StringUtf8Coder.of());
+ final PCollection<Integer> two =
+ PCollection.createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, VarIntCoder.of());
+ final PDone done = PDone.in(pipeline);
+ final TupleTag<String> oneTag = new TupleTag<String>() {};
+ final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
+ final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
+
+ hierarchy.pushNode(
+ "consumes_both",
+ one,
+ new PTransform<PCollection<String>, PDone>() {
+ @Override
+ public PDone expand(PCollection<String> input) {
+ return done;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
+ }
+ });
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final PTransform<PBegin, PCollectionTuple> producer =
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return oneAndTwo;
+ }
+ };
+ final Node enclosing =
+ hierarchy.pushNode(
+ "encloses_producer",
+ PBegin.in(pipeline),
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return input.apply(producer);
+ }
+ });
+ Node enclosed = hierarchy.pushNode("creates_one_and_two", PBegin.in(pipeline), producer);
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+
+ final Set<Node> visitedNodes = new HashSet<>();
+ hierarchy.visit(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public CompositeBehavior enterCompositeTransform(Node node) {
+ visitedNodes.add(node);
+ return node.equals(enclosing)
+ ? CompositeBehavior.DO_NOT_ENTER_TRANSFORM
+ : CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ visitedNodes.add(node);
+ }
+ });
+
+ assertThat(visitedNodes, hasItem(enclosing));
+ assertThat(visitedNodes, not(hasItem(enclosed)));
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
index e7b680a..9956d5c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
@@ -27,8 +27,7 @@
import java.util.Arrays;
import java.util.EnumSet;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.WriteFiles;
@@ -85,10 +84,13 @@
// Issue below: PCollection.createPrimitiveOutput should not be used
// from within a composite transform.
return PCollectionList.of(
- Arrays.asList(result, PCollection.<String>createPrimitiveOutputInternal(
- b.getPipeline(),
- WindowingStrategy.globalDefault(),
- result.isBounded())));
+ Arrays.asList(
+ result,
+ PCollection.createPrimitiveOutputInternal(
+ b.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ result.isBounded(),
+ StringUtf8Coder.of())));
}
}
@@ -105,11 +107,6 @@
return PDone.in(input.getPipeline());
}
-
- @Override
- protected Coder<?> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
}
@Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/dataflow/TestCountingSource.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/dataflow/TestCountingSource.java
index 9fcc3c5..3111dd8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/dataflow/TestCountingSource.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/dataflow/TestCountingSource.java
@@ -245,10 +245,7 @@
}
@Override
- public void validate() {}
-
- @Override
- public Coder<KV<Integer, Integer>> getDefaultOutputCoder() {
+ public Coder<KV<Integer, Integer>> getOutputCoder() {
return KvCoder.of(VarIntCoder.of(), VarIntCoder.of());
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java
new file mode 100644
index 0000000..15198b2
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.testing;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Sum;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link CombineFnTester}.
+ */
+@RunWith(JUnit4.class)
+public class CombineFnTesterTest {
+ @Test
+ public void checksMergeWithEmptyAccumulators() {
+ final AtomicBoolean sawEmpty = new AtomicBoolean(false);
+ CombineFn<Integer, Integer, Integer> combineFn =
+ new CombineFn<Integer, Integer, Integer>() {
+ @Override
+ public Integer createAccumulator() {
+ return 0;
+ }
+
+ @Override
+ public Integer addInput(Integer accumulator, Integer input) {
+ return accumulator + input;
+ }
+
+ @Override
+ public Integer mergeAccumulators(Iterable<Integer> accumulators) {
+ int result = 0;
+ for (int accum : accumulators) {
+ if (accum == 0) {
+ sawEmpty.set(true);
+ }
+ result += accum;
+ }
+ return result;
+ }
+
+ @Override
+ public Integer extractOutput(Integer accumulator) {
+ return accumulator;
+ }
+ };
+
+ CombineFnTester.testCombineFn(combineFn, Arrays.asList(1, 2, 3, 4, 5), 15);
+ assertThat(sawEmpty.get(), is(true));
+ }
+
+ @Test
+ public void checksWithSingleShard() {
+ final AtomicBoolean sawSingleShard = new AtomicBoolean();
+ CombineFn<Integer, Integer, Integer> combineFn =
+ new CombineFn<Integer, Integer, Integer>() {
+ int accumCount = 0;
+
+ @Override
+ public Integer createAccumulator() {
+ accumCount++;
+ return 0;
+ }
+
+ @Override
+ public Integer addInput(Integer accumulator, Integer input) {
+ return accumulator + input;
+ }
+
+ @Override
+ public Integer mergeAccumulators(Iterable<Integer> accumulators) {
+ int result = 0;
+ for (int accum : accumulators) {
+ result += accum;
+ }
+ return result;
+ }
+
+ @Override
+ public Integer extractOutput(Integer accumulator) {
+ if (accumCount == 1) {
+ sawSingleShard.set(true);
+ }
+ accumCount = 0;
+ return accumulator;
+ }
+ };
+
+ CombineFnTester.testCombineFn(combineFn, Arrays.asList(1, 2, 3, 4, 5), 15);
+ assertThat(sawSingleShard.get(), is(true));
+ }
+
+ @Test
+ public void checksWithShards() {
+ final AtomicBoolean sawManyShards = new AtomicBoolean();
+ CombineFn<Integer, Integer, Integer> combineFn =
+ new CombineFn<Integer, Integer, Integer>() {
+
+ @Override
+ public Integer createAccumulator() {
+ return 0;
+ }
+
+ @Override
+ public Integer addInput(Integer accumulator, Integer input) {
+ return accumulator + input;
+ }
+
+ @Override
+ public Integer mergeAccumulators(Iterable<Integer> accumulators) {
+ if (Iterables.size(accumulators) > 2) {
+ sawManyShards.set(true);
+ }
+ int result = 0;
+ for (int accum : accumulators) {
+ result += accum;
+ }
+ return result;
+ }
+
+ @Override
+ public Integer extractOutput(Integer accumulator) {
+ return accumulator;
+ }
+ };
+
+ CombineFnTester.testCombineFn(
+ combineFn, Arrays.asList(1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3), 30);
+ assertThat(sawManyShards.get(), is(true));
+ }
+
+ @Test
+ public void checksWithMultipleMerges() {
+ final AtomicBoolean sawMultipleMerges = new AtomicBoolean();
+ CombineFn<Integer, Integer, Integer> combineFn =
+ new CombineFn<Integer, Integer, Integer>() {
+ int mergeCalls = 0;
+
+ @Override
+ public Integer createAccumulator() {
+ return 0;
+ }
+
+ @Override
+ public Integer addInput(Integer accumulator, Integer input) {
+ return accumulator + input;
+ }
+
+ @Override
+ public Integer mergeAccumulators(Iterable<Integer> accumulators) {
+ mergeCalls++;
+ int result = 0;
+ for (int accum : accumulators) {
+ result += accum;
+ }
+ return result;
+ }
+
+ @Override
+ public Integer extractOutput(Integer accumulator) {
+ if (mergeCalls > 1) {
+ sawMultipleMerges.set(true);
+ }
+ mergeCalls = 0;
+ return accumulator;
+ }
+ };
+
+ CombineFnTester.testCombineFn(combineFn, Arrays.asList(1, 1, 2, 2, 3, 3, 4, 4, 5, 5), 30);
+ assertThat(sawMultipleMerges.get(), is(true));
+ }
+
+ @Test
+ public void checksAlternateOrder() {
+ final AtomicBoolean sawOutOfOrder = new AtomicBoolean();
+ CombineFn<Integer, List<Integer>, Integer> combineFn =
+ new CombineFn<Integer, List<Integer>, Integer>() {
+ @Override
+ public List<Integer> createAccumulator() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<Integer> addInput(List<Integer> accumulator, Integer input) {
+ // If the input is being added to an empty accumulator, it's not known to be
+ // out of order, and it cannot be compared to the previous element. If the elements
+ // are out of order (relative to the input) a greater element will be added before
+ // a smaller one.
+ if (!accumulator.isEmpty() && accumulator.get(accumulator.size() - 1) > input) {
+ sawOutOfOrder.set(true);
+ }
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<Integer> mergeAccumulators(Iterable<List<Integer>> accumulators) {
+ List<Integer> result = new ArrayList<>();
+ for (List<Integer> accum : accumulators) {
+ result.addAll(accum);
+ }
+ return result;
+ }
+
+ @Override
+ public Integer extractOutput(List<Integer> accumulator) {
+ int value = 0;
+ for (int i : accumulator) {
+ value += i;
+ }
+ return value;
+ }
+ };
+
+ CombineFnTester.testCombineFn(
+ combineFn, Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14), 105);
+ assertThat(sawOutOfOrder.get(), is(true));
+ }
+
+ @Test
+ public void usesMatcher() {
+ final AtomicBoolean matcherUsed = new AtomicBoolean();
+ Matcher<Integer> matcher =
+ new TypeSafeMatcher<Integer>() {
+ @Override
+ public void describeTo(Description description) {}
+
+ @Override
+ protected boolean matchesSafely(Integer item) {
+ matcherUsed.set(true);
+ return item == 30;
+ }
+ };
+ CombineFnTester.testCombineFn(
+ Sum.ofIntegers(), Arrays.asList(1, 1, 2, 2, 3, 3, 4, 4, 5, 5), matcher);
+ assertThat(matcherUsed.get(), is(true));
+ try {
+ CombineFnTester.testCombineFn(
+ Sum.ofIntegers(), Arrays.asList(1, 2, 3, 4, 5), Matchers.not(Matchers.equalTo(15)));
+ } catch (AssertionError ignored) {
+ // Success! Return to avoid the call to fail();
+ return;
+ }
+ fail("The matcher should have failed, throwing an error");
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java
new file mode 100644
index 0000000..b5adcb5
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.testing;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * A classloader that intercepts loading of specifically named classes. This classloader copies
+ * the original classes definition and is useful for testing code which needs to validate usage
+ * with multiple classloaders..
+ */
+public class InterceptingUrlClassLoader extends ClassLoader {
+ private final Set<String> ownedClasses;
+
+ public InterceptingUrlClassLoader(final ClassLoader parent, final String... ownedClasses) {
+ super(parent);
+ this.ownedClasses = Sets.newHashSet(ownedClasses);
+ }
+
+ @Override
+ public Class<?> loadClass(final String name) throws ClassNotFoundException {
+ final Class<?> alreadyLoaded = super.findLoadedClass(name);
+ if (alreadyLoaded != null) {
+ return alreadyLoaded;
+ }
+
+ if (name != null && ownedClasses.contains(name)) {
+ try {
+ final String classAsResource = name.replace('.', '/') + ".class";
+ final byte[] classBytes =
+ ByteStreams.toByteArray(getParent().getResourceAsStream(classAsResource));
+ return defineClass(name, classBytes, 0, classBytes.length);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return getParent().loadClass(name);
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
index 491f001..2a79060 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
@@ -121,27 +121,34 @@
}
}
- @Test
- public void testFailureEncodedDecoded() throws IOException {
- AssertionError error = null;
+ private void throwNestedError() {
+ throw new RuntimeException("Nested error");
+ }
+
+ private void throwWrappedError() {
try {
- assertEquals(0, 1);
- } catch (AssertionError e) {
+ throwNestedError();
+ } catch (Exception e) {
+ throw new RuntimeException("Wrapped error", e);
+ }
+ }
+
+ @Test
+ public void testFailureWithExceptionEncodedDecoded() throws IOException {
+ Throwable error;
+ try {
+ throwWrappedError();
+ throw new IllegalStateException("Should have failed");
+ } catch (Throwable e) {
error = e;
}
- SuccessOrFailure failure = SuccessOrFailure.failure(
- new PAssert.PAssertionSite(error.getMessage(), error.getStackTrace()));
- SerializableCoder<SuccessOrFailure> coder = SerializableCoder.of(SuccessOrFailure.class);
-
- byte[] encoded = CoderUtils.encodeToByteArray(coder, failure);
- SuccessOrFailure res = CoderUtils.decodeFromByteArray(coder, encoded);
-
- // Should compare strings, because throwables are not directly comparable.
- assertEquals("Encode-decode failed SuccessOrFailure",
- failure.assertionError().toString(), res.assertionError().toString());
- String resultStacktrace = Throwables.getStackTraceAsString(res.assertionError());
- String failureStacktrace = Throwables.getStackTraceAsString(failure.assertionError());
- assertThat(resultStacktrace, is(failureStacktrace));
+ SuccessOrFailure failure =
+ SuccessOrFailure.failure(PAssert.PAssertionSite.capture("here"), error);
+ SuccessOrFailure res = CoderUtils.clone(SerializableCoder.of(SuccessOrFailure.class), failure);
+ assertEquals(
+ "Encode-decode failed SuccessOrFailure",
+ Throwables.getStackTraceAsString(failure.assertionError()),
+ Throwables.getStackTraceAsString(res.assertionError()));
}
@Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
index adf27f8..aaf8b91 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
@@ -22,7 +22,9 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
@@ -37,6 +39,7 @@
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.PValueBase;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -349,5 +352,10 @@
.add("viewFn", viewFn)
.toString();
}
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, pCollection);
+ }
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java
index 8801bde..1d8390e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java
@@ -43,7 +43,7 @@
@Test
public void onlyPaneNoFiring() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
- PaneExtractors.onlyPane();
+ PaneExtractors.onlyPane(PAssert.PAssertionSite.capture(""));
Iterable<ValueInSingleWindow<Integer>> noFiring =
ImmutableList.of(
ValueInSingleWindow.of(
@@ -56,7 +56,7 @@
@Test
public void onlyPaneOnlyOneFiring() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
- PaneExtractors.onlyPane();
+ PaneExtractors.onlyPane(PAssert.PAssertionSite.capture(""));
Iterable<ValueInSingleWindow<Integer>> onlyFiring =
ImmutableList.of(
ValueInSingleWindow.of(
@@ -70,7 +70,7 @@
@Test
public void onlyPaneMultiplePanesFails() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
- PaneExtractors.onlyPane();
+ PaneExtractors.onlyPane(PAssert.PAssertionSite.capture(""));
Iterable<ValueInSingleWindow<Integer>> multipleFiring =
ImmutableList.of(
ValueInSingleWindow.of(
@@ -89,7 +89,6 @@
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.LATE, 2L, 1L)));
- thrown.expect(IllegalStateException.class);
thrown.expectMessage("trigger that fires at most once");
extractor.apply(multipleFiring);
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
index 7ee48c8..2969ca6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
@@ -22,6 +22,7 @@
import com.google.common.collect.ImmutableList;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.hamcrest.Matchers;
@@ -93,4 +94,15 @@
thrown.expectMessage("may not be empty");
StaticWindows.of(GlobalWindow.Coder.INSTANCE, ImmutableList.<GlobalWindow>of());
}
+
+ @Test
+ public void testCompatibility() throws IncompatibleWindowException {
+ StaticWindows staticWindows =
+ StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first, second));
+ staticWindows.verifyCompatibility(
+ StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first, second)));
+ thrown.expect(IncompatibleWindowException.class);
+ staticWindows.verifyCompatibility(
+ StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first)));
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
index 05abb59..ec681ea 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
@@ -21,6 +21,7 @@
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
@@ -37,8 +38,10 @@
import org.apache.beam.sdk.options.ApplicationNameOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.apache.beam.sdk.values.PCollection;
@@ -59,7 +62,8 @@
@Suite.SuiteClasses({
TestPipelineTest.TestPipelineCreationTest.class,
TestPipelineTest.TestPipelineEnforcementsTest.WithRealPipelineRunner.class,
- TestPipelineTest.TestPipelineEnforcementsTest.WithCrashingPipelineRunner.class
+ TestPipelineTest.TestPipelineEnforcementsTest.WithCrashingPipelineRunner.class,
+ TestPipelineTest.NewProviderTest.class
})
public class TestPipelineTest implements Serializable {
private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
@@ -100,7 +104,7 @@
@Test
public void testCreationOfPipelineOptionsFromReallyVerboselyNamedTestCase() throws Exception {
- PipelineOptions options = TestPipeline.testingPipelineOptions();
+ PipelineOptions options = pipeline.getOptions();
assertThat(
options.as(ApplicationNameOptions.class).getAppName(),
startsWith(
@@ -112,23 +116,7 @@
public void testToString() {
assertEquals(
"TestPipeline#TestPipelineTest$TestPipelineCreationTest-testToString",
- TestPipeline.create().toString());
- }
-
- @Test
- public void testToStringNestedMethod() {
- TestPipeline p = nestedMethod();
-
- assertEquals(
- "TestPipeline#TestPipelineTest$TestPipelineCreationTest-testToStringNestedMethod",
- p.toString());
- assertEquals(
- "TestPipelineTest$TestPipelineCreationTest-testToStringNestedMethod",
- p.getOptions().as(ApplicationNameOptions.class).getAppName());
- }
-
- private TestPipeline nestedMethod() {
- return TestPipeline.create();
+ pipeline.toString());
}
@Test
@@ -144,24 +132,6 @@
}
@Test
- public void testToStringNestedClassMethod() {
- TestPipeline p = new NestedTester().p();
-
- assertEquals(
- "TestPipeline#TestPipelineTest$TestPipelineCreationTest-testToStringNestedClassMethod",
- p.toString());
- assertEquals(
- "TestPipelineTest$TestPipelineCreationTest-testToStringNestedClassMethod",
- p.getOptions().as(ApplicationNameOptions.class).getAppName());
- }
-
- private static class NestedTester {
- public TestPipeline p() {
- return TestPipeline.create();
- }
- }
-
- @Test
public void testRunWithDummyEnvironmentVariableFails() {
System.getProperties()
.setProperty(TestPipeline.PROPERTY_USE_DEFAULT_DUMMY_RUNNER, Boolean.toString(true));
@@ -371,4 +341,35 @@
}
}
}
+
+ /** Tests for {@link TestPipeline#newProvider}. */
+ @RunWith(JUnit4.class)
+ public static class NewProviderTest implements Serializable {
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testNewProvider() {
+ ValueProvider<String> foo = pipeline.newProvider("foo");
+ ValueProvider<String> foobar =
+ ValueProvider.NestedValueProvider.of(
+ foo,
+ new SerializableFunction<String, String>() {
+ @Override
+ public String apply(String input) {
+ return input + "bar";
+ }
+ });
+
+ assertFalse(foo.isAccessible());
+ assertFalse(foobar.isAccessible());
+
+ PAssert.that(pipeline.apply("create foo", Create.ofProvider(foo, StringUtf8Coder.of())))
+ .containsInAnyOrder("foo");
+ PAssert.that(pipeline.apply("create foobar", Create.ofProvider(foobar, StringUtf8Coder.of())))
+ .containsInAnyOrder("foobar");
+
+ pipeline.run();
+ }
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
index cd7898b..2657e07 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
@@ -17,14 +17,17 @@
*/
package org.apache.beam.sdk.transforms;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
+import static org.junit.Assert.assertEquals;
+import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import org.apache.beam.sdk.Pipeline;
@@ -41,270 +44,371 @@
import org.hamcrest.CoreMatchers;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.junit.runners.Parameterized;
/**
* Tests for {@link ApproximateQuantiles}.
*/
-@RunWith(JUnit4.class)
+@RunWith(Enclosed.class)
public class ApproximateQuantilesTest {
- static final List<KV<String, Integer>> TABLE = Arrays.asList(
- KV.of("a", 1),
- KV.of("a", 2),
- KV.of("a", 3),
- KV.of("b", 1),
- KV.of("b", 10),
- KV.of("b", 10),
- KV.of("b", 100)
- );
+ /** Tests for the overall combiner behavior. */
+ @RunWith(JUnit4.class)
+ public static class CombinerTests {
+ static final List<KV<String, Integer>> TABLE = Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 2),
+ KV.of("a", 3),
+ KV.of("b", 1),
+ KV.of("b", 10),
+ KV.of("b", 10),
+ KV.of("b", 100)
+ );
- @Rule
- public TestPipeline p = TestPipeline.create();
+ @Rule
+ public TestPipeline p = TestPipeline.create();
- public PCollection<KV<String, Integer>> createInputTable(Pipeline p) {
- return p.apply(Create.of(TABLE).withCoder(
- KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testQuantilesGlobally() {
- PCollection<Integer> input = intRangeCollection(p, 101);
- PCollection<List<Integer>> quantiles =
- input.apply(ApproximateQuantiles.<Integer>globally(5));
-
- PAssert.that(quantiles)
- .containsInAnyOrder(Arrays.asList(0, 25, 50, 75, 100));
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testQuantilesGobally_comparable() {
- PCollection<Integer> input = intRangeCollection(p, 101);
- PCollection<List<Integer>> quantiles =
- input.apply(
- ApproximateQuantiles.globally(5, new DescendingIntComparator()));
-
- PAssert.that(quantiles)
- .containsInAnyOrder(Arrays.asList(100, 75, 50, 25, 0));
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testQuantilesPerKey() {
- PCollection<KV<String, Integer>> input = createInputTable(p);
- PCollection<KV<String, List<Integer>>> quantiles = input.apply(
- ApproximateQuantiles.<String, Integer>perKey(2));
-
- PAssert.that(quantiles)
- .containsInAnyOrder(
- KV.of("a", Arrays.asList(1, 3)),
- KV.of("b", Arrays.asList(1, 100)));
- p.run();
-
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testQuantilesPerKey_reversed() {
- PCollection<KV<String, Integer>> input = createInputTable(p);
- PCollection<KV<String, List<Integer>>> quantiles = input.apply(
- ApproximateQuantiles.<String, Integer, DescendingIntComparator>perKey(
- 2, new DescendingIntComparator()));
-
- PAssert.that(quantiles)
- .containsInAnyOrder(
- KV.of("a", Arrays.asList(3, 1)),
- KV.of("b", Arrays.asList(100, 1)));
- p.run();
- }
-
- @Test
- public void testSingleton() {
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- Arrays.asList(389),
- Arrays.asList(389, 389, 389, 389, 389));
- }
-
- @Test
- public void testSimpleQuantiles() {
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- intRange(101),
- Arrays.asList(0, 25, 50, 75, 100));
- }
-
- @Test
- public void testUnevenQuantiles() {
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(37),
- intRange(5000),
- quantileMatcher(5000, 37, 20 /* tolerance */));
- }
-
- @Test
- public void testLargerQuantiles() {
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(50),
- intRange(10001),
- quantileMatcher(10001, 50, 20 /* tolerance */));
- }
-
- @Test
- public void testTightEpsilon() {
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(10).withEpsilon(0.01),
- intRange(10001),
- quantileMatcher(10001, 10, 5 /* tolerance */));
- }
-
- @Test
- public void testDuplicates() {
- int size = 101;
- List<Integer> all = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- all.addAll(intRange(size));
- }
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- all,
- Arrays.asList(0, 25, 50, 75, 100));
- }
-
- @Test
- public void testLotsOfDuplicates() {
- List<Integer> all = new ArrayList<>();
- all.add(1);
- for (int i = 1; i < 300; i++) {
- all.add(2);
- }
- for (int i = 300; i < 1000; i++) {
- all.add(3);
- }
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- all,
- Arrays.asList(1, 2, 3, 3, 3));
- }
-
- @Test
- public void testLogDistribution() {
- List<Integer> all = new ArrayList<>();
- for (int i = 1; i < 1000; i++) {
- all.add((int) Math.log(i));
- }
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- all,
- Arrays.asList(0, 5, 6, 6, 6));
- }
-
- @Test
- public void testZipfianDistribution() {
- List<Integer> all = new ArrayList<>();
- for (int i = 1; i < 1000; i++) {
- all.add(1000 / i);
- }
- checkCombineFn(
- ApproximateQuantilesCombineFn.<Integer>create(5),
- all,
- Arrays.asList(1, 1, 2, 4, 1000));
- }
-
- @Test
- public void testAlternateComparator() {
- List<String> inputs = Arrays.asList(
- "aa", "aaa", "aaaa", "b", "ccccc", "dddd", "zz");
- checkCombineFn(
- ApproximateQuantilesCombineFn.<String>create(3),
- inputs,
- Arrays.asList("aa", "b", "zz"));
- checkCombineFn(
- ApproximateQuantilesCombineFn.create(3, new OrderByLength()),
- inputs,
- Arrays.asList("b", "aaa", "ccccc"));
- }
-
- @Test
- public void testDisplayData() {
- Top.Largest<Integer> comparer = new Top.Largest<Integer>();
- PTransform<?, ?> approxQuanitiles = ApproximateQuantiles.globally(20, comparer);
- DisplayData displayData = DisplayData.from(approxQuanitiles);
-
- assertThat(displayData, hasDisplayItem("numQuantiles", 20));
- assertThat(displayData, hasDisplayItem("comparer", comparer.getClass()));
- }
-
- private Matcher<Iterable<? extends Integer>> quantileMatcher(
- int size, int numQuantiles, int absoluteError) {
- List<Matcher<? super Integer>> quantiles = new ArrayList<>();
- quantiles.add(CoreMatchers.is(0));
- for (int k = 1; k < numQuantiles - 1; k++) {
- int expected = (int) (((double) (size - 1)) * k / (numQuantiles - 1));
- quantiles.add(new Between<>(
- expected - absoluteError, expected + absoluteError));
- }
- quantiles.add(CoreMatchers.is(size - 1));
- return contains(quantiles);
- }
-
- private static class Between<T extends Comparable<T>>
- extends TypeSafeDiagnosingMatcher<T> {
- private final T min;
- private final T max;
- private Between(T min, T max) {
- this.min = min;
- this.max = max;
- }
- @Override
- public void describeTo(Description description) {
- description.appendText("is between " + min + " and " + max);
+ public PCollection<KV<String, Integer>> createInputTable(Pipeline p) {
+ return p.apply(Create.of(TABLE).withCoder(
+ KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
}
- @Override
- protected boolean matchesSafely(T item, Description mismatchDescription) {
- return min.compareTo(item) <= 0 && item.compareTo(max) <= 0;
- }
- }
+ @Test
+ @Category(NeedsRunner.class)
+ public void testQuantilesGlobally() {
+ PCollection<Integer> input = intRangeCollection(p, 101);
+ PCollection<List<Integer>> quantiles =
+ input.apply(ApproximateQuantiles.<Integer>globally(5));
- private static class DescendingIntComparator implements
- SerializableComparator<Integer> {
- @Override
- public int compare(Integer o1, Integer o2) {
- return o2.compareTo(o1);
+ PAssert.that(quantiles)
+ .containsInAnyOrder(Arrays.asList(0, 25, 50, 75, 100));
+ p.run();
}
- }
- private static class OrderByLength implements Comparator<String>, Serializable {
- @Override
- public int compare(String a, String b) {
- if (a.length() != b.length()) {
- return a.length() - b.length();
- } else {
- return a.compareTo(b);
+ @Test
+ @Category(NeedsRunner.class)
+ public void testQuantilesGobally_comparable() {
+ PCollection<Integer> input = intRangeCollection(p, 101);
+ PCollection<List<Integer>> quantiles =
+ input.apply(
+ ApproximateQuantiles.globally(5, new DescendingIntComparator()));
+
+ PAssert.that(quantiles)
+ .containsInAnyOrder(Arrays.asList(100, 75, 50, 25, 0));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testQuantilesPerKey() {
+ PCollection<KV<String, Integer>> input = createInputTable(p);
+ PCollection<KV<String, List<Integer>>> quantiles = input.apply(
+ ApproximateQuantiles.<String, Integer>perKey(2));
+
+ PAssert.that(quantiles)
+ .containsInAnyOrder(
+ KV.of("a", Arrays.asList(1, 3)),
+ KV.of("b", Arrays.asList(1, 100)));
+ p.run();
+
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testQuantilesPerKey_reversed() {
+ PCollection<KV<String, Integer>> input = createInputTable(p);
+ PCollection<KV<String, List<Integer>>> quantiles = input.apply(
+ ApproximateQuantiles.<String, Integer, DescendingIntComparator>perKey(
+ 2, new DescendingIntComparator()));
+
+ PAssert.that(quantiles)
+ .containsInAnyOrder(
+ KV.of("a", Arrays.asList(3, 1)),
+ KV.of("b", Arrays.asList(100, 1)));
+ p.run();
+ }
+
+ @Test
+ public void testSingleton() {
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ Arrays.asList(389),
+ Arrays.asList(389, 389, 389, 389, 389));
+ }
+
+ @Test
+ public void testSimpleQuantiles() {
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ intRange(101),
+ Arrays.asList(0, 25, 50, 75, 100));
+ }
+
+ @Test
+ public void testUnevenQuantiles() {
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(37),
+ intRange(5000),
+ quantileMatcher(5000, 37, 20 /* tolerance */));
+ }
+
+ @Test
+ public void testLargerQuantiles() {
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(50),
+ intRange(10001),
+ quantileMatcher(10001, 50, 20 /* tolerance */));
+ }
+
+ @Test
+ public void testTightEpsilon() {
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(10).withEpsilon(0.01),
+ intRange(10001),
+ quantileMatcher(10001, 10, 5 /* tolerance */));
+ }
+
+ @Test
+ public void testDuplicates() {
+ int size = 101;
+ List<Integer> all = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ all.addAll(intRange(size));
+ }
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ all,
+ Arrays.asList(0, 25, 50, 75, 100));
+ }
+
+ @Test
+ public void testLotsOfDuplicates() {
+ List<Integer> all = new ArrayList<>();
+ all.add(1);
+ for (int i = 1; i < 300; i++) {
+ all.add(2);
+ }
+ for (int i = 300; i < 1000; i++) {
+ all.add(3);
+ }
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ all,
+ Arrays.asList(1, 2, 3, 3, 3));
+ }
+
+ @Test
+ public void testLogDistribution() {
+ List<Integer> all = new ArrayList<>();
+ for (int i = 1; i < 1000; i++) {
+ all.add((int) Math.log(i));
+ }
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ all,
+ Arrays.asList(0, 5, 6, 6, 6));
+ }
+
+ @Test
+ public void testZipfianDistribution() {
+ List<Integer> all = new ArrayList<>();
+ for (int i = 1; i < 1000; i++) {
+ all.add(1000 / i);
+ }
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<Integer>create(5),
+ all,
+ Arrays.asList(1, 1, 2, 4, 1000));
+ }
+
+ @Test
+ public void testAlternateComparator() {
+ List<String> inputs = Arrays.asList(
+ "aa", "aaa", "aaaa", "b", "ccccc", "dddd", "zz");
+ testCombineFn(
+ ApproximateQuantilesCombineFn.<String>create(3),
+ inputs,
+ Arrays.asList("aa", "b", "zz"));
+ testCombineFn(
+ ApproximateQuantilesCombineFn.create(3, new OrderByLength()),
+ inputs,
+ Arrays.asList("b", "aaa", "ccccc"));
+ }
+
+ @Test
+ public void testDisplayData() {
+ Top.Natural<Integer> comparer = new Top.Natural<Integer>();
+ PTransform<?, ?> approxQuanitiles = ApproximateQuantiles.globally(20, comparer);
+ DisplayData displayData = DisplayData.from(approxQuanitiles);
+
+ assertThat(displayData, hasDisplayItem("numQuantiles", 20));
+ assertThat(displayData, hasDisplayItem("comparer", comparer.getClass()));
+ }
+
+ private Matcher<Iterable<? extends Integer>> quantileMatcher(
+ int size, int numQuantiles, int absoluteError) {
+ List<Matcher<? super Integer>> quantiles = new ArrayList<>();
+ quantiles.add(CoreMatchers.is(0));
+ for (int k = 1; k < numQuantiles - 1; k++) {
+ int expected = (int) (((double) (size - 1)) * k / (numQuantiles - 1));
+ quantiles.add(new Between<>(
+ expected - absoluteError, expected + absoluteError));
+ }
+ quantiles.add(CoreMatchers.is(size - 1));
+ return contains(quantiles);
+ }
+
+ private static class Between<T extends Comparable<T>>
+ extends TypeSafeDiagnosingMatcher<T> {
+ private final T min;
+ private final T max;
+
+ private Between(T min, T max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("is between " + min + " and " + max);
+ }
+
+ @Override
+ protected boolean matchesSafely(T item, Description mismatchDescription) {
+ return min.compareTo(item) <= 0 && item.compareTo(max) <= 0;
}
}
- }
-
- private PCollection<Integer> intRangeCollection(Pipeline p, int size) {
- return p.apply("CreateIntsUpTo(" + size + ")", Create.of(intRange(size)));
- }
-
- private List<Integer> intRange(int size) {
- List<Integer> all = new ArrayList<>(size);
- for (int i = 0; i < size; i++) {
- all.add(i);
+ private static class DescendingIntComparator implements
+ SerializableComparator<Integer> {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ return o2.compareTo(o1);
+ }
}
- return all;
+
+ private static class OrderByLength implements Comparator<String>, Serializable {
+ @Override
+ public int compare(String a, String b) {
+ if (a.length() != b.length()) {
+ return a.length() - b.length();
+ } else {
+ return a.compareTo(b);
+ }
+ }
+ }
+
+
+ private PCollection<Integer> intRangeCollection(Pipeline p, int size) {
+ return p.apply("CreateIntsUpTo(" + size + ")", Create.of(intRange(size)));
+ }
+
+ private List<Integer> intRange(int size) {
+ List<Integer> all = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ all.add(i);
+ }
+ return all;
+ }
+ }
+
+ /** Tests to ensure we are calculating the optimal buffers. */
+ @RunWith(Parameterized.class)
+ public static class BufferTests {
+
+ private final double epsilon;
+ private final long maxInputSize;
+ private final int expectedNumBuffers;
+ private final int expectedBufferSize;
+ private final ApproximateQuantilesCombineFn<?, ?> combineFn;
+
+ /**
+ * Test data taken from "Munro-Paterson Algorithm" reference values table of "Approximate
+ * Medians and other Quantiles in One Pass and with Limited Memory" paper.
+ *
+ * @see ApproximateQuantilesCombineFn for paper reference.
+ */
+ private static final double[] epsilons = new double[]{0.1, 0.05, 0.01, 0.005, 0.001};
+ private static final int[] maxElementExponents = new int[]{5, 6, 7, 8, 9};
+
+ private static final int[][] expectedNumBuffersValues = new int[][]{
+ {11, 14, 17, 21, 24},
+ {11, 14, 17, 20, 23},
+ {9, 11, 14, 17, 21},
+ {8, 11, 14, 17, 20},
+ {6, 9, 11, 14, 17},
+ };
+
+ private static final int[][] expectedBufferSizeValues = new int[][]{
+ {98, 123, 153, 96, 120},
+ {98, 123, 153, 191, 239},
+ {391, 977, 1221, 1526, 954},
+ {782, 977, 1221, 1526, 1908},
+ {3125, 3907, 9766, 12208, 15259},
+ };
+
+ @Parameterized.Parameters(name = "{index}: epsilon = {0}, maxInputSize = {1}")
+ public static Collection<Object[]> data() {
+ Collection<Object[]> testData = Lists.newArrayList();
+ for (int i = 0; i < epsilons.length; i++) {
+ for (int j = 0; j < maxElementExponents.length; j++) {
+ testData.add(new Object[]{
+ epsilons[i],
+ (long) Math.pow(10, maxElementExponents[j]),
+ expectedNumBuffersValues[i][j],
+ expectedBufferSizeValues[i][j]
+ });
+ }
+ }
+
+ return testData;
+ }
+
+ public BufferTests(
+ Double epsilon, Long maxInputSize, Integer expectedNumBuffers, Integer expectedBufferSize) {
+ this.epsilon = epsilon;
+ this.maxInputSize = maxInputSize;
+ this.expectedNumBuffers = expectedNumBuffers;
+ this.expectedBufferSize = expectedBufferSize;
+
+ this.combineFn = ApproximateQuantilesCombineFn.create(
+ 10, new Top.Natural<Long>(), maxInputSize, epsilon);
+ }
+
+ /**
+ * Verify the buffers are efficiently calculated according to the reference table values.
+ */
+ @Test
+ public void testEfficiency() {
+ assertEquals("Number of buffers", expectedNumBuffers, combineFn.getNumBuffers());
+ assertEquals("Buffer size", expectedBufferSize, combineFn.getBufferSize());
+ }
+
+ /**
+ * Verify that buffers are correct according to the two constraint equations.
+ */
+ @Test
+ public void testCorrectness() {
+ int b = combineFn.getNumBuffers();
+ int k = combineFn.getBufferSize();
+ long n = this.maxInputSize;
+
+ assertThat(
+ "(b-2)2^(b-2) + 1/2 <= eN",
+ (b - 2) * (1 << (b - 2)) + 0.5,
+ Matchers.lessThanOrEqualTo(this.epsilon * n));
+ assertThat(
+ "k2^(b-1) >= N",
+ Math.pow(k * 2, b - 1),
+ Matchers.greaterThanOrEqualTo((double) n));
+ }
}
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index dc9788f..52fedc6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -17,9 +17,9 @@
*/
package org.apache.beam.sdk.transforms;
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasNamespace;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFor;
@@ -29,11 +29,13 @@
import static org.junit.Assert.assertThat;
import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
@@ -45,7 +47,6 @@
import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.DoubleCoder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -85,7 +86,6 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import org.mockito.Mock;
/**
* Tests for Combine transforms.
@@ -95,18 +95,8 @@
// This test is Serializable, just so that it's easy to have
// anonymous inner classes inside the non-static test methods.
- static final List<KV<String, Integer>> TABLE = Arrays.asList(
- KV.of("a", 1),
- KV.of("a", 1),
- KV.of("a", 4),
- KV.of("b", 1),
- KV.of("b", 13)
- );
-
static final List<KV<String, Integer>> EMPTY_TABLE = Collections.emptyList();
- @Mock private DoFn<?, ?>.ProcessContext processContext;
-
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
@@ -150,12 +140,12 @@
PCollection<KV<String, String>> combinePerKey =
perKeyInput.apply(
Combine.<String, Integer, String>perKey(new TestCombineFnWithContext(globallySumView))
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PCollection<String> combineGlobally = globallyInput
.apply(Combine.globally(new TestCombineFnWithContext(globallySumView))
.withoutDefaults()
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PAssert.that(sum).containsInAnyOrder(globalSum);
PAssert.that(combinePerKey).containsInAnyOrder(perKeyCombines);
@@ -168,16 +158,28 @@
@Category(ValidatesRunner.class)
@SuppressWarnings({"rawtypes", "unchecked"})
public void testSimpleCombine() {
- runTestSimpleCombine(TABLE, 20, Arrays.asList(KV.of("a", "114"), KV.of("b", "113")));
+ runTestSimpleCombine(Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ ), 20, Arrays.asList(KV.of("a", "114"), KV.of("b", "113")));
}
@Test
@Category(ValidatesRunner.class)
@SuppressWarnings({"rawtypes", "unchecked"})
public void testSimpleCombineWithContext() {
- runTestSimpleCombineWithContext(TABLE, 20,
- Arrays.asList(KV.of("a", "01124"), KV.of("b", "01123")),
- new String[] {"01111234"});
+ runTestSimpleCombineWithContext(Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ ), 20,
+ Arrays.asList(KV.of("a", "20:114"), KV.of("b", "20:113")),
+ new String[] {"20:111134"});
}
@Test
@@ -216,7 +218,13 @@
@Test
@Category(ValidatesRunner.class)
public void testBasicCombine() {
- runTestBasicCombine(TABLE, ImmutableSet.of(1, 13, 4), Arrays.asList(
+ runTestBasicCombine(Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ ), ImmutableSet.of(1, 13, 4), Arrays.asList(
KV.of("a", (Set<Integer>) ImmutableSet.of(1, 4)),
KV.of("b", (Set<Integer>) ImmutableSet.of(1, 13))));
}
@@ -251,9 +259,16 @@
@Category(ValidatesRunner.class)
public void testFixedWindowsCombine() {
PCollection<KV<String, Integer>> input =
- pipeline.apply(Create.timestamped(TABLE, Arrays.asList(0L, 1L, 6L, 7L, 8L))
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
- .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.millis(2))));
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
+ TimestampedValue.of(KV.of("a", 1), new Instant(1L)),
+ TimestampedValue.of(KV.of("a", 4), new Instant(6L)),
+ TimestampedValue.of(KV.of("b", 1), new Instant(7L)),
+ TimestampedValue.of(KV.of("b", 13), new Instant(8L)))
+ .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
+ .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.millis(2))));
PCollection<Integer> sum = input
.apply(Values.<Integer>create())
@@ -263,11 +278,9 @@
.apply(Combine.<String, Integer, String>perKey(new TestCombineFn()));
PAssert.that(sum).containsInAnyOrder(2, 5, 13);
- PAssert.that(sumPerKey).containsInAnyOrder(
- KV.of("a", "11"),
- KV.of("a", "4"),
- KV.of("b", "1"),
- KV.of("b", "13"));
+ PAssert.that(sumPerKey)
+ .containsInAnyOrder(
+ Arrays.asList(KV.of("a", "11"), KV.of("a", "4"), KV.of("b", "1"), KV.of("b", "13")));
pipeline.run();
}
@@ -275,9 +288,16 @@
@Category(ValidatesRunner.class)
public void testFixedWindowsCombineWithContext() {
PCollection<KV<String, Integer>> perKeyInput =
- pipeline.apply(Create.timestamped(TABLE, Arrays.asList(0L, 1L, 6L, 7L, 8L))
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
- .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.millis(2))));
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
+ TimestampedValue.of(KV.of("a", 1), new Instant(1L)),
+ TimestampedValue.of(KV.of("a", 4), new Instant(6L)),
+ TimestampedValue.of(KV.of("b", 1), new Instant(7L)),
+ TimestampedValue.of(KV.of("b", 13), new Instant(8L)))
+ .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
+ .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.millis(2))));
PCollection<Integer> globallyInput = perKeyInput.apply(Values.<Integer>create());
@@ -289,60 +309,129 @@
PCollection<KV<String, String>> combinePerKeyWithContext =
perKeyInput.apply(
Combine.<String, Integer, String>perKey(new TestCombineFnWithContext(globallySumView))
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PCollection<String> combineGloballyWithContext = globallyInput
.apply(Combine.globally(new TestCombineFnWithContext(globallySumView))
.withoutDefaults()
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PAssert.that(sum).containsInAnyOrder(2, 5, 13);
- PAssert.that(combinePerKeyWithContext).containsInAnyOrder(
- KV.of("a", "112"),
- KV.of("a", "45"),
- KV.of("b", "15"),
- KV.of("b", "1133"));
- PAssert.that(combineGloballyWithContext).containsInAnyOrder("112", "145", "1133");
+ PAssert.that(combinePerKeyWithContext)
+ .containsInAnyOrder(
+ Arrays.asList(
+ KV.of("a", "2:11"), KV.of("a", "5:4"), KV.of("b", "5:1"), KV.of("b", "13:13")));
+ PAssert.that(combineGloballyWithContext).containsInAnyOrder("2:11", "5:14", "13:13");
+ pipeline.run();
+ }
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testSlidingWindowsCombine() {
+ PCollection<String> input =
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of("a", new Instant(1L)),
+ TimestampedValue.of("b", new Instant(2L)),
+ TimestampedValue.of("c", new Instant(3L))))
+ .apply(
+ Window.<String>into(
+ SlidingWindows.of(Duration.millis(3)).every(Duration.millis(1L))));
+ PCollection<List<String>> combined =
+ input.apply(
+ Combine.globally(
+ new CombineFn<String, List<String>, List<String>>() {
+ @Override
+ public List<String> createAccumulator() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<String> addInput(List<String> accumulator, String input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<String> mergeAccumulators(Iterable<List<String>> accumulators) {
+ // Mutate all of the accumulators. Instances should be used in only one
+ // place, and not
+ // reused after merging.
+ List<String> cur = createAccumulator();
+ for (List<String> accumulator : accumulators) {
+ accumulator.addAll(cur);
+ cur = accumulator;
+ }
+ return cur;
+ }
+
+ @Override
+ public List<String> extractOutput(List<String> accumulator) {
+ List<String> result = new ArrayList<>(accumulator);
+ Collections.sort(result);
+ return result;
+ }
+ })
+ .withoutDefaults());
+
+ PAssert.that(combined)
+ .containsInAnyOrder(
+ ImmutableList.of("a"),
+ ImmutableList.of("a", "b"),
+ ImmutableList.of("a", "b", "c"),
+ ImmutableList.of("b", "c"),
+ ImmutableList.of("c"));
+
pipeline.run();
}
@Test
@Category(ValidatesRunner.class)
public void testSlidingWindowsCombineWithContext() {
+ // [a: 1, 1], [a: 4; b: 1], [b: 13]
PCollection<KV<String, Integer>> perKeyInput =
- pipeline.apply(Create.timestamped(TABLE, Arrays.asList(2L, 3L, 8L, 9L, 10L))
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
- .apply(Window.<KV<String, Integer>>into(SlidingWindows.of(Duration.millis(2))));
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("a", 1), new Instant(2L)),
+ TimestampedValue.of(KV.of("a", 1), new Instant(3L)),
+ TimestampedValue.of(KV.of("a", 4), new Instant(8L)),
+ TimestampedValue.of(KV.of("b", 1), new Instant(9L)),
+ TimestampedValue.of(KV.of("b", 13), new Instant(10L)))
+ .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
+ .apply(Window.<KV<String, Integer>>into(SlidingWindows.of(Duration.millis(2))));
PCollection<Integer> globallyInput = perKeyInput.apply(Values.<Integer>create());
- PCollection<Integer> sum = globallyInput
- .apply("Sum", Combine.globally(new SumInts()).withoutDefaults());
+ PCollection<Integer> sum = globallyInput.apply("Sum", Sum.integersGlobally().withoutDefaults());
PCollectionView<Integer> globallySumView = sum.apply(View.<Integer>asSingleton());
PCollection<KV<String, String>> combinePerKeyWithContext =
perKeyInput.apply(
Combine.<String, Integer, String>perKey(new TestCombineFnWithContext(globallySumView))
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PCollection<String> combineGloballyWithContext = globallyInput
.apply(Combine.globally(new TestCombineFnWithContext(globallySumView))
.withoutDefaults()
- .withSideInputs(Arrays.asList(globallySumView)));
+ .withSideInputs(globallySumView));
PAssert.that(sum).containsInAnyOrder(1, 2, 1, 4, 5, 14, 13);
- PAssert.that(combinePerKeyWithContext).containsInAnyOrder(
- KV.of("a", "11"),
- KV.of("a", "112"),
- KV.of("a", "11"),
- KV.of("a", "44"),
- KV.of("a", "45"),
- KV.of("b", "15"),
- KV.of("b", "11134"),
- KV.of("b", "1133"));
+ PAssert.that(combinePerKeyWithContext)
+ .containsInAnyOrder(
+ Arrays.asList(
+ KV.of("a", "1:1"),
+ KV.of("a", "2:11"),
+ KV.of("a", "1:1"),
+ KV.of("a", "4:4"),
+ KV.of("a", "5:4"),
+ KV.of("b", "5:1"),
+ KV.of("b", "14:113"),
+ KV.of("b", "13:13")));
PAssert.that(combineGloballyWithContext).containsInAnyOrder(
- "11", "112", "11", "44", "145", "11134", "1133");
+ "1:1", "2:11", "1:1", "4:4", "5:14", "14:113", "13:13");
pipeline.run();
}
@@ -383,9 +472,16 @@
@Category(ValidatesRunner.class)
public void testSessionsCombine() {
PCollection<KV<String, Integer>> input =
- pipeline.apply(Create.timestamped(TABLE, Arrays.asList(0L, 4L, 7L, 10L, 16L))
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
- .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(Duration.millis(5))));
+ pipeline
+ .apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
+ TimestampedValue.of(KV.of("a", 1), new Instant(4L)),
+ TimestampedValue.of(KV.of("a", 4), new Instant(7L)),
+ TimestampedValue.of(KV.of("b", 1), new Instant(10L)),
+ TimestampedValue.of(KV.of("b", 13), new Instant(16L)))
+ .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
+ .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(Duration.millis(5))));
PCollection<Integer> sum = input
.apply(Values.<Integer>create())
@@ -395,10 +491,8 @@
.apply(Combine.<String, Integer, String>perKey(new TestCombineFn()));
PAssert.that(sum).containsInAnyOrder(7, 13);
- PAssert.that(sumPerKey).containsInAnyOrder(
- KV.of("a", "114"),
- KV.of("b", "1"),
- KV.of("b", "13"));
+ PAssert.that(sumPerKey)
+ .containsInAnyOrder(Arrays.asList(KV.of("a", "114"), KV.of("b", "1"), KV.of("b", "13")));
pipeline.run();
}
@@ -406,7 +500,13 @@
@Category(ValidatesRunner.class)
public void testSessionsCombineWithContext() {
PCollection<KV<String, Integer>> perKeyInput =
- pipeline.apply(Create.timestamped(TABLE, Arrays.asList(0L, 4L, 7L, 10L, 16L))
+ pipeline.apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
+ TimestampedValue.of(KV.of("a", 1), new Instant(4L)),
+ TimestampedValue.of(KV.of("a", 4), new Instant(7L)),
+ TimestampedValue.of(KV.of("b", 1), new Instant(10L)),
+ TimestampedValue.of(KV.of("b", 13), new Instant(16L)))
.withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
PCollection<Integer> globallyInput = perKeyInput.apply(Values.<Integer>create());
@@ -427,21 +527,23 @@
.apply(
Combine.<String, Integer, String>perKey(
new TestCombineFnWithContext(globallyFixedWindowsView))
- .withSideInputs(Arrays.asList(globallyFixedWindowsView)));
+ .withSideInputs(globallyFixedWindowsView));
- PCollection<String> sessionsCombineGlobally = globallyInput
- .apply("Globally Input Sessions",
- Window.<Integer>into(Sessions.withGapDuration(Duration.millis(5))))
- .apply(Combine.globally(new TestCombineFnWithContext(globallyFixedWindowsView))
- .withoutDefaults()
- .withSideInputs(Arrays.asList(globallyFixedWindowsView)));
+ PCollection<String> sessionsCombineGlobally =
+ globallyInput
+ .apply(
+ "Globally Input Sessions",
+ Window.<Integer>into(Sessions.withGapDuration(Duration.millis(5))))
+ .apply(
+ Combine.globally(new TestCombineFnWithContext(globallyFixedWindowsView))
+ .withoutDefaults()
+ .withSideInputs(globallyFixedWindowsView));
PAssert.that(fixedWindowsSum).containsInAnyOrder(2, 4, 1, 13);
- PAssert.that(sessionsCombinePerKey).containsInAnyOrder(
- KV.of("a", "1114"),
- KV.of("b", "11"),
- KV.of("b", "013"));
- PAssert.that(sessionsCombineGlobally).containsInAnyOrder("11114", "013");
+ PAssert.that(sessionsCombinePerKey)
+ .containsInAnyOrder(
+ Arrays.asList(KV.of("a", "1:114"), KV.of("b", "1:1"), KV.of("b", "0:13")));
+ PAssert.that(sessionsCombineGlobally).containsInAnyOrder("1:1114", "0:13");
pipeline.run();
}
@@ -461,7 +563,13 @@
@Test
@Category(ValidatesRunner.class)
public void testAccumulatingCombine() {
- runTestAccumulatingCombine(TABLE, 4.0, Arrays.asList(KV.of("a", 2.0), KV.of("b", 7.0)));
+ runTestAccumulatingCombine(Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ ), 4.0, Arrays.asList(KV.of("a", 2.0), KV.of("b", 7.0)));
}
@Test
@@ -503,7 +611,13 @@
@Test
@Category(ValidatesRunner.class)
public void testHotKeyCombining() {
- PCollection<KV<String, Integer>> input = copy(createInput(pipeline, TABLE), 10);
+ PCollection<KV<String, Integer>> input = copy(createInput(pipeline, Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ )), 10);
CombineFn<Integer, ?, Double> mean = new MeanInts();
PCollection<KV<String, Double>> coldMean = input.apply("ColdMean",
@@ -560,7 +674,13 @@
@Test
@Category(NeedsRunner.class)
public void testBinaryCombineFn() {
- PCollection<KV<String, Integer>> input = copy(createInput(pipeline, TABLE), 2);
+ PCollection<KV<String, Integer>> input = copy(createInput(pipeline, Arrays.asList(
+ KV.of("a", 1),
+ KV.of("a", 1),
+ KV.of("a", 4),
+ KV.of("b", 1),
+ KV.of("b", 13)
+ )), 2);
PCollection<KV<String, Integer>> intProduct = input
.apply("IntProduct", Combine.<String, Integer, Integer>perKey(new TestProdInt()));
PCollection<KV<String, Integer>> objProduct = input
@@ -575,11 +695,11 @@
@Test
public void testBinaryCombineFnWithNulls() {
- checkCombineFn(new NullCombiner(), Arrays.asList(3, 3, 5), 45);
- checkCombineFn(new NullCombiner(), Arrays.asList(null, 3, 5), 30);
- checkCombineFn(new NullCombiner(), Arrays.asList(3, 3, null), 18);
- checkCombineFn(new NullCombiner(), Arrays.asList(null, 3, null), 12);
- checkCombineFn(new NullCombiner(), Arrays.<Integer>asList(null, null, null), 8);
+ testCombineFn(new NullCombiner(), Arrays.asList(3, 3, 5), 45);
+ testCombineFn(new NullCombiner(), Arrays.asList(null, 3, 5), 30);
+ testCombineFn(new NullCombiner(), Arrays.asList(3, 3, null), 18);
+ testCombineFn(new NullCombiner(), Arrays.asList(null, 3, null), 12);
+ testCombineFn(new NullCombiner(), Arrays.<Integer>asList(null, null, null), 8);
}
private static final class TestProdInt extends Combine.BinaryCombineIntegerFn {
@@ -651,7 +771,7 @@
pipeline
.apply(
"CreateMainInput",
- Create.<Void>timestamped(nonEmptyElement, emptyElement).withCoder(VoidCoder.of()))
+ Create.timestamped(nonEmptyElement, emptyElement).withCoder(VoidCoder.of()))
.apply("WindowMainInput", Window.<Void>into(windowFn))
.apply(
"OutputSideInput",
@@ -876,15 +996,13 @@
*/
private class CountSumCoder extends AtomicCoder<CountSum> {
@Override
- public void encode(CountSum value, OutputStream outStream)
- throws CoderException, IOException {
+ public void encode(CountSum value, OutputStream outStream) throws IOException {
LONG_CODER.encode(value.count, outStream);
DOUBLE_CODER.encode(value.sum, outStream);
}
@Override
- public CountSum decode(InputStream inStream)
- throws CoderException, IOException {
+ public CountSum decode(InputStream inStream) throws IOException {
long count = LONG_CODER.decode(inStream);
double sum = DOUBLE_CODER.decode(inStream);
return new CountSum(count, sum);
@@ -917,34 +1035,26 @@
// Not serializable.
static class Accumulator {
+ final String seed;
String value;
- public Accumulator(String value) {
+ public Accumulator(String seed, String value) {
+ this.seed = seed;
this.value = value;
}
public static Coder<Accumulator> getCoder() {
return new AtomicCoder<Accumulator>() {
@Override
- public void encode(Accumulator accumulator, OutputStream outStream)
- throws CoderException, IOException {
- encode(accumulator, outStream, Coder.Context.NESTED);
+ public void encode(Accumulator accumulator, OutputStream outStream) throws IOException {
+ StringUtf8Coder.of().encode(accumulator.seed, outStream);
+ StringUtf8Coder.of().encode(accumulator.value, outStream);
}
@Override
- public void encode(Accumulator accumulator, OutputStream outStream, Coder.Context context)
- throws CoderException, IOException {
- StringUtf8Coder.of().encode(accumulator.value, outStream, context);
- }
-
- @Override
- public Accumulator decode(InputStream inStream) throws CoderException, IOException {
- return decode(inStream, Coder.Context.NESTED);
- }
-
- @Override
- public Accumulator decode(InputStream inStream, Coder.Context context)
- throws CoderException, IOException {
- return new Accumulator(StringUtf8Coder.of().decode(inStream, context));
+ public Accumulator decode(InputStream inStream) throws IOException {
+ String seed = StringUtf8Coder.of().decode(inStream);
+ String value = StringUtf8Coder.of().decode(inStream);
+ return new Accumulator(seed, value);
}
};
}
@@ -958,13 +1068,13 @@
@Override
public Accumulator createAccumulator() {
- return new Accumulator("");
+ return new Accumulator("", "");
}
@Override
public Accumulator addInput(Accumulator accumulator, Integer value) {
try {
- return new Accumulator(accumulator.value + String.valueOf(value));
+ return new Accumulator(accumulator.seed, accumulator.value + String.valueOf(value));
} finally {
accumulator.value = "cleared in addInput";
}
@@ -972,12 +1082,22 @@
@Override
public Accumulator mergeAccumulators(Iterable<Accumulator> accumulators) {
- String all = "";
+ Accumulator seedAccumulator = null;
+ StringBuilder all = new StringBuilder();
for (Accumulator accumulator : accumulators) {
- all += accumulator.value;
+ if (seedAccumulator == null) {
+ seedAccumulator = accumulator;
+ } else {
+ assertEquals(
+ String.format(
+ "Different seed values in accumulator: %s vs. %s", seedAccumulator, accumulator),
+ seedAccumulator.seed,
+ accumulator.seed);
+ }
+ all.append(accumulator.value);
accumulator.value = "cleared in mergeAccumulators";
}
- return new Accumulator(all);
+ return new Accumulator(checkNotNull(seedAccumulator).seed, all.toString());
}
@Override
@@ -1007,40 +1127,47 @@
@Override
public TestCombineFn.Accumulator createAccumulator(Context c) {
- return new TestCombineFn.Accumulator(c.sideInput(view).toString());
+ Integer sideInputValue = c.sideInput(view);
+ return new TestCombineFn.Accumulator(sideInputValue.toString(), "");
}
@Override
public TestCombineFn.Accumulator addInput(
TestCombineFn.Accumulator accumulator, Integer value, Context c) {
try {
- assertThat(accumulator.value, Matchers.startsWith(c.sideInput(view).toString()));
- return new TestCombineFn.Accumulator(accumulator.value + String.valueOf(value));
+ assertThat(
+ "Not expecting view contents to change",
+ accumulator.seed,
+ Matchers.equalTo(Integer.toString(c.sideInput(view))));
+ return new TestCombineFn.Accumulator(
+ accumulator.seed, accumulator.value + String.valueOf(value));
} finally {
accumulator.value = "cleared in addInput";
}
-
}
@Override
public TestCombineFn.Accumulator mergeAccumulators(
Iterable<TestCombineFn.Accumulator> accumulators, Context c) {
- String prefix = c.sideInput(view).toString();
- String all = prefix;
+ String sideInputValue = c.sideInput(view).toString();
+ StringBuilder all = new StringBuilder();
for (TestCombineFn.Accumulator accumulator : accumulators) {
- assertThat(accumulator.value, Matchers.startsWith(prefix));
- all += accumulator.value.substring(prefix.length());
+ assertThat(
+ "Accumulators should all have the same Side Input Value",
+ accumulator.seed,
+ Matchers.equalTo(sideInputValue));
+ all.append(accumulator.value);
accumulator.value = "cleared in mergeAccumulators";
}
- return new TestCombineFn.Accumulator(all);
+ return new TestCombineFn.Accumulator(sideInputValue, all.toString());
}
@Override
public String extractOutput(TestCombineFn.Accumulator accumulator, Context c) {
- assertThat(accumulator.value, Matchers.startsWith(c.sideInput(view).toString()));
+ assertThat(accumulator.seed, Matchers.startsWith(c.sideInput(view).toString()));
char[] chars = accumulator.value.toCharArray();
Arrays.sort(chars);
- return new String(chars);
+ return accumulator.seed + ":" + new String(chars);
}
}
@@ -1078,7 +1205,7 @@
@Override
public void mergeAccumulator(Counter accumulator) {
checkState(outputs == 0);
- checkArgument(accumulator.outputs == 0);
+ assertEquals(0, accumulator.outputs);
merges += accumulator.merges + 1;
inputs += accumulator.inputs;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index a05d31c..1c7e1af 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -47,6 +47,9 @@
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.SourceTestUtils;
@@ -55,7 +58,6 @@
import org.apache.beam.sdk.transforms.Create.Values.CreateSource;
import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TimestampedValue;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -307,28 +309,24 @@
@Test
public void testCreateTimestampedDefaultOutputCoderUsingCoder() throws Exception {
Coder<Record> coder = new RecordCoder();
- PBegin pBegin = PBegin.in(p);
Create.TimestampedValues<Record> values =
Create.timestamped(
TimestampedValue.of(new Record(), new Instant(0)),
TimestampedValue.<Record>of(new Record2(), new Instant(0)))
.withCoder(coder);
- Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
- assertThat(defaultCoder, equalTo(coder));
+ assertThat(p.apply(values).getCoder(), equalTo(coder));
}
@Test
public void testCreateTimestampedDefaultOutputCoderUsingTypeDescriptor() throws Exception {
Coder<Record> coder = new RecordCoder();
p.getCoderRegistry().registerCoderForClass(Record.class, coder);
- PBegin pBegin = PBegin.in(p);
Create.TimestampedValues<Record> values =
Create.timestamped(
TimestampedValue.of(new Record(), new Instant(0)),
TimestampedValue.<Record>of(new Record2(), new Instant(0)))
.withType(new TypeDescriptor<Record>() {});
- Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
- assertThat(defaultCoder, equalTo(coder));
+ assertThat(p.apply(values).getCoder(), equalTo(coder));
}
@Test
@@ -353,6 +351,42 @@
p.run();
}
+ /** Testing options for {@link #testCreateOfProvider()}. */
+ public interface CreateOfProviderOptions extends PipelineOptions {
+ ValueProvider<String> getFoo();
+ void setFoo(ValueProvider<String> value);
+ }
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testCreateOfProvider() throws Exception {
+ PAssert.that(
+ p.apply(
+ "Static", Create.ofProvider(StaticValueProvider.of("foo"), StringUtf8Coder.of())))
+ .containsInAnyOrder("foo");
+ PAssert.that(
+ p.apply(
+ "Static nested",
+ Create.ofProvider(
+ NestedValueProvider.of(
+ StaticValueProvider.of("foo"),
+ new SerializableFunction<String, String>() {
+ @Override
+ public String apply(String input) {
+ return input + "bar";
+ }
+ }),
+ StringUtf8Coder.of())))
+ .containsInAnyOrder("foobar");
+ PAssert.that(
+ p.apply(
+ "Runtime", Create.ofProvider(p.newProvider("runtimeFoo"), StringUtf8Coder.of())))
+ .containsInAnyOrder("runtimeFoo");
+
+ p.run();
+ }
+
+
@Test
public void testCreateGetName() {
assertEquals("Create.Values", Create.of(1, 2, 3).getName());
@@ -364,31 +398,25 @@
public void testCreateDefaultOutputCoderUsingInference() throws Exception {
Coder<Record> coder = new RecordCoder();
p.getCoderRegistry().registerCoderForClass(Record.class, coder);
- PBegin pBegin = PBegin.in(p);
- Create.Values<Record> values = Create.of(new Record(), new Record(), new Record());
- Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
- assertThat(defaultCoder, equalTo(coder));
+ assertThat(
+ p.apply(Create.of(new Record(), new Record(), new Record())).getCoder(), equalTo(coder));
}
@Test
public void testCreateDefaultOutputCoderUsingCoder() throws Exception {
Coder<Record> coder = new RecordCoder();
- PBegin pBegin = PBegin.in(p);
- Create.Values<Record> values =
- Create.of(new Record(), new Record2()).withCoder(coder);
- Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
- assertThat(defaultCoder, equalTo(coder));
+ assertThat(
+ p.apply(Create.of(new Record(), new Record2()).withCoder(coder)).getCoder(),
+ equalTo(coder));
}
@Test
public void testCreateDefaultOutputCoderUsingTypeDescriptor() throws Exception {
Coder<Record> coder = new RecordCoder();
p.getCoderRegistry().registerCoderForClass(Record.class, coder);
- PBegin pBegin = PBegin.in(p);
Create.Values<Record> values =
Create.of(new Record(), new Record2()).withType(new TypeDescriptor<Record>() {});
- Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
- assertThat(defaultCoder, equalTo(coder));
+ assertThat(p.apply(values).getCoder(), equalTo(coder));
}
@Test
@@ -434,12 +462,12 @@
}
@Test
- public void testSourceGetDefaultOutputCoderReturnsConstructorCoder() throws Exception {
+ public void testSourceGetOutputCoderReturnsConstructorCoder() throws Exception {
Coder<Integer> coder = VarIntCoder.of();
CreateSource<Integer> source =
CreateSource.fromIterable(ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8), coder);
- Coder<Integer> defaultCoder = source.getDefaultOutputCoder();
+ Coder<Integer> defaultCoder = source.getOutputCoder();
assertThat(defaultCoder, equalTo(coder));
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java
index 17bbed6..b9810c1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java
@@ -24,12 +24,25 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.testing.UsesTestStream;
import org.apache.beam.sdk.testing.ValidatesRunner;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -85,9 +98,9 @@
p.run();
}
- private static class Keys implements SerializableFunction<KV<String, String>, String> {
+ private static class Keys<T> implements SerializableFunction<KV<T, String>, T> {
@Override
- public String apply(KV<String, String> input) {
+ public T apply(KV<T, String> input) {
return input.getKey();
}
}
@@ -118,11 +131,122 @@
PCollection<KV<String, String>> input = p.apply(Create.of(strings));
PCollection<KV<String, String>> output =
- input.apply(Distinct.withRepresentativeValueFn(new Keys()));
+ input.apply(Distinct.withRepresentativeValueFn(new Keys<String>())
+ .withRepresentativeType(TypeDescriptor.of(String.class)));
PAssert.that(output).satisfies(new Checker());
p.run();
}
+
+ @Rule
+ public TestPipeline windowedDistinctPipeline = TestPipeline.create();
+
+ @Test
+ @Category({ValidatesRunner.class, UsesTestStream.class})
+ public void testWindowedDistinct() {
+ Instant base = new Instant(0);
+ TestStream<String> values = TestStream.create(StringUtf8Coder.of())
+ .advanceWatermarkTo(base)
+ .addElements(
+ TimestampedValue.of("k1", base),
+ TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
+ TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))),
+ TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
+ TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
+ TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))),
+ TimestampedValue.of("k4", base.plus(Duration.standardSeconds(60))),
+ TimestampedValue.of("k5", base.plus(Duration.standardSeconds(70))),
+ TimestampedValue.of("k6", base.plus(Duration.standardSeconds(80))))
+ .advanceWatermarkToInfinity();
+
+ PCollection<String> distinctValues = windowedDistinctPipeline
+ .apply(values)
+ .apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(30))))
+ .apply(Distinct.<String>create());
+ PAssert.that(distinctValues)
+ .inWindow(new IntervalWindow(base, base.plus(Duration.standardSeconds(30))))
+ .containsInAnyOrder("k1", "k2", "k3");
+ PAssert.that(distinctValues)
+ .inWindow(new IntervalWindow(base.plus(
+ Duration.standardSeconds(30)), base.plus(Duration.standardSeconds(60))))
+ .containsInAnyOrder("k1", "k2", "k3");
+ PAssert.that(distinctValues)
+ .inWindow(new IntervalWindow(base.plus(
+ Duration.standardSeconds(60)), base.plus(Duration.standardSeconds(90))))
+ .containsInAnyOrder("k4", "k5", "k6");
+ windowedDistinctPipeline.run();
+ }
+
+ @Rule
+ public TestPipeline triggeredDistinctPipeline = TestPipeline.create();
+
+ @Test
+ @Category({ValidatesRunner.class, UsesTestStream.class})
+ public void testTriggeredDistinct() {
+ Instant base = new Instant(0);
+ TestStream<String> values = TestStream.create(StringUtf8Coder.of())
+ .advanceWatermarkTo(base)
+ .addElements(
+ TimestampedValue.of("k1", base),
+ TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
+ TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))))
+ .advanceProcessingTime(Duration.standardMinutes(1))
+ .addElements(
+ TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
+ TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
+ TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))))
+ .advanceWatermarkToInfinity();
+
+ PCollection<String> distinctValues = triggeredDistinctPipeline
+ .apply(values)
+ .apply(Window.<String>into(FixedWindows.of(Duration.standardMinutes(1)))
+ .triggering(Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(
+ Duration.standardSeconds(30))))
+ .withAllowedLateness(Duration.ZERO)
+ .accumulatingFiredPanes())
+ .apply(Distinct.<String>create());
+ PAssert.that(distinctValues).containsInAnyOrder("k1", "k2", "k3");
+ triggeredDistinctPipeline.run();
+ }
+
+ @Rule
+ public TestPipeline triggeredDistinctRepresentativePipeline = TestPipeline.create();
+
+ @Test
+ @Category({ValidatesRunner.class, UsesTestStream.class})
+ public void testTriggeredDistinctRepresentativeValues() {
+ Instant base = new Instant(0);
+ TestStream<KV<Integer, String>> values = TestStream.create(
+ KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of()))
+ .advanceWatermarkTo(base)
+ .addElements(
+ TimestampedValue.of(KV.of(1, "k1"), base),
+ TimestampedValue.of(KV.of(2, "k2"), base.plus(Duration.standardSeconds(10))),
+ TimestampedValue.of(KV.of(3, "k3"), base.plus(Duration.standardSeconds(20))))
+ .advanceProcessingTime(Duration.standardMinutes(1))
+ .addElements(
+ TimestampedValue.of(KV.of(1, "k1"), base.plus(Duration.standardSeconds(30))),
+ TimestampedValue.of(KV.of(2, "k2"), base.plus(Duration.standardSeconds(40))),
+ TimestampedValue.of(KV.of(3, "k3"), base.plus(Duration.standardSeconds(50))))
+ .advanceWatermarkToInfinity();
+
+ PCollection<KV<Integer, String>> distinctValues = triggeredDistinctRepresentativePipeline
+ .apply(values)
+ .apply(Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(1)))
+ .triggering(Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane().plusDelayOf(
+ Duration.standardSeconds(30))))
+ .withAllowedLateness(Duration.ZERO)
+ .accumulatingFiredPanes())
+ .apply(Distinct.withRepresentativeValueFn(new Keys<Integer>())
+ .withRepresentativeType(TypeDescriptor.of(Integer.class)));
+
+
+ PAssert.that(distinctValues).containsInAnyOrder(
+ KV.of(1, "k1"), KV.of(2, "k2"), KV.of(3, "k3"));
+ triggeredDistinctRepresentativePipeline.run();
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index 1bb71bb..5cb9e18 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -360,6 +360,38 @@
}
}
+ @Test
+ public void testSupportsFinishBundleOutput() throws Exception {
+ for (DoFnTester.CloningBehavior cloning : DoFnTester.CloningBehavior.values()) {
+ try (DoFnTester<Integer, Integer> tester = DoFnTester.of(new BundleCounterDoFn())) {
+ tester.setCloningBehavior(cloning);
+
+ assertThat(tester.processBundle(1, 2, 3, 4), contains(4));
+ assertThat(tester.processBundle(5, 6, 7), contains(3));
+ assertThat(tester.processBundle(8, 9), contains(2));
+ }
+ }
+ }
+
+ private static class BundleCounterDoFn extends DoFn<Integer, Integer> {
+ private int elements;
+
+ @StartBundle
+ public void startBundle() {
+ elements = 0;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ elements++;
+ }
+
+ @FinishBundle
+ public void finishBundle(FinishBundleContext c) {
+ c.output(elements, Instant.now(), GlobalWindow.INSTANCE);
+ }
+ }
+
private static class SideInputDoFn extends DoFn<Integer, Integer> {
private final PCollectionView<Integer> value;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
index 11f284f..68ceafb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
@@ -17,7 +17,10 @@
*/
package org.apache.beam.sdk.transforms;
+import static org.apache.beam.sdk.transforms.Contextful.fn;
+import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.values.TypeDescriptors.integers;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
@@ -30,9 +33,11 @@
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Contextful.Fn;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.junit.Rule;
import org.junit.Test;
@@ -77,6 +82,32 @@
}
/**
+ * Basic test of {@link FlatMapElements} with a {@link Fn} and a side input.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFlatMapBasicWithSideInput() throws Exception {
+ final PCollectionView<Integer> view =
+ pipeline.apply("Create base", Create.of(40)).apply(View.<Integer>asSingleton());
+ PCollection<Integer> output =
+ pipeline
+ .apply(Create.of(0, 1, 2))
+ .apply(
+ FlatMapElements.into(integers()).via(fn(
+ new Fn<Integer, Iterable<Integer>>() {
+ @Override
+ public List<Integer> apply(Integer input, Context c) {
+ return ImmutableList.of(
+ c.sideInput(view) - input, c.sideInput(view) + input);
+ }
+ },
+ requiresSideInputs(view))));
+
+ PAssert.that(output).containsInAnyOrder(38, 39, 40, 40, 41, 42);
+ pipeline.run();
+ }
+
+ /**
* Tests that when built with a concrete subclass of {@link SimpleFunction}, the type descriptor
* of the output reflects its static type.
*/
@@ -144,7 +175,7 @@
};
FlatMapElements<?, ?> simpleMap = FlatMapElements.via(simpleFn);
- assertThat(DisplayData.from(simpleMap), hasDisplayItem("flatMapFn", simpleFn.getClass()));
+ assertThat(DisplayData.from(simpleMap), hasDisplayItem("class", simpleFn.getClass()));
}
@Test
@@ -162,7 +193,7 @@
};
FlatMapElements<?, ?> simpleFlatMap = FlatMapElements.via(simpleFn);
- assertThat(DisplayData.from(simpleFlatMap), hasDisplayItem("flatMapFn", simpleFn.getClass()));
+ assertThat(DisplayData.from(simpleFlatMap), hasDisplayItem("class", simpleFn.getClass()));
assertThat(DisplayData.from(simpleFlatMap), hasDisplayItem("foo", "baz"));
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java
index a8cb843..0a21716 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java
@@ -56,7 +56,10 @@
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.joda.time.Duration;
import org.junit.Assert;
import org.junit.Rule;
@@ -228,7 +231,7 @@
public void testFlattenNoListsNoCoder() {
// not ValidatesRunner because it should fail at pipeline construction time anyhow.
thrown.expect(IllegalStateException.class);
- thrown.expectMessage("cannot provide a Coder for empty");
+ thrown.expectMessage("Unable to return a default Coder");
PCollectionList.<ClassWithoutCoder>empty(p)
.apply(Flatten.<ClassWithoutCoder>pCollections());
@@ -314,6 +317,38 @@
p.run();
}
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testFlattenMultiplePCollectionsHavingMultipleConsumers() {
+ PCollection<String> input = p.apply(Create.of("AA", "BBB", "CC"));
+ final TupleTag<String> outputEvenLengthTag = new TupleTag<String>() {};
+ final TupleTag<String> outputOddLengthTag = new TupleTag<String>() {};
+
+ PCollectionTuple tuple = input.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (c.element().length() % 2 == 0) {
+ c.output(c.element());
+ } else {
+ c.output(outputOddLengthTag, c.element());
+ }
+ }
+ }).withOutputTags(outputEvenLengthTag, TupleTagList.of(outputOddLengthTag)));
+
+ PCollection<String> outputEvenLength = tuple.get(outputEvenLengthTag);
+ PCollection<String> outputOddLength = tuple.get(outputOddLengthTag);
+
+ PCollection<String> outputMerged = PCollectionList.of(outputEvenLength)
+ .and(outputOddLength)
+ .apply(Flatten.<String>pCollections());
+
+ PAssert.that(outputMerged).containsInAnyOrder("AA", "BBB", "CC");
+ PAssert.that(outputEvenLength).containsInAnyOrder("AA", "CC");
+ PAssert.that(outputOddLength).containsInAnyOrder("BBB");
+
+ p.run();
+ }
+
/////////////////////////////////////////////////////////////////////////////
@Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 0cd885c..a76714c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -23,18 +23,20 @@
import static org.hamcrest.CoreMatchers.hasItem;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
@@ -45,22 +47,31 @@
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.MapCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.testing.LargeKeys;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.testing.UsesTestStream;
import org.apache.beam.sdk.testing.ValidatesRunner;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TimestampedValue;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matcher;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Assert;
@@ -76,13 +87,13 @@
*/
@RunWith(JUnit4.class)
@SuppressWarnings({"rawtypes", "unchecked"})
-public class GroupByKeyTest {
+public class GroupByKeyTest implements Serializable {
@Rule
- public final TestPipeline p = TestPipeline.create();
+ public transient TestPipeline p = TestPipeline.create();
@Rule
- public ExpectedException thrown = ExpectedException.none();
+ public transient ExpectedException thrown = ExpectedException.none();
@Test
@Category(ValidatesRunner.class)
@@ -103,27 +114,18 @@
PCollection<KV<String, Iterable<Integer>>> output =
input.apply(GroupByKey.<String, Integer>create());
- PAssert.that(output)
- .satisfies(new AssertThatHasExpectedContentsForTestGroupByKey());
+ SerializableFunction<Iterable<KV<String, Iterable<Integer>>>, Void> checker =
+ containsKvs(
+ kv("k1", 3, 4),
+ kv("k5", Integer.MIN_VALUE, Integer.MAX_VALUE),
+ kv("k2", 66, -33),
+ kv("k3", 0));
+ PAssert.that(output).satisfies(checker);
+ PAssert.that(output).inWindow(GlobalWindow.INSTANCE).satisfies(checker);
p.run();
}
- static class AssertThatHasExpectedContentsForTestGroupByKey
- implements SerializableFunction<Iterable<KV<String, Iterable<Integer>>>,
- Void> {
- @Override
- public Void apply(Iterable<KV<String, Iterable<Integer>>> actual) {
- assertThat(actual, containsInAnyOrder(
- isKv(is("k1"), containsInAnyOrder(3, 4)),
- isKv(is("k5"), containsInAnyOrder(Integer.MAX_VALUE,
- Integer.MIN_VALUE)),
- isKv(is("k2"), containsInAnyOrder(66, -33)),
- isKv(is("k3"), containsInAnyOrder(0))));
- return null;
- }
- }
-
@Test
@Category(ValidatesRunner.class)
public void testGroupByKeyAndWindows() {
@@ -144,24 +146,115 @@
.apply(GroupByKey.<String, Integer>create());
PAssert.that(output)
- .satisfies(new AssertThatHasExpectedContentsForTestGroupByKeyAndWindows());
+ .satisfies(
+ containsKvs(
+ kv("k1", 3),
+ kv("k1", 4),
+ kv("k5", Integer.MAX_VALUE, Integer.MIN_VALUE),
+ kv("k2", 66),
+ kv("k2", -33),
+ kv("k3", 0)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(0L), Duration.millis(5L)))
+ .satisfies(
+ containsKvs(kv("k1", 3), kv("k5", Integer.MIN_VALUE, Integer.MAX_VALUE), kv("k2", 66)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(5L), Duration.millis(5L)))
+ .satisfies(containsKvs(kv("k1", 4), kv("k2", -33), kv("k3", 0)));
p.run();
}
- static class AssertThatHasExpectedContentsForTestGroupByKeyAndWindows
- implements SerializableFunction<Iterable<KV<String, Iterable<Integer>>>,
- Void> {
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testGroupByKeyMultipleWindows() {
+ PCollection<KV<String, Integer>> windowedInput =
+ p.apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("foo", 1), new Instant(1)),
+ TimestampedValue.of(KV.of("foo", 4), new Instant(4)),
+ TimestampedValue.of(KV.of("bar", 3), new Instant(3))))
+ .apply(
+ Window.<KV<String, Integer>>into(
+ SlidingWindows.of(Duration.millis(5L)).every(Duration.millis(3L))));
+
+ PCollection<KV<String, Iterable<Integer>>> output =
+ windowedInput.apply(GroupByKey.<String, Integer>create());
+
+ PAssert.that(output)
+ .satisfies(
+ containsKvs(kv("foo", 1, 4), kv("foo", 1), kv("foo", 4), kv("bar", 3), kv("bar", 3)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(-3L), Duration.millis(5L)))
+ .satisfies(containsKvs(kv("foo", 1)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(0L), Duration.millis(5L)))
+ .satisfies(containsKvs(kv("foo", 1, 4), kv("bar", 3)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(3L), Duration.millis(5L)))
+ .satisfies(containsKvs(kv("foo", 4), kv("bar", 3)));
+
+ p.run();
+ }
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testGroupByKeyMergingWindows() {
+ PCollection<KV<String, Integer>> windowedInput =
+ p.apply(
+ Create.timestamped(
+ TimestampedValue.of(KV.of("foo", 1), new Instant(1)),
+ TimestampedValue.of(KV.of("foo", 4), new Instant(4)),
+ TimestampedValue.of(KV.of("bar", 3), new Instant(3)),
+ TimestampedValue.of(KV.of("foo", 9), new Instant(9))))
+ .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(Duration.millis(4L))));
+
+ PCollection<KV<String, Iterable<Integer>>> output =
+ windowedInput.apply(GroupByKey.<String, Integer>create());
+
+ PAssert.that(output).satisfies(containsKvs(kv("foo", 1, 4), kv("foo", 9), kv("bar", 3)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(1L), new Instant(8L)))
+ .satisfies(containsKvs(kv("foo", 1, 4)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(3L), new Instant(7L)))
+ .satisfies(containsKvs(kv("bar", 3)));
+ PAssert.that(output)
+ .inWindow(new IntervalWindow(new Instant(9L), new Instant(13L)))
+ .satisfies(containsKvs(kv("foo", 9)));
+
+ p.run();
+ }
+
+ private static KV<String, Collection<Integer>> kv(String key, Integer... values) {
+ return KV.<String, Collection<Integer>>of(key, ImmutableList.copyOf(values));
+ }
+
+ private static SerializableFunction<Iterable<KV<String, Iterable<Integer>>>, Void> containsKvs(
+ KV<String, Collection<Integer>>... kvs) {
+ return new ContainsKVs(ImmutableList.copyOf(kvs));
+ }
+
+ /**
+ * A function that asserts that the input element contains the expected {@link KV KVs} in any
+ * order, where values appear in any order.
+ */
+ private static class ContainsKVs
+ implements SerializableFunction<Iterable<KV<String, Iterable<Integer>>>, Void> {
+ private final List<KV<String, Collection<Integer>>> expectedKvs;
+
+ private ContainsKVs(List<KV<String, Collection<Integer>>> expectedKvs) {
+ this.expectedKvs = expectedKvs;
+ }
+
@Override
- public Void apply(Iterable<KV<String, Iterable<Integer>>> actual) {
- assertThat(actual, containsInAnyOrder(
- isKv(is("k1"), containsInAnyOrder(3)),
- isKv(is("k1"), containsInAnyOrder(4)),
- isKv(is("k5"), containsInAnyOrder(Integer.MAX_VALUE,
- Integer.MIN_VALUE)),
- isKv(is("k2"), containsInAnyOrder(66)),
- isKv(is("k2"), containsInAnyOrder(-33)),
- isKv(is("k3"), containsInAnyOrder(0))));
+ public Void apply(Iterable<KV<String, Iterable<Integer>>> input) {
+ List<Matcher<? super KV<String, Iterable<Integer>>>> matchers = new ArrayList<>();
+ for (KV<String, Collection<Integer>> expected : expectedKvs) {
+ Integer[] values = expected.getValue().toArray(new Integer[0]);
+ matchers.add(isKv(equalTo(expected.getKey()), containsInAnyOrder(values)));
+ }
+ assertThat(input, containsInAnyOrder(matchers.toArray(new Matcher[0])));
return null;
}
}
@@ -183,6 +276,40 @@
p.run();
}
+ /**
+ * Tests that when a processing time timers comes in after a window is expired it does not cause a
+ * spurious output.
+ */
+ @Test
+ @Category({ValidatesRunner.class, UsesTestStream.class})
+ public void testCombiningAccumulatingProcessingTime() throws Exception {
+ PCollection<Integer> triggeredSums =
+ p.apply(
+ TestStream.create(VarIntCoder.of())
+ .advanceWatermarkTo(new Instant(0))
+ .addElements(
+ TimestampedValue.of(2, new Instant(2)),
+ TimestampedValue.of(5, new Instant(5)))
+ .advanceWatermarkTo(new Instant(100))
+ .advanceProcessingTime(Duration.millis(10))
+ .advanceWatermarkToInfinity())
+ .apply(
+ Window.<Integer>into(FixedWindows.of(Duration.millis(100)))
+ .withTimestampCombiner(TimestampCombiner.EARLIEST)
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.ZERO)
+ .triggering(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(Duration.millis(10)))))
+ .apply(Sum.integersGlobally().withoutDefaults());
+
+ PAssert.that(triggeredSums)
+ .containsInAnyOrder(7);
+
+ p.run();
+ }
+
@Test
public void testGroupByKeyNonDeterministic() throws Exception {
@@ -295,11 +422,11 @@
new PTransform<PBegin, PCollection<KV<String, Integer>>>() {
@Override
public PCollection<KV<String, Integer>> expand(PBegin input) {
- return PCollection.<KV<String, Integer>>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.UNBOUNDED)
- .setTypeDescriptor(new TypeDescriptor<KV<String, Integer>>() {});
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
}
});
@@ -427,6 +554,79 @@
p.run();
}
+ private static String bigString(char c, int size) {
+ char[] buf = new char[size];
+ for (int i = 0; i < size; i++) {
+ buf[i] = c;
+ }
+ return new String(buf);
+ }
+
+ private static void runLargeKeysTest(TestPipeline p, final int keySize) throws Exception {
+ PCollection<KV<String, Integer>> result = p
+ .apply(Create.of("a", "a", "b"))
+ .apply("Expand", ParDo.of(new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ c.output(KV.of(bigString(c.element().charAt(0), keySize), c.element()));
+ }
+ }))
+ .apply(GroupByKey.<String, String>create())
+ .apply("Count", ParDo.of(new DoFn<KV<String, Iterable<String>>, KV<String, Integer>>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ int size = 0;
+ for (String value : c.element().getValue()) {
+ size++;
+ }
+ c.output(KV.of(c.element().getKey(), size));
+ }
+ }));
+
+ PAssert.that(result).satisfies(
+ new SerializableFunction<Iterable<KV<String, Integer>>, Void>() {
+ @Override
+ public Void apply(Iterable<KV<String, Integer>> values) {
+ assertThat(values,
+ containsInAnyOrder(
+ KV.of(bigString('a', keySize), 2), KV.of(bigString('b', keySize), 1)));
+ return null;
+ }
+ });
+
+ p.run();
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, LargeKeys.Above10KB.class})
+ public void testLargeKeys10KB() throws Exception {
+ runLargeKeysTest(p, 10 << 10);
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, LargeKeys.Above100KB.class})
+ public void testLargeKeys100KB() throws Exception {
+ runLargeKeysTest(p, 100 << 10);
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, LargeKeys.Above1MB.class})
+ public void testLargeKeys1MB() throws Exception {
+ runLargeKeysTest(p, 1 << 20);
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, LargeKeys.Above10MB.class})
+ public void testLargeKeys10MB() throws Exception {
+ runLargeKeysTest(p, 10 << 20);
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, LargeKeys.Above100MB.class})
+ public void testLargeKeys100MB() throws Exception {
+ runLargeKeysTest(p, 100 << 20);
+ }
+
/**
* This is a bogus key class that returns random hash values from {@link #hashCode()} and always
* returns {@code false} for {@link #equals(Object)}. The results of the test are correct if
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java
index 54e2d5a..c213d6a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java
@@ -56,7 +56,7 @@
private static final int BATCH_SIZE = 5;
private static final long NUM_ELEMENTS = 10;
private static final int ALLOWED_LATENESS = 0;
- private static final Logger LOGGER = LoggerFactory.getLogger(GroupIntoBatchesTest.class);
+ private static final Logger LOG = LoggerFactory.getLogger(GroupIntoBatchesTest.class);
@Rule public transient TestPipeline pipeline = TestPipeline.create();
private transient ArrayList<KV<String, String>> data = createTestData();
@@ -159,7 +159,7 @@
new DoFn<KV<String, String>, Void>() {
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) {
- LOGGER.debug(
+ LOG.debug(
"*** ELEMENT: ({},{}) *** with timestamp %s in window %s",
c.element().getKey(),
c.element().getValue(),
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
index 241b60e..2c24f10 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
@@ -17,7 +17,10 @@
*/
package org.apache.beam.sdk.transforms;
+import static org.apache.beam.sdk.transforms.Contextful.fn;
+import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.values.TypeDescriptors.integers;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.junit.Assert.assertThat;
@@ -28,12 +31,13 @@
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.ValidatesRunner;
+import org.apache.beam.sdk.transforms.Contextful.Fn;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -96,6 +100,30 @@
}
/**
+ * Basic test of {@link MapElements} with a {@link Fn} and a side input.
+ */
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMapBasicWithSideInput() throws Exception {
+ final PCollectionView<Integer> view =
+ pipeline.apply("Create base", Create.of(40)).apply(View.<Integer>asSingleton());
+ PCollection<Integer> output =
+ pipeline
+ .apply(Create.of(0, 1, 2))
+ .apply(MapElements.into(integers())
+ .via(fn(new Fn<Integer, Integer>() {
+ @Override
+ public Integer apply(Integer element, Context c) {
+ return element + c.sideInput(view);
+ }
+ },
+ requiresSideInputs(view))));
+
+ PAssert.that(output).containsInAnyOrder(40, 41, 42);
+ pipeline.run();
+ }
+
+ /**
* Basic test of {@link MapElements} coder propagation with a parametric {@link SimpleFunction}.
*/
@Test
@@ -157,7 +185,7 @@
pipeline
.apply(Create.of(1, 2, 3))
.apply(
- MapElements.into(TypeDescriptors.integers())
+ MapElements.into(integers())
.via(
new SerializableFunction<Integer, Integer>() {
@Override
@@ -216,9 +244,9 @@
};
MapElements<?, ?> serializableMap =
- MapElements.into(TypeDescriptors.integers()).via(serializableFn);
+ MapElements.into(integers()).via(serializableFn);
assertThat(DisplayData.from(serializableMap),
- hasDisplayItem("mapFn", serializableFn.getClass()));
+ hasDisplayItem("class", serializableFn.getClass()));
}
@Test
@@ -231,7 +259,7 @@
};
MapElements<?, ?> simpleMap = MapElements.via(simpleFn);
- assertThat(DisplayData.from(simpleMap), hasDisplayItem("mapFn", simpleFn.getClass()));
+ assertThat(DisplayData.from(simpleMap), hasDisplayItem("class", simpleFn.getClass()));
}
@Test
public void testSimpleFunctionDisplayData() {
@@ -250,7 +278,7 @@
MapElements<?, ?> simpleMap = MapElements.via(simpleFn);
- assertThat(DisplayData.from(simpleMap), hasDisplayItem("mapFn", simpleFn.getClass()));
+ assertThat(DisplayData.from(simpleMap), hasDisplayItem("class", simpleFn.getClass()));
assertThat(DisplayData.from(simpleMap), hasDisplayItem("foo", "baz"));
}
@@ -269,7 +297,7 @@
Set<DisplayData> displayData = evaluator.<Integer>displayDataForPrimitiveTransforms(map);
assertThat("MapElements should include the mapFn in its primitive display data",
- displayData, hasItem(hasDisplayItem("mapFn", mapFn.getClass())));
+ displayData, hasItem(hasDisplayItem("class", mapFn.getClass())));
}
static class VoidValues<K, V>
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
index 2b43560..a298a5e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.sdk.transforms;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
@@ -45,7 +45,7 @@
@Test
public void testMaxIntegerFn() {
- checkCombineFn(
+ testCombineFn(
Max.ofIntegers(),
Lists.newArrayList(1, 2, 3, 4),
4);
@@ -53,7 +53,7 @@
@Test
public void testMaxLongFn() {
- checkCombineFn(
+ testCombineFn(
Max.ofLongs(),
Lists.newArrayList(1L, 2L, 3L, 4L),
4L);
@@ -61,7 +61,7 @@
@Test
public void testMaxDoubleFn() {
- checkCombineFn(
+ testCombineFn(
Max.ofDoubles(),
Lists.newArrayList(1.0, 2.0, 3.0, 4.0),
4.0);
@@ -69,7 +69,7 @@
@Test
public void testDisplayData() {
- Top.Largest<Integer> comparer = new Top.Largest<>();
+ Top.Natural<Integer> comparer = new Top.Natural<>();
Combine.Globally<Integer, Integer> max = Max.globally(comparer);
assertThat(DisplayData.from(max), hasDisplayItem("comparer", comparer.getClass()));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java
index 79ebc25..e138135 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.sdk.transforms;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.junit.Assert.assertEquals;
import com.google.common.collect.Lists;
@@ -64,7 +64,7 @@
@Test
public void testMeanFn() throws Exception {
- checkCombineFn(
+ testCombineFn(
Mean.<Integer>of(),
Lists.newArrayList(1, 2, 3, 4),
2.5);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
index e89b223..a515b63 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
@@ -18,7 +18,7 @@
package org.apache.beam.sdk.transforms;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
@@ -45,7 +45,7 @@
}
@Test
public void testMinIntegerFn() {
- checkCombineFn(
+ testCombineFn(
Min.ofIntegers(),
Lists.newArrayList(1, 2, 3, 4),
1);
@@ -53,7 +53,7 @@
@Test
public void testMinLongFn() {
- checkCombineFn(
+ testCombineFn(
Min.ofLongs(),
Lists.newArrayList(1L, 2L, 3L, 4L),
1L);
@@ -61,7 +61,7 @@
@Test
public void testMinDoubleFn() {
- checkCombineFn(
+ testCombineFn(
Min.ofDoubles(),
Lists.newArrayList(1.0, 2.0, 3.0, 4.0),
1.0);
@@ -69,7 +69,7 @@
@Test
public void testDisplayData() {
- Top.Smallest<Integer> comparer = new Top.Smallest<>();
+ Top.Reversed<Integer> comparer = new Top.Reversed<>();
Combine.Globally<Integer, Integer> min = Min.globally(comparer);
assertThat(DisplayData.from(min), hasDisplayItem("comparer", comparer.getClass()));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 35c02ba..03e3104 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -32,9 +32,9 @@
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -53,6 +53,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
@@ -63,9 +64,12 @@
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
import org.apache.beam.sdk.state.SetState;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
@@ -515,58 +519,18 @@
@Test
@Category(NeedsRunner.class)
public void testParDoWritingToUndeclaredTag() {
-
List<Integer> inputs = Arrays.asList(3, -42, 666);
TupleTag<String> notOutputTag = new TupleTag<String>("additional"){};
- PCollection<String> output = pipeline
+ pipeline
.apply(Create.of(inputs))
.apply(ParDo.of(new TestDoFn(
Arrays.<PCollectionView<Integer>>asList(),
- Arrays.asList(notOutputTag))));
+ Arrays.asList(notOutputTag)))
+ /* No call to .withOutputTags - should cause error */);
- PAssert.that(output)
- .satisfies(ParDoTest.HasExpectedOutput.forInput(inputs));
-
- pipeline.run();
- }
-
- @Test
- // TODO: The exception thrown is runner-specific, even if the behavior is general
- @Category(NeedsRunner.class)
- public void testParDoUndeclaredTagLimit() {
-
- PCollection<Integer> input = pipeline.apply(Create.of(Arrays.asList(3)));
-
- // Success for a total of 1000 outputs.
- input
- .apply("Success1000", ParDo.of(new DoFn<Integer, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) {
- TupleTag<String> specialOutputTag = new TupleTag<String>(){};
- c.output(specialOutputTag, "special");
- c.output(specialOutputTag, "special");
- c.output(specialOutputTag, "special");
-
- for (int i = 0; i < 998; i++) {
- c.output(new TupleTag<String>(){}, "tag" + i);
- }
- }}));
- pipeline.run();
-
- // Failure for a total of 1001 outputs.
- input
- .apply("Failure1001", ParDo.of(new DoFn<Integer, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) {
- for (int i = 0; i < 1000; i++) {
- c.output(new TupleTag<String>(){}, "output" + i);
- }
- }}));
-
- thrown.expect(RuntimeException.class);
- thrown.expectMessage("the number of outputs has exceeded a limit");
+ thrown.expectMessage("additional");
pipeline.run();
}
@@ -1107,43 +1071,32 @@
private final List<Integer> inputs;
private final List<Integer> sideInputs;
private final String additionalOutput;
- private final boolean ordered;
public static HasExpectedOutput forInput(List<Integer> inputs) {
return new HasExpectedOutput(
new ArrayList<Integer>(inputs),
new ArrayList<Integer>(),
- "",
- false);
+ "");
}
private HasExpectedOutput(List<Integer> inputs,
List<Integer> sideInputs,
- String additionalOutput,
- boolean ordered) {
+ String additionalOutput) {
this.inputs = inputs;
this.sideInputs = sideInputs;
this.additionalOutput = additionalOutput;
- this.ordered = ordered;
}
public HasExpectedOutput andSideInputs(Integer... sideInputValues) {
- List<Integer> sideInputs = new ArrayList<>();
- for (Integer sideInputValue : sideInputValues) {
- sideInputs.add(sideInputValue);
- }
- return new HasExpectedOutput(inputs, sideInputs, additionalOutput, ordered);
+ return new HasExpectedOutput(
+ inputs, Arrays.asList(sideInputValues), additionalOutput);
}
public HasExpectedOutput fromOutput(TupleTag<String> outputTag) {
return fromOutput(outputTag.getId());
}
public HasExpectedOutput fromOutput(String outputId) {
- return new HasExpectedOutput(inputs, sideInputs, outputId, ordered);
- }
-
- public HasExpectedOutput inOrder() {
- return new HasExpectedOutput(inputs, sideInputs, additionalOutput, true);
+ return new HasExpectedOutput(inputs, sideInputs, outputId);
}
@Override
@@ -1179,11 +1132,7 @@
}
String[] expectedProcessedsArray =
expectedProcesseds.toArray(new String[expectedProcesseds.size()]);
- if (!ordered || expectedProcesseds.isEmpty()) {
- assertThat(processeds, containsInAnyOrder(expectedProcessedsArray));
- } else {
- assertThat(processeds, contains(expectedProcessedsArray));
- }
+ assertThat(processeds, containsInAnyOrder(expectedProcessedsArray));
for (String finished : finisheds) {
assertEquals(additionalOutputPrefix + "finished", finished);
@@ -1647,6 +1596,108 @@
}
@Test
+ public void testStateNotKeyed() {
+ final String stateId = "foo";
+
+ DoFn<String, Integer> fn =
+ new DoFn<String, Integer>() {
+
+ @StateId(stateId)
+ private final StateSpec<ValueState<Integer>> intState =
+ StateSpecs.value();
+
+ @ProcessElement
+ public void processElement(
+ ProcessContext c, @StateId(stateId) ValueState<Integer> state) {}
+ };
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("state");
+ thrown.expectMessage("KvCoder");
+
+ pipeline.apply(Create.of("hello", "goodbye", "hello again")).apply(ParDo.of(fn));
+ }
+
+ @Test
+ public void testStateNotDeterministic() {
+ final String stateId = "foo";
+
+ // DoubleCoder is not deterministic, so this should crash
+ DoFn<KV<Double, String>, Integer> fn =
+ new DoFn<KV<Double, String>, Integer>() {
+
+ @StateId(stateId)
+ private final StateSpec<ValueState<Integer>> intState =
+ StateSpecs.value();
+
+ @ProcessElement
+ public void processElement(
+ ProcessContext c, @StateId(stateId) ValueState<Integer> state) {}
+ };
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("state");
+ thrown.expectMessage("deterministic");
+
+ pipeline
+ .apply(Create.of(KV.of(1.0, "hello"), KV.of(5.4, "goodbye"), KV.of(7.2, "hello again")))
+ .apply(ParDo.of(fn));
+ }
+
+ @Test
+ public void testTimerNotKeyed() {
+ final String timerId = "foo";
+
+ DoFn<String, Integer> fn =
+ new DoFn<String, Integer>() {
+
+ @TimerId(timerId)
+ private final TimerSpec timer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void processElement(
+ ProcessContext c, @TimerId(timerId) Timer timer) {}
+
+ @OnTimer(timerId)
+ public void onTimer() {}
+ };
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("timer");
+ thrown.expectMessage("KvCoder");
+
+ pipeline.apply(Create.of("hello", "goodbye", "hello again")).apply(ParDo.of(fn));
+ }
+
+ @Test
+ public void testTimerNotDeterministic() {
+ final String timerId = "foo";
+
+ // DoubleCoder is not deterministic, so this should crash
+ DoFn<KV<Double, String>, Integer> fn =
+ new DoFn<KV<Double, String>, Integer>() {
+
+ @TimerId(timerId)
+ private final TimerSpec timer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void processElement(
+ ProcessContext c, @TimerId(timerId) Timer timer) {}
+
+ @OnTimer(timerId)
+ public void onTimer() {}
+ };
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("timer");
+ thrown.expectMessage("deterministic");
+
+ pipeline
+ .apply(Create.of(KV.of(1.0, "hello"), KV.of(5.4, "goodbye"), KV.of(7.2, "hello again")))
+ .apply(ParDo.of(fn));
+ }
+
+ @Test
@Category({ValidatesRunner.class, UsesStatefulParDo.class})
public void testValueStateCoderInference() {
final String stateId = "foo";
@@ -1935,9 +1986,16 @@
@ProcessElement
public void processElement(
ProcessContext c, @StateId(stateId) BagState<Integer> state) {
- Iterable<Integer> currentValue = state.read();
+ ReadableState<Boolean> isEmpty = state.isEmpty();
state.add(c.element().getValue());
- if (Iterables.size(state.read()) >= 4) {
+ assertFalse(isEmpty.read());
+ Iterable<Integer> currentValue = state.read();
+ if (Iterables.size(currentValue) >= 4) {
+ // Make sure that the cached Iterable doesn't change when new elements are added.
+ state.add(-1);
+ assertEquals(4, Iterables.size(currentValue));
+ assertEquals(5, Iterables.size(state.read()));
+
List<Integer> sorted = Lists.newArrayList(currentValue);
Collections.sort(sorted);
c.output(sorted);
@@ -1972,9 +2030,9 @@
@ProcessElement
public void processElement(
ProcessContext c, @StateId(stateId) BagState<MyInteger> state) {
- Iterable<MyInteger> currentValue = state.read();
state.add(new MyInteger(c.element().getValue()));
- if (Iterables.size(state.read()) >= 4) {
+ Iterable<MyInteger> currentValue = state.read();
+ if (Iterables.size(currentValue) >= 4) {
List<MyInteger> sorted = Lists.newArrayList(currentValue);
Collections.sort(sorted);
c.output(sorted);
@@ -2010,9 +2068,9 @@
@ProcessElement
public void processElement(
ProcessContext c, @StateId(stateId) BagState<MyInteger> state) {
- Iterable<MyInteger> currentValue = state.read();
state.add(new MyInteger(c.element().getValue()));
- if (Iterables.size(state.read()) >= 4) {
+ Iterable<MyInteger> currentValue = state.read();
+ if (Iterables.size(currentValue) >= 4) {
List<MyInteger> sorted = Lists.newArrayList(currentValue);
Collections.sort(sorted);
c.output(sorted);
@@ -2054,10 +2112,18 @@
@StateId(stateId) SetState<Integer> state,
@StateId(countStateId) CombiningState<Integer, int[], Integer>
count) {
+ ReadableState<Boolean> isEmpty = state.isEmpty();
state.add(c.element().getValue());
+ assertFalse(isEmpty.read());
count.add(1);
if (count.read() >= 4) {
- Set<Integer> set = Sets.newHashSet(state.read());
+ // Make sure that the cached Iterable doesn't change when new elements are added.
+ Iterable<Integer> ints = state.read();
+ state.add(-1);
+ assertEquals(3, Iterables.size(ints));
+ assertEquals(4, Iterables.size(state.read()));
+
+ Set<Integer> set = Sets.newHashSet(ints);
c.output(set);
}
}
@@ -2183,10 +2249,18 @@
@StateId(countStateId) CombiningState<Integer, int[], Integer>
count) {
KV<String, Integer> value = c.element().getValue();
+ ReadableState<Iterable<Entry<String, Integer>>> entriesView = state.entries();
state.put(value.getKey(), value.getValue());
count.add(1);
if (count.read() >= 4) {
Iterable<Map.Entry<String, Integer>> iterate = state.entries().read();
+ // Make sure that the cached Iterable doesn't change when new elements are added, but
+ // that cached ReadableState views of the state do change.
+ state.put("BadKey", -1);
+ assertEquals(3, Iterables.size(iterate));
+ assertEquals(4, Iterables.size(entriesView.read()));
+ assertEquals(4, Iterables.size(state.entries().read()));
+
for (Map.Entry<String, Integer> entry : iterate) {
c.output(KV.of(entry.getKey(), entry.getValue()));
}
@@ -2477,9 +2551,9 @@
@ProcessElement
public void processElement(
ProcessContext c, @StateId(stateId) BagState<Integer> state) {
- Iterable<Integer> currentValue = state.read();
state.add(c.element().getValue());
- if (Iterables.size(state.read()) >= 4) {
+ Iterable<Integer> currentValue = state.read();
+ if (Iterables.size(currentValue) >= 4) {
List<Integer> sorted = Lists.newArrayList(currentValue);
Collections.sort(sorted);
c.output(sorted);
@@ -2548,6 +2622,43 @@
pipeline.run();
}
+ /**
+ * Tests a GBK followed immediately by a {@link ParDo} that users timers. This checks a common
+ * case where both GBK and the user code share a timer delivery bundle.
+ */
+ @Test
+ @Category({ValidatesRunner.class, UsesTimersInParDo.class})
+ public void testGbkFollowedByUserTimers() throws Exception {
+
+ DoFn<KV<String, Iterable<Integer>>, Integer> fn =
+ new DoFn<KV<String, Iterable<Integer>>, Integer>() {
+
+ public static final String TIMER_ID = "foo";
+
+ @TimerId(TIMER_ID)
+ private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void processElement(ProcessContext context, @TimerId(TIMER_ID) Timer timer) {
+ timer.offset(Duration.standardSeconds(1)).setRelative();
+ context.output(3);
+ }
+
+ @OnTimer(TIMER_ID)
+ public void onTimer(OnTimerContext context) {
+ context.output(42);
+ }
+ };
+
+ PCollection<Integer> output =
+ pipeline
+ .apply(Create.of(KV.of("hello", 37)))
+ .apply(GroupByKey.<String, Integer>create())
+ .apply(ParDo.of(fn));
+ PAssert.that(output).containsInAnyOrder(3, 42);
+ pipeline.run();
+ }
+
@Test
@Category({ValidatesRunner.class, UsesTimersInParDo.class})
public void testEventTimeTimerAlignBounded() throws Exception {
@@ -2998,4 +3109,65 @@
// If it doesn't crash, we made it!
}
+
+ /** A {@link PipelineOptions} subclass for testing passing to a {@link DoFn}. */
+ public interface MyOptions extends PipelineOptions {
+ @Default.String("fake option")
+ String getFakeOption();
+ void setFakeOption(String value);
+ }
+
+ @Test
+ @Category(ValidatesRunner.class)
+ public void testPipelineOptionsParameter() {
+ PCollection<String> results = pipeline
+ .apply(Create.of(1))
+ .apply(
+ ParDo.of(
+ new DoFn<Integer, String>() {
+ @ProcessElement
+ public void process(ProcessContext c, PipelineOptions options) {
+ c.output(options.as(MyOptions.class).getFakeOption());
+ }
+ }));
+
+ String testOptionValue = "not fake anymore";
+ pipeline.getOptions().as(MyOptions.class).setFakeOption(testOptionValue);
+ PAssert.that(results).containsInAnyOrder("not fake anymore");
+
+ pipeline.run();
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, UsesTimersInParDo.class})
+ public void testPipelineOptionsParameterOnTimer() {
+ final String timerId = "thisTimer";
+
+ PCollection<String> results =
+ pipeline
+ .apply(Create.of(KV.of(0, 0)))
+ .apply(
+ ParDo.of(
+ new DoFn<KV<Integer, Integer>, String>() {
+ @TimerId(timerId)
+ private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void process(
+ ProcessContext c, BoundedWindow w, @TimerId(timerId) Timer timer) {
+ timer.set(w.maxTimestamp());
+ }
+
+ @OnTimer(timerId)
+ public void onTimer(OnTimerContext c, PipelineOptions options) {
+ c.output(options.as(MyOptions.class).getFakeOption());
+ }
+ }));
+
+ String testOptionValue = "not fake anymore";
+ pipeline.getOptions().as(MyOptions.class).setFakeOption(testOptionValue);
+ PAssert.that(results).containsInAnyOrder("not fake anymore");
+
+ pipeline.run();
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
index 02a44d2..d2d2529 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
@@ -18,10 +18,14 @@
package org.apache.beam.sdk.transforms;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.sdk.testing.TestPipeline.testingPipelineOptions;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import com.google.common.collect.Ordering;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,6 +33,9 @@
import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestStream;
@@ -37,7 +44,6 @@
import org.apache.beam.sdk.testing.UsesTestStream;
import org.apache.beam.sdk.testing.ValidatesRunner;
import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
-import org.apache.beam.sdk.transforms.splittabledofn.OffsetRange;
import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -67,10 +73,16 @@
static class PairStringWithIndexToLength extends DoFn<String, KV<String, Integer>> {
@ProcessElement
- public void process(ProcessContext c, OffsetRangeTracker tracker) {
- for (long i = tracker.currentRestriction().getFrom(); tracker.tryClaim(i); ++i) {
+ public ProcessContinuation process(ProcessContext c, OffsetRangeTracker tracker) {
+ for (long i = tracker.currentRestriction().getFrom(), numIterations = 0;
+ tracker.tryClaim(i);
+ ++i, ++numIterations) {
c.output(KV.of(c.element(), (int) i));
+ if (numIterations % 3 == 0) {
+ return resume();
+ }
}
+ return stop();
}
@GetInitialRestriction
@@ -93,8 +105,25 @@
}
}
+ private static PipelineOptions streamingTestPipelineOptions() {
+ // Using testing options with streaming=true makes it possible to enable UsesSplittableParDo
+ // tests in Dataflow runner, because as of writing, it can run Splittable DoFn only in
+ // streaming mode.
+ // This is a no-op for other runners currently (Direct runner doesn't care, and other
+ // runners don't implement SDF at all yet).
+ //
+ // This is a workaround until https://issues.apache.org/jira/browse/BEAM-1620
+ // is properly implemented and supports marking tests as streaming-only.
+ //
+ // https://issues.apache.org/jira/browse/BEAM-2483 specifically tracks the removal of the
+ // current workaround.
+ PipelineOptions options = testingPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(true);
+ return options;
+ }
+
@Rule
- public final transient TestPipeline p = TestPipeline.create();
+ public final transient TestPipeline p = TestPipeline.fromOptions(streamingTestPipelineOptions());
@Test
@Category({ValidatesRunner.class, UsesSplittableParDo.class})
@@ -182,6 +211,12 @@
private static class SDFWithMultipleOutputsPerBlock extends DoFn<String, Integer> {
private static final int MAX_INDEX = 98765;
+ private final int numClaimsPerCall;
+
+ private SDFWithMultipleOutputsPerBlock(int numClaimsPerCall) {
+ this.numClaimsPerCall = numClaimsPerCall;
+ }
+
private static int snapToNextBlock(int index, int[] blockStarts) {
for (int i = 1; i < blockStarts.length; ++i) {
if (index > blockStarts[i - 1] && index <= blockStarts[i]) {
@@ -192,14 +227,20 @@
}
@ProcessElement
- public void processElement(ProcessContext c, OffsetRangeTracker tracker) {
+ public ProcessContinuation processElement(ProcessContext c, OffsetRangeTracker tracker) {
int[] blockStarts = {-1, 0, 12, 123, 1234, 12345, 34567, MAX_INDEX};
int trueStart = snapToNextBlock((int) tracker.currentRestriction().getFrom(), blockStarts);
- for (int i = trueStart; tracker.tryClaim(blockStarts[i]); ++i) {
+ for (int i = trueStart, numIterations = 1;
+ tracker.tryClaim(blockStarts[i]);
+ ++i, ++numIterations) {
for (int index = blockStarts[i]; index < blockStarts[i + 1]; ++index) {
c.output(index);
}
+ if (numIterations == numClaimsPerCall) {
+ return resume();
+ }
}
+ return stop();
}
@GetInitialRestriction
@@ -212,7 +253,7 @@
@Category({ValidatesRunner.class, UsesSplittableParDo.class})
public void testOutputAfterCheckpoint() throws Exception {
PCollection<Integer> outputs = p.apply(Create.of("foo"))
- .apply(ParDo.of(new SDFWithMultipleOutputsPerBlock()));
+ .apply(ParDo.of(new SDFWithMultipleOutputsPerBlock(3)));
PAssert.thatSingleton(outputs.apply(Count.<Integer>globally()))
.isEqualTo((long) SDFWithMultipleOutputsPerBlock.MAX_INDEX);
p.run();
@@ -287,9 +328,105 @@
PAssert.that(res).containsInAnyOrder("a:0", "a:1", "a:2", "a:3", "b:4", "b:5", "b:6", "b:7");
p.run();
+ }
- // TODO: also add test coverage when the SDF checkpoints - the resumed call should also
- // properly access side inputs.
+ @BoundedPerElement
+ private static class SDFWithMultipleOutputsPerBlockAndSideInput
+ extends DoFn<Integer, KV<String, Integer>> {
+ private static final int MAX_INDEX = 98765;
+ private final PCollectionView<String> sideInput;
+ private final int numClaimsPerCall;
+
+ public SDFWithMultipleOutputsPerBlockAndSideInput(
+ PCollectionView<String> sideInput, int numClaimsPerCall) {
+ this.sideInput = sideInput;
+ this.numClaimsPerCall = numClaimsPerCall;
+ }
+
+ private static int snapToNextBlock(int index, int[] blockStarts) {
+ for (int i = 1; i < blockStarts.length; ++i) {
+ if (index > blockStarts[i - 1] && index <= blockStarts[i]) {
+ return i;
+ }
+ }
+ throw new IllegalStateException("Shouldn't get here");
+ }
+
+ @ProcessElement
+ public ProcessContinuation processElement(ProcessContext c, OffsetRangeTracker tracker) {
+ int[] blockStarts = {-1, 0, 12, 123, 1234, 12345, 34567, MAX_INDEX};
+ int trueStart = snapToNextBlock((int) tracker.currentRestriction().getFrom(), blockStarts);
+ for (int i = trueStart, numIterations = 1;
+ tracker.tryClaim(blockStarts[i]);
+ ++i, ++numIterations) {
+ for (int index = blockStarts[i]; index < blockStarts[i + 1]; ++index) {
+ c.output(KV.of(c.sideInput(sideInput) + ":" + c.element(), index));
+ }
+ if (numIterations == numClaimsPerCall) {
+ return resume();
+ }
+ }
+ return stop();
+ }
+
+ @GetInitialRestriction
+ public OffsetRange getInitialRange(Integer element) {
+ return new OffsetRange(0, MAX_INDEX);
+ }
+ }
+
+ @Test
+ @Category({
+ ValidatesRunner.class,
+ UsesSplittableParDo.class,
+ UsesSplittableParDoWithWindowedSideInputs.class
+ })
+ public void testWindowedSideInputWithCheckpoints() throws Exception {
+ PCollection<Integer> mainInput =
+ p.apply("main",
+ Create.timestamped(
+ TimestampedValue.of(0, new Instant(0)),
+ TimestampedValue.of(1, new Instant(1)),
+ TimestampedValue.of(2, new Instant(2)),
+ TimestampedValue.of(3, new Instant(3))))
+ .apply("window 1", Window.<Integer>into(FixedWindows.of(Duration.millis(1))));
+
+ PCollectionView<String> sideInput =
+ p.apply("side",
+ Create.timestamped(
+ TimestampedValue.of("a", new Instant(0)),
+ TimestampedValue.of("b", new Instant(2))))
+ .apply("window 2", Window.<String>into(FixedWindows.of(Duration.millis(2))))
+ .apply("singleton", View.<String>asSingleton());
+
+ PCollection<KV<String, Integer>> res =
+ mainInput.apply(
+ ParDo.of(
+ new SDFWithMultipleOutputsPerBlockAndSideInput(
+ sideInput, 3 /* numClaimsPerCall */))
+ .withSideInputs(sideInput));
+ PCollection<KV<String, Iterable<Integer>>> grouped =
+ res.apply(GroupByKey.<String, Integer>create());
+
+ PAssert.that(grouped.apply(Keys.<String>create()))
+ .containsInAnyOrder("a:0", "a:1", "b:2", "b:3");
+ PAssert.that(grouped)
+ .satisfies(
+ new SerializableFunction<Iterable<KV<String, Iterable<Integer>>>, Void>() {
+ @Override
+ public Void apply(Iterable<KV<String, Iterable<Integer>>> input) {
+ List<Integer> expected = new ArrayList<>();
+ for (int i = 0; i < SDFWithMultipleOutputsPerBlockAndSideInput.MAX_INDEX; ++i) {
+ expected.add(i);
+ }
+ for (KV<String, Iterable<Integer>> kv : input) {
+ assertEquals(expected, Ordering.<Integer>natural().sortedCopy(kv.getValue()));
+ }
+ return null;
+ }
+ });
+ p.run();
+
// TODO: also test coverage when some of the windows of the side input are not ready.
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java
index 9d2c6f6..e5bf904 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.sdk.transforms;
-import static org.apache.beam.sdk.TestUtils.checkCombineFn;
+import static org.apache.beam.sdk.testing.CombineFnTester.testCombineFn;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
@@ -51,7 +51,7 @@
@Test
public void testSumIntegerFn() {
- checkCombineFn(
+ testCombineFn(
Sum.ofIntegers(),
Lists.newArrayList(1, 2, 3, 4),
10);
@@ -59,7 +59,7 @@
@Test
public void testSumLongFn() {
- checkCombineFn(
+ testCombineFn(
Sum.ofLongs(),
Lists.newArrayList(1L, 2L, 3L, 4L),
10L);
@@ -67,7 +67,7 @@
@Test
public void testSumDoubleFn() {
- checkCombineFn(
+ testCombineFn(
Sum.ofDoubles(),
Lists.newArrayList(1.0, 2.0, 3.0, 4.0),
10.0);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
index 9b0b27d..a7f439c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
@@ -231,17 +231,17 @@
@Test
public void testTopGetNames() {
assertEquals("Combine.globally(Top(OrderByLength))", Top.of(1, new OrderByLength()).getName());
- assertEquals("Combine.globally(Top(Smallest))", Top.smallest(1).getName());
- assertEquals("Combine.globally(Top(Largest))", Top.largest(2).getName());
+ assertEquals("Combine.globally(Top(Reversed))", Top.smallest(1).getName());
+ assertEquals("Combine.globally(Top(Natural))", Top.largest(2).getName());
assertEquals("Combine.perKey(Top(IntegerComparator))",
Top.perKey(1, new IntegerComparator()).getName());
- assertEquals("Combine.perKey(Top(Smallest))", Top.<String, Integer>smallestPerKey(1).getName());
- assertEquals("Combine.perKey(Top(Largest))", Top.<String, Integer>largestPerKey(2).getName());
+ assertEquals("Combine.perKey(Top(Reversed))", Top.<String, Integer>smallestPerKey(1).getName());
+ assertEquals("Combine.perKey(Top(Natural))", Top.<String, Integer>largestPerKey(2).getName());
}
@Test
public void testDisplayData() {
- Top.Largest<Integer> comparer = new Top.Largest<Integer>();
+ Top.Natural<Integer> comparer = new Top.Natural<Integer>();
Combine.Globally<Integer, List<Integer>> top = Top.of(1234, comparer);
DisplayData displayData = DisplayData.from(top);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
index cdd03d9..bfb8b5a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
@@ -60,7 +60,6 @@
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TimestampedValue;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.hamcrest.Matchers;
import org.joda.time.Duration;
@@ -1340,11 +1339,11 @@
new PTransform<PBegin, PCollection<KV<String, Integer>>>() {
@Override
public PCollection<KV<String, Integer>> expand(PBegin input) {
- return PCollection.<KV<String, Integer>>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.UNBOUNDED)
- .setTypeDescriptor(new TypeDescriptor<KV<String, Integer>>() {});
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(),
+ WindowingStrategy.globalDefault(),
+ PCollection.IsBounded.UNBOUNDED,
+ KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
}
})
.apply(view);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java
new file mode 100644
index 0000000..113e8fe
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java
@@ -0,0 +1,797 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs;
+import static org.apache.beam.sdk.transforms.Watch.Growth.afterTimeSinceNewOutput;
+import static org.apache.beam.sdk.transforms.Watch.Growth.afterTotalOf;
+import static org.apache.beam.sdk.transforms.Watch.Growth.allOf;
+import static org.apache.beam.sdk.transforms.Watch.Growth.eitherOf;
+import static org.apache.beam.sdk.transforms.Watch.Growth.never;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.joda.time.Duration.standardSeconds;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+import com.google.common.hash.HashCode;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesSplittableParDo;
+import org.apache.beam.sdk.transforms.Watch.Growth.PollFn;
+import org.apache.beam.sdk.transforms.Watch.Growth.PollResult;
+import org.apache.beam.sdk.transforms.Watch.GrowthState;
+import org.apache.beam.sdk.transforms.Watch.GrowthTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.joda.time.ReadableDuration;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link Watch}. */
+@RunWith(JUnit4.class)
+public class WatchTest implements Serializable {
+ @Rule public transient TestPipeline p = TestPipeline.create();
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testSinglePollMultipleInputs() {
+ PCollection<KV<String, String>> res =
+ p.apply(Create.of("a", "b"))
+ .apply(
+ Watch.growthOf(
+ new PollFn<String, String>() {
+ @Override
+ public PollResult<String> apply(String element, Context c)
+ throws Exception {
+ return PollResult.complete(
+ Instant.now(), Arrays.asList(element + ".foo", element + ".bar"));
+ }
+ })
+ .withPollInterval(Duration.ZERO));
+
+ PAssert.that(res)
+ .containsInAnyOrder(
+ Arrays.asList(
+ KV.of("a", "a.foo"), KV.of("a", "a.bar"),
+ KV.of("b", "b.foo"), KV.of("b", "b.bar")));
+
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testSinglePollMultipleInputsWithSideInput() {
+ final PCollectionView<String> moo =
+ p.apply("moo", Create.of("moo")).apply("moo singleton", View.<String>asSingleton());
+ final PCollectionView<String> zoo =
+ p.apply("zoo", Create.of("zoo")).apply("zoo singleton", View.<String>asSingleton());
+ PCollection<KV<String, String>> res =
+ p.apply("input", Create.of("a", "b"))
+ .apply(
+ Watch.growthOf(
+ new PollFn<String, String>() {
+ @Override
+ public PollResult<String> apply(String element, Context c)
+ throws Exception {
+ return PollResult.complete(
+ Instant.now(),
+ Arrays.asList(
+ element + " " + c.sideInput(moo) + " " + c.sideInput(zoo)));
+ }
+ },
+ requiresSideInputs(moo, zoo))
+ .withPollInterval(Duration.ZERO));
+
+ PAssert.that(res)
+ .containsInAnyOrder(Arrays.asList(KV.of("a", "a moo zoo"), KV.of("b", "b moo zoo")));
+
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testMultiplePollsWithTerminationBecauseOutputIsFinal() {
+ testMultiplePolls(false);
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testMultiplePollsWithTerminationDueToTerminationCondition() {
+ testMultiplePolls(true);
+ }
+
+ private void testMultiplePolls(boolean terminationConditionElapsesBeforeOutputIsFinal) {
+ List<Integer> all = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ PCollection<Integer> res =
+ p.apply(Create.of("a"))
+ .apply(
+ Watch.growthOf(
+ new TimedPollFn<String, Integer>(
+ all,
+ standardSeconds(1) /* timeToOutputEverything */,
+ standardSeconds(3) /* timeToDeclareOutputFinal */,
+ standardSeconds(30) /* timeToFail */))
+ .withTerminationPerInput(
+ Watch.Growth.<String>afterTotalOf(
+ standardSeconds(
+ // At 2 seconds, all output has been yielded, but not yet
+ // declared final - so polling should terminate per termination
+ // condition.
+ // At 3 seconds, all output has been yielded (and declared final),
+ // so polling should terminate because of that without waiting for
+ // 100 seconds.
+ terminationConditionElapsesBeforeOutputIsFinal ? 2 : 100)))
+ .withPollInterval(Duration.millis(300))
+ .withOutputCoder(VarIntCoder.of()))
+ .apply("Drop input", Values.<Integer>create());
+
+ PAssert.that(res).containsInAnyOrder(all);
+
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testMultiplePollsStopAfterTimeSinceNewOutput() {
+ List<Integer> all = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ PCollection<Integer> res =
+ p.apply(Create.of("a"))
+ .apply(
+ Watch.growthOf(
+ new TimedPollFn<String, Integer>(
+ all,
+ standardSeconds(1) /* timeToOutputEverything */,
+ // Never declare output final
+ standardSeconds(1000) /* timeToDeclareOutputFinal */,
+ standardSeconds(30) /* timeToFail */))
+ // Should terminate after 4 seconds - earlier than timeToFail
+ .withTerminationPerInput(
+ Watch.Growth.<String>afterTimeSinceNewOutput(standardSeconds(3)))
+ .withPollInterval(Duration.millis(300))
+ .withOutputCoder(VarIntCoder.of()))
+ .apply("Drop input", Values.<Integer>create());
+
+ PAssert.that(res).containsInAnyOrder(all);
+
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testSinglePollWithManyResults() {
+ // More than the default 100 elements per checkpoint for direct runner.
+ final long numResults = 3000;
+ PCollection<KV<String, Integer>> res =
+ p.apply(Create.of("a"))
+ .apply(
+ Watch.growthOf(
+ new PollFn<String, KV<String, Integer>>() {
+ @Override
+ public PollResult<KV<String, Integer>> apply(String element, Context c)
+ throws Exception {
+ String pollId = UUID.randomUUID().toString();
+ List<KV<String, Integer>> output = Lists.newArrayList();
+ for (int i = 0; i < numResults; ++i) {
+ output.add(KV.of(pollId, i));
+ }
+ return PollResult.complete(Instant.now(), output);
+ }
+ })
+ .withTerminationPerInput(Watch.Growth.<String>afterTotalOf(standardSeconds(1)))
+ .withPollInterval(Duration.millis(1))
+ .withOutputCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
+ .apply("Drop input", Values.<KV<String, Integer>>create());
+
+ PAssert.that("Poll called only once", res.apply(Keys.<String>create()))
+ .satisfies(
+ new SerializableFunction<Iterable<String>, Void>() {
+ @Override
+ public Void apply(Iterable<String> pollIds) {
+ assertEquals(1, Sets.newHashSet(pollIds).size());
+ return null;
+ }
+ });
+ PAssert.that("Yields all expected results", res.apply("Drop poll id", Values.<Integer>create()))
+ .satisfies(
+ new SerializableFunction<Iterable<Integer>, Void>() {
+ @Override
+ public Void apply(Iterable<Integer> input) {
+ assertEquals(
+ "Total number of results mismatches",
+ numResults,
+ Lists.newArrayList(input).size());
+ assertEquals("Results are not unique", numResults, Sets.newHashSet(input).size());
+ return null;
+ }
+ });
+
+ p.run();
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesSplittableParDo.class})
+ public void testMultiplePollsWithManyResults() {
+ final long numResults = 3000;
+ List<Integer> all = Lists.newArrayList();
+ for (int i = 0; i < numResults; ++i) {
+ all.add(i);
+ }
+
+ PCollection<TimestampedValue<Integer>> res =
+ p.apply(Create.of("a"))
+ .apply(
+ Watch.growthOf(
+ new TimedPollFn<String, Integer>(
+ all,
+ standardSeconds(1) /* timeToOutputEverything */,
+ standardSeconds(3) /* timeToDeclareOutputFinal */,
+ standardSeconds(30) /* timeToFail */))
+ .withPollInterval(Duration.millis(500))
+ .withOutputCoder(VarIntCoder.of()))
+ .apply(ReifyTimestamps.<String, Integer>inValues())
+ .apply("Drop timestamped input", Values.<TimestampedValue<Integer>>create());
+
+ PAssert.that(res)
+ .satisfies(
+ new SerializableFunction<Iterable<TimestampedValue<Integer>>, Void>() {
+ @Override
+ public Void apply(Iterable<TimestampedValue<Integer>> outputs) {
+ Function<TimestampedValue<Integer>, Integer> extractValueFn =
+ new Function<TimestampedValue<Integer>, Integer>() {
+ @Nullable
+ @Override
+ public Integer apply(@Nullable TimestampedValue<Integer> input) {
+ return input.getValue();
+ }
+ };
+ Function<TimestampedValue<Integer>, Instant> extractTimestampFn =
+ new Function<TimestampedValue<Integer>, Instant>() {
+ @Nullable
+ @Override
+ public Instant apply(@Nullable TimestampedValue<Integer> input) {
+ return input.getTimestamp();
+ }
+ };
+
+ Ordering<TimestampedValue<Integer>> byValue =
+ Ordering.natural().onResultOf(extractValueFn);
+ Ordering<TimestampedValue<Integer>> byTimestamp =
+ Ordering.natural().onResultOf(extractTimestampFn);
+ // New outputs appear in timestamp order because each output's assigned timestamp
+ // is Instant.now() at the time of poll.
+ assertTrue(
+ "Outputs must be in timestamp order",
+ byTimestamp.isOrdered(byValue.sortedCopy(outputs)));
+ assertEquals(
+ "Yields all expected values",
+ Sets.newHashSet(Iterables.transform(outputs, extractValueFn)).size(),
+ numResults);
+ assertThat(
+ "Poll called more than once",
+ Sets.newHashSet(Iterables.transform(outputs, extractTimestampFn)).size(),
+ greaterThan(1));
+ return null;
+ }
+ });
+
+ p.run();
+ }
+
+ /**
+ * Gradually emits all items from the given list, pairing each one with a UUID that identifies the
+ * round of polling, so a client can check how many rounds of polling there were.
+ */
+ private static class TimedPollFn<InputT, OutputT> extends PollFn<InputT, OutputT> {
+ private final Instant baseTime;
+ private final List<OutputT> outputs;
+ private final Duration timeToOutputEverything;
+ private final Duration timeToDeclareOutputFinal;
+ private final Duration timeToFail;
+
+ public TimedPollFn(
+ List<OutputT> outputs,
+ Duration timeToOutputEverything,
+ Duration timeToDeclareOutputFinal,
+ Duration timeToFail) {
+ this.baseTime = Instant.now();
+ this.outputs = outputs;
+ this.timeToOutputEverything = timeToOutputEverything;
+ this.timeToDeclareOutputFinal = timeToDeclareOutputFinal;
+ this.timeToFail = timeToFail;
+ }
+
+ @Override
+ public PollResult<OutputT> apply(InputT element, Context c) throws Exception {
+ Instant now = Instant.now();
+ Duration elapsed = new Duration(baseTime, Instant.now());
+ if (elapsed.isLongerThan(timeToFail)) {
+ fail(
+ String.format(
+ "Poll called %s after base time, which is longer than the threshold of %s",
+ elapsed, timeToFail));
+ }
+
+ double fractionElapsed = 1.0 * elapsed.getMillis() / timeToOutputEverything.getMillis();
+ int numToEmit = (int) Math.min(outputs.size(), fractionElapsed * outputs.size());
+ List<TimestampedValue<OutputT>> toEmit = Lists.newArrayList();
+ for (int i = 0; i < numToEmit; ++i) {
+ toEmit.add(TimestampedValue.of(outputs.get(i), now));
+ }
+ return elapsed.isLongerThan(timeToDeclareOutputFinal)
+ ? PollResult.complete(toEmit)
+ : PollResult.incomplete(toEmit).withWatermark(now);
+ }
+ }
+
+ @Test
+ public void testTerminationConditionsNever() {
+ Watch.Growth.Never<Object> c = never();
+ Integer state = c.forNewInput(Instant.now(), null);
+ assertFalse(c.canStopPolling(Instant.now(), state));
+ }
+
+ @Test
+ public void testTerminationConditionsAfterTotalOf() {
+ Instant now = Instant.now();
+ Watch.Growth.AfterTotalOf<Object> c = afterTotalOf(standardSeconds(5));
+ KV<Instant, ReadableDuration> state = c.forNewInput(now, null);
+ assertFalse(c.canStopPolling(now, state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(6)), state));
+ }
+
+ @Test
+ public void testTerminationConditionsAfterTimeSinceNewOutput() {
+ Instant now = Instant.now();
+ Watch.Growth.AfterTimeSinceNewOutput<Object> c = afterTimeSinceNewOutput(standardSeconds(5));
+ KV<Instant, ReadableDuration> state = c.forNewInput(now, null);
+ assertFalse(c.canStopPolling(now, state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(6)), state));
+
+ state = c.onSeenNewOutput(now.plus(standardSeconds(3)), state);
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(6)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(9)), state));
+
+ state = c.onSeenNewOutput(now.plus(standardSeconds(5)), state);
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(6)), state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(9)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(11)), state));
+ }
+
+ @Test
+ public void testTerminationConditionsEitherOf() {
+ Instant now = Instant.now();
+ Watch.Growth.AfterTotalOf<Object> a = afterTotalOf(standardSeconds(5));
+ Watch.Growth.AfterTotalOf<Object> b = afterTotalOf(standardSeconds(10));
+
+ Watch.Growth.BinaryCombined<
+ Object, KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>>
+ c = eitherOf(a, b);
+ KV<KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>> state =
+ c.forNewInput(now, null);
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(7)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(12)), state));
+ }
+
+ @Test
+ public void testTerminationConditionsAllOf() {
+ Instant now = Instant.now();
+ Watch.Growth.AfterTotalOf<Object> a = afterTotalOf(standardSeconds(5));
+ Watch.Growth.AfterTotalOf<Object> b = afterTotalOf(standardSeconds(10));
+
+ Watch.Growth.BinaryCombined<
+ Object, KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>>
+ c = allOf(a, b);
+ KV<KV<Instant, ReadableDuration>, KV<Instant, ReadableDuration>> state =
+ c.forNewInput(now, null);
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(3)), state));
+ assertFalse(c.canStopPolling(now.plus(standardSeconds(7)), state));
+ assertTrue(c.canStopPolling(now.plus(standardSeconds(12)), state));
+ }
+
+ private static GrowthTracker<String, Integer> newTracker(GrowthState<String, Integer> state) {
+ return new GrowthTracker<>(StringUtf8Coder.of(), state, never());
+ }
+
+ private static GrowthTracker<String, Integer> newTracker() {
+ return newTracker(new GrowthState<String, Integer>(never().forNewInput(Instant.now(), null)));
+ }
+
+ @Test
+ public void testGrowthTrackerCheckpointEmpty() {
+ // Checkpoint an empty tracker.
+ GrowthTracker<String, Integer> tracker = newTracker();
+ GrowthState<String, Integer> residual = tracker.checkpoint();
+ GrowthState<String, Integer> primary = tracker.currentRestriction();
+ Watch.Growth.Never<String> condition = never();
+ assertEquals(
+ primary.toString(condition),
+ new GrowthState<>(
+ Collections.<HashCode, Instant>emptyMap() /* completed */,
+ Collections.<TimestampedValue<String>>emptyList() /* pending */,
+ true /* isOutputFinal */,
+ (Integer) null /* terminationState */,
+ BoundedWindow.TIMESTAMP_MAX_VALUE /* pollWatermark */)
+ .toString(condition));
+ assertEquals(
+ residual.toString(condition),
+ new GrowthState<>(
+ Collections.<HashCode, Instant>emptyMap() /* completed */,
+ Collections.<TimestampedValue<String>>emptyList() /* pending */,
+ false /* isOutputFinal */,
+ 0 /* terminationState */,
+ BoundedWindow.TIMESTAMP_MIN_VALUE /* pollWatermark */)
+ .toString(condition));
+ }
+
+ @Test
+ public void testGrowthTrackerCheckpointNonEmpty() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2)))))
+ .withWatermark(now.plus(standardSeconds(7))));
+
+ assertEquals(now.plus(standardSeconds(1)), tracker.getWatermark());
+ assertTrue(tracker.hasPending());
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertTrue(tracker.hasPending());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertTrue(tracker.hasPending());
+ assertEquals(now.plus(standardSeconds(3)), tracker.getWatermark());
+
+ GrowthTracker<String, Integer> residualTracker = newTracker(tracker.checkpoint());
+ GrowthTracker<String, Integer> primaryTracker = newTracker(tracker.currentRestriction());
+
+ // Verify primary: should contain what the current tracker claimed, and nothing else.
+ assertEquals(now.plus(standardSeconds(1)), primaryTracker.getWatermark());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("a", primaryTracker.tryClaimNextPending().getValue());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("b", primaryTracker.tryClaimNextPending().getValue());
+ assertFalse(primaryTracker.hasPending());
+ assertFalse(primaryTracker.shouldPollMore());
+ // No more pending elements in primary restriction, and no polling.
+ primaryTracker.checkDone();
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, primaryTracker.getWatermark());
+
+ // Verify residual: should contain what the current tracker didn't claim.
+ assertEquals(now.plus(standardSeconds(3)), residualTracker.getWatermark());
+ assertTrue(residualTracker.hasPending());
+ assertEquals("c", residualTracker.tryClaimNextPending().getValue());
+ assertTrue(residualTracker.hasPending());
+ assertEquals("d", residualTracker.tryClaimNextPending().getValue());
+ assertFalse(residualTracker.hasPending());
+ assertTrue(residualTracker.shouldPollMore());
+ // No more pending elements in residual restriction, but poll watermark still holds.
+ assertEquals(now.plus(standardSeconds(7)), residualTracker.getWatermark());
+
+ // Verify current tracker: it was checkpointed, so should contain nothing else.
+ assertNull(tracker.tryClaimNextPending());
+ tracker.checkDone();
+ assertFalse(tracker.hasPending());
+ assertFalse(tracker.shouldPollMore());
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, tracker.getWatermark());
+ }
+
+ @Test
+ public void testGrowthTrackerOutputFullyBeforeCheckpointIncomplete() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2)))))
+ .withWatermark(now.plus(standardSeconds(7))));
+
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertEquals("c", tracker.tryClaimNextPending().getValue());
+ assertEquals("d", tracker.tryClaimNextPending().getValue());
+ assertFalse(tracker.hasPending());
+ assertEquals(now.plus(standardSeconds(7)), tracker.getWatermark());
+
+ GrowthTracker<String, Integer> residualTracker = newTracker(tracker.checkpoint());
+ GrowthTracker<String, Integer> primaryTracker = newTracker(tracker.currentRestriction());
+
+ // Verify primary: should contain what the current tracker claimed, and nothing else.
+ assertEquals(now.plus(standardSeconds(1)), primaryTracker.getWatermark());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("a", primaryTracker.tryClaimNextPending().getValue());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("b", primaryTracker.tryClaimNextPending().getValue());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("c", primaryTracker.tryClaimNextPending().getValue());
+ assertTrue(primaryTracker.hasPending());
+ assertEquals("d", primaryTracker.tryClaimNextPending().getValue());
+ assertFalse(primaryTracker.hasPending());
+ assertFalse(primaryTracker.shouldPollMore());
+ // No more pending elements in primary restriction, and no polling.
+ primaryTracker.checkDone();
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, primaryTracker.getWatermark());
+
+ // Verify residual: should contain what the current tracker didn't claim.
+ assertFalse(residualTracker.hasPending());
+ assertTrue(residualTracker.shouldPollMore());
+ // No more pending elements in residual restriction, but poll watermark still holds.
+ assertEquals(now.plus(standardSeconds(7)), residualTracker.getWatermark());
+
+ // Verify current tracker: it was checkpointed, so should contain nothing else.
+ tracker.checkDone();
+ assertFalse(tracker.hasPending());
+ assertFalse(tracker.shouldPollMore());
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, tracker.getWatermark());
+ }
+
+ @Test
+ public void testGrowthTrackerPollAfterCheckpointIncompleteWithNewOutputs() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2)))))
+ .withWatermark(now.plus(standardSeconds(7))));
+
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertEquals("c", tracker.tryClaimNextPending().getValue());
+ assertEquals("d", tracker.tryClaimNextPending().getValue());
+
+ GrowthState<String, Integer> checkpoint = tracker.checkpoint();
+ // Simulate resuming from the checkpoint and adding more elements.
+ {
+ GrowthTracker<String, Integer> residualTracker = newTracker(checkpoint);
+ residualTracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("e", now.plus(standardSeconds(5))),
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))),
+ TimestampedValue.of("f", now.plus(standardSeconds(8)))))
+ .withWatermark(now.plus(standardSeconds(12))));
+
+ assertEquals(now.plus(standardSeconds(5)), residualTracker.getWatermark());
+ assertEquals("e", residualTracker.tryClaimNextPending().getValue());
+ assertEquals(now.plus(standardSeconds(8)), residualTracker.getWatermark());
+ assertEquals("f", residualTracker.tryClaimNextPending().getValue());
+
+ assertFalse(residualTracker.hasPending());
+ assertTrue(residualTracker.shouldPollMore());
+ assertEquals(now.plus(standardSeconds(12)), residualTracker.getWatermark());
+ }
+ // Try same without an explicitly specified watermark.
+ {
+ GrowthTracker<String, Integer> residualTracker = newTracker(checkpoint);
+ residualTracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("e", now.plus(standardSeconds(5))),
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))),
+ TimestampedValue.of("f", now.plus(standardSeconds(8))))));
+
+ assertEquals(now.plus(standardSeconds(5)), residualTracker.getWatermark());
+ assertEquals("e", residualTracker.tryClaimNextPending().getValue());
+ assertEquals(now.plus(standardSeconds(5)), residualTracker.getWatermark());
+ assertEquals("f", residualTracker.tryClaimNextPending().getValue());
+
+ assertFalse(residualTracker.hasPending());
+ assertTrue(residualTracker.shouldPollMore());
+ assertEquals(now.plus(standardSeconds(5)), residualTracker.getWatermark());
+ }
+ }
+
+ @Test
+ public void testGrowthTrackerPollAfterCheckpointWithoutNewOutputs() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2)))))
+ .withWatermark(now.plus(standardSeconds(7))));
+
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertEquals("c", tracker.tryClaimNextPending().getValue());
+ assertEquals("d", tracker.tryClaimNextPending().getValue());
+
+ // Simulate resuming from the checkpoint but there are no new elements.
+ GrowthState<String, Integer> checkpoint = tracker.checkpoint();
+ {
+ GrowthTracker<String, Integer> residualTracker = newTracker(checkpoint);
+ residualTracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2)))))
+ .withWatermark(now.plus(standardSeconds(12))));
+
+ assertFalse(residualTracker.hasPending());
+ assertTrue(residualTracker.shouldPollMore());
+ assertEquals(now.plus(standardSeconds(12)), residualTracker.getWatermark());
+ }
+ // Try the same without an explicitly specified watermark
+ {
+ GrowthTracker<String, Integer> residualTracker = newTracker(checkpoint);
+ residualTracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))))));
+ // No new elements and no explicit watermark supplied - should reuse old watermark.
+ assertEquals(now.plus(standardSeconds(7)), residualTracker.getWatermark());
+ }
+ }
+
+ @Test
+ public void testGrowthTrackerPollAfterCheckpointWithoutNewOutputsNoWatermark() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))))));
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertEquals("c", tracker.tryClaimNextPending().getValue());
+ assertEquals("d", tracker.tryClaimNextPending().getValue());
+ assertEquals(now.plus(standardSeconds(1)), tracker.getWatermark());
+
+ // Simulate resuming from the checkpoint but there are no new elements.
+ GrowthState<String, Integer> checkpoint = tracker.checkpoint();
+ GrowthTracker<String, Integer> residualTracker = newTracker(checkpoint);
+ residualTracker.addNewAsPending(
+ PollResult.incomplete(
+ Arrays.asList(
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))))));
+ // No new elements and no explicit watermark supplied - should keep old watermark.
+ assertEquals(now.plus(standardSeconds(1)), residualTracker.getWatermark());
+ }
+
+ @Test
+ public void testGrowthTrackerRepeatedEmptyPollWatermark() {
+ // Empty poll result with no watermark
+ {
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(Collections.<TimestampedValue<String>>emptyList()));
+ assertEquals(BoundedWindow.TIMESTAMP_MIN_VALUE, tracker.getWatermark());
+
+ // Simulate resuming from the checkpoint but there are still no new elements.
+ GrowthTracker<String, Integer> residualTracker = newTracker(tracker.checkpoint());
+ tracker.addNewAsPending(
+ PollResult.incomplete(Collections.<TimestampedValue<String>>emptyList()));
+ // No new elements and no explicit watermark supplied - still no watermark.
+ assertEquals(BoundedWindow.TIMESTAMP_MIN_VALUE, residualTracker.getWatermark());
+ }
+ // Empty poll result with watermark
+ {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.incomplete(Collections.<TimestampedValue<String>>emptyList())
+ .withWatermark(now));
+ assertEquals(now, tracker.getWatermark());
+
+ // Simulate resuming from the checkpoint but there are still no new elements.
+ GrowthTracker<String, Integer> residualTracker = newTracker(tracker.checkpoint());
+ tracker.addNewAsPending(
+ PollResult.incomplete(Collections.<TimestampedValue<String>>emptyList()));
+ // No new elements and no explicit watermark supplied - should keep old watermark.
+ assertEquals(now, residualTracker.getWatermark());
+ }
+ }
+
+ @Test
+ public void testGrowthTrackerOutputFullyBeforeCheckpointComplete() {
+ Instant now = Instant.now();
+ GrowthTracker<String, Integer> tracker = newTracker();
+ tracker.addNewAsPending(
+ PollResult.complete(
+ Arrays.asList(
+ TimestampedValue.of("d", now.plus(standardSeconds(4))),
+ TimestampedValue.of("c", now.plus(standardSeconds(3))),
+ TimestampedValue.of("a", now.plus(standardSeconds(1))),
+ TimestampedValue.of("b", now.plus(standardSeconds(2))))));
+
+ assertEquals("a", tracker.tryClaimNextPending().getValue());
+ assertEquals("b", tracker.tryClaimNextPending().getValue());
+ assertEquals("c", tracker.tryClaimNextPending().getValue());
+ assertEquals("d", tracker.tryClaimNextPending().getValue());
+ assertFalse(tracker.hasPending());
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, tracker.getWatermark());
+
+ GrowthTracker<String, Integer> residualTracker = newTracker(tracker.checkpoint());
+
+ // Verify residual: should be empty, since output was final.
+ residualTracker.checkDone();
+ assertFalse(residualTracker.hasPending());
+ assertFalse(residualTracker.shouldPollMore());
+ // No more pending elements in residual restriction, but poll watermark still holds.
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, residualTracker.getWatermark());
+
+ // Verify current tracker: it was checkpointed, so should contain nothing else.
+ tracker.checkDone();
+ assertFalse(tracker.hasPending());
+ assertFalse(tracker.shouldPollMore());
+ assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, tracker.getWatermark());
+ }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WithKeysTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WithKeysTest.java
index 8abbf1a..444979e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WithKeysTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WithKeysTest.java
@@ -39,7 +39,7 @@
*/
@RunWith(JUnit4.class)
public class WithKeysTest {
- static final String[] COLLECTION = new String[] {
+ private static final String[] COLLECTION = new String[] {
"a",
"aa",
"b",
@@ -47,7 +47,7 @@
"bbb"
};
- static final List<KV<Integer, String>> WITH_KEYS = Arrays.asList(
+ private static final List<KV<Integer, String>> WITH_KEYS = Arrays.asList(
KV.of(1, "a"),
KV.of(2, "aa"),
KV.of(1, "b"),
@@ -55,7 +55,7 @@
KV.of(3, "bbb")
);
- static final List<KV<Integer, String>> WITH_CONST_KEYS = Arrays.asList(
+ private static final List<KV<Integer, String>> WITH_CONST_KEYS = Arrays.asList(
KV.of(100, "a"),
KV.of(100, "aa"),
KV.of(100, "b"),
@@ -63,6 +63,14 @@
KV.of(100, "bbb")
);
+ private static final List<KV<Void, String>> WITH_CONST_NULL_KEYS = Arrays.asList(
+ KV.of((Void) null, "a"),
+ KV.of((Void) null, "aa"),
+ KV.of((Void) null, "b"),
+ KV.of((Void) null, "bb"),
+ KV.of((Void) null, "bbb")
+ );
+
@Rule
public final TestPipeline p = TestPipeline.create();
@@ -99,6 +107,22 @@
}
@Test
+ @Category(NeedsRunner.class)
+ public void testConstantVoidKeys() {
+
+ PCollection<String> input =
+ p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(
+ StringUtf8Coder.of()));
+
+ PCollection<KV<Void, String>> output =
+ input.apply(WithKeys.<Void, String>of((Void) null));
+ PAssert.that(output)
+ .containsInAnyOrder(WITH_CONST_NULL_KEYS);
+
+ p.run();
+ }
+
+ @Test
public void testWithKeysGetName() {
assertEquals("WithKeys", WithKeys.<Integer, String>of(100).getName());
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 3edb194..72883ff 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.sdk.transforms.reflect;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume;
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertEquals;
@@ -89,8 +91,8 @@
when(mockArgumentProvider.processContext(Matchers.<DoFn>any())).thenReturn(mockProcessContext);
}
- private void invokeProcessElement(DoFn<String, String> fn) {
- DoFnInvokers.invokerFor(fn).invokeProcessElement(mockArgumentProvider);
+ private DoFn.ProcessContinuation invokeProcessElement(DoFn<String, String> fn) {
+ return DoFnInvokers.invokerFor(fn).invokeProcessElement(mockArgumentProvider);
}
private void invokeOnTimer(String timerId, DoFn<String, String> fn) {
@@ -119,7 +121,7 @@
public void processElement(ProcessContext c) throws Exception {}
}
MockFn mockFn = mock(MockFn.class);
- invokeProcessElement(mockFn);
+ assertEquals(stop(), invokeProcessElement(mockFn));
verify(mockFn).processElement(mockProcessContext);
}
@@ -140,7 +142,7 @@
public void testDoFnWithProcessElementInterface() throws Exception {
IdentityUsingInterfaceWithProcessElement fn =
mock(IdentityUsingInterfaceWithProcessElement.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).processElement(mockProcessContext);
}
@@ -161,14 +163,14 @@
@Test
public void testDoFnWithMethodInSuperclass() throws Exception {
IdentityChildWithoutOverride fn = mock(IdentityChildWithoutOverride.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).process(mockProcessContext);
}
@Test
public void testDoFnWithMethodInSubclass() throws Exception {
IdentityChildWithOverride fn = mock(IdentityChildWithOverride.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).process(mockProcessContext);
}
@@ -179,7 +181,7 @@
public void processElement(ProcessContext c, IntervalWindow w) throws Exception {}
}
MockFn fn = mock(MockFn.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).processElement(mockProcessContext, mockWindow);
}
@@ -203,7 +205,7 @@
throws Exception {}
}
MockFn fn = mock(MockFn.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).processElement(mockProcessContext, mockState);
}
@@ -229,11 +231,35 @@
public void onTimer() {}
}
MockFn fn = mock(MockFn.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).processElement(mockProcessContext, mockTimer);
}
@Test
+ public void testDoFnWithReturn() throws Exception {
+ class MockFn extends DoFn<String, String> {
+ @DoFn.ProcessElement
+ public ProcessContinuation processElement(ProcessContext c, SomeRestrictionTracker tracker)
+ throws Exception {
+ return null;
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(String element) {
+ return null;
+ }
+
+ @NewTracker
+ public SomeRestrictionTracker newTracker(SomeRestriction restriction) {
+ return null;
+ }
+ }
+ MockFn fn = mock(MockFn.class);
+ when(fn.processElement(mockProcessContext, null)).thenReturn(resume());
+ assertEquals(resume(), invokeProcessElement(fn));
+ }
+
+ @Test
public void testDoFnWithStartBundleSetupTeardown() throws Exception {
class MockFn extends DoFn<String, String> {
@ProcessElement
@@ -288,7 +314,9 @@
/** Public so Mockito can do "delegatesTo()" in the test below. */
public static class MockFn extends DoFn<String, String> {
@ProcessElement
- public void processElement(ProcessContext c, SomeRestrictionTracker tracker) {}
+ public ProcessContinuation processElement(ProcessContext c, SomeRestrictionTracker tracker) {
+ return null;
+ }
@GetInitialRestriction
public SomeRestriction getInitialRestriction(String element) {
@@ -340,7 +368,7 @@
.splitRestriction(
eq("blah"), same(restriction), Mockito.<DoFn.OutputReceiver<SomeRestriction>>any());
when(fn.newTracker(restriction)).thenReturn(tracker);
- fn.processElement(mockProcessContext, tracker);
+ when(fn.processElement(mockProcessContext, tracker)).thenReturn(resume());
assertEquals(coder, invoker.invokeGetRestrictionCoder(CoderRegistry.createDefault()));
assertEquals(restriction, invoker.invokeGetInitialRestriction("blah"));
@@ -356,6 +384,8 @@
});
assertEquals(Arrays.asList(part1, part2, part3), outputs);
assertEquals(tracker, invoker.invokeNewTracker(restriction));
+ assertEquals(
+ resume(),
invoker.invokeProcessElement(
new FakeArgumentProvider<String, String>() {
@Override
@@ -367,7 +397,7 @@
public RestrictionTracker<?> restrictionTracker() {
return tracker;
}
- });
+ }));
}
private static class RestrictionWithDefaultTracker
@@ -441,7 +471,7 @@
assertEquals("foo", output);
}
});
- invoker.invokeProcessElement(mockArgumentProvider);
+ assertEquals(stop(), invoker.invokeProcessElement(mockArgumentProvider));
assertThat(
invoker.invokeNewTracker(new RestrictionWithDefaultTracker()),
instanceOf(DefaultTracker.class));
@@ -531,14 +561,14 @@
@Test
public void testLocalPrivateDoFnClass() throws Exception {
PrivateDoFnClass fn = mock(PrivateDoFnClass.class);
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
verify(fn).processThis(mockProcessContext);
}
@Test
public void testStaticPackagePrivateDoFnClass() throws Exception {
DoFn<String, String> fn = mock(DoFnInvokersTestHelper.newStaticPackagePrivateDoFn().getClass());
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
DoFnInvokersTestHelper.verifyStaticPackagePrivateDoFn(fn, mockProcessContext);
}
@@ -546,28 +576,28 @@
public void testInnerPackagePrivateDoFnClass() throws Exception {
DoFn<String, String> fn =
mock(new DoFnInvokersTestHelper().newInnerPackagePrivateDoFn().getClass());
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
DoFnInvokersTestHelper.verifyInnerPackagePrivateDoFn(fn, mockProcessContext);
}
@Test
public void testStaticPrivateDoFnClass() throws Exception {
DoFn<String, String> fn = mock(DoFnInvokersTestHelper.newStaticPrivateDoFn().getClass());
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
DoFnInvokersTestHelper.verifyStaticPrivateDoFn(fn, mockProcessContext);
}
@Test
public void testInnerPrivateDoFnClass() throws Exception {
DoFn<String, String> fn = mock(new DoFnInvokersTestHelper().newInnerPrivateDoFn().getClass());
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
DoFnInvokersTestHelper.verifyInnerPrivateDoFn(fn, mockProcessContext);
}
@Test
public void testAnonymousInnerDoFn() throws Exception {
DoFn<String, String> fn = mock(new DoFnInvokersTestHelper().newInnerAnonymousDoFn().getClass());
- invokeProcessElement(fn);
+ assertEquals(stop(), invokeProcessElement(fn));
DoFnInvokersTestHelper.verifyInnerAnonymousDoFn(fn, mockProcessContext);
}
@@ -604,6 +634,41 @@
}
@Test
+ public void testProcessElementExceptionWithReturn() throws Exception {
+ thrown.expect(UserCodeException.class);
+ thrown.expectMessage("bogus");
+ DoFnInvokers.invokerFor(
+ new DoFn<Integer, Integer>() {
+ @ProcessElement
+ public ProcessContinuation processElement(
+ @SuppressWarnings("unused") ProcessContext c, SomeRestrictionTracker tracker) {
+ throw new IllegalArgumentException("bogus");
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return null;
+ }
+
+ @NewTracker
+ public SomeRestrictionTracker newTracker(SomeRestriction restriction) {
+ return null;
+ }
+ })
+ .invokeProcessElement(new FakeArgumentProvider<Integer, Integer>() {
+ @Override
+ public DoFn.ProcessContext processContext(DoFn<Integer, Integer> doFn) {
+ return null; // will not be touched
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ return null; // will not be touched
+ }
+ });
+ }
+
+ @Test
public void testStartBundleException() throws Exception {
DoFnInvoker<Integer, Integer> invoker =
DoFnInvokers.invokerFor(
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesProcessElementTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesProcessElementTest.java
index d321f54..44ae5c4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesProcessElementTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesProcessElementTest.java
@@ -50,7 +50,7 @@
@Test
public void testBadReturnType() throws Exception {
thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("Must return void");
+ thrown.expectMessage("Must return void or ProcessContinuation");
analyzeProcessElementMethod(
new AnonymousMethod() {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
index 07b3348..08af65e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
@@ -52,7 +52,8 @@
public class DoFnSignaturesSplittableDoFnTest {
@Rule public ExpectedException thrown = ExpectedException.none();
- private static class SomeRestriction {}
+ private abstract static class SomeRestriction
+ implements HasDefaultTracker<SomeRestriction, SomeRestrictionTracker> {}
private abstract static class SomeRestrictionTracker
implements RestrictionTracker<SomeRestriction> {}
@@ -60,6 +61,20 @@
private abstract static class SomeRestrictionCoder extends StructuredCoder<SomeRestriction> {}
@Test
+ public void testReturnsProcessContinuation() throws Exception {
+ DoFnSignature.ProcessElementMethod signature =
+ analyzeProcessElementMethod(
+ new AnonymousMethod() {
+ private DoFn.ProcessContinuation method(
+ DoFn<Integer, String>.ProcessContext context) {
+ return null;
+ }
+ });
+
+ assertTrue(signature.hasReturnValue());
+ }
+
+ @Test
public void testHasRestrictionTracker() throws Exception {
DoFnSignature.ProcessElementMethod signature =
analyzeProcessElementMethod(
@@ -100,11 +115,6 @@
public SomeRestriction getInitialRestriction(Integer element) {
return null;
}
-
- @NewTracker
- public SomeRestrictionTracker newTracker(SomeRestriction restriction) {
- return null;
- }
}
@BoundedPerElement
@@ -130,6 +140,55 @@
.isBoundedPerElement());
}
+ private static class BaseFnWithoutContinuation extends DoFn<Integer, String> {
+ @ProcessElement
+ public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return null;
+ }
+ }
+
+ private static class BaseFnWithContinuation extends DoFn<Integer, String> {
+ @ProcessElement
+ public ProcessContinuation processElement(
+ ProcessContext context, SomeRestrictionTracker tracker) {
+ return null;
+ }
+
+ @GetInitialRestriction
+ public SomeRestriction getInitialRestriction(Integer element) {
+ return null;
+ }
+ }
+
+ @Test
+ public void testSplittableBoundednessInferredFromReturnValue() throws Exception {
+ assertEquals(
+ PCollection.IsBounded.BOUNDED,
+ DoFnSignatures.getSignature(BaseFnWithoutContinuation.class).isBoundedPerElement());
+ assertEquals(
+ PCollection.IsBounded.UNBOUNDED,
+ DoFnSignatures.getSignature(BaseFnWithContinuation.class).isBoundedPerElement());
+ }
+
+ @Test
+ public void testSplittableRespectsBoundednessAnnotation() throws Exception {
+ @BoundedPerElement
+ class BoundedFnWithContinuation extends BaseFnWithContinuation {}
+
+ assertEquals(
+ PCollection.IsBounded.BOUNDED,
+ DoFnSignatures.getSignature(BoundedFnWithContinuation.class).isBoundedPerElement());
+
+ @UnboundedPerElement
+ class UnboundedFnWithContinuation extends BaseFnWithContinuation {}
+
+ assertEquals(
+ PCollection.IsBounded.UNBOUNDED,
+ DoFnSignatures.getSignature(UnboundedFnWithContinuation.class).isBoundedPerElement());
+ }
@Test
public void testUnsplittableIsBounded() throws Exception {
class UnsplittableFn extends DoFn<Integer, String> {
@@ -172,8 +231,10 @@
public void testSplittableWithAllFunctions() throws Exception {
class GoodSplittableDoFn extends DoFn<Integer, String> {
@ProcessElement
- public void processElement(
- ProcessContext context, SomeRestrictionTracker tracker) {}
+ public ProcessContinuation processElement(
+ ProcessContext context, SomeRestrictionTracker tracker) {
+ return null;
+ }
@GetInitialRestriction
public SomeRestriction getInitialRestriction(Integer element) {
@@ -198,6 +259,7 @@
DoFnSignature signature = DoFnSignatures.getSignature(GoodSplittableDoFn.class);
assertEquals(SomeRestrictionTracker.class, signature.processElement().trackerT().getRawType());
assertTrue(signature.processElement().isSplittable());
+ assertTrue(signature.processElement().hasReturnValue());
assertEquals(
SomeRestriction.class, signature.getInitialRestriction().restrictionT().getRawType());
assertEquals(SomeRestriction.class, signature.splitRestriction().restrictionT().getRawType());
@@ -214,7 +276,9 @@
public void testSplittableWithAllFunctionsGeneric() throws Exception {
class GoodGenericSplittableDoFn<RestrictionT, TrackerT, CoderT> extends DoFn<Integer, String> {
@ProcessElement
- public void processElement(ProcessContext context, TrackerT tracker) {}
+ public ProcessContinuation processElement(ProcessContext context, TrackerT tracker) {
+ return null;
+ }
@GetInitialRestriction
public RestrictionT getInitialRestriction(Integer element) {
@@ -242,6 +306,7 @@
SomeRestriction, SomeRestrictionTracker, SomeRestrictionCoder>() {}.getClass());
assertEquals(SomeRestrictionTracker.class, signature.processElement().trackerT().getRawType());
assertTrue(signature.processElement().isSplittable());
+ assertTrue(signature.processElement().hasReturnValue());
assertEquals(
SomeRestriction.class, signature.getInitialRestriction().restrictionT().getRawType());
assertEquals(SomeRestriction.class, signature.splitRestriction().restrictionT().getRawType());
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index cffb0ad..70c8dfd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -29,6 +29,7 @@
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
import org.apache.beam.sdk.state.TimeDomain;
@@ -329,6 +330,19 @@
}
@Test
+ public void testPipelineOptionsParameter() throws Exception {
+ DoFnSignature sig =
+ DoFnSignatures.getSignature(new DoFn<String, String>() {
+ @ProcessElement
+ public void process(ProcessContext c, PipelineOptions options) {}
+ }.getClass());
+
+ assertThat(
+ sig.processElement().extraParameters(),
+ Matchers.<Parameter>hasItem(instanceOf(Parameter.PipelineOptionsParameter.class)));
+ }
+
+ @Test
public void testDeclAndUsageOfTimerInSuperclass() throws Exception {
DoFnSignature sig =
DoFnSignatures.getSignature(new DoFnOverridingAbstractTimerUse().getClass());
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java
index 831894c..8aed6b9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTrackerTest.java
@@ -21,6 +21,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.beam.sdk.io.range.OffsetRange;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
index cd562e9..c8c01f5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
@@ -91,6 +91,14 @@
}
@Test
+ public void testDaysCompatibility() throws IncompatibleWindowException {
+ CalendarWindows.DaysWindows daysWindows = CalendarWindows.days(10);
+ daysWindows.verifyCompatibility(CalendarWindows.days(10));
+ thrown.expect(IncompatibleWindowException.class);
+ daysWindows.verifyCompatibility(CalendarWindows.days(9));
+ }
+
+ @Test
public void testWeeks() throws Exception {
Map<IntervalWindow, Set<String>> expected = new HashMap<>();
@@ -165,6 +173,14 @@
}
@Test
+ public void testMonthsCompatibility() throws IncompatibleWindowException {
+ CalendarWindows.MonthsWindows monthsWindows = CalendarWindows.months(10).beginningOnDay(15);
+ monthsWindows.verifyCompatibility(CalendarWindows.months(10).beginningOnDay(15));
+ thrown.expect(IncompatibleWindowException.class);
+ monthsWindows.verifyCompatibility(CalendarWindows.months(10).beginningOnDay(30));
+ }
+
+ @Test
public void testMultiMonths() throws Exception {
Map<IntervalWindow, Set<String>> expected = new HashMap<>();
@@ -239,6 +255,14 @@
}
@Test
+ public void testYearsCompatibility() throws IncompatibleWindowException {
+ CalendarWindows.YearsWindows yearsWindows = CalendarWindows.years(2017).beginningOnDay(1, 1);
+ yearsWindows.verifyCompatibility(CalendarWindows.years(2017).beginningOnDay(1, 1));
+ thrown.expect(IncompatibleWindowException.class);
+ yearsWindows.verifyCompatibility(CalendarWindows.years(2017).beginningOnDay(1, 2));
+ }
+
+ @Test
public void testTimeZone() throws Exception {
Map<IntervalWindow, Set<String>> expected = new HashMap<>();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
index 47c273a..80a534c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
@@ -149,6 +149,13 @@
}
@Test
+ public void testVerifyCompatibility() throws IncompatibleWindowException {
+ FixedWindows.of(new Duration(10)).verifyCompatibility(FixedWindows.of(new Duration(10)));
+ thrown.expect(IncompatibleWindowException.class);
+ FixedWindows.of(new Duration(10)).verifyCompatibility(FixedWindows.of(new Duration(20)));
+ }
+
+ @Test
public void testValidOutputTimes() throws Exception {
for (long timestamp : Arrays.asList(200, 800, 700)) {
WindowFnTestUtils.validateGetOutputTimestamp(
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
index 9d94928..42c15b5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
@@ -36,7 +36,9 @@
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -45,6 +47,8 @@
*/
@RunWith(JUnit4.class)
public class SessionsTest {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
@Test
public void testSimple() throws Exception {
@@ -106,6 +110,16 @@
Sessions.withGapDuration(new Duration(20))));
}
+ @Test
+ public void testVerifyCompatibility() throws IncompatibleWindowException {
+ Sessions.withGapDuration(new Duration(10))
+ .verifyCompatibility(Sessions.withGapDuration(new Duration(10)));
+
+ thrown.expect(IncompatibleWindowException.class);
+ Sessions.withGapDuration(new Duration(10))
+ .verifyCompatibility(FixedWindows.of(new Duration(10)));
+ }
+
/**
* Validates that the output timestamp for aggregate data falls within the acceptable range.
*/
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
index dd673d3..bfd01f0 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
@@ -21,6 +21,7 @@
import static org.apache.beam.sdk.testing.WindowFnTestUtils.set;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -34,7 +35,9 @@
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -43,6 +46,8 @@
*/
@RunWith(JUnit4.class)
public class SlidingWindowsTest {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
@Test
public void testSimple() throws Exception {
@@ -51,11 +56,12 @@
expected.put(new IntervalWindow(new Instant(0), new Instant(10)), set(1, 2, 5, 9));
expected.put(new IntervalWindow(new Instant(5), new Instant(15)), set(5, 9, 10, 11));
expected.put(new IntervalWindow(new Instant(10), new Instant(20)), set(10, 11));
+ SlidingWindows windowFn = SlidingWindows.of(new Duration(10)).every(new Duration(5));
assertEquals(
expected,
- runWindowFn(
- SlidingWindows.of(new Duration(10)).every(new Duration(5)),
+ runWindowFn(windowFn,
Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L)));
+ assertThat(windowFn.assignsToOneWindow(), is(false));
}
@Test
@@ -65,11 +71,27 @@
expected.put(new IntervalWindow(new Instant(0), new Instant(7)), set(1, 2, 5));
expected.put(new IntervalWindow(new Instant(5), new Instant(12)), set(5, 9, 10, 11));
expected.put(new IntervalWindow(new Instant(10), new Instant(17)), set(10, 11));
+ SlidingWindows windowFn = SlidingWindows.of(new Duration(7)).every(new Duration(5));
+ assertEquals(
+ expected,
+ runWindowFn(windowFn,
+ Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L)));
+ assertThat(windowFn.assignsToOneWindow(), is(false));
+ }
+
+ @Test
+ public void testEqualSize() throws Exception {
+ Map<IntervalWindow, Set<String>> expected = new HashMap<>();
+ expected.put(new IntervalWindow(new Instant(0), new Instant(3)), set(1, 2));
+ expected.put(new IntervalWindow(new Instant(3), new Instant(6)), set(3, 4, 5));
+ expected.put(new IntervalWindow(new Instant(6), new Instant(9)), set(6, 7));
+ SlidingWindows windowFn = SlidingWindows.of(new Duration(3)).every(new Duration(3));
assertEquals(
expected,
runWindowFn(
- SlidingWindows.of(new Duration(7)).every(new Duration(5)),
- Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L)));
+ windowFn,
+ Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L)));
+ assertThat(windowFn.assignsToOneWindow(), is(true));
}
@Test
@@ -78,12 +100,14 @@
expected.put(new IntervalWindow(new Instant(0), new Instant(3)), set(1, 2));
expected.put(new IntervalWindow(new Instant(10), new Instant(13)), set(10, 11));
expected.put(new IntervalWindow(new Instant(100), new Instant(103)), set(100));
+ SlidingWindows windowFn = SlidingWindows.of(new Duration(3)).every(new Duration(10));
assertEquals(
expected,
runWindowFn(
// Only look at the first 3 millisecs of every 10-millisec interval.
- SlidingWindows.of(new Duration(3)).every(new Duration(10)),
+ windowFn,
Arrays.asList(1L, 2L, 3L, 5L, 9L, 10L, 11L, 100L)));
+ assertThat(windowFn.assignsToOneWindow(), is(true));
}
@Test
@@ -153,6 +177,13 @@
}
@Test
+ public void testVerifyCompatibility() throws IncompatibleWindowException {
+ SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(10)));
+ thrown.expect(IncompatibleWindowException.class);
+ SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(20)));
+ }
+
+ @Test
public void testDefaultWindowMappingFn() {
// [40, 1040), [340, 1340), [640, 1640) ...
SlidingWindows slidingWindows = SlidingWindows.of(new Duration(1000))
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 92f6a9c..e2f8c26 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -31,19 +31,34 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
+import org.apache.beam.sdk.coders.CustomCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesCustomWindowMerging;
import org.apache.beam.sdk.testing.ValidatesRunner;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.GroupByKey;
@@ -165,6 +180,42 @@
assertEquals(fixed25, strategy.getWindowFn());
}
+ @Test
+ public void testWindowIntoAssignesLongerAllowedLateness() {
+
+ FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10));
+ FixedWindows fixed25 = FixedWindows.of(Duration.standardMinutes(25));
+
+ PCollection<String> notChanged = pipeline
+ .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of()))
+ .apply("WindowInto25", Window.<String>into(fixed25)
+ .withAllowedLateness(Duration.standardDays(1))
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(5)))
+ .accumulatingFiredPanes())
+ .apply("WindowInto10", Window.<String>into(fixed10)
+ .withAllowedLateness(Duration.standardDays(2)));
+
+ assertEquals(Duration.standardDays(2), notChanged.getWindowingStrategy()
+ .getAllowedLateness());
+
+ PCollection<String> data = pipeline
+ .apply("createChanged", Create.of("hello", "world").withCoder(StringUtf8Coder.of()));
+
+ PCollection<String> longWindow = data.apply("WindowInto25c", Window.<String>into(fixed25)
+ .withAllowedLateness(Duration.standardDays(1))
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(5)))
+ .accumulatingFiredPanes());
+
+ assertEquals(Duration.standardDays(1), longWindow.getWindowingStrategy()
+ .getAllowedLateness());
+
+ PCollection<String> autoCorrectedWindow = longWindow.apply("WindowInto10c",
+ Window.<String>into(fixed10).withAllowedLateness(Duration.standardHours(1)));
+
+ assertEquals(Duration.standardDays(1), autoCorrectedWindow.getWindowingStrategy()
+ .getAllowedLateness());
+ }
+
/**
* With {@link #testWindowIntoNullWindowFnNoAssign()}, demonstrates that the expansions of the
* {@link Window} transform depends on if it actually assigns elements to windows.
@@ -301,6 +352,14 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other, "WindowOddEvenBuckets is only compatible with WindowOddEvenBuckets.");
+ }
+ }
+
+ @Override
public Coder<IntervalWindow> windowCoder() {
return new IntervalWindow.IntervalWindowCoder();
}
@@ -526,4 +585,170 @@
assertThat(data, not(hasDisplayItem("trigger")));
assertThat(data, not(hasDisplayItem("allowedLateness")));
}
+
+ @Test
+ @Category({ValidatesRunner.class, UsesCustomWindowMerging.class})
+ public void testMergingCustomWindows() {
+ Instant startInstant = new Instant(0L);
+ List<TimestampedValue<String>> input = new ArrayList<>();
+ PCollection<String> inputCollection =
+ pipeline.apply(
+ Create.timestamped(
+ TimestampedValue.of("big", startInstant.plus(Duration.standardSeconds(10))),
+ TimestampedValue.of("small1", startInstant.plus(Duration.standardSeconds(20))),
+ // This one will be outside of bigWindow thus not merged
+ TimestampedValue.of("small2", startInstant.plus(Duration.standardSeconds(39)))));
+ PCollection<String> windowedCollection =
+ inputCollection.apply(Window.into(new CustomWindowFn<String>()));
+ PCollection<Long> count =
+ windowedCollection.apply(Combine.globally(Count.<String>combineFn()).withoutDefaults());
+ // "small1" and "big" elements merged into bigWindow "small2" not merged
+ // because timestamp is not in bigWindow
+ PAssert.that("Wrong number of elements in output collection", count).containsInAnyOrder(2L, 1L);
+ pipeline.run();
+ }
+
+ // This test is usefull because some runners have a special merge implementation
+ // for keyed collections
+ @Test
+ @Category({ValidatesRunner.class, UsesCustomWindowMerging.class})
+ public void testMergingCustomWindowsKeyedCollection() {
+ Instant startInstant = new Instant(0L);
+ PCollection<KV<Integer, String>> inputCollection =
+ pipeline.apply(
+ Create.timestamped(
+ TimestampedValue.of(
+ KV.of(0, "big"), startInstant.plus(Duration.standardSeconds(10))),
+ TimestampedValue.of(
+ KV.of(1, "small1"), startInstant.plus(Duration.standardSeconds(20))),
+ // This element is not contained within the bigWindow and not merged
+ TimestampedValue.of(
+ KV.of(2, "small2"), startInstant.plus(Duration.standardSeconds(39)))));
+ PCollection<KV<Integer, String>> windowedCollection =
+ inputCollection.apply(Window.into(new CustomWindowFn<KV<Integer, String>>()));
+ PCollection<Long> count =
+ windowedCollection.apply(
+ Combine.globally(Count.<KV<Integer, String>>combineFn()).withoutDefaults());
+ // "small1" and "big" elements merged into bigWindow "small2" not merged
+ // because it is not contained in bigWindow
+ PAssert.that("Wrong number of elements in output collection", count).containsInAnyOrder(2L, 1L);
+ pipeline.run();
+ }
+
+ private static class CustomWindow extends IntervalWindow {
+ private boolean isBig;
+
+ CustomWindow(Instant start, Instant end, boolean isBig) {
+ super(start, end);
+ this.isBig = isBig;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ CustomWindow that = (CustomWindow) o;
+ return super.equals(o) && this.isBig == that.isBig;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), isBig);
+ }
+ }
+
+ private static class CustomWindowCoder extends CustomCoder<CustomWindow> {
+
+ private static final CustomWindowCoder INSTANCE = new CustomWindowCoder();
+ private static final Coder<IntervalWindow> INTERVAL_WINDOW_CODER = IntervalWindow.getCoder();
+ private static final VarIntCoder VAR_INT_CODER = VarIntCoder.of();
+
+ public static CustomWindowCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(CustomWindow window, OutputStream outStream) throws IOException {
+ INTERVAL_WINDOW_CODER.encode(window, outStream);
+ VAR_INT_CODER.encode(window.isBig ? 1 : 0, outStream);
+ }
+
+ @Override
+ public CustomWindow decode(InputStream inStream) throws IOException {
+ IntervalWindow superWindow = INTERVAL_WINDOW_CODER.decode(inStream);
+ boolean isBig = VAR_INT_CODER.decode(inStream) != 0;
+ return new CustomWindow(superWindow.start(), superWindow.end(), isBig);
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ INTERVAL_WINDOW_CODER.verifyDeterministic();
+ VAR_INT_CODER.verifyDeterministic();
+ }
+ }
+
+ private static class CustomWindowFn<T> extends WindowFn<T, CustomWindow> {
+ @Override
+ public Collection<CustomWindow> assignWindows(AssignContext c) throws Exception {
+ String element;
+ // It loses genericity of type T but this is not a big deal for a test.
+ // And it allows to avoid duplicating CustomWindowFn to support PCollection<KV>
+ if (c.element() instanceof KV) {
+ element = ((KV<Integer, String>) c.element()).getValue();
+ } else {
+ element = (String) c.element();
+ }
+ // put big elements in windows of 30s and small ones in windows of 5s
+ if ("big".equals(element)) {
+ return Collections.singletonList(
+ new CustomWindow(
+ c.timestamp(), c.timestamp().plus(Duration.standardSeconds(30)), true));
+ } else {
+ return Collections.singletonList(
+ new CustomWindow(
+ c.timestamp(), c.timestamp().plus(Duration.standardSeconds(5)), false));
+ }
+ }
+
+ @Override
+ public void mergeWindows(MergeContext c) throws Exception {
+ Map<CustomWindow, Set<CustomWindow>> windowsToMerge = new HashMap<>();
+ for (CustomWindow window : c.windows()) {
+ if (window.isBig) {
+ HashSet<CustomWindow> windows = new HashSet<>();
+ windows.add(window);
+ windowsToMerge.put(window, windows);
+ }
+ }
+ for (CustomWindow window : c.windows()) {
+ for (Map.Entry<CustomWindow, Set<CustomWindow>> bigWindow : windowsToMerge.entrySet()) {
+ if (bigWindow.getKey().contains(window)) {
+ bigWindow.getValue().add(window);
+ }
+ }
+ }
+ for (Map.Entry<CustomWindow, Set<CustomWindow>> mergeEntry : windowsToMerge.entrySet()) {
+ c.merge(mergeEntry.getValue(), mergeEntry.getKey());
+ }
+ }
+
+ @Override
+ public boolean isCompatible(WindowFn<?, ?> other) {
+ return other instanceof CustomWindowFn;
+ }
+
+ @Override
+ public Coder<CustomWindow> windowCoder() {
+ return CustomWindowCoder.of();
+ }
+
+ @Override
+ public WindowMappingFn<CustomWindow> getDefaultWindowMappingFn() {
+ throw new UnsupportedOperationException("side inputs not supported");
+ }
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
index 2171466..32e23da 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
@@ -22,6 +22,7 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
@@ -43,6 +44,9 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {}
+
+ @Override
public Coder<BoundedWindow> windowCoder() {
// not used
return (Coder) GlobalWindow.Coder.INSTANCE;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java
index ebd8297..29e727b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java
@@ -20,11 +20,17 @@
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
@@ -39,10 +45,48 @@
*/
@RunWith(JUnit4.class)
public class MutationDetectorsTest {
+ /**
+ * Solely used to test that immutability is enforced from the SDK's perspective and not from
+ * Java's {@link Object#equals} method. Note that we do not expect users to create such
+ * an implementation.
+ */
+ private class ForSDKMutationDetectionTestCoder extends AtomicCoder<Object> {
+ // Use a unique instance that is returned as the structural value making all structural
+ // values of this coder equivalent to each other.
+ private final Object uniqueInstance = new Object();
+
+ @Override
+ public void encode(Object value, OutputStream outStream) throws IOException {
+ }
+
+ @Override
+ public Object decode(InputStream inStream) throws IOException {
+ return new AtomicInteger();
+ }
+
+ @Override
+ public Object structuralValue(Object value) {
+ return uniqueInstance;
+ }
+ }
@Rule public ExpectedException thrown = ExpectedException.none();
/**
+ * Tests that mutation detection is enforced from the SDK point of view
+ * (Based on the {@link Coder#structuralValue}) and not from the Java's equals method.
+ */
+ @Test
+ public void testMutationBasedOnStructuralValue() throws Exception {
+ AtomicInteger value = new AtomicInteger();
+ MutationDetector detector =
+ MutationDetectors.forValueWithCoder(value, new ForSDKMutationDetectionTestCoder());
+ // Even though we modified the value, we are relying on the fact that the structural
+ // value will be used to compare equality
+ value.incrementAndGet();
+ detector.verifyUnmodified();
+ }
+ /**
* Tests that {@link MutationDetectors#forValueWithCoder} detects a mutation to a list.
*/
@Test
@@ -93,6 +137,18 @@
}
/**
+ * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on a
+ * {@link Set} coded as an {@link Iterable}.
+ */
+ @Test
+ public void testStructuralValue() throws Exception {
+ Set<Integer> value = Sets.newHashSet(Arrays.asList(1, 2, 3, 4));
+ MutationDetector detector =
+ MutationDetectors.forValueWithCoder(value, IterableCoder.of(VarIntCoder.of()));
+ detector.verifyUnmodified();
+ }
+
+ /**
* Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on an
* {@link Iterable} that is not known to be bounded; after coder-based cloning the bound
* will be known and it will be a {@link List} so it will encode more compactly the second
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
index 9a80730..c3b0171 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
@@ -18,8 +18,10 @@
package org.apache.beam.sdk.util;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
import com.google.common.collect.ImmutableList;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -28,6 +30,9 @@
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.testing.InterceptingUrlClassLoader;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -51,6 +56,30 @@
}
@Test
+ public void customClassLoader() throws Exception {
+ // define a classloader with test-classes in it
+ final ClassLoader testLoader = Thread.currentThread().getContextClassLoader();
+ final ClassLoader loader = new InterceptingUrlClassLoader(testLoader, MySource.class.getName());
+ final Class<?> source = loader.loadClass(
+ "org.apache.beam.sdk.util.SerializableUtilsTest$MySource");
+ assertNotSame(source.getClassLoader(), MySource.class.getClassLoader());
+
+ // validate if the caller set the classloader that it works well
+ final Serializable customLoaderSourceInstance = Serializable.class.cast(
+ source.getConstructor().newInstance());
+ final Thread thread = Thread.currentThread();
+ thread.setContextClassLoader(loader);
+ try {
+ assertSerializationClassLoader(loader, customLoaderSourceInstance);
+ } finally {
+ thread.setContextClassLoader(testLoader);
+ }
+
+ // now let beam be a little be more fancy and try to ensure it by itself from the incoming value
+ assertSerializationClassLoader(loader, customLoaderSourceInstance);
+ }
+
+ @Test
public void testTranscode() {
String stringValue = "hi bob";
int intValue = 42;
@@ -114,4 +143,35 @@
expectedException.expectMessage("unable to serialize");
SerializableUtils.ensureSerializable(new UnserializableCoderByJava());
}
+
+ private void assertSerializationClassLoader(
+ final ClassLoader loader, final Serializable customLoaderSourceInstance) {
+ final Serializable copy = SerializableUtils.ensureSerializable(customLoaderSourceInstance);
+ assertEquals(loader, copy.getClass().getClassLoader());
+ assertEquals(
+ copy.getClass().getClassLoader(),
+ customLoaderSourceInstance.getClass().getClassLoader());
+ }
+
+ /**
+ * a sample class to test framework serialization,
+ * {@see SerializableUtilsTest#customClassLoader}.
+ */
+ public static class MySource extends BoundedSource<String> {
+ @Override
+ public List<? extends BoundedSource<String>> split(
+ final long desiredBundleSizeBytes, final PipelineOptions options) throws Exception {
+ return null;
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(final PipelineOptions options) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public BoundedReader<String> createReader(final PipelineOptions options) throws IOException {
+ return null;
+ }
+ }
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
index 58e2bbd..33503b6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
@@ -31,6 +31,7 @@
import java.util.Map;
import java.util.Map.Entry;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -59,9 +60,9 @@
@Test
public void testOfThenHas() {
- PCollection<Object> pCollection = PCollection.createPrimitiveOutputInternal(
- pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
- TupleTag<Object> tag = new TupleTag<>();
+ PCollection<Integer> pCollection = PCollection.createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
+ TupleTag<Integer> tag = new TupleTag<>();
assertTrue(PCollectionTuple.of(tag, pCollection).has(tag));
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorsTest.java
index 1bf0fc9..645da5e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorsTest.java
@@ -25,9 +25,12 @@
import static org.apache.beam.sdk.values.TypeDescriptors.strings;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
import java.util.List;
import java.util.Set;
+import org.hamcrest.CoreMatchers;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -70,4 +73,61 @@
assertNotEquals(descriptor, new TypeDescriptor<List<String>>() {});
assertNotEquals(descriptor, new TypeDescriptor<List<Boolean>>() {});
}
+
+ private interface Generic<FooT, BarT> {}
+
+ private static <ActualFooT> Generic<ActualFooT, String> typeErasedGeneric() {
+ return new Generic<ActualFooT, String>() {};
+ }
+
+ private static <ActualFooT, ActualBarT> TypeDescriptor<ActualFooT> extractFooT(
+ Generic<ActualFooT, ActualBarT> instance) {
+ return TypeDescriptors.extractFromTypeParameters(
+ instance,
+ Generic.class,
+ new TypeDescriptors.TypeVariableExtractor<
+ Generic<ActualFooT, ActualBarT>, ActualFooT>() {});
+ }
+
+ private static <ActualFooT, ActualBarT> TypeDescriptor<ActualBarT> extractBarT(
+ Generic<ActualFooT, ActualBarT> instance) {
+ return TypeDescriptors.extractFromTypeParameters(
+ instance,
+ Generic.class,
+ new TypeDescriptors.TypeVariableExtractor<
+ Generic<ActualFooT, ActualBarT>, ActualBarT>() {});
+ }
+
+ private static <ActualFooT, ActualBarT> TypeDescriptor<KV<ActualFooT, ActualBarT>> extractKV(
+ Generic<ActualFooT, ActualBarT> instance) {
+ return TypeDescriptors.extractFromTypeParameters(
+ instance,
+ Generic.class,
+ new TypeDescriptors.TypeVariableExtractor<
+ Generic<ActualFooT, ActualBarT>, KV<ActualFooT, ActualBarT>>() {});
+ }
+
+ @Test
+ public void testTypeDescriptorsTypeParameterOf() throws Exception {
+ assertEquals(strings(), extractFooT(new Generic<String, Integer>() {}));
+ assertEquals(integers(), extractBarT(new Generic<String, Integer>() {}));
+ assertEquals(kvs(strings(), integers()), extractKV(new Generic<String, Integer>() {}));
+ }
+
+ @Test
+ public void testTypeDescriptorsTypeParameterOfErased() throws Exception {
+ Generic<Integer, String> instance = TypeDescriptorsTest.typeErasedGeneric();
+
+ TypeDescriptor<Integer> fooT = extractFooT(instance);
+ assertNotNull(fooT);
+ // Using toString() assertions because verifying the contents of a Type is very cumbersome,
+ // and the expected types can not be easily constructed directly.
+ assertEquals("ActualFooT", fooT.toString());
+
+ assertEquals(strings(), extractBarT(instance));
+
+ TypeDescriptor<KV<Integer, String>> kvT = extractKV(instance);
+ assertNotNull(kvT);
+ assertThat(kvT.toString(), CoreMatchers.containsString("KV<ActualFooT, java.lang.String>"));
+ }
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/pom.xml b/sdks/java/extensions/google-cloud-platform-core/pom.xml
index e4e951b..8599a8e 100644
--- a/sdks/java/extensions/google-cloud-platform-core/pom.xml
+++ b/sdks/java/extensions/google-cloud-platform-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
index 6db0a01..f35c62a 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
@@ -88,10 +88,11 @@
ImmutableList.Builder<MatchResult> ret = ImmutableList.builder();
for (Boolean isGlob : isGlobBooleans) {
if (isGlob) {
- checkState(globsMatchResults.hasNext(), "Expect globsMatchResults has next.");
+ checkState(globsMatchResults.hasNext(), "Expect globsMatchResults has next: %s", globs);
ret.add(globsMatchResults.next());
} else {
- checkState(nonGlobsMatchResults.hasNext(), "Expect nonGlobsMatchResults has next.");
+ checkState(
+ nonGlobsMatchResults.hasNext(), "Expect nonGlobsMatchResults has next: %s", nonGlobs);
ret.add(nonGlobsMatchResults.next());
}
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index 94b733a..da4a1e0 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -135,7 +135,7 @@
private static final int MAX_CONCURRENT_BATCHES = 256;
private static final FluentBackoff BACKOFF_FACTORY =
- FluentBackoff.DEFAULT.withMaxRetries(3).withInitialBackoff(Duration.millis(200));
+ FluentBackoff.DEFAULT.withMaxRetries(10).withInitialBackoff(Duration.standardSeconds(1));
/////////////////////////////////////////////////////////////////////////////
@@ -176,7 +176,13 @@
char c = src[i++];
switch (c) {
case '*':
- dst.append(".*");
+ // One char lookahead for **
+ if (i < src.length && src[i] == '*') {
+ dst.append(".*");
+ ++i;
+ } else {
+ dst.append("[^/]*");
+ }
break;
case '?':
dst.append("[^/]");
@@ -761,7 +767,11 @@
@Override
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException {
- throw new IOException(String.format("Error trying to delete %s: %s", file, e));
+ if (e.getCode() == 404) {
+ LOG.info("Ignoring failed deletion of file {} which already does not exist: {}", file, e);
+ } else {
+ throw new IOException(String.format("Error trying to delete %s: %s", file, e));
+ }
}
});
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/RetryHttpRequestInitializer.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/RetryHttpRequestInitializer.java
index 2b7135e..fd908cf 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/RetryHttpRequestInitializer.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/RetryHttpRequestInitializer.java
@@ -17,8 +17,9 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.http.HttpBackOffIOExceptionHandler;
-import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
+import static com.google.api.client.util.BackOffUtils.next;
+
+import com.google.api.client.http.HttpIOExceptionHandler;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
@@ -60,60 +61,106 @@
*/
private static final int HANGING_GET_TIMEOUT_SEC = 80;
- private static class LoggingHttpBackOffIOExceptionHandler
- extends HttpBackOffIOExceptionHandler {
- public LoggingHttpBackOffIOExceptionHandler(BackOff backOff) {
- super(backOff);
+ /** Handlers used to provide additional logging information on unsuccessful HTTP requests. */
+ private static class LoggingHttpBackOffHandler
+ implements HttpIOExceptionHandler, HttpUnsuccessfulResponseHandler {
+
+ private final Sleeper sleeper;
+ private final BackOff ioExceptionBackOff;
+ private final BackOff unsuccessfulResponseBackOff;
+ private final Set<Integer> ignoredResponseCodes;
+ private int ioExceptionRetries;
+ private int unsuccessfulResponseRetries;
+
+ private LoggingHttpBackOffHandler(
+ Sleeper sleeper,
+ BackOff ioExceptionBackOff,
+ BackOff unsucessfulResponseBackOff,
+ Set<Integer> ignoredResponseCodes) {
+ this.sleeper = sleeper;
+ this.ioExceptionBackOff = ioExceptionBackOff;
+ this.unsuccessfulResponseBackOff = unsucessfulResponseBackOff;
+ this.ignoredResponseCodes = ignoredResponseCodes;
}
@Override
public boolean handleIOException(HttpRequest request, boolean supportsRetry)
throws IOException {
- boolean willRetry = super.handleIOException(request, supportsRetry);
+ // We will retry if the request supports retry or the backoff was successful.
+ // Note that the order of these checks is important since
+ // backOffWasSuccessful will perform a sleep.
+ boolean willRetry = supportsRetry && backOffWasSuccessful(ioExceptionBackOff);
if (willRetry) {
+ ioExceptionRetries += 1;
LOG.debug("Request failed with IOException, will retry: {}", request.getUrl());
} else {
- LOG.warn("Request failed with IOException, will NOT retry: {}", request.getUrl());
+ String message = "Request failed with IOException, "
+ + "performed {} retries due to IOExceptions, "
+ + "performed {} retries due to unsuccessful status codes, "
+ + "HTTP framework says request {} be retried, "
+ + "(caller responsible for retrying): {}";
+ LOG.warn(message,
+ ioExceptionRetries,
+ unsuccessfulResponseRetries,
+ supportsRetry ? "can" : "cannot",
+ request.getUrl());
}
return willRetry;
}
- }
-
- private static class LoggingHttpBackoffUnsuccessfulResponseHandler
- implements HttpUnsuccessfulResponseHandler {
- private final HttpBackOffUnsuccessfulResponseHandler handler;
- private final Set<Integer> ignoredResponseCodes;
-
- public LoggingHttpBackoffUnsuccessfulResponseHandler(BackOff backoff,
- Sleeper sleeper, Set<Integer> ignoredResponseCodes) {
- this.ignoredResponseCodes = ignoredResponseCodes;
- handler = new HttpBackOffUnsuccessfulResponseHandler(backoff);
- handler.setSleeper(sleeper);
- handler.setBackOffRequired(
- new HttpBackOffUnsuccessfulResponseHandler.BackOffRequired() {
- @Override
- public boolean isRequired(HttpResponse response) {
- int statusCode = response.getStatusCode();
- return (statusCode / 100 == 5) || // 5xx: server error
- statusCode == 429; // 429: Too many requests
- }
- });
- }
@Override
- public boolean handleResponse(HttpRequest request, HttpResponse response,
- boolean supportsRetry) throws IOException {
- boolean retry = handler.handleResponse(request, response, supportsRetry);
- if (retry) {
- LOG.debug("Request failed with code {} will retry: {}",
+ public boolean handleResponse(HttpRequest request, HttpResponse response, boolean supportsRetry)
+ throws IOException {
+ // We will retry if the request supports retry and the status code requires a backoff
+ // and the backoff was successful. Note that the order of these checks is important since
+ // backOffWasSuccessful will perform a sleep.
+ boolean willRetry = supportsRetry
+ && retryOnStatusCode(response.getStatusCode())
+ && backOffWasSuccessful(unsuccessfulResponseBackOff);
+ if (willRetry) {
+ unsuccessfulResponseRetries += 1;
+ LOG.debug("Request failed with code {}, will retry: {}",
response.getStatusCode(), request.getUrl());
-
- } else if (!ignoredResponseCodes.contains(response.getStatusCode())) {
- LOG.warn("Request failed with code {}, will NOT retry: {}",
- response.getStatusCode(), request.getUrl());
+ } else {
+ String message = "Request failed with code {}, "
+ + "performed {} retries due to IOExceptions, "
+ + "performed {} retries due to unsuccessful status codes, "
+ + "HTTP framework says request {} be retried, "
+ + "(caller responsible for retrying): {}";
+ if (ignoredResponseCodes.contains(response.getStatusCode())) {
+ // Log ignored response codes at a lower level
+ LOG.debug(message,
+ response.getStatusCode(),
+ ioExceptionRetries,
+ unsuccessfulResponseRetries,
+ supportsRetry ? "can" : "cannot",
+ request.getUrl());
+ } else {
+ LOG.warn(message,
+ response.getStatusCode(),
+ ioExceptionRetries,
+ unsuccessfulResponseRetries,
+ supportsRetry ? "can" : "cannot",
+ request.getUrl());
+ }
}
+ return willRetry;
+ }
- return retry;
+ /** Returns true iff performing the backoff was successful. */
+ private boolean backOffWasSuccessful(BackOff backOff) {
+ try {
+ return next(sleeper, backOff);
+ } catch (InterruptedException | IOException e) {
+ return false;
+ }
+ }
+
+ /** Returns true iff the {@code statusCode} represents an error that should be retried. */
+ private boolean retryOnStatusCode(int statusCode) {
+ return (statusCode == 0) // Code 0 usually means no response / network error
+ || (statusCode / 100 == 5) // 5xx: server error
+ || statusCode == 429; // 429: Too many requests
}
}
@@ -169,20 +216,20 @@
// TODO: Do this exclusively for work requests.
request.setReadTimeout(HANGING_GET_TIMEOUT_SEC * 1000);
- // Back off on retryable http errors.
- request.setUnsuccessfulResponseHandler(
+ LoggingHttpBackOffHandler loggingHttpBackOffHandler = new LoggingHttpBackOffHandler(
+ sleeper,
+ // Retry immediately on IOExceptions.
+ BackOff.ZERO_BACKOFF,
+ // Back off on retryable http errors.
// A back-off multiplier of 2 raises the maximum request retrying time
// to approximately 5 minutes (keeping other back-off parameters to
// their default values).
- new LoggingHttpBackoffUnsuccessfulResponseHandler(
- new ExponentialBackOff.Builder().setNanoClock(nanoClock)
- .setMultiplier(2).build(),
- sleeper, ignoredResponseCodes));
+ new ExponentialBackOff.Builder().setNanoClock(nanoClock).setMultiplier(2).build(),
+ ignoredResponseCodes
+ );
- // Retry immediately on IOExceptions.
- LoggingHttpBackOffIOExceptionHandler loggingBackoffHandler =
- new LoggingHttpBackOffIOExceptionHandler(BackOff.ZERO_BACKOFF);
- request.setIOExceptionHandler(loggingBackoffHandler);
+ request.setUnsuccessfulResponseHandler(loggingHttpBackOffHandler);
+ request.setIOExceptionHandler(loggingHttpBackOffHandler);
// Set response initializer
if (responseInterceptor != null) {
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
index 625c248..a0d9e4b 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
@@ -15,14 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam;
+package org.apache.beam.sdk.extensions.gcp;
-import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages;
+import static org.apache.beam.sdk.util.ApiSurface.classesInPackage;
+import static org.apache.beam.sdk.util.ApiSurface.containsOnlyClassesMatching;
import static org.hamcrest.MatcherAssert.assertThat;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.apache.beam.sdk.util.ApiSurface;
+import org.hamcrest.Matcher;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -32,28 +34,32 @@
public class GcpCoreApiSurfaceTest {
@Test
- public void testApiSurface() throws Exception {
+ public void testGcpCoreApiSurface() throws Exception {
+ final Package thisPackage = getClass().getPackage();
+ final ClassLoader thisClassLoader = getClass().getClassLoader();
+ final ApiSurface apiSurface =
+ ApiSurface.ofPackage(thisPackage, thisClassLoader)
+ .pruningPattern("org[.]apache[.]beam[.].*Test.*")
+ .pruningPattern("org[.]apache[.]beam[.].*IT")
+ .pruningPattern("java[.]lang.*")
+ .pruningPattern("java[.]util.*");
@SuppressWarnings("unchecked")
- final Set<String> allowed =
+ final Set<Matcher<Class<?>>> allowedClasses =
ImmutableSet.of(
- "org.apache.beam",
- "com.google.api.client",
- "com.google.api.services.storage",
- "com.google.auth",
- "com.fasterxml.jackson.annotation",
- "com.fasterxml.jackson.core",
- "com.fasterxml.jackson.databind",
- "org.apache.avro",
- "org.hamcrest",
- // via DataflowMatchers
- "org.codehaus.jackson",
- // via Avro
- "org.joda.time",
- "org.junit",
- "sun.reflect");
+ classesInPackage("com.google.api.client.googleapis"),
+ classesInPackage("com.google.api.client.http"),
+ classesInPackage("com.google.api.client.json"),
+ classesInPackage("com.google.api.client.util"),
+ classesInPackage("com.google.api.services.storage"),
+ classesInPackage("com.google.auth"),
+ classesInPackage("com.fasterxml.jackson.annotation"),
+ classesInPackage("java"),
+ classesInPackage("javax"),
+ classesInPackage("org.apache.beam.sdk"),
+ classesInPackage("org.joda.time")
+ );
- assertThat(
- ApiSurface.getSdkApiSurface(getClass().getClassLoader()), containsOnlyPackages(allowed));
+ assertThat(apiSurface, containsOnlyClassesMatching(allowedClasses));
}
}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
index 5326450..897cd53 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
@@ -93,11 +93,17 @@
@Test
public void testGlobTranslation() {
assertEquals("foo", GcsUtil.wildcardToRegexp("foo"));
- assertEquals("fo.*o", GcsUtil.wildcardToRegexp("fo*o"));
- assertEquals("f.*o\\.[^/]", GcsUtil.wildcardToRegexp("f*o.?"));
- assertEquals("foo-[0-9].*", GcsUtil.wildcardToRegexp("foo-[0-9]*"));
- assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**/*foo"));
- assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**foo"));
+ assertEquals("fo[^/]*o", GcsUtil.wildcardToRegexp("fo*o"));
+ assertEquals("f[^/]*o\\.[^/]", GcsUtil.wildcardToRegexp("f*o.?"));
+ assertEquals("foo-[0-9][^/]*", GcsUtil.wildcardToRegexp("foo-[0-9]*"));
+ assertEquals("foo-[0-9].*", GcsUtil.wildcardToRegexp("foo-[0-9]**"));
+ assertEquals(".*foo", GcsUtil.wildcardToRegexp("**/*foo"));
+ assertEquals(".*foo", GcsUtil.wildcardToRegexp("**foo"));
+ assertEquals("foo/[^/]*", GcsUtil.wildcardToRegexp("foo/*"));
+ assertEquals("foo[^/]*", GcsUtil.wildcardToRegexp("foo*"));
+ assertEquals("foo/[^/]*/[^/]*/[^/]*", GcsUtil.wildcardToRegexp("foo/*/*/*"));
+ assertEquals("foo/[^/]*/.*", GcsUtil.wildcardToRegexp("foo/*/**"));
+ assertEquals("foo.*baz", GcsUtil.wildcardToRegexp("foo**baz"));
}
private static GcsOptions gcsOptionsWithTestCredential() {
@@ -518,6 +524,51 @@
}
@Test
+ public void testRemoveWhenFileNotFound() throws Exception {
+ JsonFactory jsonFactory = new JacksonFactory();
+
+ String contentBoundary = "batch_foobarbaz";
+ String contentBoundaryLine = "--" + contentBoundary;
+ String endOfContentBoundaryLine = "--" + contentBoundary + "--";
+
+ GenericJson error = new GenericJson()
+ .set("error", new GenericJson().set("code", 404));
+ error.setFactory(jsonFactory);
+
+ String content = contentBoundaryLine + "\n"
+ + "Content-Type: application/http\n"
+ + "\n"
+ + "HTTP/1.1 404 Not Found\n"
+ + "Content-Length: -1\n"
+ + "\n"
+ + error.toString()
+ + "\n"
+ + "\n"
+ + endOfContentBoundaryLine
+ + "\n";
+
+ final LowLevelHttpResponse mockResponse = Mockito.mock(LowLevelHttpResponse.class);
+ when(mockResponse.getContentType()).thenReturn("multipart/mixed; boundary=" + contentBoundary);
+ when(mockResponse.getStatusCode()).thenReturn(200);
+ when(mockResponse.getContent()).thenReturn(toStream(content));
+
+ // A mock transport that lets us mock the API responses.
+ MockLowLevelHttpRequest request = new MockLowLevelHttpRequest() {
+ @Override
+ public LowLevelHttpResponse execute() throws IOException {
+ return mockResponse;
+ }
+ };
+ MockHttpTransport mockTransport =
+ new MockHttpTransport.Builder().setLowLevelHttpRequest(request).build();
+
+ GcsUtil gcsUtil = gcsOptionsWithTestCredential().getGcsUtil();
+ gcsUtil.setStorageClient(
+ new Storage(mockTransport, Transport.getJsonFactory(), new RetryHttpRequestInitializer()));
+ gcsUtil.remove(Arrays.asList("gs://some-bucket/already-deleted"));
+ }
+
+ @Test
public void testCreateBucket() throws IOException {
GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/RetryHttpRequestInitializerTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/RetryHttpRequestInitializerTest.java
index 37551a4..13a9309 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/RetryHttpRequestInitializerTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/RetryHttpRequestInitializerTest.java
@@ -49,10 +49,11 @@
import java.security.PrivateKey;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.beam.sdk.testing.ExpectedLogs;
import org.hamcrest.Matchers;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -67,6 +68,8 @@
@RunWith(JUnit4.class)
public class RetryHttpRequestInitializerTest {
+ @Rule public ExpectedLogs expectedLogs = ExpectedLogs.none(RetryHttpRequestInitializer.class);
+
@Mock private PrivateKey mockPrivateKey;
@Mock private LowLevelHttpRequest mockLowLevelRequest;
@Mock private LowLevelHttpResponse mockLowLevelResponse;
@@ -135,6 +138,7 @@
verify(mockLowLevelRequest).setTimeout(anyInt(), anyInt());
verify(mockLowLevelRequest).execute();
verify(mockLowLevelResponse).getStatusCode();
+ expectedLogs.verifyNotLogged("Request failed");
}
/**
@@ -153,7 +157,7 @@
HttpResponse response = result.executeUnparsed();
assertNotNull(response);
} catch (HttpResponseException e) {
- Assert.assertThat(e.getMessage(), Matchers.containsString("403"));
+ assertThat(e.getMessage(), Matchers.containsString("403"));
}
verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class));
@@ -162,6 +166,7 @@
verify(mockLowLevelRequest).setTimeout(anyInt(), anyInt());
verify(mockLowLevelRequest).execute();
verify(mockLowLevelResponse).getStatusCode();
+ expectedLogs.verifyWarn("Request failed with code 403");
}
/**
@@ -188,6 +193,7 @@
verify(mockLowLevelRequest, times(3)).setTimeout(anyInt(), anyInt());
verify(mockLowLevelRequest, times(3)).execute();
verify(mockLowLevelResponse, times(3)).getStatusCode();
+ expectedLogs.verifyDebug("Request failed with code 503");
}
/**
@@ -211,6 +217,7 @@
verify(mockLowLevelRequest, times(2)).setTimeout(anyInt(), anyInt());
verify(mockLowLevelRequest, times(2)).execute();
verify(mockLowLevelResponse).getStatusCode();
+ expectedLogs.verifyDebug("Request failed with IOException");
}
/**
@@ -224,19 +231,22 @@
int n = 0;
@Override
public Integer answer(InvocationOnMock invocation) {
- return (n++ < retries - 1) ? 503 : 200;
+ return n++ < retries ? 503 : 9999;
}});
Storage.Buckets.Get result = storage.buckets().get("test");
- HttpResponse response = result.executeUnparsed();
- assertNotNull(response);
+ try {
+ result.executeUnparsed();
+ fail();
+ } catch (Throwable t) {
+ }
verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class));
- verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(),
- anyString());
- verify(mockLowLevelRequest, times(retries)).setTimeout(anyInt(), anyInt());
- verify(mockLowLevelRequest, times(retries)).execute();
- verify(mockLowLevelResponse, times(retries)).getStatusCode();
+ verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString());
+ verify(mockLowLevelRequest, times(retries + 1)).setTimeout(anyInt(), anyInt());
+ verify(mockLowLevelRequest, times(retries + 1)).execute();
+ verify(mockLowLevelResponse, times(retries + 1)).getStatusCode();
+ expectedLogs.verifyWarn("performed 10 retries due to unsuccessful status codes");
}
/**
@@ -276,6 +286,7 @@
} catch (Throwable e) {
assertThat(e, Matchers.<Throwable>instanceOf(SocketTimeoutException.class));
assertEquals(1 + defaultNumberOfRetries, executeCount.get());
+ expectedLogs.verifyWarn("performed 10 retries due to IOExceptions");
}
}
}
diff --git a/sdks/java/extensions/jackson/pom.xml b/sdks/java/extensions/jackson/pom.xml
index 4b09c11..844a092 100644
--- a/sdks/java/extensions/jackson/pom.xml
+++ b/sdks/java/extensions/jackson/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/extensions/join-library/pom.xml b/sdks/java/extensions/join-library/pom.xml
index 556ec40..15954e6 100644
--- a/sdks/java/extensions/join-library/pom.xml
+++ b/sdks/java/extensions/join-library/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java b/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java
index f4e6ccb..9acb048 100644
--- a/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java
+++ b/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java
@@ -141,7 +141,7 @@
* @param <V1> Type of the values for the left collection.
* @param <V2> Type of the values for the right collection.
* @return A joined collection of KV where Key is the key and value is a
- * KV where Key is of type V1 and Value is type V2. Keys that
+ * KV where Key is of type V1 and Value is type V2. Values that
* should be null or empty is replaced with nullValue.
*/
public static <K, V1, V2> PCollection<KV<K, KV<V1, V2>>> rightOuterJoin(
@@ -184,4 +184,67 @@
KvCoder.of(((KvCoder) leftCollection.getCoder()).getValueCoder(),
((KvCoder) rightCollection.getCoder()).getValueCoder())));
}
+
+ /**
+ * Full Outer Join of two collections of KV elements.
+ * @param leftCollection Left side collection to join.
+ * @param rightCollection Right side collection to join.
+ * @param leftNullValue Value to use as null value when left side do not match right side.
+ * @param rightNullValue Value to use as null value when right side do not match right side.
+ * @param <K> Type of the key for both collections
+ * @param <V1> Type of the values for the left collection.
+ * @param <V2> Type of the values for the right collection.
+ * @return A joined collection of KV where Key is the key and value is a
+ * KV where Key is of type V1 and Value is type V2. Values that
+ * should be null or empty is replaced with leftNullValue/rightNullValue.
+ */
+ public static <K, V1, V2> PCollection<KV<K, KV<V1, V2>>> fullOuterJoin(
+ final PCollection<KV<K, V1>> leftCollection,
+ final PCollection<KV<K, V2>> rightCollection,
+ final V1 leftNullValue, final V2 rightNullValue) {
+ checkNotNull(leftCollection);
+ checkNotNull(rightCollection);
+ checkNotNull(leftNullValue);
+ checkNotNull(rightNullValue);
+
+ final TupleTag<V1> v1Tuple = new TupleTag<>();
+ final TupleTag<V2> v2Tuple = new TupleTag<>();
+
+ PCollection<KV<K, CoGbkResult>> coGbkResultCollection =
+ KeyedPCollectionTuple.of(v1Tuple, leftCollection)
+ .and(v2Tuple, rightCollection)
+ .apply(CoGroupByKey.<K>create());
+
+ return coGbkResultCollection.apply(ParDo.of(
+ new DoFn<KV<K, CoGbkResult>, KV<K, KV<V1, V2>>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ KV<K, CoGbkResult> e = c.element();
+
+ Iterable<V1> leftValuesIterable = e.getValue().getAll(v1Tuple);
+ Iterable<V2> rightValuesIterable = e.getValue().getAll(v2Tuple);
+ if (leftValuesIterable.iterator().hasNext()
+ && rightValuesIterable.iterator().hasNext()) {
+ for (V2 rightValue : rightValuesIterable) {
+ for (V1 leftValue : leftValuesIterable) {
+ c.output(KV.of(e.getKey(), KV.of(leftValue, rightValue)));
+ }
+ }
+ } else if (leftValuesIterable.iterator().hasNext()
+ && !rightValuesIterable.iterator().hasNext()) {
+ for (V1 leftValue : leftValuesIterable) {
+ c.output(KV.of(e.getKey(), KV.of(leftValue, rightNullValue)));
+ }
+ } else if (!leftValuesIterable.iterator().hasNext()
+ && rightValuesIterable.iterator().hasNext()) {
+ for (V2 rightValue : rightValuesIterable) {
+ c.output(KV.of(e.getKey(), KV.of(leftNullValue, rightValue)));
+ }
+ }
+ }
+ }))
+ .setCoder(KvCoder.of(((KvCoder) leftCollection.getCoder()).getKeyCoder(),
+ KvCoder.of(((KvCoder) leftCollection.getCoder()).getValueCoder(),
+ ((KvCoder) rightCollection.getCoder()).getValueCoder())));
+ }
}
diff --git a/sdks/java/extensions/join-library/src/test/java/org/apache/beam/sdk/extensions/joinlibrary/OuterFullJoinTest.java b/sdks/java/extensions/join-library/src/test/java/org/apache/beam/sdk/extensions/joinlibrary/OuterFullJoinTest.java
new file mode 100644
index 0000000..cdf4f4f
--- /dev/null
+++ b/sdks/java/extensions/join-library/src/test/java/org/apache/beam/sdk/extensions/joinlibrary/OuterFullJoinTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.joinlibrary;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * This test Outer Full Join functionality.
+ */
+public class OuterFullJoinTest {
+
+ List<KV<String, Long>> leftListOfKv;
+ List<KV<String, String>> listRightOfKv;
+ List<KV<String, KV<Long, String>>> expectedResult;
+
+ @Rule
+ public final transient TestPipeline p = TestPipeline.create();
+
+ @Before
+ public void setup() {
+
+ leftListOfKv = new ArrayList<>();
+ listRightOfKv = new ArrayList<>();
+
+ expectedResult = new ArrayList<>();
+ }
+
+ @Test
+ public void testJoinOneToOneMapping() {
+ leftListOfKv.add(KV.of("Key1", 5L));
+ leftListOfKv.add(KV.of("Key2", 4L));
+ PCollection<KV<String, Long>> leftCollection = p
+ .apply("CreateLeft", Create.of(leftListOfKv));
+
+ listRightOfKv.add(KV.of("Key1", "foo"));
+ listRightOfKv.add(KV.of("Key2", "bar"));
+ PCollection<KV<String, String>> rightCollection = p
+ .apply("CreateRight", Create.of(listRightOfKv));
+
+ PCollection<KV<String, KV<Long, String>>> output = Join.fullOuterJoin(
+ leftCollection, rightCollection, -1L, "");
+
+ expectedResult.add(KV.of("Key1", KV.of(5L, "foo")));
+ expectedResult.add(KV.of("Key2", KV.of(4L, "bar")));
+ PAssert.that(output).containsInAnyOrder(expectedResult);
+
+ p.run();
+ }
+
+ @Test
+ public void testJoinOneToManyMapping() {
+ leftListOfKv.add(KV.of("Key2", 4L));
+ PCollection<KV<String, Long>> leftCollection = p
+ .apply("CreateLeft", Create.of(leftListOfKv));
+
+ listRightOfKv.add(KV.of("Key2", "bar"));
+ listRightOfKv.add(KV.of("Key2", "gazonk"));
+ PCollection<KV<String, String>> rightCollection = p
+ .apply("CreateRight", Create.of(listRightOfKv));
+
+ PCollection<KV<String, KV<Long, String>>> output = Join.fullOuterJoin(
+ leftCollection, rightCollection, -1L, "");
+
+ expectedResult.add(KV.of("Key2", KV.of(4L, "bar")));
+ expectedResult.add(KV.of("Key2", KV.of(4L, "gazonk")));
+ PAssert.that(output).containsInAnyOrder(expectedResult);
+
+ p.run();
+ }
+
+ @Test
+ public void testJoinManyToOneMapping() {
+ leftListOfKv.add(KV.of("Key2", 4L));
+ leftListOfKv.add(KV.of("Key2", 6L));
+ PCollection<KV<String, Long>> leftCollection = p
+ .apply("CreateLeft", Create.of(leftListOfKv));
+
+ listRightOfKv.add(KV.of("Key2", "bar"));
+ PCollection<KV<String, String>> rightCollection = p
+ .apply("CreateRight", Create.of(listRightOfKv));
+
+ PCollection<KV<String, KV<Long, String>>> output = Join.fullOuterJoin(
+ leftCollection, rightCollection, -1L, "");
+
+ expectedResult.add(KV.of("Key2", KV.of(4L, "bar")));
+ expectedResult.add(KV.of("Key2", KV.of(6L, "bar")));
+ PAssert.that(output).containsInAnyOrder(expectedResult);
+
+ p.run();
+ }
+
+ @Test
+ public void testJoinNoneToNoneMapping() {
+ leftListOfKv.add(KV.of("Key2", 4L));
+ PCollection<KV<String, Long>> leftCollection = p
+ .apply("CreateLeft", Create.of(leftListOfKv));
+
+ listRightOfKv.add(KV.of("Key3", "bar"));
+ PCollection<KV<String, String>> rightCollection = p
+ .apply("CreateRight", Create.of(listRightOfKv));
+
+ PCollection<KV<String, KV<Long, String>>> output = Join.fullOuterJoin(
+ leftCollection, rightCollection, -1L, "");
+
+ expectedResult.add(KV.of("Key2", KV.of(4L, "")));
+ expectedResult.add(KV.of("Key3", KV.of(-1L, "bar")));
+ PAssert.that(output).containsInAnyOrder(expectedResult);
+ p.run();
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJoinLeftCollectionNull() {
+ p.enableAbandonedNodeEnforcement(false);
+ Join.fullOuterJoin(
+ null,
+ p.apply(
+ Create.of(listRightOfKv)
+ .withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))),
+ "", "");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJoinRightCollectionNull() {
+ p.enableAbandonedNodeEnforcement(false);
+ Join.fullOuterJoin(
+ p.apply(
+ Create.of(leftListOfKv).withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))),
+ null,
+ -1L, -1L);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJoinLeftNullValueIsNull() {
+ p.enableAbandonedNodeEnforcement(false);
+ Join.fullOuterJoin(
+ p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))),
+ p.apply(
+ "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))),
+ null, "");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJoinRightNullValueIsNull() {
+ p.enableAbandonedNodeEnforcement(false);
+ Join.fullOuterJoin(
+ p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))),
+ p.apply(
+ "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))),
+ -1L, null);
+ }
+}
diff --git a/sdks/java/extensions/pom.xml b/sdks/java/extensions/pom.xml
index 3d63626..5e8d495 100644
--- a/sdks/java/extensions/pom.xml
+++ b/sdks/java/extensions/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -36,7 +36,9 @@
<module>jackson</module>
<module>join-library</module>
<module>protobuf</module>
+ <module>sketching</module>
<module>sorter</module>
+ <module>sql</module>
</modules>
</project>
diff --git a/sdks/java/extensions/protobuf/pom.xml b/sdks/java/extensions/protobuf/pom.xml
index ae909ab..2316571 100644
--- a/sdks/java/extensions/protobuf/pom.xml
+++ b/sdks/java/extensions/protobuf/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/extensions/sketching/pom.xml b/sdks/java/extensions/sketching/pom.xml
new file mode 100755
index 0000000..f0538ae
--- /dev/null
+++ b/sdks/java/extensions/sketching/pom.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-extensions-sketching</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: Extensions :: Sketching</name>
+
+ <properties>
+ <streamlib.version>2.9.5</streamlib.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ <version>${streamlib.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test dependencies -->
+ <!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
diff --git a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
new file mode 100644
index 0000000..1da0cc3
--- /dev/null
+++ b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sketching;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import com.google.auto.value.AutoValue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * {@link PTransform}s for computing the approximate number of distinct elements in a stream.
+ *
+ * <p>This class relies on the HyperLogLog algorithm, and more precisely HyperLogLog+, the improved
+ * version of Google.
+ *
+ * <h2>References</h2>
+ *
+ * <p>The implementation comes from <a href="https://github.com/addthis/stream-lib">Addthis'
+ * Stream-lib library</a>. <br>
+ * The original paper of the HyperLogLog is available <a
+ * href="http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf">here</a>. <br>
+ * A paper from the same authors to have a clearer view of the algorithm is available <a
+ * href="http://cscubs.cs.uni-bonn.de/2016/proceedings/paper-03.pdf">here</a>. <br>
+ * Google's HyperLogLog+ version is detailed in <a
+ * href="https://research.google.com/pubs/pub40671.html">this paper</a>.
+ *
+ * <h2>Parameters</h2>
+ *
+ * <p>Two parameters can be tuned in order to control the computation's accuracy:
+ *
+ * <ul>
+ * <li><b>Precision: {@code p}</b> <br>
+ * Controls the accuracy of the estimation. The precision value will have an impact on the
+ * number of buckets used to store information about the distinct elements. <br>
+ * In general one can expect a relative error of about {@code 1.1 / sqrt(2^p)}. The value
+ * should be of at least 4 to guarantee a minimal accuracy. <br>
+ * By default, the precision is set to {@code 12} for a relative error of around {@code 2%}.
+ * <li><b>Sparse Precision: {@code sp}</b> <br>
+ * Used to create a sparse representation in order to optimize memory and improve accuracy at
+ * small cardinalities. <br>
+ * The value of {@code sp} should be greater than {@code p}, but lower than 32. <br>
+ * By default, the sparse representation is not used ({@code sp = 0}). One should use it if
+ * the cardinality may be less than {@code 12000}.
+ * </ul>
+ *
+ * <h2>Examples</h2>
+ *
+ * <p>There are 2 ways of using this class:
+ *
+ * <ul>
+ * <li>Use the {@link PTransform}s that return {@code PCollection<Long>} corresponding to the
+ * estimate number of distinct elements in the input {@link PCollection} of objects or for
+ * each key in a {@link PCollection} of {@link KV}s.
+ * <li>Use the {@link ApproximateDistinctFn} {@code CombineFn} that is exposed in order to make
+ * advanced processing involving the {@link HyperLogLogPlus} structure which resumes the
+ * stream.
+ * </ul>
+ *
+ * <h3>Using the Transforms</h3>
+ *
+ * <h4>Example 1: globally default use</h4>
+ *
+ * <pre>{@code
+ * PCollection<Integer> input = ...;
+ * PCollection<Long> hllSketch = input.apply(ApproximateDistinct.<Integer>globally());
+ * }</pre>
+ *
+ * <h4>Example 2: per key default use</h4>
+ *
+ * <pre>{@code
+ * PCollection<Integer, String> input = ...;
+ * PCollection<Integer, Long> hllSketches = input.apply(ApproximateDistinct
+ * .<Integer, String>perKey());
+ * }</pre>
+ *
+ * <h4>Example 3: tune precision and use sparse representation</h4>
+ *
+ * <p>One can tune the precision and sparse precision parameters in order to control the accuracy
+ * and the memory. The tuning works exactly the same for {@link #globally()} and {@link #perKey()}.
+ *
+ * <pre>{@code
+ * int precision = 15;
+ * int sparsePrecision = 25;
+ * PCollection<Double> input = ...;
+ * PCollection<Long> hllSketch = input.apply(ApproximateDistinct
+ * .<Double>globally()
+ * .withPrecision(precision)
+ * .withSparsePrecision(sparsePrecision));
+ * }</pre>
+ *
+ * <h3>Using the {@link ApproximateDistinctFn} CombineFn</h3>
+ *
+ * <p>The CombineFn does the same thing as the transform but it can be used in cases where you want
+ * to manipulate the {@link HyperLogLogPlus} sketch, for example if you want to store it in a
+ * database to have a backup. It can also be used in stateful processing or in {@link
+ * org.apache.beam.sdk.transforms.CombineFns.ComposedCombineFn}.
+ *
+ * <h4>Example 1: basic use</h4>
+ *
+ * <p>This example is not really interesting but show how you can properly create an {@link
+ * ApproximateDistinctFn}. One must always specify a coder using the {@link
+ * ApproximateDistinctFn#create(Coder)} method.
+ *
+ * <pre>{@code
+ * PCollection<Integer> input = ...;
+ * PCollection<HyperLogLogPlus> output = input.apply(Combine.globally(ApproximateDistinctFn
+ * .<Integer>create(BigEndianIntegerCoder.of()));
+ * }</pre>
+ *
+ * <h4>Example 2: use the {@link CombineFn} in a stateful {@link ParDo}</h4>
+ *
+ * <p>One may want to use the {@link ApproximateDistinctFn} in a stateful ParDo in order to make
+ * some processing depending on the current cardinality of the stream. <br>
+ * For more information about stateful processing see the blog spot on this topic <a
+ * href="https://beam.apache.org/blog/2017/02/13/stateful-processing.html">here</a>.
+ *
+ * <p>Here is an example of {@link DoFn} using an {@link ApproximateDistinctFn} as a {@link
+ * org.apache.beam.sdk.state.CombiningState}:
+ *
+ * <pre><code>
+ * {@literal class StatefulCardinality<V> extends DoFn<V, OutputT>} {
+ * {@literal @StateId}("hyperloglog")
+ * {@literal private final StateSpec<CombiningState<V, HyperLogLogPlus, HyperLogLogPlus>>}
+ * indexSpec;
+ *
+ * {@literal public StatefulCardinality(ApproximateDistinctFn<V> fn)} {
+ * indexSpec = StateSpecs.combining(fn);
+ * }
+ *
+ * {@literal @ProcessElement}
+ * public void processElement(
+ * ProcessContext context,
+ * {@literal @StateId}("hllSketch")
+ * {@literal CombiningState<V, HyperLogLogPlus, HyperLogLogPlus> hllSketch)} {
+ * long current = MoreObjects.firstNonNull(hllSketch.getAccum().cardinality(), 0L);
+ * hllSketch.add(context.element());
+ * context.output(...);
+ * }
+ * }
+ * </code></pre>
+ *
+ * <p>Then the {@link DoFn} can be called like this:
+ *
+ * <pre>{@code
+ * PCollection<V> input = ...;
+ * ApproximateDistinctFn<V> myFn = ApproximateDistinctFn.create(input.getCoder());
+ * PCollection<V> = input.apply(ParDo.of(new StatefulCardinality<>(myFn)));
+ * }</pre>
+ *
+ * <h4>Example 3: use the {@link RetrieveCardinality} utility class</h4>
+ *
+ * <p>One may want to retrieve the cardinality as a long after making some advanced processing using
+ * the {@link HyperLogLogPlus} structure. <br>
+ * The {@link RetrieveCardinality} utility class provides an easy way to do so:
+ *
+ * <pre>{@code
+ * PCollection<MyObject> input = ...;
+ * PCollection<HyperLogLogPlus> hll = input.apply(Combine.globally(ApproximateDistinctFn
+ * .<MyObject>create(new MyObjectCoder())
+ * .withSparseRepresentation(20)));
+ *
+ * // Some advanced processing
+ * PCollection<SomeObject> advancedResult = hll.apply(...);
+ *
+ * PCollection<Long> cardinality = hll.apply(ApproximateDistinct.RetrieveCardinality.globally());
+ *
+ * }</pre>
+ *
+ * <p><b>Warning: this class is experimental.</b> Its API is subject to change in future versions of
+ * Beam. For example, it may be merged with the {@link
+ * org.apache.beam.sdk.transforms.ApproximateUnique} transform.
+ */
+@Experimental
+public final class ApproximateDistinct {
+
+ /**
+ * Computes the approximate number of distinct elements in the input {@code PCollection<InputT>}
+ * and returns a {@code PCollection<Long>}.
+ *
+ * @param <InputT> the type of the elements in the input {@link PCollection}
+ */
+ public static <InputT> GloballyDistinct<InputT> globally() {
+ return GloballyDistinct.<InputT>builder().build();
+ }
+
+ /**
+ * Like {@link #globally} but per key, i.e computes the approximate number of distinct values per
+ * key in a {@code PCollection<KV<K, V>>} and returns {@code PCollection<KV<K, Long>>}.
+ *
+ * @param <K> type of the keys mapping the elements
+ * @param <V> type of the values being combined per key
+ */
+ public static <K, V> PerKeyDistinct<K, V> perKey() {
+ return PerKeyDistinct.<K, V>builder().build();
+ }
+
+ /**
+ * Implementation of {@link #globally()}.
+ *
+ * @param <InputT>
+ */
+ @AutoValue
+ public abstract static class GloballyDistinct<InputT>
+ extends PTransform<PCollection<InputT>, PCollection<Long>> {
+
+ abstract int precision();
+
+ abstract int sparsePrecision();
+
+ abstract Builder<InputT> toBuilder();
+
+ static <InputT> Builder<InputT> builder() {
+ return new AutoValue_ApproximateDistinct_GloballyDistinct.Builder<InputT>()
+ .setPrecision(12)
+ .setSparsePrecision(0);
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder<InputT> {
+ abstract Builder<InputT> setPrecision(int p);
+
+ abstract Builder<InputT> setSparsePrecision(int sp);
+
+ abstract GloballyDistinct<InputT> build();
+ }
+
+ public GloballyDistinct<InputT> withPrecision(int p) {
+ return toBuilder().setPrecision(p).build();
+ }
+
+ public GloballyDistinct<InputT> withSparsePrecision(int sp) {
+ return toBuilder().setSparsePrecision(sp).build();
+ }
+
+ @Override
+ public PCollection<Long> expand(PCollection<InputT> input) {
+ return input
+ .apply(
+ "Compute HyperLogLog Structure",
+ Combine.globally(
+ ApproximateDistinctFn.<InputT>create(input.getCoder())
+ .withPrecision(this.precision())
+ .withSparseRepresentation(this.sparsePrecision())))
+ .apply("Retrieve Cardinality", ParDo.of(RetrieveCardinality.globally()));
+ }
+ }
+
+ /**
+ * Implementation of {@link #perKey()}.
+ *
+ * @param <K>
+ * @param <V>
+ */
+ @AutoValue
+ public abstract static class PerKeyDistinct<K, V>
+ extends PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Long>>> {
+
+ abstract int precision();
+
+ abstract int sparsePrecision();
+
+ abstract Builder<K, V> toBuilder();
+
+ static <K, V> Builder<K, V> builder() {
+ return new AutoValue_ApproximateDistinct_PerKeyDistinct.Builder<K, V>()
+ .setPrecision(12)
+ .setSparsePrecision(0);
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder<K, V> {
+ abstract Builder<K, V> setPrecision(int p);
+
+ abstract Builder<K, V> setSparsePrecision(int sp);
+
+ abstract PerKeyDistinct<K, V> build();
+ }
+
+ public PerKeyDistinct<K, V> withPrecision(int p) {
+ return toBuilder().setPrecision(p).build();
+ }
+
+ public PerKeyDistinct<K, V> withSparsePrecision(int sp) {
+ return toBuilder().setSparsePrecision(sp).build();
+ }
+
+ @Override
+ public PCollection<KV<K, Long>> expand(PCollection<KV<K, V>> input) {
+ KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();
+ return input
+ .apply(
+ Combine.<K, V, HyperLogLogPlus>perKey(
+ ApproximateDistinctFn.<V>create(inputCoder.getValueCoder())
+ .withPrecision(this.precision())
+ .withSparseRepresentation(this.sparsePrecision())))
+ .apply("Retrieve Cardinality", ParDo.of(RetrieveCardinality.<K>perKey()));
+ }
+ }
+
+ /**
+ * Implements the {@link CombineFn} of {@link ApproximateDistinct} transforms.
+ *
+ * @param <InputT> the type of the elements in the input {@link PCollection}
+ */
+ public static class ApproximateDistinctFn<InputT>
+ extends CombineFn<InputT, HyperLogLogPlus, HyperLogLogPlus> {
+
+ private final int p;
+
+ private final int sp;
+
+ private final Coder<InputT> inputCoder;
+
+ private ApproximateDistinctFn(int p, int sp, Coder<InputT> coder) {
+ this.p = p;
+ this.sp = sp;
+ inputCoder = coder;
+ }
+
+ /**
+ * Returns an {@link ApproximateDistinctFn} combiner with the given input coder.
+ *
+ * @param coder the coder that encodes the elements' type
+ */
+ public static <InputT> ApproximateDistinctFn<InputT> create(Coder<InputT> coder) {
+ try {
+ coder.verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ throw new IllegalArgumentException("Coder is not deterministic ! " + e.getMessage(), e);
+ }
+ return new ApproximateDistinctFn<>(12, 0, coder);
+ }
+
+ /**
+ * Returns a new {@link ApproximateDistinctFn} combiner with a new precision {@code p}.
+ *
+ * <p>Keep in mind that {@code p} cannot be lower than 4, because the estimation would be too
+ * inaccurate.
+ *
+ * <p>See {@link ApproximateDistinct#precisionForRelativeError(double)} and {@link
+ * ApproximateDistinct#relativeErrorForPrecision(int)} to have more information about the
+ * relationship between precision and relative error.
+ *
+ * @param p the precision value for the normal representation
+ */
+ public ApproximateDistinctFn<InputT> withPrecision(int p) {
+ checkArgument(p >= 4, "Expected: p >= 4. Actual: p = %s", p);
+ return new ApproximateDistinctFn<>(p, this.sp, this.inputCoder);
+ }
+
+ /**
+ * Returns a new {@link ApproximateDistinctFn} combiner with a sparse representation of
+ * precision {@code sp}.
+ *
+ * <p>Values above 32 are not yet supported by the AddThis version of HyperLogLog+.
+ *
+ * <p>Fore more information about the sparse representation, read Google's paper available <a
+ * href="https://research.google.com/pubs/pub40671.html">here</a>.
+ *
+ * @param sp the precision of HyperLogLog+' sparse representation
+ */
+ public ApproximateDistinctFn<InputT> withSparseRepresentation(int sp) {
+ checkArgument(
+ (sp > this.p && sp < 32) || (sp == 0),
+ "Expected: p <= sp <= 32." + "Actual: p = %s, sp = %s",
+ this.p,
+ sp);
+ return new ApproximateDistinctFn<>(this.p, sp, this.inputCoder);
+ }
+
+ @Override
+ public HyperLogLogPlus createAccumulator() {
+ return new HyperLogLogPlus(p, sp);
+ }
+
+ @Override
+ public HyperLogLogPlus addInput(HyperLogLogPlus acc, InputT record) {
+ try {
+ acc.offer(CoderUtils.encodeToByteArray(inputCoder, record));
+ } catch (CoderException e) {
+ throw new IllegalStateException("The input value cannot be encoded: " + e.getMessage(), e);
+ }
+ return acc;
+ }
+
+ /** Output the whole structure so it can be queried, reused or stored easily. */
+ @Override
+ public HyperLogLogPlus extractOutput(HyperLogLogPlus accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public HyperLogLogPlus mergeAccumulators(Iterable<HyperLogLogPlus> accumulators) {
+ HyperLogLogPlus mergedAccum = createAccumulator();
+ for (HyperLogLogPlus accum : accumulators) {
+ try {
+ mergedAccum.addAll(accum);
+ } catch (CardinalityMergeException e) {
+ // Should never happen because only HyperLogLogPlus accumulators are instantiated.
+ throw new IllegalStateException(
+ "The accumulators cannot be merged: " + e.getMessage(), e);
+ }
+ }
+ return mergedAccum;
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder
+ .add(DisplayData.item("p", p).withLabel("precision"))
+ .add(DisplayData.item("sp", sp).withLabel("sparse representation precision"));
+ }
+ }
+
+ /** Coder for {@link HyperLogLogPlus} class. */
+ public static class HyperLogLogPlusCoder extends CustomCoder<HyperLogLogPlus> {
+
+ private static final HyperLogLogPlusCoder INSTANCE = new HyperLogLogPlusCoder();
+
+ private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
+
+ public static HyperLogLogPlusCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(HyperLogLogPlus value, OutputStream outStream) throws IOException {
+ if (value == null) {
+ throw new CoderException("cannot encode a null HyperLogLogPlus sketch");
+ }
+ BYTE_ARRAY_CODER.encode(value.getBytes(), outStream);
+ }
+
+ @Override
+ public HyperLogLogPlus decode(InputStream inStream) throws IOException {
+ return HyperLogLogPlus.Builder.build(BYTE_ARRAY_CODER.decode(inStream));
+ }
+
+ @Override
+ public boolean isRegisterByteSizeObserverCheap(HyperLogLogPlus value) {
+ return true;
+ }
+
+ @Override
+ protected long getEncodedElementByteSize(HyperLogLogPlus value) throws IOException {
+ if (value == null) {
+ throw new CoderException("cannot encode a null HyperLogLogPlus sketch");
+ }
+ return value.sizeof();
+ }
+ }
+
+ /**
+ * Utility class that provides {@link DoFn}s to retrieve the cardinality from a {@link
+ * HyperLogLogPlus} structure in a global or perKey context.
+ */
+ public static class RetrieveCardinality {
+
+ public static <K> DoFn<KV<K, HyperLogLogPlus>, KV<K, Long>> perKey() {
+ return new DoFn<KV<K, HyperLogLogPlus>, KV<K, Long>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ KV<K, HyperLogLogPlus> kv = c.element();
+ c.output(KV.of(kv.getKey(), kv.getValue().cardinality()));
+ }
+ };
+ }
+
+ public static DoFn<HyperLogLogPlus, Long> globally() {
+ return new DoFn<HyperLogLogPlus, Long>() {
+ @ProcessElement
+ public void apply(ProcessContext c) {
+ c.output(c.element().cardinality());
+ }
+ };
+ }
+ }
+
+ /**
+ * Computes the precision based on the desired relative error.
+ *
+ * <p>According to the paper, the mean squared error is bounded by the following formula:
+ *
+ * <pre>b(m) / sqrt(m)
+ * Where m is the number of buckets used ({@code p = log2(m)})
+ * and {@code b(m) < 1.106} for {@code m > 16 (and p > 4)}.
+ * </pre>
+ *
+ * <br>
+ * <b>WARNING:</b> <br>
+ * This does not mean relative error in the estimation <b>can't</b> be higher. <br>
+ * This only means that on average the relative error will be lower than the desired relative
+ * error. <br>
+ * Nevertheless, the more elements arrive in the {@link PCollection}, the lower the variation will
+ * be. <br>
+ * Indeed, this is like when you throw a dice millions of time: the relative frequency of each
+ * different result <code>{1,2,3,4,5,6}</code> will get closer to {@code 1/6}.
+ *
+ * @param relativeError the mean squared error should be in the interval ]0,1]
+ * @return the minimum precision p in order to have the desired relative error on average.
+ */
+ public static long precisionForRelativeError(double relativeError) {
+ return Math.round(
+ Math.ceil(Math.log(Math.pow(1.106, 2.0) / Math.pow(relativeError, 2.0)) / Math.log(2)));
+ }
+
+ /**
+ * @param p the precision i.e. the number of bits used for indexing the buckets
+ * @return the Mean squared error of the Estimation of cardinality to expect for the given value
+ * of p.
+ */
+ public static double relativeErrorForPrecision(int p) {
+ if (p < 4) {
+ return 1.0;
+ }
+ double betaM;
+ switch (p) {
+ case 4:
+ betaM = 1.156;
+ break;
+ case 5:
+ betaM = 1.2;
+ break;
+ case 6:
+ betaM = 1.104;
+ break;
+ case 7:
+ betaM = 1.096;
+ break;
+ default:
+ betaM = 1.05;
+ break;
+ }
+ return betaM / Math.sqrt(Math.exp(p * Math.log(2)));
+ }
+}
diff --git a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/package-info.java b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/package-info.java
new file mode 100755
index 0000000..2e8d60e
--- /dev/null
+++ b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for computing statistical indicators using probabilistic sketches.
+ */
+package org.apache.beam.sdk.extensions.sketching;
diff --git a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
new file mode 100644
index 0000000..cdbcc45
--- /dev/null
+++ b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sketching;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.extensions.sketching.ApproximateDistinct.ApproximateDistinctFn;
+import org.apache.beam.sdk.testing.CoderProperties;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Tests for {@link ApproximateDistinct}. */
+@RunWith(JUnit4.class)
+public class ApproximateDistinctTest implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ApproximateDistinctTest.class);
+
+ @Rule public final transient TestPipeline tp = TestPipeline.create();
+
+ @Test
+ public void smallCardinality() {
+ final int smallCard = 1000;
+ final int p = 6;
+ final double expectedErr = 1.104 / Math.sqrt(p);
+
+ List<Integer> small = new ArrayList<>();
+ for (int i = 0; i < smallCard; i++) {
+ small.add(i);
+ }
+
+ PCollection<Long> cardinality =
+ tp.apply("small stream", Create.<Integer>of(small))
+ .apply("small cardinality", ApproximateDistinct.<Integer>globally().withPrecision(p));
+
+ PAssert.that("Not Accurate Enough", cardinality)
+ .satisfies(new VerifyAccuracy(smallCard, expectedErr));
+
+ tp.run();
+ }
+
+ @Test
+ public void bigCardinality() {
+ final int cardinality = 15000;
+ final int p = 15;
+ final int sp = 20;
+ final double expectedErr = 1.04 / Math.sqrt(p);
+
+ List<Integer> stream = new ArrayList<>();
+ for (int i = 1; i <= cardinality; i++) {
+ stream.addAll(Collections.nCopies(2, i));
+ }
+ Collections.shuffle(stream);
+
+ PCollection<Long> res =
+ tp.apply("big stream", Create.<Integer>of(stream))
+ .apply(
+ "big cardinality",
+ ApproximateDistinct.<Integer>globally().withPrecision(p).withSparsePrecision(sp));
+
+ PAssert.that("Verify Accuracy for big cardinality", res)
+ .satisfies(new VerifyAccuracy(cardinality, expectedErr));
+
+ tp.run();
+ }
+
+ @Test
+ public void perKey() {
+ final int cardinality = 1000;
+ final int p = 15;
+ final double expectedErr = 1.04 / Math.sqrt(p);
+
+ List<Integer> stream = new ArrayList<>();
+ for (int i = 1; i <= cardinality; i++) {
+ stream.addAll(Collections.nCopies(2, i));
+ }
+ Collections.shuffle(stream);
+
+ PCollection<Long> results =
+ tp.apply("per key stream", Create.of(stream))
+ .apply("create keys", WithKeys.<Integer, Integer>of(1))
+ .apply(
+ "per key cardinality",
+ ApproximateDistinct.<Integer, Integer>perKey().withPrecision(p))
+ .apply("extract values", Values.<Long>create());
+
+ PAssert.that("Verify Accuracy for cardinality per key", results)
+ .satisfies(new VerifyAccuracy(cardinality, expectedErr));
+
+ tp.run();
+ }
+
+ @Test
+ public void customObject() {
+ final int cardinality = 500;
+ final int p = 15;
+ final double expectedErr = 1.04 / Math.sqrt(p);
+
+ Schema schema =
+ SchemaBuilder.record("User")
+ .fields()
+ .requiredString("Pseudo")
+ .requiredInt("Age")
+ .endRecord();
+ List<GenericRecord> users = new ArrayList<>();
+ for (int i = 1; i <= cardinality; i++) {
+ GenericData.Record newRecord = new GenericData.Record(schema);
+ newRecord.put("Pseudo", "User" + i);
+ newRecord.put("Age", i);
+ users.add(newRecord);
+ }
+ PCollection<Long> results =
+ tp.apply("Create stream", Create.of(users).withCoder(AvroCoder.of(schema)))
+ .apply(
+ "Test custom object",
+ ApproximateDistinct.<GenericRecord>globally().withPrecision(p));
+
+ PAssert.that("Verify Accuracy for custom object", results)
+ .satisfies(new VerifyAccuracy(cardinality, expectedErr));
+
+ tp.run();
+ }
+
+ @Test
+ public void testCoder() throws Exception {
+ HyperLogLogPlus hllp = new HyperLogLogPlus(12, 18);
+ for (int i = 0; i < 10; i++) {
+ hllp.offer(i);
+ }
+ CoderProperties.<HyperLogLogPlus>coderDecodeEncodeEqual(
+ ApproximateDistinct.HyperLogLogPlusCoder.of(), hllp);
+ }
+
+ @Test
+ public void testDisplayData() {
+ final ApproximateDistinctFn<Integer> fnWithPrecision =
+ ApproximateDistinctFn.create(BigEndianIntegerCoder.of()).withPrecision(23);
+
+ assertThat(DisplayData.from(fnWithPrecision), hasDisplayItem("p", 23));
+ assertThat(DisplayData.from(fnWithPrecision), hasDisplayItem("sp", 0));
+ }
+
+ class VerifyAccuracy implements SerializableFunction<Iterable<Long>, Void> {
+
+ private final int expectedCard;
+
+ private final double expectedError;
+
+ VerifyAccuracy(int expectedCard, double expectedError) {
+ this.expectedCard = expectedCard;
+ this.expectedError = expectedError;
+ }
+
+ @Override
+ public Void apply(Iterable<Long> input) {
+ for (Long estimate : input) {
+ boolean isAccurate = Math.abs(estimate - expectedCard) / expectedCard < expectedError;
+ Assert.assertTrue(
+ "not accurate enough : \nExpected Cardinality : "
+ + expectedCard
+ + "\nComputed Cardinality : "
+ + estimate,
+ isAccurate);
+ }
+ return null;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sorter/pom.xml b/sdks/java/extensions/sorter/pom.xml
index 9d25f9d..b5e1a51 100644
--- a/sdks/java/extensions/sorter/pom.xml
+++ b/sdks/java/extensions/sorter/pom.xml
@@ -22,17 +22,13 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>beam-sdks-java-extensions-sorter</artifactId>
<name>Apache Beam :: SDKs :: Java :: Extensions :: Sorter</name>
- <properties>
- <hadoop.version>2.7.1</hadoop.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
@@ -42,14 +38,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/SortValues.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/SortValues.java
index d1b4d07..cb9d984 100644
--- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/SortValues.java
+++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/SortValues.java
@@ -76,18 +76,14 @@
@Override
public PCollection<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> expand(
PCollection<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> input) {
- return input.apply(
- ParDo.of(
- new SortValuesDoFn<PrimaryKeyT, SecondaryKeyT, ValueT>(
- sorterOptions,
- getSecondaryKeyCoder(input.getCoder()),
- getValueCoder(input.getCoder()))));
- }
-
- @Override
- protected Coder<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> getDefaultOutputCoder(
- PCollection<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> input) {
- return input.getCoder();
+ return input
+ .apply(
+ ParDo.of(
+ new SortValuesDoFn<PrimaryKeyT, SecondaryKeyT, ValueT>(
+ sorterOptions,
+ getSecondaryKeyCoder(input.getCoder()),
+ getValueCoder(input.getCoder()))))
+ .setCoder(input.getCoder());
}
/** Retrieves the {@link Coder} for the secondary key-value pairs. */
diff --git a/sdks/java/extensions/sql/NOTICE b/sdks/java/extensions/sql/NOTICE
new file mode 100644
index 0000000..112b1e1
--- /dev/null
+++ b/sdks/java/extensions/sql/NOTICE
@@ -0,0 +1,45 @@
+Apache Beam :: SDKs :: Java :: Extensions :: SQL
+Copyright 2016-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software developed at
+Google (http://www.google.com/).
+
+This product contains subcomponents with separate copyright notices and
+license terms. Your use of the source code for the these subcomponents
+is subject to the terms and conditions of their respective licenses.
+
+=======================================================================
+Janino - An embedded Java[TM] compiler
+
+Copyright (c) 2001-2016, Arno Unkrig
+Copyright (c) 2015-2016 TIBCO Software Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials
+ provided with the distribution.
+ 3. Neither the name of JANINO nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/sdks/java/extensions/sql/pom.xml b/sdks/java/extensions/sql/pom.xml
new file mode 100644
index 0000000..6ffa3ab
--- /dev/null
+++ b/sdks/java/extensions/sql/pom.xml
@@ -0,0 +1,425 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-extensions-sql</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: Extensions :: SQL</name>
+ <description>Beam SQL provides a new interface to generate a Beam pipeline from SQL statement</description>
+
+ <packaging>jar</packaging>
+
+ <properties>
+ <timestamp>${maven.build.timestamp}</timestamp>
+ <maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
+ <calcite.version>1.13.0</calcite.version>
+ <avatica.version>1.10.0</avatica.version>
+ <mockito.version>1.9.5</mockito.version>
+ </properties>
+
+ <profiles>
+ <!--
+ The direct runner is available by default.
+ You can also include it on the classpath explicitly with -P direct-runner
+ -->
+ <profile>
+ <id>direct-runner</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <!-- Set testSourceDirectory in order to exclude generated-test-sources -->
+ <testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
+ <sourceDirectory>${project.build.sourceDirectory}</sourceDirectory>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ <compilerArgs>
+ <!-- Generated calcite code has some deprecation warning -->
+ <arg>-Xlint:-deprecation</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-fmpp-resources</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/codegen</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/codegen</directory>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <!-- Extract parser grammar template from calcite-core.jar and put
+ it under ${project.build.directory} where all freemarker templates are. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>unpack-parser-template</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ <type>jar</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/</outputDirectory>
+ <includes>**/Parser.jj</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.googlecode.fmpp-maven-plugin</groupId>
+ <artifactId>fmpp-maven-plugin</artifactId>
+ <version>1.0</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ <version>2.3.25-incubating</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>generate-fmpp-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ <configuration>
+ <cfgFile>${project.build.directory}/codegen/config.fmpp</cfgFile>
+ <outputDirectory>target/generated-sources</outputDirectory>
+ <templateDirectory>${project.build.directory}/codegen/templates</templateDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9</version>
+ <executions>
+ <execution>
+ <id>add-generated-sources</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <id>javacc</id>
+ <goals>
+ <goal>javacc</goal>
+ </goals>
+ <configuration>
+ <sourceDirectory>${project.build.directory}/generated-sources/</sourceDirectory>
+ <includes>
+ <include>**/Parser.jj</include>
+ </includes>
+ <lookAhead>2</lookAhead>
+ <isStatic>false</isStatic>
+ <outputDirectory>${project.build.directory}/generated-sources/</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>-da</argLine> <!-- disable assert in Calcite converter validation -->
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <!-- include Apache Calcite and related dependencies -->
+ <include>org.apache.calcite:*</include>
+ <include>org.apache.calcite.avatica:*</include>
+ <include>org.codehaus.janino:*</include>
+ <include>com.google.protobuf:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <excludes>
+ <!-- com.google.common is too generic, need to exclude guava-testlib -->
+ <exclude>com.google.common.**.testing.*</exclude>
+ </excludes>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.${renderedArtifactId}.repackaged.com.google.common
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.${renderedArtifactId}.repackaged.com.google.thirdparty
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>
+ org.apache.${renderedArtifactId}.repackaged.com.google.protobuf
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.calcite</pattern>
+ <shadedPattern>
+ org.apache.${renderedArtifactId}.repackaged.org.apache.calcite
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.codehaus</pattern>
+ <shadedPattern>
+ org.apache.${renderedArtifactId}.repackaged.org.codehaus
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Coverage analysis for unit tests. -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ <version>${calcite.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-linq4j</artifactId>
+ <version>${calcite.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ <version>${avatica.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-lite</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-join-library</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <!-- this is a hack to make it available at compile time but not bundled.-->
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-kafka</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- for tests -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>1.2.12</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/sdks/java/extensions/sql/src/main/codegen/config.fmpp b/sdks/java/extensions/sql/src/main/codegen/config.fmpp
new file mode 100644
index 0000000..61645e2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/codegen/config.fmpp
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data: {
+ parser: tdd(../data/Parser.tdd)
+}
+
+freemarkerLinks: {
+ includes: includes/
+}
diff --git a/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd b/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
new file mode 100644
index 0000000..09a5379
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ # Generated parser implementation class package and name
+ package: "org.apache.beam.sdk.extensions.sql.impl.parser.impl",
+ class: "BeamSqlParserImpl",
+
+ # List of import statements.
+ imports: [
+ "org.apache.calcite.sql.validate.*",
+ "org.apache.calcite.util.*",
+ "org.apache.beam.sdk.extensions.sql.impl.parser.*",
+ "java.util.*"
+ ]
+
+ # List of keywords.
+ keywords: [
+ "LOCATION",
+ "TBLPROPERTIES",
+ "COMMENT"
+ ]
+
+ # List of methods for parsing custom SQL statements.
+ statementParserMethods: [
+ "SqlCreateTable()"
+ ]
+
+ # List of methods for parsing custom literals.
+ # Example: ParseJsonLiteral().
+ literalParserMethods: [
+ ]
+
+ # List of methods for parsing custom data types.
+ dataTypeParserMethods: [
+ ]
+
+ nonReservedKeywords: [
+ ]
+
+ createStatementParserMethods: [
+ ]
+
+ alterStatementParserMethods: [
+ ]
+
+ dropStatementParserMethods: [
+ ]
+
+ # List of files in @includes directory that have parser method
+ # implementations for custom SQL statements, literals or types
+ # given as part of "statementParserMethods", "literalParserMethods" or
+ # "dataTypeParserMethods".
+ implementationFiles: [
+ "parserImpls.ftl"
+ ]
+
+ includeCompoundIdentifier: true,
+ includeBraces: true,
+ includeAdditionalDeclarations: false,
+ allowBangEqual: false
+}
diff --git a/sdks/java/extensions/sql/src/main/codegen/includes/license.ftl b/sdks/java/extensions/sql/src/main/codegen/includes/license.ftl
new file mode 100644
index 0000000..7e66353
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/codegen/includes/license.ftl
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
diff --git a/sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl b/sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
new file mode 100644
index 0000000..136c728
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
@@ -0,0 +1,89 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+
+private void ColumnDef(List<ColumnDefinition> list) :
+{
+ SqlParserPos pos;
+ SqlIdentifier name;
+ SqlDataTypeSpec type;
+ ColumnConstraint constraint = null;
+ SqlNode comment = null;
+}
+{
+ name = SimpleIdentifier() { pos = getPos(); }
+ type = DataType()
+ [
+ <PRIMARY> <KEY>
+ { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
+ ]
+ [
+ <COMMENT> comment = StringLiteral()
+ ]
+ {
+ list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ }
+}
+
+SqlNodeList ColumnDefinitionList() :
+{
+ SqlParserPos pos;
+ List<ColumnDefinition> list = Lists.newArrayList();
+}
+{
+ <LPAREN> { pos = getPos(); }
+ ColumnDef(list)
+ ( <COMMA> ColumnDef(list) )*
+ <RPAREN> {
+ return new SqlNodeList(list, pos.plus(getPos()));
+ }
+}
+
+/**
+ * CREATE TABLE ( IF NOT EXISTS )?
+ * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
+ * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT output_format_classname )?
+ * LOCATION location_uri
+ * ( TBLPROPERTIES tbl_properties )?
+ * ( AS select_stmt )
+ */
+SqlNode SqlCreateTable() :
+{
+ SqlParserPos pos;
+ SqlIdentifier tblName;
+ SqlNodeList fieldList;
+ SqlNode type = null;
+ SqlNode comment = null;
+ SqlNode location = null;
+ SqlNode tbl_properties = null;
+ SqlNode select = null;
+}
+{
+ <CREATE> { pos = getPos(); }
+ <TABLE>
+ tblName = CompoundIdentifier()
+ fieldList = ColumnDefinitionList()
+ <TYPE>
+ type = StringLiteral()
+ [
+ <COMMENT>
+ comment = StringLiteral()
+ ]
+ [
+ <LOCATION>
+ location = StringLiteral()
+ ]
+ [ <TBLPROPERTIES> tbl_properties = StringLiteral() ]
+ [ <AS> select = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) ] {
+ return new SqlCreateTable(pos, tblName, fieldList, type, comment,
+ location, tbl_properties, select);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamRecordSqlType.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamRecordSqlType.java
new file mode 100644
index 0000000..982494a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamRecordSqlType.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.coders.BigDecimalCoder;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.ByteCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.BooleanCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.DateCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.DoubleCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.FloatCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.ShortCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper.TimeCoder;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.BeamRecordType;
+
+/**
+ * Type provider for {@link BeamRecord} with SQL types.
+ *
+ * <p>Limited SQL types are supported now, visit
+ * <a href="https://beam.apache.org/blog/2017/07/21/sql-dsl.html#data-type">data types</a>
+ * for more details.
+ *
+ */
+public class BeamRecordSqlType extends BeamRecordType {
+ private static final Map<Integer, Class> SQL_TYPE_TO_JAVA_CLASS = new HashMap<>();
+ static {
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.TINYINT, Byte.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.SMALLINT, Short.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.INTEGER, Integer.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.BIGINT, Long.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.FLOAT, Float.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.DOUBLE, Double.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.DECIMAL, BigDecimal.class);
+
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.BOOLEAN, Boolean.class);
+
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.CHAR, String.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.VARCHAR, String.class);
+
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.TIME, GregorianCalendar.class);
+
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.DATE, Date.class);
+ SQL_TYPE_TO_JAVA_CLASS.put(Types.TIMESTAMP, Date.class);
+ }
+
+ public List<Integer> fieldTypes;
+
+ protected BeamRecordSqlType(List<String> fieldsName, List<Coder> fieldsCoder) {
+ super(fieldsName, fieldsCoder);
+ }
+
+ private BeamRecordSqlType(List<String> fieldsName, List<Integer> fieldTypes
+ , List<Coder> fieldsCoder) {
+ super(fieldsName, fieldsCoder);
+ this.fieldTypes = fieldTypes;
+ }
+
+ public static BeamRecordSqlType create(List<String> fieldNames,
+ List<Integer> fieldTypes) {
+ if (fieldNames.size() != fieldTypes.size()) {
+ throw new IllegalStateException("the sizes of 'dataType' and 'fieldTypes' must match.");
+ }
+ List<Coder> fieldCoders = new ArrayList<>(fieldTypes.size());
+ for (int idx = 0; idx < fieldTypes.size(); ++idx) {
+ switch (fieldTypes.get(idx)) {
+ case Types.INTEGER:
+ fieldCoders.add(BigEndianIntegerCoder.of());
+ break;
+ case Types.SMALLINT:
+ fieldCoders.add(ShortCoder.of());
+ break;
+ case Types.TINYINT:
+ fieldCoders.add(ByteCoder.of());
+ break;
+ case Types.DOUBLE:
+ fieldCoders.add(DoubleCoder.of());
+ break;
+ case Types.FLOAT:
+ fieldCoders.add(FloatCoder.of());
+ break;
+ case Types.DECIMAL:
+ fieldCoders.add(BigDecimalCoder.of());
+ break;
+ case Types.BIGINT:
+ fieldCoders.add(BigEndianLongCoder.of());
+ break;
+ case Types.VARCHAR:
+ case Types.CHAR:
+ fieldCoders.add(StringUtf8Coder.of());
+ break;
+ case Types.TIME:
+ fieldCoders.add(TimeCoder.of());
+ break;
+ case Types.DATE:
+ case Types.TIMESTAMP:
+ fieldCoders.add(DateCoder.of());
+ break;
+ case Types.BOOLEAN:
+ fieldCoders.add(BooleanCoder.of());
+ break;
+
+ default:
+ throw new UnsupportedOperationException(
+ "Data type: " + fieldTypes.get(idx) + " not supported yet!");
+ }
+ }
+ return new BeamRecordSqlType(fieldNames, fieldTypes, fieldCoders);
+ }
+
+ @Override
+ public void validateValueType(int index, Object fieldValue) throws IllegalArgumentException {
+ if (null == fieldValue) {// no need to do type check for NULL value
+ return;
+ }
+
+ int fieldType = fieldTypes.get(index);
+ Class javaClazz = SQL_TYPE_TO_JAVA_CLASS.get(fieldType);
+ if (javaClazz == null) {
+ throw new IllegalArgumentException("Data type: " + fieldType + " not supported yet!");
+ }
+
+ if (!fieldValue.getClass().equals(javaClazz)) {
+ throw new IllegalArgumentException(
+ String.format("[%s](%s) doesn't match type [%s]",
+ fieldValue, fieldValue.getClass(), fieldType)
+ );
+ }
+ }
+
+ public List<Integer> getFieldTypes() {
+ return Collections.unmodifiableList(fieldTypes);
+ }
+
+ public Integer getFieldTypeByIndex(int index){
+ return fieldTypes.get(index);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof BeamRecordSqlType) {
+ BeamRecordSqlType ins = (BeamRecordSqlType) obj;
+ return fieldTypes.equals(ins.getFieldTypes()) && getFieldNames().equals(ins.getFieldNames());
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * getFieldNames().hashCode() + getFieldTypes().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "BeamRecordSqlType [fieldNames=" + getFieldNames()
+ + ", fieldTypes=" + fieldTypes + "]";
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
new file mode 100644
index 0000000..fc80df5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamPCollectionTable;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+
+/**
+ * {@code BeamSql} is the DSL interface of BeamSQL. It translates a SQL query as a
+ * {@link PTransform}, so developers can use standard SQL queries in a Beam pipeline.
+ *
+ * <h1>Beam SQL DSL usage:</h1>
+ * A typical pipeline with Beam SQL DSL is:
+ * <pre>
+ *{@code
+PipelineOptions options = PipelineOptionsFactory.create();
+Pipeline p = Pipeline.create(options);
+
+//create table from TextIO;
+PCollection<BeamSqlRow> inputTableA = p.apply(TextIO.read().from("/my/input/patha"))
+ .apply(...);
+PCollection<BeamSqlRow> inputTableB = p.apply(TextIO.read().from("/my/input/pathb"))
+ .apply(...);
+
+//run a simple query, and register the output as a table in BeamSql;
+String sql1 = "select MY_FUNC(c1), c2 from PCOLLECTION";
+PCollection<BeamSqlRow> outputTableA = inputTableA.apply(
+ BeamSql.query(sql1)
+ .withUdf("MY_FUNC", MY_FUNC.class, "FUNC"));
+
+//run a JOIN with one table from TextIO, and one table from another query
+PCollection<BeamSqlRow> outputTableB = PCollectionTuple.of(
+ new TupleTag<BeamSqlRow>("TABLE_O_A"), outputTableA)
+ .and(new TupleTag<BeamSqlRow>("TABLE_B"), inputTableB)
+ .apply(BeamSql.queryMulti("select * from TABLE_O_A JOIN TABLE_B where ..."));
+
+//output the final result with TextIO
+outputTableB.apply(...).apply(TextIO.write().to("/my/output/path"));
+
+p.run().waitUntilFinish();
+ * }
+ * </pre>
+ *
+ */
+@Experimental
+public class BeamSql {
+ /**
+ * Transforms a SQL query into a {@link PTransform} representing an equivalent execution plan.
+ *
+ * <p>The returned {@link PTransform} can be applied to a {@link PCollectionTuple} representing
+ * all the input tables and results in a {@code PCollection<BeamSqlRow>} representing the output
+ * table. The {@link PCollectionTuple} contains the mapping from {@code table names} to
+ * {@code PCollection<BeamSqlRow>}, each representing an input table.
+ *
+ * <ul>
+ * <li>If the sql query only uses a subset of tables from the upstream {@link PCollectionTuple},
+ * this is valid;</li>
+ * <li>If the sql query references a table not included in the upstream {@link PCollectionTuple},
+ * an {@code IllegalStateException} is thrown during query validation;</li>
+ * <li>Always, tables from the upstream {@link PCollectionTuple} are only valid in the scope
+ * of the current query call.</li>
+ * </ul>
+ */
+ public static QueryTransform queryMulti(String sqlQuery) {
+ return new QueryTransform(sqlQuery);
+ }
+
+ /**
+ * Transforms a SQL query into a {@link PTransform} representing an equivalent execution plan.
+ *
+ * <p>This is a simplified form of {@link #queryMulti(String)} where the query must reference
+ * a single input table.
+ *
+ * <p>Make sure to query it from a static table name <em>PCOLLECTION</em>.
+ */
+ public static SimpleQueryTransform query(String sqlQuery) {
+ return new SimpleQueryTransform(sqlQuery);
+ }
+
+ /**
+ * A {@link PTransform} representing an execution plan for a SQL query.
+ *
+ * <p>The table names in the input {@code PCollectionTuple} are only valid during the current
+ * query.
+ */
+ public static class QueryTransform extends
+ PTransform<PCollectionTuple, PCollection<BeamRecord>> {
+ private BeamSqlEnv beamSqlEnv = new BeamSqlEnv();
+ private String sqlQuery;
+
+ public QueryTransform(String sqlQuery) {
+ this.sqlQuery = sqlQuery;
+ }
+
+ /**
+ * register a UDF function used in this query.
+ *
+ * <p>Refer to {@link BeamSqlUdf} for more about how to implement a UDF in BeamSql.
+ */
+ public QueryTransform withUdf(String functionName, Class<? extends BeamSqlUdf> clazz){
+ beamSqlEnv.registerUdf(functionName, clazz);
+ return this;
+ }
+ /**
+ * register {@link SerializableFunction} as a UDF function used in this query.
+ * Note, {@link SerializableFunction} must have a constructor without arguments.
+ */
+ public QueryTransform withUdf(String functionName, SerializableFunction sfn){
+ beamSqlEnv.registerUdf(functionName, sfn);
+ return this;
+ }
+
+ /**
+ * register a {@link CombineFn} as UDAF function used in this query.
+ */
+ public QueryTransform withUdaf(String functionName, CombineFn combineFn){
+ beamSqlEnv.registerUdaf(functionName, combineFn);
+ return this;
+ }
+
+ @Override
+ public PCollection<BeamRecord> expand(PCollectionTuple input) {
+ registerTables(input);
+
+ BeamRelNode beamRelNode = null;
+ try {
+ beamRelNode = beamSqlEnv.getPlanner().convertToBeamRel(sqlQuery);
+ } catch (ValidationException | RelConversionException | SqlParseException e) {
+ throw new IllegalStateException(e);
+ }
+
+ try {
+ return beamRelNode.buildBeamPipeline(input, beamSqlEnv);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ //register tables, related with input PCollections.
+ private void registerTables(PCollectionTuple input){
+ for (TupleTag<?> sourceTag : input.getAll().keySet()) {
+ PCollection<BeamRecord> sourceStream = (PCollection<BeamRecord>) input.get(sourceTag);
+ BeamRecordCoder sourceCoder = (BeamRecordCoder) sourceStream.getCoder();
+
+ beamSqlEnv.registerTable(sourceTag.getId(),
+ new BeamPCollectionTable(sourceStream,
+ (BeamRecordSqlType) sourceCoder.getRecordType()));
+ }
+ }
+ }
+
+ /**
+ * A {@link PTransform} representing an execution plan for a SQL query referencing
+ * a single table.
+ */
+ public static class SimpleQueryTransform
+ extends PTransform<PCollection<BeamRecord>, PCollection<BeamRecord>> {
+ private static final String PCOLLECTION_TABLE_NAME = "PCOLLECTION";
+ private QueryTransform delegate;
+
+ public SimpleQueryTransform(String sqlQuery) {
+ this.delegate = new QueryTransform(sqlQuery);
+ }
+
+ /**
+ * register a UDF function used in this query.
+ *
+ * <p>Refer to {@link BeamSqlUdf} for more about how to implement a UDAF in BeamSql.
+ */
+ public SimpleQueryTransform withUdf(String functionName, Class<? extends BeamSqlUdf> clazz){
+ delegate.withUdf(functionName, clazz);
+ return this;
+ }
+
+ /**
+ * register {@link SerializableFunction} as a UDF function used in this query.
+ * Note, {@link SerializableFunction} must have a constructor without arguments.
+ */
+ public SimpleQueryTransform withUdf(String functionName, SerializableFunction sfn){
+ delegate.withUdf(functionName, sfn);
+ return this;
+ }
+
+ /**
+ * register a {@link CombineFn} as UDAF function used in this query.
+ */
+ public SimpleQueryTransform withUdaf(String functionName, CombineFn combineFn){
+ delegate.withUdaf(functionName, combineFn);
+ return this;
+ }
+
+ private void validateQuery() {
+ SqlNode sqlNode;
+ try {
+ sqlNode = delegate.beamSqlEnv.getPlanner().parseQuery(delegate.sqlQuery);
+ delegate.beamSqlEnv.getPlanner().getPlanner().close();
+ } catch (SqlParseException e) {
+ throw new IllegalStateException(e);
+ }
+
+ if (sqlNode instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) sqlNode;
+ String tableName = select.getFrom().toString();
+ if (!tableName.equalsIgnoreCase(PCOLLECTION_TABLE_NAME)) {
+ throw new IllegalStateException("Use fixed table name " + PCOLLECTION_TABLE_NAME);
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "Sql operation: " + sqlNode.toString() + " is not supported!");
+ }
+ }
+
+ @Override
+ public PCollection<BeamRecord> expand(PCollection<BeamRecord> input) {
+ validateQuery();
+ return PCollectionTuple.of(new TupleTag<BeamRecord>(PCOLLECTION_TABLE_NAME), input)
+ .apply(delegate);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
new file mode 100644
index 0000000..2cadb0e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.parser.BeamSqlParser;
+import org.apache.beam.sdk.extensions.sql.impl.parser.ParserUtils;
+import org.apache.beam.sdk.extensions.sql.impl.parser.SqlCreateTable;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.store.MetaStore;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.sql.SqlNode;
+
+/**
+ * {@link BeamSqlCli} provides methods to execute Beam SQL with an interactive client.
+ */
+@Experimental
+public class BeamSqlCli {
+ private BeamSqlEnv env;
+ /**
+ * The store which persists all the table meta data.
+ */
+ private MetaStore metaStore;
+
+ public BeamSqlCli metaStore(MetaStore metaStore) {
+ this.metaStore = metaStore;
+ this.env = new BeamSqlEnv();
+
+ // dump tables in metaStore into schema
+ List<Table> tables = this.metaStore.listTables();
+ for (Table table : tables) {
+ env.registerTable(table.getName(), metaStore.buildBeamSqlTable(table.getName()));
+ }
+
+ return this;
+ }
+
+ public MetaStore getMetaStore() {
+ return metaStore;
+ }
+
+ /**
+ * Returns a human readable representation of the query execution plan.
+ */
+ public String explainQuery(String sqlString) throws Exception {
+ BeamRelNode exeTree = env.getPlanner().convertToBeamRel(sqlString);
+ String beamPlan = RelOptUtil.toString(exeTree);
+ return beamPlan;
+ }
+
+ /**
+ * Executes the given sql.
+ */
+ public void execute(String sqlString) throws Exception {
+ BeamSqlParser parser = new BeamSqlParser(sqlString);
+ SqlNode sqlNode = parser.impl().parseSqlStmtEof();
+
+ if (sqlNode instanceof SqlCreateTable) {
+ handleCreateTable((SqlCreateTable) sqlNode, metaStore);
+ } else {
+ PipelineOptions options = PipelineOptionsFactory.fromArgs(new String[] {}).withValidation()
+ .as(PipelineOptions.class);
+ options.setJobName("BeamPlanCreator");
+ Pipeline pipeline = Pipeline.create(options);
+ compilePipeline(sqlString, pipeline, env);
+ pipeline.run();
+ }
+ }
+
+ private void handleCreateTable(SqlCreateTable stmt, MetaStore store) {
+ Table table = ParserUtils.convertCreateTableStmtToTable(stmt);
+ if (table.getType() == null) {
+ throw new IllegalStateException("Table type is not specified and BeamSqlCli#defaultTableType"
+ + "is not configured!");
+ }
+
+ store.createTable(table);
+
+ // register the new table into the schema
+ env.registerTable(table.getName(), metaStore.buildBeamSqlTable(table.getName()));
+ }
+
+ /**
+ * compile SQL, and return a {@link Pipeline}.
+ */
+ private static PCollection<BeamRecord> compilePipeline(String sqlStatement, Pipeline basePipeline,
+ BeamSqlEnv sqlEnv) throws Exception {
+ PCollection<BeamRecord> resultStream =
+ sqlEnv.getPlanner().compileBeamPipeline(sqlStatement, basePipeline, sqlEnv);
+ return resultStream;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlRecordHelper.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlRecordHelper.java
new file mode 100644
index 0000000..870165d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlRecordHelper.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.BigDecimalCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ * A {@link Coder} encodes {@link BeamRecord}.
+ */
+@Experimental
+public class BeamSqlRecordHelper {
+
+ public static BeamRecordSqlType getSqlRecordType(BeamRecord record) {
+ return (BeamRecordSqlType) record.getDataType();
+ }
+
+ /**
+ * {@link Coder} for Java type {@link Short}.
+ */
+ public static class ShortCoder extends CustomCoder<Short> {
+ private static final ShortCoder INSTANCE = new ShortCoder();
+
+ public static ShortCoder of() {
+ return INSTANCE;
+ }
+
+ private ShortCoder() {
+ }
+
+ @Override
+ public void encode(Short value, OutputStream outStream) throws CoderException, IOException {
+ new DataOutputStream(outStream).writeShort(value);
+ }
+
+ @Override
+ public Short decode(InputStream inStream) throws CoderException, IOException {
+ return new DataInputStream(inStream).readShort();
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+ /**
+ * {@link Coder} for Java type {@link Float}, it's stored as {@link BigDecimal}.
+ */
+ public static class FloatCoder extends CustomCoder<Float> {
+ private static final FloatCoder INSTANCE = new FloatCoder();
+ private static final BigDecimalCoder CODER = BigDecimalCoder.of();
+
+ public static FloatCoder of() {
+ return INSTANCE;
+ }
+
+ private FloatCoder() {
+ }
+
+ @Override
+ public void encode(Float value, OutputStream outStream) throws CoderException, IOException {
+ CODER.encode(new BigDecimal(value), outStream);
+ }
+
+ @Override
+ public Float decode(InputStream inStream) throws CoderException, IOException {
+ return CODER.decode(inStream).floatValue();
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+ /**
+ * {@link Coder} for Java type {@link Double}, it's stored as {@link BigDecimal}.
+ */
+ public static class DoubleCoder extends CustomCoder<Double> {
+ private static final DoubleCoder INSTANCE = new DoubleCoder();
+ private static final BigDecimalCoder CODER = BigDecimalCoder.of();
+
+ public static DoubleCoder of() {
+ return INSTANCE;
+ }
+
+ private DoubleCoder() {
+ }
+
+ @Override
+ public void encode(Double value, OutputStream outStream) throws CoderException, IOException {
+ CODER.encode(new BigDecimal(value), outStream);
+ }
+
+ @Override
+ public Double decode(InputStream inStream) throws CoderException, IOException {
+ return CODER.decode(inStream).doubleValue();
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+
+ /**
+ * {@link Coder} for Java type {@link GregorianCalendar}, it's stored as {@link Long}.
+ */
+ public static class TimeCoder extends CustomCoder<GregorianCalendar> {
+ private static final BigEndianLongCoder longCoder = BigEndianLongCoder.of();
+ private static final TimeCoder INSTANCE = new TimeCoder();
+
+ public static TimeCoder of() {
+ return INSTANCE;
+ }
+
+ private TimeCoder() {
+ }
+
+ @Override
+ public void encode(GregorianCalendar value, OutputStream outStream)
+ throws CoderException, IOException {
+ longCoder.encode(value.getTime().getTime(), outStream);
+ }
+
+ @Override
+ public GregorianCalendar decode(InputStream inStream) throws CoderException, IOException {
+ GregorianCalendar calendar = new GregorianCalendar();
+ calendar.setTime(new Date(longCoder.decode(inStream)));
+ return calendar;
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+ /**
+ * {@link Coder} for Java type {@link Date}, it's stored as {@link Long}.
+ */
+ public static class DateCoder extends CustomCoder<Date> {
+ private static final BigEndianLongCoder longCoder = BigEndianLongCoder.of();
+ private static final DateCoder INSTANCE = new DateCoder();
+
+ public static DateCoder of() {
+ return INSTANCE;
+ }
+
+ private DateCoder() {
+ }
+
+ @Override
+ public void encode(Date value, OutputStream outStream) throws CoderException, IOException {
+ longCoder.encode(value.getTime(), outStream);
+ }
+
+ @Override
+ public Date decode(InputStream inStream) throws CoderException, IOException {
+ return new Date(longCoder.decode(inStream));
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+
+ /**
+ * {@link Coder} for Java type {@link Boolean}.
+ */
+ public static class BooleanCoder extends CustomCoder<Boolean> {
+ private static final BooleanCoder INSTANCE = new BooleanCoder();
+
+ public static BooleanCoder of() {
+ return INSTANCE;
+ }
+
+ private BooleanCoder() {
+ }
+
+ @Override
+ public void encode(Boolean value, OutputStream outStream) throws CoderException, IOException {
+ new DataOutputStream(outStream).writeBoolean(value);
+ }
+
+ @Override
+ public Boolean decode(InputStream inStream) throws CoderException, IOException {
+ return new DataInputStream(inStream).readBoolean();
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlTable.java
new file mode 100644
index 0000000..df1e162
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlTable.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+ * This interface defines a Beam Sql Table.
+ */
+public interface BeamSqlTable {
+ /**
+ * In Beam SQL, there's no difference between a batch query and a streaming
+ * query. {@link BeamIOType} is used to validate the sources.
+ */
+ BeamIOType getSourceType();
+
+ /**
+ * create a {@code PCollection<BeamSqlRow>} from source.
+ *
+ */
+ PCollection<BeamRecord> buildIOReader(Pipeline pipeline);
+
+ /**
+ * create a {@code IO.write()} instance to write to target.
+ *
+ */
+ PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter();
+
+ /**
+ * Get the schema info of the table.
+ */
+ BeamRecordSqlType getRowType();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlUdf.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlUdf.java
new file mode 100644
index 0000000..91bad20
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlUdf.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Experimental;
+
+/**
+ * Interface to create a UDF in Beam SQL.
+ *
+ * <p>A static method {@code eval} is required. Here is an example:
+ *
+ * <blockquote><pre>
+ * public static class MyLeftFunction {
+ * public String eval(
+ * @Parameter(name = "s") String s,
+ * @Parameter(name = "n", optional = true) Integer n) {
+ * return s.substring(0, n == null ? 1 : n);
+ * }
+ * }</pre></blockquote>
+ *
+ * <p>The first parameter is named "s" and is mandatory,
+ * and the second parameter is named "n" and is optional.
+ */
+@Experimental
+public interface BeamSqlUdf extends Serializable {
+ String UDF_METHOD = "eval";
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
new file mode 100644
index 0000000..0154e1e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.example;
+
+import java.sql.Types;
+import java.util.Arrays;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSql;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * This is a quick example, which uses Beam SQL DSL to create a data pipeline.
+ *
+ * <p>Run the example with
+ * <pre>
+ * mvn -pl sdks/java/extensions/sql \
+ * compile exec:java -Dexec.mainClass=org.apache.beam.sdk.extensions.sql.example.BeamSqlExample \
+ * -Dexec.args="--runner=DirectRunner" -Pdirect-runner
+ * </pre>
+ */
+class BeamSqlExample {
+ public static void main(String[] args) throws Exception {
+ PipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(PipelineOptions.class);
+ Pipeline p = Pipeline.create(options);
+
+ //define the input row format
+ List<String> fieldNames = Arrays.asList("c1", "c2", "c3");
+ List<Integer> fieldTypes = Arrays.asList(Types.INTEGER, Types.VARCHAR, Types.DOUBLE);
+ BeamRecordSqlType type = BeamRecordSqlType.create(fieldNames, fieldTypes);
+ BeamRecord row1 = new BeamRecord(type, 1, "row", 1.0);
+ BeamRecord row2 = new BeamRecord(type, 2, "row", 2.0);
+ BeamRecord row3 = new BeamRecord(type, 3, "row", 3.0);
+
+ //create a source PCollection with Create.of();
+ PCollection<BeamRecord> inputTable = PBegin.in(p).apply(Create.of(row1, row2, row3)
+ .withCoder(type.getRecordCoder()));
+
+ //Case 1. run a simple SQL query over input PCollection with BeamSql.simpleQuery;
+ PCollection<BeamRecord> outputStream = inputTable.apply(
+ BeamSql.query("select c1, c2, c3 from PCOLLECTION where c1 > 1"));
+
+ //print the output record of case 1;
+ outputStream.apply(
+ "log_result",
+ MapElements.<BeamRecord, Void>via(
+ new SimpleFunction<BeamRecord, Void>() {
+ public @Nullable Void apply(BeamRecord input) {
+ //expect output:
+ // PCOLLECTION: [3, row, 3.0]
+ // PCOLLECTION: [2, row, 2.0]
+ System.out.println("PCOLLECTION: " + input.getDataValues());
+ return null;
+ }
+ }));
+
+ //Case 2. run the query with BeamSql.query over result PCollection of case 1.
+ PCollection<BeamRecord> outputStream2 =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("CASE1_RESULT"), outputStream)
+ .apply(BeamSql.queryMulti("select c2, sum(c3) from CASE1_RESULT group by c2"));
+
+ //print the output record of case 2;
+ outputStream2.apply(
+ "log_result",
+ MapElements.<BeamRecord, Void>via(
+ new SimpleFunction<BeamRecord, Void>() {
+ @Override
+ public @Nullable Void apply(BeamRecord input) {
+ //expect output:
+ // CASE1_RESULT: [row, 5.0]
+ System.out.println("CASE1_RESULT: " + input.getDataValues());
+ return null;
+ }
+ }));
+
+ p.run().waitUntilFinish();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/package-info.java
new file mode 100644
index 0000000..f1569178
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * examples on how to use BeamSQL.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.example;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
new file mode 100644
index 0000000..405bedf
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSql;
+import org.apache.beam.sdk.extensions.sql.BeamSqlCli;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.BeamSqlUdf;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.UdafImpl;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Statistic;
+import org.apache.calcite.schema.Statistics;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
+import org.apache.calcite.tools.Frameworks;
+
+/**
+ * {@link BeamSqlEnv} prepares the execution context for {@link BeamSql} and
+ * {@link BeamSqlCli}.
+ *
+ * <p>It contains a {@link SchemaPlus} which holds the metadata of tables/UDF functions,
+ * and a {@link BeamQueryPlanner} which parse/validate/optimize/translate input SQL queries.
+ */
+public class BeamSqlEnv implements Serializable{
+ transient SchemaPlus schema;
+ transient BeamQueryPlanner planner;
+
+ public BeamSqlEnv() {
+ schema = Frameworks.createRootSchema(true);
+ planner = new BeamQueryPlanner(schema);
+ }
+
+ /**
+ * Register a UDF function which can be used in SQL expression.
+ */
+ public void registerUdf(String functionName, Class<? extends BeamSqlUdf> clazz) {
+ schema.add(functionName, ScalarFunctionImpl.create(clazz, BeamSqlUdf.UDF_METHOD));
+ }
+
+ /**
+ * Register {@link SerializableFunction} as a UDF function which can be used in SQL expression.
+ * Note, {@link SerializableFunction} must have a constructor without arguments.
+ */
+ public void registerUdf(String functionName, SerializableFunction sfn) {
+ schema.add(functionName, ScalarFunctionImpl.create(sfn.getClass(), "apply"));
+ }
+
+ /**
+ * Register a UDAF function which can be used in GROUP-BY expression.
+ * See {@link org.apache.beam.sdk.transforms.Combine.CombineFn} on how to implement a UDAF.
+ */
+ public void registerUdaf(String functionName, Combine.CombineFn combineFn) {
+ schema.add(functionName, new UdafImpl(combineFn));
+ }
+
+ /**
+ * Registers a {@link BaseBeamTable} which can be used for all subsequent queries.
+ *
+ */
+ public void registerTable(String tableName, BeamSqlTable table) {
+ schema.add(tableName, new BeamCalciteTable(table.getRowType()));
+ planner.getSourceTables().put(tableName, table);
+ }
+
+ /**
+ * Find {@link BaseBeamTable} by table name.
+ */
+ public BeamSqlTable findTable(String tableName){
+ return planner.getSourceTables().get(tableName);
+ }
+
+ private static class BeamCalciteTable implements ScannableTable, Serializable {
+ private BeamRecordSqlType beamSqlRowType;
+ public BeamCalciteTable(BeamRecordSqlType beamSqlRowType) {
+ this.beamSqlRowType = beamSqlRowType;
+ }
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ return CalciteUtils.toCalciteRowType(this.beamSqlRowType)
+ .apply(BeamQueryPlanner.TYPE_FACTORY);
+ }
+
+ @Override
+ public Enumerable<Object[]> scan(DataContext root) {
+ // not used as Beam SQL uses its own execution engine
+ return null;
+ }
+
+ /**
+ * Not used {@link Statistic} to optimize the plan.
+ */
+ @Override
+ public Statistic getStatistic() {
+ return Statistics.UNKNOWN;
+ }
+
+ /**
+ * all sources are treated as TABLE in Beam SQL.
+ */
+ @Override
+ public Schema.TableType getJdbcTableType() {
+ return Schema.TableType.TABLE;
+ }
+ }
+
+ public BeamQueryPlanner getPlanner() {
+ return planner;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlExpressionExecutor.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlExpressionExecutor.java
new file mode 100644
index 0000000..3aaf505
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlExpressionExecutor.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ * {@code BeamSqlExpressionExecutor} fills the gap between relational
+ * expressions in Calcite SQL and executable code.
+ *
+ */
+public interface BeamSqlExpressionExecutor extends Serializable {
+
+ /**
+ * invoked before data processing.
+ */
+ void prepare();
+
+ /**
+ * apply transformation to input record {@link BeamRecord} with {@link BoundedWindow}.
+ *
+ */
+ List<Object> execute(BeamRecord inputRow, BoundedWindow window);
+
+ void close();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutor.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutor.java
new file mode 100644
index 0000000..31d5022
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutor.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlCaseExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlCastExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlUdfExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlWindowEndExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlWindowExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlWindowStartExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlDivideExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMinusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlModExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlPlusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlGreaterThanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlGreaterThanOrEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlIsNotNullExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlIsNullExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlNotEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentDateExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimeExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimestampExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateCeilExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateFloorExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimeMinusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimePlusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlExtractExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlIntervalMultiplyExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlAndExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlNotExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlOrExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlAbsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlAcosExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlAsinExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlAtan2Expression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlAtanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlCeilExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlCosExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlCotExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlDegreesExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlExpExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlFloorExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlLnExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlLogExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlPiExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlPowerExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlRadiansExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlRandExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlRandIntegerExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlRoundExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlSignExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlSinExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlTanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math.BeamSqlTruncateExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret.BeamSqlReinterpretExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlCharLengthExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlConcatExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlInitCapExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlLowerExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlOverlayExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlPositionExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlSubstringExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlTrimExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlUpperExpression;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamFilterRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamProjectRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.apache.calcite.util.NlsString;
+
+/**
+ * Executor based on {@link BeamSqlExpression} and {@link BeamSqlPrimitive}.
+ * {@code BeamSqlFnExecutor} converts a {@link BeamRelNode} to a {@link BeamSqlExpression},
+ * which can be evaluated against the {@link BeamRecord}.
+ *
+ */
+public class BeamSqlFnExecutor implements BeamSqlExpressionExecutor {
+ protected List<BeamSqlExpression> exps;
+
+ public BeamSqlFnExecutor(BeamRelNode relNode) {
+ this.exps = new ArrayList<>();
+ if (relNode instanceof BeamFilterRel) {
+ BeamFilterRel filterNode = (BeamFilterRel) relNode;
+ RexNode condition = filterNode.getCondition();
+ exps.add(buildExpression(condition));
+ } else if (relNode instanceof BeamProjectRel) {
+ BeamProjectRel projectNode = (BeamProjectRel) relNode;
+ List<RexNode> projects = projectNode.getProjects();
+ for (RexNode rexNode : projects) {
+ exps.add(buildExpression(rexNode));
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ String.format("%s is not supported yet!", relNode.getClass().toString()));
+ }
+ }
+
+ /**
+ * {@link #buildExpression(RexNode)} visits the operands of {@link RexNode} recursively,
+ * and represent each {@link SqlOperator} with a corresponding {@link BeamSqlExpression}.
+ */
+ static BeamSqlExpression buildExpression(RexNode rexNode) {
+ BeamSqlExpression ret = null;
+ if (rexNode instanceof RexLiteral) {
+ RexLiteral node = (RexLiteral) rexNode;
+ SqlTypeName type = node.getTypeName();
+ Object value = node.getValue();
+
+ if (SqlTypeName.CHAR_TYPES.contains(type)
+ && node.getValue() instanceof NlsString) {
+ // NlsString is not serializable, we need to convert
+ // it to string explicitly.
+ return BeamSqlPrimitive.of(type, ((NlsString) value).getValue());
+ } else if (isDateNode(type, value)) {
+ // does this actually make sense?
+ // Calcite actually treat Calendar as the java type of Date Literal
+ return BeamSqlPrimitive.of(type, ((Calendar) value).getTime());
+ } else {
+ // node.getType().getSqlTypeName() and node.getSqlTypeName() can be different
+ // e.g. sql: "select 1"
+ // here the literal 1 will be parsed as a RexLiteral where:
+ // node.getType().getSqlTypeName() = INTEGER (the display type)
+ // node.getSqlTypeName() = DECIMAL (the actual internal storage format)
+ // So we need to do a convert here.
+ // check RexBuilder#makeLiteral for more information.
+ SqlTypeName realType = node.getType().getSqlTypeName();
+ Object realValue = value;
+ if (type == SqlTypeName.DECIMAL) {
+ BigDecimal rawValue = (BigDecimal) value;
+ switch (realType) {
+ case TINYINT:
+ realValue = (byte) rawValue.intValue();
+ break;
+ case SMALLINT:
+ realValue = (short) rawValue.intValue();
+ break;
+ case INTEGER:
+ realValue = rawValue.intValue();
+ break;
+ case BIGINT:
+ realValue = rawValue.longValue();
+ break;
+ case DECIMAL:
+ realValue = rawValue;
+ break;
+ default:
+ throw new IllegalStateException("type/realType mismatch: "
+ + type + " VS " + realType);
+ }
+ } else if (type == SqlTypeName.DOUBLE) {
+ Double rawValue = (Double) value;
+ if (realType == SqlTypeName.FLOAT) {
+ realValue = rawValue.floatValue();
+ }
+ }
+ return BeamSqlPrimitive.of(realType, realValue);
+ }
+ } else if (rexNode instanceof RexInputRef) {
+ RexInputRef node = (RexInputRef) rexNode;
+ ret = new BeamSqlInputRefExpression(node.getType().getSqlTypeName(), node.getIndex());
+ } else if (rexNode instanceof RexCall) {
+ RexCall node = (RexCall) rexNode;
+ String opName = node.op.getName();
+ List<BeamSqlExpression> subExps = new ArrayList<>();
+ for (RexNode subNode : node.getOperands()) {
+ subExps.add(buildExpression(subNode));
+ }
+ switch (opName) {
+ // logical operators
+ case "AND":
+ ret = new BeamSqlAndExpression(subExps);
+ break;
+ case "OR":
+ ret = new BeamSqlOrExpression(subExps);
+ break;
+ case "NOT":
+ ret = new BeamSqlNotExpression(subExps);
+ break;
+ case "=":
+ ret = new BeamSqlEqualsExpression(subExps);
+ break;
+ case "<>":
+ ret = new BeamSqlNotEqualsExpression(subExps);
+ break;
+ case ">":
+ ret = new BeamSqlGreaterThanExpression(subExps);
+ break;
+ case ">=":
+ ret = new BeamSqlGreaterThanOrEqualsExpression(subExps);
+ break;
+ case "<":
+ ret = new BeamSqlLessThanExpression(subExps);
+ break;
+ case "<=":
+ ret = new BeamSqlLessThanOrEqualsExpression(subExps);
+ break;
+
+ // arithmetic operators
+ case "+":
+ ret = new BeamSqlPlusExpression(subExps);
+ break;
+ case "-":
+ if (SqlTypeName.NUMERIC_TYPES.contains(node.type.getSqlTypeName())) {
+ ret = new BeamSqlMinusExpression(subExps);
+ } else {
+ ret = new BeamSqlDatetimeMinusExpression(subExps, node.type.getSqlTypeName());
+ }
+ break;
+ case "*":
+ if (SqlTypeName.NUMERIC_TYPES.contains(node.type.getSqlTypeName())) {
+ ret = new BeamSqlMultiplyExpression(subExps);
+ } else {
+ ret = new BeamSqlIntervalMultiplyExpression(subExps);
+ }
+ break;
+ case "/":
+ case "/INT":
+ ret = new BeamSqlDivideExpression(subExps);
+ break;
+ case "MOD":
+ ret = new BeamSqlModExpression(subExps);
+ break;
+
+ case "ABS":
+ ret = new BeamSqlAbsExpression(subExps);
+ break;
+ case "ROUND":
+ ret = new BeamSqlRoundExpression(subExps);
+ break;
+ case "LN":
+ ret = new BeamSqlLnExpression(subExps);
+ break;
+ case "LOG10":
+ ret = new BeamSqlLogExpression(subExps);
+ break;
+ case "EXP":
+ ret = new BeamSqlExpExpression(subExps);
+ break;
+ case "ACOS":
+ ret = new BeamSqlAcosExpression(subExps);
+ break;
+ case "ASIN":
+ ret = new BeamSqlAsinExpression(subExps);
+ break;
+ case "ATAN":
+ ret = new BeamSqlAtanExpression(subExps);
+ break;
+ case "COT":
+ ret = new BeamSqlCotExpression(subExps);
+ break;
+ case "DEGREES":
+ ret = new BeamSqlDegreesExpression(subExps);
+ break;
+ case "RADIANS":
+ ret = new BeamSqlRadiansExpression(subExps);
+ break;
+ case "COS":
+ ret = new BeamSqlCosExpression(subExps);
+ break;
+ case "SIN":
+ ret = new BeamSqlSinExpression(subExps);
+ break;
+ case "TAN":
+ ret = new BeamSqlTanExpression(subExps);
+ break;
+ case "SIGN":
+ ret = new BeamSqlSignExpression(subExps);
+ break;
+ case "POWER":
+ ret = new BeamSqlPowerExpression(subExps);
+ break;
+ case "PI":
+ ret = new BeamSqlPiExpression();
+ break;
+ case "ATAN2":
+ ret = new BeamSqlAtan2Expression(subExps);
+ break;
+ case "TRUNCATE":
+ ret = new BeamSqlTruncateExpression(subExps);
+ break;
+ case "RAND":
+ ret = new BeamSqlRandExpression(subExps);
+ break;
+ case "RAND_INTEGER":
+ ret = new BeamSqlRandIntegerExpression(subExps);
+ break;
+
+ // string operators
+ case "||":
+ ret = new BeamSqlConcatExpression(subExps);
+ break;
+ case "POSITION":
+ ret = new BeamSqlPositionExpression(subExps);
+ break;
+ case "CHAR_LENGTH":
+ case "CHARACTER_LENGTH":
+ ret = new BeamSqlCharLengthExpression(subExps);
+ break;
+ case "UPPER":
+ ret = new BeamSqlUpperExpression(subExps);
+ break;
+ case "LOWER":
+ ret = new BeamSqlLowerExpression(subExps);
+ break;
+ case "TRIM":
+ ret = new BeamSqlTrimExpression(subExps);
+ break;
+ case "SUBSTRING":
+ ret = new BeamSqlSubstringExpression(subExps);
+ break;
+ case "OVERLAY":
+ ret = new BeamSqlOverlayExpression(subExps);
+ break;
+ case "INITCAP":
+ ret = new BeamSqlInitCapExpression(subExps);
+ break;
+
+ // date functions
+ case "Reinterpret":
+ return new BeamSqlReinterpretExpression(subExps, node.type.getSqlTypeName());
+ case "CEIL":
+ if (SqlTypeName.NUMERIC_TYPES.contains(node.type.getSqlTypeName())) {
+ return new BeamSqlCeilExpression(subExps);
+ } else {
+ return new BeamSqlDateCeilExpression(subExps);
+ }
+ case "FLOOR":
+ if (SqlTypeName.NUMERIC_TYPES.contains(node.type.getSqlTypeName())) {
+ return new BeamSqlFloorExpression(subExps);
+ } else {
+ return new BeamSqlDateFloorExpression(subExps);
+ }
+ case "EXTRACT_DATE":
+ case "EXTRACT":
+ return new BeamSqlExtractExpression(subExps);
+
+ case "LOCALTIME":
+ case "CURRENT_TIME":
+ return new BeamSqlCurrentTimeExpression(subExps);
+
+ case "CURRENT_TIMESTAMP":
+ case "LOCALTIMESTAMP":
+ return new BeamSqlCurrentTimestampExpression(subExps);
+
+ case "CURRENT_DATE":
+ return new BeamSqlCurrentDateExpression();
+
+ case "DATETIME_PLUS":
+ return new BeamSqlDatetimePlusExpression(subExps);
+
+
+ case "CASE":
+ ret = new BeamSqlCaseExpression(subExps);
+ break;
+ case "CAST":
+ ret = new BeamSqlCastExpression(subExps, node.type.getSqlTypeName());
+ break;
+
+ case "IS NULL":
+ ret = new BeamSqlIsNullExpression(subExps.get(0));
+ break;
+ case "IS NOT NULL":
+ ret = new BeamSqlIsNotNullExpression(subExps.get(0));
+ break;
+
+ case "HOP":
+ case "TUMBLE":
+ case "SESSION":
+ ret = new BeamSqlWindowExpression(subExps, node.type.getSqlTypeName());
+ break;
+ case "HOP_START":
+ case "TUMBLE_START":
+ case "SESSION_START":
+ ret = new BeamSqlWindowStartExpression();
+ break;
+ case "HOP_END":
+ case "TUMBLE_END":
+ case "SESSION_END":
+ ret = new BeamSqlWindowEndExpression();
+ break;
+ default:
+ //handle UDF
+ if (((RexCall) rexNode).getOperator() instanceof SqlUserDefinedFunction) {
+ SqlUserDefinedFunction udf = (SqlUserDefinedFunction) ((RexCall) rexNode).getOperator();
+ ScalarFunctionImpl fn = (ScalarFunctionImpl) udf.getFunction();
+ ret = new BeamSqlUdfExpression(fn.method, subExps,
+ ((RexCall) rexNode).type.getSqlTypeName());
+ } else {
+ throw new UnsupportedOperationException("Operator: " + opName + " is not supported yet!");
+ }
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ String.format("%s is not supported yet!", rexNode.getClass().toString()));
+ }
+
+ if (ret != null && !ret.accept()) {
+ throw new IllegalStateException(ret.getClass().getSimpleName()
+ + " does not accept the operands.(" + rexNode + ")");
+ }
+
+ return ret;
+ }
+
+ private static boolean isDateNode(SqlTypeName type, Object value) {
+ return (type == SqlTypeName.DATE || type == SqlTypeName.TIMESTAMP)
+ && value instanceof Calendar;
+ }
+
+ @Override
+ public void prepare() {
+ }
+
+ @Override
+ public List<Object> execute(BeamRecord inputRow, BoundedWindow window) {
+ List<Object> results = new ArrayList<>();
+ for (BeamSqlExpression exp : exps) {
+ results.add(exp.evaluate(inputRow, window).getValue());
+ }
+ return results;
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
new file mode 100644
index 0000000..955444f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlCaseExpression} represents CASE, NULLIF, COALESCE in SQL.
+ */
+public class BeamSqlCaseExpression extends BeamSqlExpression {
+ public BeamSqlCaseExpression(List<BeamSqlExpression> operands) {
+ // the return type of CASE is the type of the `else` condition
+ super(operands, operands.get(operands.size() - 1).getOutputType());
+ }
+
+ @Override public boolean accept() {
+ // `when`-`then` pair + `else`
+ if (operands.size() % 2 != 1) {
+ return false;
+ }
+
+ for (int i = 0; i < operands.size() - 1; i += 2) {
+ if (opType(i) != SqlTypeName.BOOLEAN) {
+ return false;
+ } else if (opType(i + 1) != outputType) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ for (int i = 0; i < operands.size() - 1; i += 2) {
+ if (opValueEvaluated(i, inputRow, window)) {
+ return BeamSqlPrimitive.of(
+ outputType,
+ opValueEvaluated(i + 1, inputRow, window)
+ );
+ }
+ }
+ return BeamSqlPrimitive.of(outputType,
+ opValueEvaluated(operands.size() - 1, inputRow, window));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpression.java
new file mode 100644
index 0000000..9ea66c1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpression.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.DateTimeFormatterBuilder;
+import org.joda.time.format.DateTimeParser;
+
+/**
+ * Base class to support 'CAST' operations for all {@link SqlTypeName}.
+ */
+public class BeamSqlCastExpression extends BeamSqlExpression {
+
+ private static final int index = 0;
+ private static final String outputTimestampFormat = "yyyy-MM-dd HH:mm:ss";
+ private static final String outputDateFormat = "yyyy-MM-dd";
+ /**
+ * Date and Timestamp formats used to parse
+ * {@link SqlTypeName#DATE}, {@link SqlTypeName#TIMESTAMP}.
+ */
+ private static final DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
+ .append(null/*printer*/, new DateTimeParser[] {
+ // date formats
+ DateTimeFormat.forPattern("yy-MM-dd").getParser(),
+ DateTimeFormat.forPattern("yy/MM/dd").getParser(),
+ DateTimeFormat.forPattern("yy.MM.dd").getParser(),
+ DateTimeFormat.forPattern("yyMMdd").getParser(),
+ DateTimeFormat.forPattern("yyyyMMdd").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd").getParser(),
+ DateTimeFormat.forPattern("yyyy/MM/dd").getParser(),
+ DateTimeFormat.forPattern("yyyy.MM.dd").getParser(),
+ // datetime formats
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ssz").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss z").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSSz").getParser(),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS z").getParser() }).toFormatter()
+ .withPivotYear(2020);
+
+ public BeamSqlCastExpression(List<BeamSqlExpression> operands, SqlTypeName castType) {
+ super(operands, castType);
+ }
+
+ @Override
+ public boolean accept() {
+ return numberOfOperands() == 1;
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ SqlTypeName castOutputType = getOutputType();
+ switch (castOutputType) {
+ case INTEGER:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.INTEGER, SqlFunctions.toInt(opValueEvaluated(index, inputRow, window)));
+ case DOUBLE:
+ return BeamSqlPrimitive.of(SqlTypeName.DOUBLE,
+ SqlFunctions.toDouble(opValueEvaluated(index, inputRow, window)));
+ case SMALLINT:
+ return BeamSqlPrimitive.of(SqlTypeName.SMALLINT,
+ SqlFunctions.toShort(opValueEvaluated(index, inputRow, window)));
+ case TINYINT:
+ return BeamSqlPrimitive.of(SqlTypeName.TINYINT,
+ SqlFunctions.toByte(opValueEvaluated(index, inputRow, window)));
+ case BIGINT:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.BIGINT, SqlFunctions.toLong(opValueEvaluated(index, inputRow, window)));
+ case DECIMAL:
+ return BeamSqlPrimitive.of(SqlTypeName.DECIMAL,
+ SqlFunctions.toBigDecimal(opValueEvaluated(index, inputRow, window)));
+ case FLOAT:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.FLOAT, SqlFunctions.toFloat(opValueEvaluated(index, inputRow, window)));
+ case CHAR:
+ case VARCHAR:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.VARCHAR, opValueEvaluated(index, inputRow, window).toString());
+ case DATE:
+ return BeamSqlPrimitive.of(SqlTypeName.DATE,
+ toDate(opValueEvaluated(index, inputRow, window), outputDateFormat));
+ case TIMESTAMP:
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP,
+ toTimeStamp(opValueEvaluated(index, inputRow, window), outputTimestampFormat));
+ }
+ throw new UnsupportedOperationException(
+ String.format("Cast to type %s not supported", castOutputType));
+ }
+
+ private Date toDate(Object inputDate, String outputFormat) {
+ try {
+ return Date
+ .valueOf(dateTimeFormatter.parseLocalDate(inputDate.toString()).toString(outputFormat));
+ } catch (IllegalArgumentException | UnsupportedOperationException e) {
+ throw new UnsupportedOperationException("Can't be cast to type 'Date'");
+ }
+ }
+
+ private Timestamp toTimeStamp(Object inputTimestamp, String outputFormat) {
+ try {
+ return Timestamp.valueOf(
+ dateTimeFormatter.parseDateTime(inputTimestamp.toString()).secondOfMinute()
+ .roundCeilingCopy().toString(outputFormat));
+ } catch (IllegalArgumentException | UnsupportedOperationException e) {
+ throw new UnsupportedOperationException("Can't be cast to type 'Timestamp'");
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlExpression.java
new file mode 100644
index 0000000..d18b141
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlExpression.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} is an equivalent expression in BeamSQL, of {@link RexNode} in Calcite.
+ *
+ * <p>An implementation of {@link BeamSqlExpression} takes one or more {@code BeamSqlExpression}
+ * as its operands, and return a value with type {@link SqlTypeName}.
+ *
+ */
+public abstract class BeamSqlExpression implements Serializable {
+ protected List<BeamSqlExpression> operands;
+ protected SqlTypeName outputType;
+
+ protected BeamSqlExpression(){}
+
+ public BeamSqlExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ this.operands = operands;
+ this.outputType = outputType;
+ }
+
+ public BeamSqlExpression op(int idx) {
+ return operands.get(idx);
+ }
+
+ public SqlTypeName opType(int idx) {
+ return op(idx).getOutputType();
+ }
+
+ public <T> T opValueEvaluated(int idx, BeamRecord row, BoundedWindow window) {
+ return (T) op(idx).evaluate(row, window).getValue();
+ }
+
+ /**
+ * assertion to make sure the input and output are supported in this expression.
+ */
+ public abstract boolean accept();
+
+ /**
+ * Apply input record {@link BeamRecord} with {@link BoundedWindow} to this expression,
+ * the output value is wrapped with {@link BeamSqlPrimitive}.
+ */
+ public abstract BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window);
+
+ public List<BeamSqlExpression> getOperands() {
+ return operands;
+ }
+
+ public SqlTypeName getOutputType() {
+ return outputType;
+ }
+
+ public int numberOfOperands() {
+ return operands.size();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpression.java
new file mode 100644
index 0000000..2c321f7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * An primitive operation for direct field extraction.
+ */
+public class BeamSqlInputRefExpression extends BeamSqlExpression {
+ private int inputRef;
+
+ public BeamSqlInputRefExpression(SqlTypeName sqlTypeName, int inputRef) {
+ super(null, sqlTypeName);
+ this.inputRef = inputRef;
+ }
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return BeamSqlPrimitive.of(outputType, inputRow.getFieldValue(inputRef));
+ }
+
+ public int getInputRef() {
+ return inputRef;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitive.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitive.java
new file mode 100644
index 0000000..21cbc80
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitive.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
+
+/**
+ * {@link BeamSqlPrimitive} is a special, self-reference {@link BeamSqlExpression}.
+ * It holds the value, and return it directly during {@link #evaluate(BeamRecord, BoundedWindow)}.
+ *
+ */
+public class BeamSqlPrimitive<T> extends BeamSqlExpression {
+ private T value;
+
+ private BeamSqlPrimitive() {
+ }
+
+ private BeamSqlPrimitive(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ /**
+ * A builder function to create from Type and value directly.
+ */
+ public static <T> BeamSqlPrimitive<T> of(SqlTypeName outputType, T value){
+ BeamSqlPrimitive<T> exp = new BeamSqlPrimitive<>();
+ exp.outputType = outputType;
+ exp.value = value;
+ if (!exp.accept()) {
+ throw new IllegalArgumentException(
+ String.format("value [%s] doesn't match type [%s].", value, outputType));
+ }
+ return exp;
+ }
+
+ public SqlTypeName getOutputType() {
+ return outputType;
+ }
+
+ public T getValue() {
+ return value;
+ }
+
+ public long getLong() {
+ return (Long) getValue();
+ }
+
+ public double getDouble() {
+ return (Double) getValue();
+ }
+
+ public float getFloat() {
+ return (Float) getValue();
+ }
+
+ public int getInteger() {
+ return (Integer) getValue();
+ }
+
+ public short getShort() {
+ return (Short) getValue();
+ }
+
+ public byte getByte() {
+ return (Byte) getValue();
+ }
+ public boolean getBoolean() {
+ return (Boolean) getValue();
+ }
+
+ public String getString() {
+ return (String) getValue();
+ }
+
+ public Date getDate() {
+ return (Date) getValue();
+ }
+
+ public BigDecimal getDecimal() {
+ return (BigDecimal) getValue();
+ }
+
+ @Override
+ public boolean accept() {
+ if (value == null) {
+ return true;
+ }
+
+ switch (outputType) {
+ case BIGINT:
+ return value instanceof Long;
+ case DECIMAL:
+ return value instanceof BigDecimal;
+ case DOUBLE:
+ return value instanceof Double;
+ case FLOAT:
+ return value instanceof Float;
+ case INTEGER:
+ return value instanceof Integer;
+ case SMALLINT:
+ return value instanceof Short;
+ case TINYINT:
+ return value instanceof Byte;
+ case BOOLEAN:
+ return value instanceof Boolean;
+ case CHAR:
+ case VARCHAR:
+ return value instanceof String || value instanceof NlsString;
+ case TIME:
+ return value instanceof GregorianCalendar;
+ case TIMESTAMP:
+ case DATE:
+ return value instanceof Date;
+ case INTERVAL_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_HOUR:
+ case INTERVAL_DAY:
+ case INTERVAL_MONTH:
+ case INTERVAL_YEAR:
+ return value instanceof BigDecimal;
+ case SYMBOL:
+ // for SYMBOL, it supports anything...
+ return true;
+ default:
+ throw new UnsupportedOperationException(outputType.name());
+ }
+ }
+
+ @Override
+ public BeamSqlPrimitive<T> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return this;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpression.java
new file mode 100644
index 0000000..625de2c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpression.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * invoke a UDF function.
+ */
+public class BeamSqlUdfExpression extends BeamSqlExpression {
+ //as Method is not Serializable, need to keep class/method information, and rebuild it.
+ private transient Method method;
+ private transient Object udfIns;
+ private String className;
+ private String methodName;
+ private List<String> paraClassName = new ArrayList<>();
+
+ public BeamSqlUdfExpression(Method method, List<BeamSqlExpression> subExps,
+ SqlTypeName sqlTypeName) {
+ super(subExps, sqlTypeName);
+ this.method = method;
+
+ this.className = method.getDeclaringClass().getName();
+ this.methodName = method.getName();
+ for (Class<?> c : method.getParameterTypes()) {
+ paraClassName.add(c.getName());
+ }
+ }
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ if (method == null) {
+ reConstructMethod();
+ }
+ try {
+ List<Object> paras = new ArrayList<>();
+ for (BeamSqlExpression e : getOperands()) {
+ paras.add(e.evaluate(inputRow, window).getValue());
+ }
+
+ return BeamSqlPrimitive.of(getOutputType(),
+ method.invoke(udfIns, paras.toArray(new Object[]{})));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * re-construct method from class/method.
+ */
+ private void reConstructMethod() {
+ try {
+ List<Class<?>> paraClass = new ArrayList<>();
+ for (String pc : paraClassName) {
+ paraClass.add(Class.forName(pc));
+ }
+ method = Class.forName(className).getMethod(methodName, paraClass.toArray(new Class<?>[] {}));
+ if (!Modifier.isStatic(method.getModifiers())) {
+ udfIns = Class.forName(className).newInstance();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowEndExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowEndExpression.java
new file mode 100644
index 0000000..919612e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowEndExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.Date;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP_END}, {@code TUMBLE_END}, {@code SESSION_END} operation.
+ *
+ * <p>These operators returns the <em>end</em> timestamp of window.
+ */
+public class BeamSqlWindowEndExpression extends BeamSqlExpression {
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive<Date> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ if (window instanceof IntervalWindow) {
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP, ((IntervalWindow) window).end().toDate());
+ } else {
+ throw new UnsupportedOperationException(
+ "Cannot run HOP_END|TUMBLE_END|SESSION_END on GlobalWindow.");
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowExpression.java
new file mode 100644
index 0000000..0298f26
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowExpression.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP}, {@code TUMBLE}, {@code SESSION} operation.
+ *
+ * <p>These functions don't change the timestamp field, instead it's used to indicate
+ * the event_timestamp field, and how the window is defined.
+ */
+public class BeamSqlWindowExpression extends BeamSqlExpression {
+
+ public BeamSqlWindowExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override
+ public boolean accept() {
+ return operands.get(0).getOutputType().equals(SqlTypeName.DATE)
+ || operands.get(0).getOutputType().equals(SqlTypeName.TIME)
+ || operands.get(0).getOutputType().equals(SqlTypeName.TIMESTAMP);
+ }
+
+ @Override
+ public BeamSqlPrimitive<Date> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP,
+ (Date) operands.get(0).evaluate(inputRow, window).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowStartExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowStartExpression.java
new file mode 100644
index 0000000..4b250a5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowStartExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.Date;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP_START}, {@code TUMBLE_START},
+ * {@code SESSION_START} operation.
+ *
+ * <p>These operators returns the <em>start</em> timestamp of window.
+ */
+public class BeamSqlWindowStartExpression extends BeamSqlExpression {
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive<Date> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ if (window instanceof IntervalWindow) {
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP, ((IntervalWindow) window).start().toDate());
+ } else {
+ throw new UnsupportedOperationException(
+ "Cannot run HOP_START|TUMBLE_START|SESSION_START on GlobalWindow.");
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/UdafImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/UdafImpl.java
new file mode 100644
index 0000000..83ed7f8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/UdafImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.io.Serializable;
+import java.lang.reflect.ParameterizedType;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.calcite.adapter.enumerable.AggImplementor;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.AggregateFunction;
+import org.apache.calcite.schema.FunctionParameter;
+import org.apache.calcite.schema.ImplementableAggFunction;
+
+/**
+ * Implement {@link AggregateFunction} to take a {@link CombineFn} as UDAF.
+ */
+public final class UdafImpl<InputT, AccumT, OutputT>
+ implements AggregateFunction, ImplementableAggFunction, Serializable{
+ private CombineFn<InputT, AccumT, OutputT> combineFn;
+
+ public UdafImpl(CombineFn<InputT, AccumT, OutputT> combineFn) {
+ this.combineFn = combineFn;
+ }
+
+ public CombineFn<InputT, AccumT, OutputT> getCombineFn() {
+ return combineFn;
+ }
+
+ @Override
+ public List<FunctionParameter> getParameters() {
+ List<FunctionParameter> para = new ArrayList<>();
+ para.add(new FunctionParameter() {
+ public int getOrdinal() {
+ return 0; //up to one parameter is supported in UDAF.
+ }
+
+ public String getName() {
+ // not used as Beam SQL uses its own execution engine
+ return null;
+ }
+
+ public RelDataType getType(RelDataTypeFactory typeFactory) {
+ //the first generic type of CombineFn is the input type.
+ ParameterizedType parameterizedType = (ParameterizedType) combineFn.getClass()
+ .getGenericSuperclass();
+ return typeFactory.createJavaType(
+ (Class) parameterizedType.getActualTypeArguments()[0]);
+ }
+
+ public boolean isOptional() {
+ // not used as Beam SQL uses its own execution engine
+ return false;
+ }
+ });
+ return para;
+ }
+
+ @Override
+ public AggImplementor getImplementor(boolean windowContext) {
+ // not used as Beam SQL uses its own execution engine
+ return null;
+ }
+
+ @Override
+ public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
+ return typeFactory.createJavaType((Class) combineFn.getOutputType().getType());
+ }
+}
+
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
new file mode 100644
index 0000000..cc15ff5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for all arithmetic operators.
+ */
+public abstract class BeamSqlArithmeticExpression extends BeamSqlExpression {
+ private static final List<SqlTypeName> ORDERED_APPROX_TYPES = new ArrayList<>();
+ static {
+ ORDERED_APPROX_TYPES.add(SqlTypeName.TINYINT);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.SMALLINT);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.INTEGER);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.BIGINT);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.FLOAT);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.DOUBLE);
+ ORDERED_APPROX_TYPES.add(SqlTypeName.DECIMAL);
+ }
+
+ protected BeamSqlArithmeticExpression(List<BeamSqlExpression> operands) {
+ super(operands, deduceOutputType(operands.get(0).getOutputType(),
+ operands.get(1).getOutputType()));
+ }
+
+ protected BeamSqlArithmeticExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> evaluate(BeamRecord inputRow,
+ BoundedWindow window) {
+ BigDecimal left = BigDecimal.valueOf(
+ Double.valueOf(opValueEvaluated(0, inputRow, window).toString()));
+ BigDecimal right = BigDecimal.valueOf(
+ Double.valueOf(opValueEvaluated(1, inputRow, window).toString()));
+
+ BigDecimal result = calc(left, right);
+ return getCorrectlyTypedResult(result);
+ }
+
+ protected abstract BigDecimal calc(BigDecimal left, BigDecimal right);
+
+ protected static SqlTypeName deduceOutputType(SqlTypeName left, SqlTypeName right) {
+ int leftIndex = ORDERED_APPROX_TYPES.indexOf(left);
+ int rightIndex = ORDERED_APPROX_TYPES.indexOf(right);
+ if ((left == SqlTypeName.FLOAT || right == SqlTypeName.FLOAT)
+ && (left == SqlTypeName.DECIMAL || right == SqlTypeName.DECIMAL)) {
+ return SqlTypeName.DOUBLE;
+ }
+
+ if (leftIndex < rightIndex) {
+ return right;
+ } else if (leftIndex > rightIndex) {
+ return left;
+ } else {
+ return left;
+ }
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() != 2) {
+ return false;
+ }
+
+ for (BeamSqlExpression operand : operands) {
+ if (!SqlTypeName.NUMERIC_TYPES.contains(operand.getOutputType())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ protected BeamSqlPrimitive<? extends Number> getCorrectlyTypedResult(BigDecimal rawResult) {
+ Number actualValue;
+ switch (outputType) {
+ case TINYINT:
+ actualValue = rawResult.byteValue();
+ break;
+ case SMALLINT:
+ actualValue = rawResult.shortValue();
+ break;
+ case INTEGER:
+ actualValue = rawResult.intValue();
+ break;
+ case BIGINT:
+ actualValue = rawResult.longValue();
+ break;
+ case FLOAT:
+ actualValue = rawResult.floatValue();
+ break;
+ case DOUBLE:
+ actualValue = rawResult.doubleValue();
+ break;
+ case DECIMAL:
+ default:
+ actualValue = rawResult;
+ }
+ return BeamSqlPrimitive.of(outputType, actualValue);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlDivideExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
new file mode 100644
index 0000000..d62a3f8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '/' operator.
+ */
+public class BeamSqlDivideExpression extends BeamSqlArithmeticExpression {
+ public BeamSqlDivideExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+ return left.divide(right, 10, RoundingMode.HALF_EVEN);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
new file mode 100644
index 0000000..4fc6a4b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '-' operator.
+ */
+public class BeamSqlMinusExpression extends BeamSqlArithmeticExpression {
+ public BeamSqlMinusExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+ return left.subtract(right);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
new file mode 100644
index 0000000..5c55bf4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '%' operator.
+ */
+public class BeamSqlModExpression extends BeamSqlArithmeticExpression {
+ public BeamSqlModExpression(List<BeamSqlExpression> operands) {
+ super(operands, operands.get(1).getOutputType());
+ }
+
+ @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+ return BigDecimal.valueOf(left.doubleValue() % right.doubleValue());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
new file mode 100644
index 0000000..e6cd35d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '*' operator.
+ */
+public class BeamSqlMultiplyExpression extends BeamSqlArithmeticExpression {
+ public BeamSqlMultiplyExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+ return left.multiply(right);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlPlusExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
new file mode 100644
index 0000000..87ccfe4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '+' operator.
+ */
+public class BeamSqlPlusExpression extends BeamSqlArithmeticExpression {
+ public BeamSqlPlusExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+ return left.add(right);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/package-info.java
new file mode 100644
index 0000000..78ec610
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Arithmetic operators.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlCompareExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlCompareExpression.java
new file mode 100644
index 0000000..df8bd61
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlCompareExpression.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@link BeamSqlCompareExpression} is used for compare operations.
+ *
+ * <p>See {@link BeamSqlEqualsExpression}, {@link BeamSqlLessThanExpression},
+ * {@link BeamSqlLessThanOrEqualsExpression}, {@link BeamSqlGreaterThanExpression},
+ * {@link BeamSqlGreaterThanOrEqualsExpression} and {@link BeamSqlNotEqualsExpression}
+ * for more details.
+ *
+ */
+public abstract class BeamSqlCompareExpression extends BeamSqlExpression {
+
+ private BeamSqlCompareExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ public BeamSqlCompareExpression(List<BeamSqlExpression> operands) {
+ this(operands, SqlTypeName.BOOLEAN);
+ }
+
+ /**
+ * Compare operation must have 2 operands.
+ */
+ @Override
+ public boolean accept() {
+ return operands.size() == 2;
+ }
+
+ @Override
+ public BeamSqlPrimitive<Boolean> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Object leftValue = operands.get(0).evaluate(inputRow, window).getValue();
+ Object rightValue = operands.get(1).evaluate(inputRow, window).getValue();
+ switch (operands.get(0).getOutputType()) {
+ case BIGINT:
+ case DECIMAL:
+ case DOUBLE:
+ case FLOAT:
+ case INTEGER:
+ case SMALLINT:
+ case TINYINT:
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+ compare((Number) leftValue, (Number) rightValue));
+ case BOOLEAN:
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+ compare((Boolean) leftValue, (Boolean) rightValue));
+ case VARCHAR:
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+ compare((CharSequence) leftValue, (CharSequence) rightValue));
+ default:
+ throw new UnsupportedOperationException(toString());
+ }
+ }
+
+ /**
+ * Compare between String values, mapping to {@link SqlTypeName#VARCHAR}.
+ */
+ public abstract Boolean compare(CharSequence leftValue, CharSequence rightValue);
+
+ /**
+ * Compare between Boolean values, mapping to {@link SqlTypeName#BOOLEAN}.
+ */
+ public abstract Boolean compare(Boolean leftValue, Boolean rightValue);
+
+ /**
+ * Compare between Number values, including {@link SqlTypeName#BIGINT},
+ * {@link SqlTypeName#DECIMAL}, {@link SqlTypeName#DOUBLE}, {@link SqlTypeName#FLOAT},
+ * {@link SqlTypeName#INTEGER}, {@link SqlTypeName#SMALLINT} and {@link SqlTypeName#TINYINT}.
+ */
+ public abstract Boolean compare(Number leftValue, Number rightValue);
+
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlEqualsExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlEqualsExpression.java
new file mode 100644
index 0000000..9b275ce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlEqualsExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code =} operation.
+ */
+public class BeamSqlEqualsExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlEqualsExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) == 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ return !(leftValue ^ rightValue);
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() == (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanExpression.java
new file mode 100644
index 0000000..4add258
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code >} operation.
+ */
+public class BeamSqlGreaterThanExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlGreaterThanExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) > 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ throw new IllegalArgumentException("> is not supported for Boolean.");
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() > (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
new file mode 100644
index 0000000..99c4c89
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code >=} operation.
+ */
+public class BeamSqlGreaterThanOrEqualsExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlGreaterThanOrEqualsExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) >= 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ throw new IllegalArgumentException(">= is not supported for Boolean.");
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() >= (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
new file mode 100644
index 0000000..9a9739e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'IS NOT NULL' operation.
+ */
+public class BeamSqlIsNotNullExpression extends BeamSqlExpression {
+
+ private BeamSqlIsNotNullExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ public BeamSqlIsNotNullExpression(BeamSqlExpression operand){
+ this(Arrays.asList(operand), SqlTypeName.BOOLEAN);
+ }
+
+ /**
+ * only one operand is required.
+ */
+ @Override
+ public boolean accept() {
+ return operands.size() == 1;
+ }
+
+ @Override
+ public BeamSqlPrimitive<Boolean> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Object leftValue = operands.get(0).evaluate(inputRow, window).getValue();
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, leftValue != null);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNullExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNullExpression.java
new file mode 100644
index 0000000..6034344
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNullExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'IS NULL' operation.
+ */
+public class BeamSqlIsNullExpression extends BeamSqlExpression {
+
+ private BeamSqlIsNullExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ public BeamSqlIsNullExpression(BeamSqlExpression operand){
+ this(Arrays.asList(operand), SqlTypeName.BOOLEAN);
+ }
+
+ /**
+ * only one operand is required.
+ */
+ @Override
+ public boolean accept() {
+ return operands.size() == 1;
+ }
+
+ @Override
+ public BeamSqlPrimitive<Boolean> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Object leftValue = operands.get(0).evaluate(inputRow, window).getValue();
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, leftValue == null);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanExpression.java
new file mode 100644
index 0000000..2122d93
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code <} operation.
+ */
+public class BeamSqlLessThanExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlLessThanExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) < 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ throw new IllegalArgumentException("< is not supported for Boolean.");
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() < (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanOrEqualsExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanOrEqualsExpression.java
new file mode 100644
index 0000000..8cd4402
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanOrEqualsExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code <=} operation.
+ */
+public class BeamSqlLessThanOrEqualsExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlLessThanOrEqualsExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) <= 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ throw new IllegalArgumentException("<= is not supported for Boolean.");
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() <= (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlNotEqualsExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlNotEqualsExpression.java
new file mode 100644
index 0000000..3733a26
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlNotEqualsExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * {@code BeamSqlExpression} for {@code <>} operation.
+ */
+public class BeamSqlNotEqualsExpression extends BeamSqlCompareExpression {
+
+ public BeamSqlNotEqualsExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+ return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) != 0;
+ }
+
+ @Override
+ public Boolean compare(Boolean leftValue, Boolean rightValue) {
+ return leftValue ^ rightValue;
+ }
+
+ @Override
+ public Boolean compare(Number leftValue, Number rightValue) {
+ return (leftValue == null && rightValue == null)
+ || (leftValue != null && rightValue != null
+ && leftValue.floatValue() != (rightValue).floatValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/package-info.java
new file mode 100644
index 0000000..2a400f7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Comparison operators.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpression.java
new file mode 100644
index 0000000..336772d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpression.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Collections;
+import java.util.Date;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for CURRENT_DATE and LOCALTIME.
+ *
+ * <p>Returns the current date in the session time zone, in a value of datatype DATE.
+ */
+public class BeamSqlCurrentDateExpression extends BeamSqlExpression {
+ public BeamSqlCurrentDateExpression() {
+ super(Collections.<BeamSqlExpression>emptyList(), SqlTypeName.DATE);
+ }
+ @Override public boolean accept() {
+ return getOperands().size() == 0;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return BeamSqlPrimitive.of(outputType, new Date());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpression.java
new file mode 100644
index 0000000..fe3feb8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpression.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.List;
+import java.util.TimeZone;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for LOCALTIME and CURRENT_TIME.
+ *
+ * <p>Returns the current date and time in the session time zone in a value of datatype TIME, with
+ * precision digits of precision.
+ *
+ * <p>NOTE: for simplicity, we will ignore the {@code precision} param.
+ */
+public class BeamSqlCurrentTimeExpression extends BeamSqlExpression {
+ public BeamSqlCurrentTimeExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.TIME);
+ }
+ @Override public boolean accept() {
+ int opCount = getOperands().size();
+ return opCount <= 1;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ GregorianCalendar ret = new GregorianCalendar(TimeZone.getDefault());
+ ret.setTime(new Date());
+ return BeamSqlPrimitive.of(outputType, ret);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpression.java
new file mode 100644
index 0000000..ca4b3ce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for LOCALTIMESTAMP and CURRENT_TIMESTAMP.
+ *
+ * <p>Returns the current date and time in the session time zone in a value of datatype TIMESTAMP,
+ * with precision digits of precision.
+ *
+ * <p>NOTE: for simplicity, we will ignore the {@code precision} param.
+ */
+public class BeamSqlCurrentTimestampExpression extends BeamSqlExpression {
+ public BeamSqlCurrentTimestampExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.TIMESTAMP);
+ }
+ @Override public boolean accept() {
+ int opCount = getOperands().size();
+ return opCount <= 1;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return BeamSqlPrimitive.of(outputType, new Date());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpression.java
new file mode 100644
index 0000000..0e1d3db
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpression.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.DateTimeUtils;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for CEIL(date).
+ *
+ * <p>NOTE: only support CEIL for {@link TimeUnitRange#YEAR} and {@link TimeUnitRange#MONTH}.
+ */
+public class BeamSqlDateCeilExpression extends BeamSqlExpression {
+ public BeamSqlDateCeilExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.TIMESTAMP);
+ }
+ @Override public boolean accept() {
+ return operands.size() == 2
+ && opType(1) == SqlTypeName.SYMBOL;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Date date = opValueEvaluated(0, inputRow, window);
+ long time = date.getTime();
+ TimeUnitRange unit = ((BeamSqlPrimitive<TimeUnitRange>) op(1)).getValue();
+
+ long newTime = DateTimeUtils.unixTimestampCeil(unit, time);
+ Date newDate = new Date(newTime);
+
+ return BeamSqlPrimitive.of(outputType, newDate);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpression.java
new file mode 100644
index 0000000..2593629
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpression.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.DateTimeUtils;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for FLOOR(date).
+ *
+ * <p>NOTE: only support FLOOR for {@link TimeUnitRange#YEAR} and {@link TimeUnitRange#MONTH}.
+ */
+public class BeamSqlDateFloorExpression extends BeamSqlExpression {
+ public BeamSqlDateFloorExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DATE);
+ }
+ @Override public boolean accept() {
+ return operands.size() == 2
+ && opType(1) == SqlTypeName.SYMBOL;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Date date = opValueEvaluated(0, inputRow, window);
+ long time = date.getTime();
+ TimeUnitRange unit = ((BeamSqlPrimitive<TimeUnitRange>) op(1)).getValue();
+
+ long newTime = DateTimeUtils.unixTimestampFloor(unit, time);
+ Date newDate = new Date(newTime);
+
+ return BeamSqlPrimitive.of(outputType, newDate);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpression.java
new file mode 100644
index 0000000..6948ba1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpression.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DurationFieldType;
+
+/**
+ * Infix '-' operation for timestamps.
+ *
+ * <p>Implements 2 SQL subtraction operations at the moment:
+ * 'timestampdiff(timeunit, timestamp, timestamp)', and 'timestamp - interval'
+ *
+ * <p>Calcite converts both of the above into infix '-' expression, with different operands and
+ * return types.
+ *
+ * <p>This class delegates evaluation to specific implementation of one of the above operations,
+ * see {@link BeamSqlTimestampMinusTimestampExpression}
+ * and {@link BeamSqlTimestampMinusIntervalExpression}
+ *
+ * <p>Calcite supports one more subtraction kind: 'interval - interval',
+ * but it is not implemented yet.
+ */
+public class BeamSqlDatetimeMinusExpression extends BeamSqlExpression {
+
+ static final Map<SqlTypeName, DurationFieldType> INTERVALS_DURATIONS_TYPES =
+ ImmutableMap.<SqlTypeName, DurationFieldType>builder()
+ .put(SqlTypeName.INTERVAL_SECOND, DurationFieldType.seconds())
+ .put(SqlTypeName.INTERVAL_MINUTE, DurationFieldType.minutes())
+ .put(SqlTypeName.INTERVAL_HOUR, DurationFieldType.hours())
+ .put(SqlTypeName.INTERVAL_DAY, DurationFieldType.days())
+ .put(SqlTypeName.INTERVAL_MONTH, DurationFieldType.months())
+ .put(SqlTypeName.INTERVAL_YEAR, DurationFieldType.years())
+ .build();
+
+ private BeamSqlExpression delegateExpression;
+
+ public BeamSqlDatetimeMinusExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+
+ this.delegateExpression = createDelegateExpression(operands, outputType);
+ }
+
+ private BeamSqlExpression createDelegateExpression(
+ List<BeamSqlExpression> operands, SqlTypeName outputType) {
+
+ if (isTimestampMinusTimestamp(operands, outputType)) {
+ return new BeamSqlTimestampMinusTimestampExpression(operands, outputType);
+ } else if (isTimestampMinusInterval(operands, outputType)) {
+ return new BeamSqlTimestampMinusIntervalExpression(operands, outputType);
+ }
+
+ return null;
+ }
+
+ private boolean isTimestampMinusTimestamp(
+ List<BeamSqlExpression> operands, SqlTypeName outputType) {
+
+ return BeamSqlTimestampMinusTimestampExpression.accept(operands, outputType);
+ }
+
+ private boolean isTimestampMinusInterval(
+ List<BeamSqlExpression> operands, SqlTypeName outputType) {
+
+ return BeamSqlTimestampMinusIntervalExpression.accept(operands, outputType);
+ }
+
+ @Override
+ public boolean accept() {
+ return delegateExpression != null && delegateExpression.accept();
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ if (delegateExpression == null) {
+ throw new IllegalStateException("Unable to execute unsupported 'datetime minus' expression");
+ }
+
+ return delegateExpression.evaluate(inputRow, window);
+ }
+}
+
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpression.java
new file mode 100644
index 0000000..426cda0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpression.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.TimeUnitUtils.timeUnitInternalMultiplier;
+import static org.apache.beam.sdk.extensions.sql.impl.utils.SqlTypeUtils.findExpressionOfType;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+
+/**
+ * DATETIME_PLUS operation.
+ * Calcite converts 'TIMESTAMPADD(..)' or 'DATE + INTERVAL' from the user input
+ * into DATETIME_PLUS.
+ *
+ * <p>Input and output are expected to be of type TIMESTAMP.
+ */
+public class BeamSqlDatetimePlusExpression extends BeamSqlExpression {
+
+ private static final Set<SqlTypeName> SUPPORTED_INTERVAL_TYPES = ImmutableSet.of(
+ SqlTypeName.INTERVAL_SECOND,
+ SqlTypeName.INTERVAL_MINUTE,
+ SqlTypeName.INTERVAL_HOUR,
+ SqlTypeName.INTERVAL_DAY,
+ SqlTypeName.INTERVAL_MONTH,
+ SqlTypeName.INTERVAL_YEAR);
+
+ public BeamSqlDatetimePlusExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.TIMESTAMP);
+ }
+
+ /**
+ * Requires exactly 2 operands. One should be a timestamp, another an interval
+ */
+ @Override
+ public boolean accept() {
+ return operands.size() == 2
+ && SqlTypeName.TIMESTAMP.equals(operands.get(0).getOutputType())
+ && SUPPORTED_INTERVAL_TYPES.contains(operands.get(1).getOutputType());
+ }
+
+ /**
+ * Adds interval to the timestamp.
+ *
+ * <p>Interval has a value of 'multiplier * TimeUnit.multiplier'.
+ *
+ * <p>For example, '3 years' is going to have a type of INTERVAL_YEAR, and a value of 36.
+ * And '2 minutes' is going to be an INTERVAL_MINUTE with a value of 120000. This is the way
+ * Calcite handles interval expressions, and {@link BeamSqlIntervalMultiplyExpression} also works
+ * the same way.
+ */
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ DateTime timestamp = getTimestampOperand(inputRow, window);
+ BeamSqlPrimitive intervalOperandPrimitive = getIntervalOperand(inputRow, window);
+ SqlTypeName intervalOperandType = intervalOperandPrimitive.getOutputType();
+ int intervalMultiplier = getIntervalMultiplier(intervalOperandPrimitive);
+
+ DateTime newDate = addInterval(timestamp, intervalOperandType, intervalMultiplier);
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP, newDate.toDate());
+ }
+
+ private int getIntervalMultiplier(BeamSqlPrimitive intervalOperandPrimitive) {
+ BigDecimal intervalOperandValue = intervalOperandPrimitive.getDecimal();
+ BigDecimal multiplier = intervalOperandValue.divide(
+ timeUnitInternalMultiplier(intervalOperandPrimitive.getOutputType()),
+ BigDecimal.ROUND_CEILING);
+ return multiplier.intValueExact();
+ }
+
+ private BeamSqlPrimitive getIntervalOperand(BeamRecord inputRow, BoundedWindow window) {
+ return findExpressionOfType(operands, SUPPORTED_INTERVAL_TYPES).get()
+ .evaluate(inputRow, window);
+ }
+
+ private DateTime getTimestampOperand(BeamRecord inputRow, BoundedWindow window) {
+ BeamSqlPrimitive timestampOperandPrimitive =
+ findExpressionOfType(operands, SqlTypeName.TIMESTAMP).get().evaluate(inputRow, window);
+ return new DateTime(timestampOperandPrimitive.getDate());
+ }
+
+ private DateTime addInterval(
+ DateTime dateTime, SqlTypeName intervalType, int numberOfIntervals) {
+
+ switch (intervalType) {
+ case INTERVAL_SECOND:
+ return dateTime.plusSeconds(numberOfIntervals);
+ case INTERVAL_MINUTE:
+ return dateTime.plusMinutes(numberOfIntervals);
+ case INTERVAL_HOUR:
+ return dateTime.plusHours(numberOfIntervals);
+ case INTERVAL_DAY:
+ return dateTime.plusDays(numberOfIntervals);
+ case INTERVAL_MONTH:
+ return dateTime.plusMonths(numberOfIntervals);
+ case INTERVAL_YEAR:
+ return dateTime.plusYears(numberOfIntervals);
+ default:
+ throw new IllegalArgumentException("Adding "
+ + intervalType.getName() + " to date is not supported");
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpression.java
new file mode 100644
index 0000000..38afd0a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpression.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.DateTimeUtils;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for EXTRACT.
+ *
+ * <p>The following date functions also implicitly converted to {@code EXTRACT}:
+ * <ul>
+ * <li>YEAR(date) => EXTRACT(YEAR FROM date)</li>
+ * <li>MONTH(date) => EXTRACT(MONTH FROM date)</li>
+ * <li>DAY(date) => EXTRACT(DAY FROM date)</li>
+ * <li>QUARTER(date) => EXTRACT(QUARTER FROM date)</li>
+ * <li>WEEK(date) => EXTRACT(WEEK FROM date)</li>
+ * <li>DAYOFYEAR(date) => EXTRACT(DOY FROM date)</li>
+ * <li>DAYOFMONTH(date) => EXTRACT(DAY FROM date)</li>
+ * <li>DAYOFWEEK(date) => EXTRACT(DOW FROM date)</li>
+ * </ul>
+ */
+public class BeamSqlExtractExpression extends BeamSqlExpression {
+ private static final Map<TimeUnitRange, Integer> typeMapping = new HashMap<>();
+ static {
+ typeMapping.put(TimeUnitRange.DOW, Calendar.DAY_OF_WEEK);
+ typeMapping.put(TimeUnitRange.DOY, Calendar.DAY_OF_YEAR);
+ typeMapping.put(TimeUnitRange.WEEK, Calendar.WEEK_OF_YEAR);
+ }
+
+ public BeamSqlExtractExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.BIGINT);
+ }
+ @Override public boolean accept() {
+ return operands.size() == 2
+ && opType(1) == SqlTypeName.BIGINT;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Long time = opValueEvaluated(1, inputRow, window);
+
+ TimeUnitRange unit = ((BeamSqlPrimitive<TimeUnitRange>) op(0)).getValue();
+
+ switch (unit) {
+ case YEAR:
+ case MONTH:
+ case DAY:
+ Long timeByDay = time / 1000 / 3600 / 24;
+ Long extracted = DateTimeUtils.unixDateExtract(
+ unit,
+ timeByDay
+ );
+ return BeamSqlPrimitive.of(outputType, extracted);
+
+ case DOY:
+ case DOW:
+ case WEEK:
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTime(new Date(time));
+ return BeamSqlPrimitive.of(outputType, (long) calendar.get(typeMapping.get(unit)));
+
+ case QUARTER:
+ calendar = Calendar.getInstance();
+ calendar.setTime(new Date(time));
+ long ret = calendar.get(Calendar.MONTH) / 3;
+ if (ret * 3 < calendar.get(Calendar.MONTH)) {
+ ret += 1;
+ }
+ return BeamSqlPrimitive.of(outputType, ret);
+
+ default:
+ throw new UnsupportedOperationException(
+ "Extract for time unit: " + unit + " not supported!");
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpression.java
new file mode 100644
index 0000000..f4ddf71
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpression.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.TimeUnitUtils.timeUnitInternalMultiplier;
+import static org.apache.beam.sdk.extensions.sql.impl.utils.SqlTypeUtils.findExpressionOfType;
+
+import com.google.common.base.Optional;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Multiplication operator for intervals.
+ * For example, allows to express things like '3 years'.
+ *
+ * <p>One use case of this is implementation of TIMESTAMPADD().
+ * Calcite converts TIMESTAMPADD(date, multiplier, inteval) into
+ * DATETIME_PLUS(date, multiplier * interval).
+ * The 'multiplier * interval' part is what this class implements. It's not a regular
+ * numerical multiplication because the return type is expected to be an interval, and the value
+ * is expected to use corresponding TimeUnit's internal value (e.g. 12 for YEAR, 60000 for MINUTE).
+ */
+public class BeamSqlIntervalMultiplyExpression extends BeamSqlExpression {
+ public BeamSqlIntervalMultiplyExpression(List<BeamSqlExpression> operands) {
+ super(operands, deduceOutputType(operands));
+ }
+
+ /**
+ * Output type is null if no operands found with matching types.
+ * Execution will later fail when calling accept()
+ */
+ private static SqlTypeName deduceOutputType(List<BeamSqlExpression> operands) {
+ Optional<BeamSqlExpression> intervalOperand =
+ findExpressionOfType(operands, SqlTypeName.INTERVAL_TYPES);
+
+ return intervalOperand.isPresent()
+ ? intervalOperand.get().getOutputType()
+ : null;
+ }
+
+ /**
+ * Requires exactly 2 operands. One should be integer, another should be interval
+ */
+ @Override
+ public boolean accept() {
+ return operands.size() == 2
+ && findExpressionOfType(operands, SqlTypeName.INTEGER).isPresent()
+ && findExpressionOfType(operands, SqlTypeName.INTERVAL_TYPES).isPresent();
+ }
+ /**
+ * Evaluates the number of times the interval should be repeated, times the TimeUnit multiplier.
+ * For example for '3 * MONTH' this will return an object with type INTERVAL_MONTH and value 36.
+ *
+ * <p>This is due to the fact that TimeUnit has different internal multipliers for each interval,
+ * e.g. MONTH is 12, but MINUTE is 60000. When Calcite parses SQL interval literals, it returns
+ * those internal multipliers. This means we need to do similar thing, so that this multiplication
+ * expression behaves the same way as literal interval expression.
+ *
+ * <p>That is, we need to make sure that this:
+ * "TIMESTAMP '1984-04-19 01:02:03' + INTERVAL '2' YEAR"
+ * is equivalent tot this:
+ * "TIMESTAMPADD(YEAR, 2, TIMESTAMP '1984-04-19 01:02:03')"
+ */
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ BeamSqlPrimitive intervalOperandPrimitive =
+ findExpressionOfType(operands, SqlTypeName.INTERVAL_TYPES).get().evaluate(inputRow, window);
+ SqlTypeName intervalOperandType = intervalOperandPrimitive.getOutputType();
+
+ BeamSqlPrimitive integerOperandPrimitive =
+ findExpressionOfType(operands, SqlTypeName.INTEGER).get().evaluate(inputRow, window);
+ BigDecimal integerOperandValue = new BigDecimal(integerOperandPrimitive.getInteger());
+
+ BigDecimal multiplicationResult =
+ integerOperandValue.multiply(
+ timeUnitInternalMultiplier(intervalOperandType));
+
+ return BeamSqlPrimitive.of(outputType, multiplicationResult);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpression.java
new file mode 100644
index 0000000..236d148
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpression.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimeMinusExpression.INTERVALS_DURATIONS_TYPES;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.joda.time.DurationFieldType;
+import org.joda.time.Period;
+
+/**
+ * '-' operator for 'timestamp - interval' expressions.
+ *
+ * <p>See {@link BeamSqlDatetimeMinusExpression} for other kinds of datetime types subtraction.
+ */
+public class BeamSqlTimestampMinusIntervalExpression extends BeamSqlExpression {
+
+ public BeamSqlTimestampMinusIntervalExpression(
+ List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override
+ public boolean accept() {
+ return accept(operands, outputType);
+ }
+
+ static boolean accept(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ return operands.size() == 2
+ && SqlTypeName.TIMESTAMP.equals(outputType)
+ && SqlTypeName.TIMESTAMP.equals(operands.get(0).getOutputType())
+ && INTERVALS_DURATIONS_TYPES.containsKey(operands.get(1).getOutputType());
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord row, BoundedWindow window) {
+ DateTime date = new DateTime(opValueEvaluated(0, row, window));
+ Period period = intervalToPeriod(op(1).evaluate(row, window));
+
+ Date subtractionResult = date.minus(period).toDate();
+
+ return BeamSqlPrimitive.of(outputType, subtractionResult);
+ }
+
+ private Period intervalToPeriod(BeamSqlPrimitive operand) {
+ BigDecimal intervalValue = operand.getDecimal();
+ SqlTypeName intervalType = operand.getOutputType();
+
+ int numberOfIntervals = intervalValue
+ .divide(TimeUnitUtils.timeUnitInternalMultiplier(intervalType)).intValueExact();
+
+ return new Period().withField(durationFieldType(intervalType), numberOfIntervals);
+ }
+
+ private static DurationFieldType durationFieldType(SqlTypeName intervalTypeToCount) {
+ return INTERVALS_DURATIONS_TYPES.get(intervalTypeToCount);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpression.java
new file mode 100644
index 0000000..64ac9c8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpression.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimeMinusExpression.INTERVALS_DURATIONS_TYPES;
+
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.joda.time.DurationFieldType;
+import org.joda.time.Period;
+import org.joda.time.PeriodType;
+
+/**
+ * Infix '-' operation for timestamps.
+ *
+ * <p>Currently this implementation is specific to how Calcite parses 'TIMESTAMPDIFF(..)'.
+ * It converts the TIMESTAMPDIFF() call into infix minus and normalizes it
+ * with corresponding TimeUnit's multiplier.
+ *
+ * <p>See {@link BeamSqlDatetimeMinusExpression} for other kinds of datetime types subtraction.
+ */
+public class BeamSqlTimestampMinusTimestampExpression extends BeamSqlExpression {
+ private SqlTypeName intervalType;
+
+ public BeamSqlTimestampMinusTimestampExpression(
+ List<BeamSqlExpression> operands, SqlTypeName intervalType) {
+ super(operands, SqlTypeName.BIGINT);
+ this.intervalType = intervalType;
+ }
+
+ /**
+ * Requires exactly 2 operands. One should be a timestamp, another an interval
+ */
+ @Override
+ public boolean accept() {
+ return accept(operands, intervalType);
+ }
+
+ static boolean accept(List<BeamSqlExpression> operands, SqlTypeName intervalType) {
+ return INTERVALS_DURATIONS_TYPES.containsKey(intervalType)
+ && operands.size() == 2
+ && SqlTypeName.TIMESTAMP.equals(operands.get(0).getOutputType())
+ && SqlTypeName.TIMESTAMP.equals(operands.get(1).getOutputType());
+ }
+
+ /**
+ * Returns the count of intervals between dates, times TimeUnit.multiplier of the interval type.
+ * Calcite deals with all intervals this way. Whenever there is an interval, its value is always
+ * multiplied by the corresponding TimeUnit.multiplier
+ */
+ public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ DateTime timestampStart = new DateTime(opValueEvaluated(1, inputRow, window));
+ DateTime timestampEnd = new DateTime(opValueEvaluated(0, inputRow, window));
+
+ long numberOfIntervals = numberOfIntervalsBetweenDates(timestampStart, timestampEnd);
+ long multiplier = TimeUnitUtils.timeUnitInternalMultiplier(intervalType).longValue();
+
+ return BeamSqlPrimitive.of(SqlTypeName.BIGINT, multiplier * numberOfIntervals);
+ }
+
+ private long numberOfIntervalsBetweenDates(DateTime timestampStart, DateTime timestampEnd) {
+ Period period = new Period(timestampStart, timestampEnd,
+ PeriodType.forFields(new DurationFieldType[] { durationFieldType(intervalType) }));
+ return period.get(durationFieldType(intervalType));
+ }
+
+ private static DurationFieldType durationFieldType(SqlTypeName intervalTypeToCount) {
+ if (!INTERVALS_DURATIONS_TYPES.containsKey(intervalTypeToCount)) {
+ throw new IllegalArgumentException("Counting "
+ + intervalTypeToCount.getName() + "s between dates is not supported");
+ }
+
+ return INTERVALS_DURATIONS_TYPES.get(intervalTypeToCount);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtils.java
new file mode 100644
index 0000000..b432d20
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.math.BigDecimal;
+
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utils to convert between Calcite's TimeUnit and Sql intervals.
+ */
+public abstract class TimeUnitUtils {
+
+ /**
+ * @return internal multiplier of a TimeUnit, e.g. YEAR is 12, MINUTE is 60000
+ * @throws IllegalArgumentException if interval type is not supported
+ */
+ public static BigDecimal timeUnitInternalMultiplier(final SqlTypeName sqlIntervalType) {
+ switch (sqlIntervalType) {
+ case INTERVAL_SECOND:
+ return TimeUnit.SECOND.multiplier;
+ case INTERVAL_MINUTE:
+ return TimeUnit.MINUTE.multiplier;
+ case INTERVAL_HOUR:
+ return TimeUnit.HOUR.multiplier;
+ case INTERVAL_DAY:
+ return TimeUnit.DAY.multiplier;
+ case INTERVAL_MONTH:
+ return TimeUnit.MONTH.multiplier;
+ case INTERVAL_YEAR:
+ return TimeUnit.YEAR.multiplier;
+ default:
+ throw new IllegalArgumentException("Interval " + sqlIntervalType
+ + " cannot be converted to TimeUnit");
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/package-info.java
new file mode 100644
index 0000000..3037296
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * date functions.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlAndExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlAndExpression.java
new file mode 100644
index 0000000..2cae22b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlAndExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'AND' operation.
+ */
+public class BeamSqlAndExpression extends BeamSqlLogicalExpression {
+ public BeamSqlAndExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public BeamSqlPrimitive<Boolean> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ boolean result = true;
+ for (BeamSqlExpression exp : operands) {
+ BeamSqlPrimitive<Boolean> expOut = exp.evaluate(inputRow, window);
+ result = result && expOut.getValue();
+ if (!result) {
+ break;
+ }
+ }
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, result);
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlLogicalExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlLogicalExpression.java
new file mode 100644
index 0000000..5691e33
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlLogicalExpression.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for Logical operators.
+ */
+public abstract class BeamSqlLogicalExpression extends BeamSqlExpression {
+ private BeamSqlLogicalExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+ public BeamSqlLogicalExpression(List<BeamSqlExpression> operands) {
+ this(operands, SqlTypeName.BOOLEAN);
+ }
+
+ @Override
+ public boolean accept() {
+ for (BeamSqlExpression exp : operands) {
+ // only accept BOOLEAN expression as operand
+ if (!exp.getOutputType().equals(SqlTypeName.BOOLEAN)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpression.java
new file mode 100644
index 0000000..72a6982
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for logical operator: NOT.
+ *
+ * <p>Whether boolean is not TRUE; returns UNKNOWN if boolean is UNKNOWN.
+ */
+public class BeamSqlNotExpression extends BeamSqlLogicalExpression {
+ public BeamSqlNotExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public boolean accept() {
+ if (numberOfOperands() != 1) {
+ return false;
+ }
+ return super.accept();
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ Boolean value = opValueEvaluated(0, inputRow, window);
+ if (value == null) {
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, window);
+ } else {
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, !value);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlOrExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlOrExpression.java
new file mode 100644
index 0000000..74dde7a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlOrExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'OR' operation.
+ */
+public class BeamSqlOrExpression extends BeamSqlLogicalExpression {
+ public BeamSqlOrExpression(List<BeamSqlExpression> operands) {
+ super(operands);
+ }
+
+ @Override
+ public BeamSqlPrimitive<Boolean> evaluate(BeamRecord inputRow, BoundedWindow window) {
+ boolean result = false;
+ for (BeamSqlExpression exp : operands) {
+ BeamSqlPrimitive<Boolean> expOut = exp.evaluate(inputRow, window);
+ result = result || expOut.getValue();
+ if (result) {
+ break;
+ }
+ }
+ return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, result);
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/package-info.java
new file mode 100644
index 0000000..42df66c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Logical operators.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAbsExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAbsExpression.java
new file mode 100644
index 0000000..01b4cc7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAbsExpression.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'ABS' function.
+ */
+public class BeamSqlAbsExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlAbsExpression(List<BeamSqlExpression> operands) {
+ super(operands, operands.get(0).getOutputType());
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ BeamSqlPrimitive result = null;
+ switch (op.getOutputType()) {
+ case INTEGER:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.INTEGER, SqlFunctions.abs(op.getInteger()));
+ break;
+ case BIGINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.BIGINT, SqlFunctions.abs(op.getLong()));
+ break;
+ case TINYINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.TINYINT, SqlFunctions.abs(op.getByte()));
+ break;
+ case SMALLINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.SMALLINT, SqlFunctions.abs(op.getShort()));
+ break;
+ case FLOAT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.FLOAT, SqlFunctions.abs(op.getFloat()));
+ break;
+ case DECIMAL:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DECIMAL, SqlFunctions.abs(new BigDecimal(op.getValue().toString())));
+ break;
+ case DOUBLE:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.abs(op.getDouble()));
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAcosExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAcosExpression.java
new file mode 100644
index 0000000..3bc10ae
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAcosExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'ACOS' function.
+ */
+public class BeamSqlAcosExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlAcosExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.acos(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAsinExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAsinExpression.java
new file mode 100644
index 0000000..950a9ee
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAsinExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'ASIN' function.
+ */
+public class BeamSqlAsinExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlAsinExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.asin(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtan2Expression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtan2Expression.java
new file mode 100644
index 0000000..33fa3b6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtan2Expression.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@link BeamSqlMathBinaryExpression} for 'ATAN2' function.
+ */
+public class BeamSqlAtan2Expression extends BeamSqlMathBinaryExpression {
+
+ public BeamSqlAtan2Expression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> calculate(BeamSqlPrimitive leftOp,
+ BeamSqlPrimitive rightOp) {
+ return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, SqlFunctions
+ .atan2(SqlFunctions.toDouble(leftOp.getValue()),
+ SqlFunctions.toDouble(rightOp.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtanExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtanExpression.java
new file mode 100644
index 0000000..2f750dd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtanExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'ATAN' function.
+ */
+public class BeamSqlAtanExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlAtanExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.atan(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCeilExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCeilExpression.java
new file mode 100644
index 0000000..c9ca2b0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCeilExpression.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'CEIL' function.
+ */
+public class BeamSqlCeilExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlCeilExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ switch (getOutputType()) {
+ case DECIMAL:
+ return BeamSqlPrimitive.of(SqlTypeName.DECIMAL, SqlFunctions.ceil(op.getDecimal()));
+ default:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.ceil(SqlFunctions.toDouble(op.getValue())));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCosExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCosExpression.java
new file mode 100644
index 0000000..e06e926
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCosExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'COS' function.
+ */
+public class BeamSqlCosExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlCosExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.cos(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
new file mode 100644
index 0000000..68d56b5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'COT' function.
+ */
+public class BeamSqlCotExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlCotExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.cot(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
new file mode 100644
index 0000000..de4eac2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'DEGREES' function.
+ */
+public class BeamSqlDegreesExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlDegreesExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.degrees(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlExpExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlExpExpression.java
new file mode 100644
index 0000000..a789355
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlExpExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'EXP' function.
+ */
+public class BeamSqlExpExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlExpExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.exp(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlFloorExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlFloorExpression.java
new file mode 100644
index 0000000..def50f9
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlFloorExpression.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'FLOOR' function.
+ */
+public class BeamSqlFloorExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlFloorExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ switch (getOutputType()) {
+ case DECIMAL:
+ return BeamSqlPrimitive.of(SqlTypeName.DECIMAL, SqlFunctions.floor(op.getDecimal()));
+ default:
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.floor(SqlFunctions.toDouble(op.getValue())));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLnExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLnExpression.java
new file mode 100644
index 0000000..ea46044
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLnExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'LN' function.
+ */
+public class BeamSqlLnExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlLnExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.ln(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLogExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLogExpression.java
new file mode 100644
index 0000000..9a99b61
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLogExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'Log10' function.
+ */
+public class BeamSqlLogExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlLogExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.log10(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpression.java
new file mode 100644
index 0000000..ed0aac0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpression.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for all binary functions such as
+ * POWER, MOD, RAND_INTEGER, ATAN2, ROUND, TRUNCATE.
+ */
+public abstract class BeamSqlMathBinaryExpression extends BeamSqlExpression {
+
+ public BeamSqlMathBinaryExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override public boolean accept() {
+ return numberOfOperands() == 2 && isOperandNumeric(opType(0)) && isOperandNumeric(opType(1));
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> evaluate(BeamRecord inputRow,
+ BoundedWindow window) {
+ BeamSqlExpression leftOp = op(0);
+ BeamSqlExpression rightOp = op(1);
+ return calculate(leftOp.evaluate(inputRow, window), rightOp.evaluate(inputRow, window));
+ }
+
+ /**
+ * The base method for implementation of math binary functions.
+ *
+ * @param leftOp {@link BeamSqlPrimitive}
+ * @param rightOp {@link BeamSqlPrimitive}
+ * @return {@link BeamSqlPrimitive}
+ */
+ public abstract BeamSqlPrimitive<? extends Number> calculate(BeamSqlPrimitive leftOp,
+ BeamSqlPrimitive rightOp);
+
+ /**
+ * The method to check whether operands are numeric or not.
+ */
+ public boolean isOperandNumeric(SqlTypeName opType) {
+ return SqlTypeName.NUMERIC_TYPES.contains(opType);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpression.java
new file mode 100644
index 0000000..b1a210e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpression.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+
+/**
+ * Base class for all unary functions such as
+ * ABS, SQRT, LN, LOG10, EXP, CEIL, FLOOR, RAND, ACOS,
+ * ASIN, ATAN, COS, COT, DEGREES, RADIANS, SIGN, SIN, TAN.
+ */
+public abstract class BeamSqlMathUnaryExpression extends BeamSqlExpression {
+
+ public BeamSqlMathUnaryExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override public boolean accept() {
+ boolean acceptance = false;
+
+ if (numberOfOperands() == 1 && SqlTypeName.NUMERIC_TYPES.contains(opType(0))) {
+ acceptance = true;
+ }
+ return acceptance;
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> evaluate(BeamRecord inputRow,
+ BoundedWindow window) {
+ BeamSqlExpression operand = op(0);
+ return calculate(operand.evaluate(inputRow, window));
+ }
+
+ /**
+ * For the operands of other type {@link SqlTypeName#NUMERIC_TYPES}.
+ * */
+
+ public abstract BeamSqlPrimitive calculate(BeamSqlPrimitive op);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPiExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPiExpression.java
new file mode 100644
index 0000000..3072ea0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPiExpression.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for the PI function.
+ */
+public class BeamSqlPiExpression extends BeamSqlExpression {
+
+ public BeamSqlPiExpression() {
+ this.outputType = SqlTypeName.DOUBLE;
+ }
+
+ @Override public boolean accept() {
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, Math.PI);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPowerExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPowerExpression.java
new file mode 100644
index 0000000..cc58679
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPowerExpression.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathBinaryExpression} for 'POWER' function.
+ */
+public class BeamSqlPowerExpression extends BeamSqlMathBinaryExpression {
+
+ public BeamSqlPowerExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override
+ public BeamSqlPrimitive<? extends Number> calculate(BeamSqlPrimitive leftOp,
+ BeamSqlPrimitive rightOp) {
+ return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, SqlFunctions
+ .power(SqlFunctions.toDouble(leftOp.getValue()),
+ SqlFunctions.toDouble(rightOp.getValue())));
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRadiansExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRadiansExpression.java
new file mode 100644
index 0000000..74c633d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRadiansExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'RADIANS' function.
+ */
+public class BeamSqlRadiansExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlRadiansExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.radians(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandExpression.java
new file mode 100644
index 0000000..00f2693
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandExpression.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import java.util.Random;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'RAND([seed])' function.
+ */
+public class BeamSqlRandExpression extends BeamSqlExpression {
+ private Random rand = new Random();
+ private Integer seed = null;
+
+ public BeamSqlRandExpression(List<BeamSqlExpression> subExps) {
+ super(subExps, SqlTypeName.DOUBLE);
+ }
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRecord, BoundedWindow window) {
+ if (operands.size() == 1) {
+ int rowSeed = opValueEvaluated(0, inputRecord, window);
+ if (seed == null || seed != rowSeed) {
+ rand.setSeed(rowSeed);
+ }
+ }
+ return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, rand.nextDouble());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandIntegerExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandIntegerExpression.java
new file mode 100644
index 0000000..d055de6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandIntegerExpression.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import java.util.Random;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'RAND_INTEGER([seed, ] numeric)'
+ * function.
+ */
+public class BeamSqlRandIntegerExpression extends BeamSqlExpression {
+ private Random rand = new Random();
+ private Integer seed = null;
+
+ public BeamSqlRandIntegerExpression(List<BeamSqlExpression> subExps) {
+ super(subExps, SqlTypeName.INTEGER);
+ }
+
+ @Override
+ public boolean accept() {
+ return true;
+ }
+
+ @Override
+ public BeamSqlPrimitive evaluate(BeamRecord inputRecord, BoundedWindow window) {
+ int numericIdx = 0;
+ if (operands.size() == 2) {
+ int rowSeed = opValueEvaluated(0, inputRecord, window);
+ if (seed == null || seed != rowSeed) {
+ rand.setSeed(rowSeed);
+ }
+ numericIdx = 1;
+ }
+ return BeamSqlPrimitive.of(SqlTypeName.INTEGER,
+ rand.nextInt((int) opValueEvaluated(numericIdx, inputRecord, window)));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRoundExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRoundExpression.java
new file mode 100644
index 0000000..1725dbb
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRoundExpression.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathBinaryExpression} for 'ROUND' function.
+ */
+public class BeamSqlRoundExpression extends BeamSqlMathBinaryExpression {
+
+ private final BeamSqlPrimitive zero = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 0);
+
+ public BeamSqlRoundExpression(List<BeamSqlExpression> operands) {
+ super(operands, operands.get(0).getOutputType());
+ checkForSecondOperand(operands);
+ }
+
+ private void checkForSecondOperand(List<BeamSqlExpression> operands) {
+ if (numberOfOperands() == 1) {
+ operands.add(1, zero);
+ }
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> calculate(BeamSqlPrimitive leftOp,
+ BeamSqlPrimitive rightOp) {
+ BeamSqlPrimitive result = null;
+ switch (leftOp.getOutputType()) {
+ case SMALLINT:
+ result = BeamSqlPrimitive.of(SqlTypeName.SMALLINT,
+ (short) roundInt(toInt(leftOp.getValue()), toInt(rightOp.getValue())));
+ break;
+ case TINYINT:
+ result = BeamSqlPrimitive.of(SqlTypeName.TINYINT,
+ (byte) roundInt(toInt(leftOp.getValue()), toInt(rightOp.getValue())));
+ break;
+ case INTEGER:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.INTEGER, roundInt(leftOp.getInteger(), toInt(rightOp.getValue())));
+ break;
+ case BIGINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.BIGINT, roundLong(leftOp.getLong(), toInt(rightOp.getValue())));
+ break;
+ case DOUBLE:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, roundDouble(leftOp.getDouble(), toInt(rightOp.getValue())));
+ break;
+ case FLOAT:
+ result = BeamSqlPrimitive.of(SqlTypeName.FLOAT,
+ (float) roundDouble(leftOp.getFloat(), toInt(rightOp.getValue())));
+ break;
+ case DECIMAL:
+ result = BeamSqlPrimitive.of(SqlTypeName.DECIMAL,
+ roundBigDecimal(toBigDecimal(leftOp.getValue()), toInt(rightOp.getValue())));
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
+ private int roundInt(int v1, int v2) {
+ return SqlFunctions.sround(v1, v2);
+ }
+
+ private double roundDouble(double v1, int v2) {
+ return SqlFunctions.sround(v1, v2);
+ }
+
+ private BigDecimal roundBigDecimal(BigDecimal v1, int v2) {
+ return SqlFunctions.sround(v1, v2);
+ }
+
+ private long roundLong(long v1, int v2) {
+ return SqlFunctions.sround(v1, v2);
+ }
+
+ private int toInt(Object value) {
+ return SqlFunctions.toInt(value);
+ }
+
+ private BigDecimal toBigDecimal(Object value) {
+ return SqlFunctions.toBigDecimal(value);
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSignExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSignExpression.java
new file mode 100644
index 0000000..6be8102
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSignExpression.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'SIGN' function.
+ */
+public class BeamSqlSignExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlSignExpression(List<BeamSqlExpression> operands) {
+ super(operands, operands.get(0).getOutputType());
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ BeamSqlPrimitive result = null;
+ switch (op.getOutputType()) {
+ case TINYINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.TINYINT, (byte) SqlFunctions.sign(SqlFunctions.toByte(op.getValue())));
+ break;
+ case SMALLINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.SMALLINT, (short) SqlFunctions.sign(SqlFunctions.toShort(op.getValue())));
+ break;
+ case INTEGER:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.INTEGER, SqlFunctions.sign(SqlFunctions.toInt(op.getValue())));
+ break;
+ case BIGINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.BIGINT, SqlFunctions.sign(SqlFunctions.toLong(op.getValue())));
+ break;
+ case FLOAT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.FLOAT, (float) SqlFunctions.sign(SqlFunctions.toFloat(op.getValue())));
+ break;
+ case DOUBLE:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.sign(SqlFunctions.toDouble(op.getValue())));
+ break;
+ case DECIMAL:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DECIMAL, SqlFunctions.sign(SqlFunctions.toBigDecimal(op.getValue())));
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSinExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSinExpression.java
new file mode 100644
index 0000000..25dc119
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSinExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'SIN' function.
+ */
+public class BeamSqlSinExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlSinExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.sin(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTanExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTanExpression.java
new file mode 100644
index 0000000..4edd570
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTanExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'TAN' function.
+ */
+public class BeamSqlTanExpression extends BeamSqlMathUnaryExpression {
+
+ public BeamSqlTanExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.DOUBLE);
+ }
+
+ @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+ return BeamSqlPrimitive
+ .of(SqlTypeName.DOUBLE, SqlFunctions.tan(SqlFunctions.toDouble(op.getValue())));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTruncateExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTruncateExpression.java
new file mode 100644
index 0000000..1060a63
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTruncateExpression.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathBinaryExpression} for 'TRUNCATE' function.
+ */
+public class BeamSqlTruncateExpression extends BeamSqlMathBinaryExpression {
+
+ public BeamSqlTruncateExpression(List<BeamSqlExpression> operands) {
+ super(operands, operands.get(0).getOutputType());
+ }
+
+ @Override public BeamSqlPrimitive<? extends Number> calculate(BeamSqlPrimitive leftOp,
+ BeamSqlPrimitive rightOp) {
+ BeamSqlPrimitive result = null;
+ int rightIntOperand = SqlFunctions.toInt(rightOp.getValue());
+ switch (leftOp.getOutputType()) {
+ case SMALLINT:
+ result = BeamSqlPrimitive.of(SqlTypeName.SMALLINT,
+ (short) SqlFunctions.struncate(SqlFunctions.toInt(leftOp.getValue()), rightIntOperand));
+ break;
+ case TINYINT:
+ result = BeamSqlPrimitive.of(SqlTypeName.TINYINT,
+ (byte) SqlFunctions.struncate(SqlFunctions.toInt(leftOp.getValue()), rightIntOperand));
+ break;
+ case INTEGER:
+ result = BeamSqlPrimitive.of(SqlTypeName.INTEGER,
+ SqlFunctions.struncate(SqlFunctions.toInt(leftOp.getValue()), rightIntOperand));
+ break;
+ case BIGINT:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.BIGINT, SqlFunctions.struncate(leftOp.getLong(), rightIntOperand));
+ break;
+ case FLOAT:
+ result = BeamSqlPrimitive.of(SqlTypeName.FLOAT,
+ (float) SqlFunctions.struncate(SqlFunctions.toFloat(leftOp.getValue()),
+ rightIntOperand));
+ break;
+ case DOUBLE:
+ result = BeamSqlPrimitive.of(SqlTypeName.DOUBLE,
+ SqlFunctions.struncate(SqlFunctions.toDouble(leftOp.getValue()), rightIntOperand));
+ break;
+ case DECIMAL:
+ result = BeamSqlPrimitive
+ .of(SqlTypeName.DECIMAL, SqlFunctions.struncate(leftOp.getDecimal(), rightIntOperand));
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/package-info.java
new file mode 100644
index 0000000..740e1b5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * MATH functions/operators.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/package-info.java
new file mode 100644
index 0000000..c420361
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation for operators in {@link org.apache.calcite.sql.fun.SqlStdOperatorTable}.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/BeamSqlReinterpretExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/BeamSqlReinterpretExpression.java
new file mode 100644
index 0000000..b22fd09
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/BeamSqlReinterpretExpression.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for Reinterpret call.
+ *
+ * <p>Currently supported conversions:
+ * - {@link SqlTypeName#DATETIME_TYPES} to {@code BIGINT};
+ * - {@link SqlTypeName#INTEGER} to {@code BIGINT};
+ */
+public class BeamSqlReinterpretExpression extends BeamSqlExpression {
+
+ private static final Reinterpreter REINTERPRETER = Reinterpreter.builder()
+ .withConversion(DatetimeReinterpretConversions.TIME_TO_BIGINT)
+ .withConversion(DatetimeReinterpretConversions.DATE_TYPES_TO_BIGINT)
+ .withConversion(IntegerReinterpretConversions.INTEGER_TYPES_TO_BIGINT)
+ .build();
+
+ public BeamSqlReinterpretExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override public boolean accept() {
+ return getOperands().size() == 1
+ && REINTERPRETER.canConvert(opType(0), SqlTypeName.BIGINT);
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ return REINTERPRETER.convert(
+ SqlTypeName.BIGINT,
+ operands.get(0).evaluate(inputRow, window));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversions.java
new file mode 100644
index 0000000..e6b405b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversions.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import com.google.common.base.Function;
+
+import java.util.Date;
+import java.util.GregorianCalendar;
+
+import javax.annotation.Nonnull;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utility class to contain implementations of datetime SQL type conversions.
+ */
+public abstract class DatetimeReinterpretConversions {
+
+ public static final ReinterpretConversion TIME_TO_BIGINT =
+ ReinterpretConversion.builder()
+ .from(SqlTypeName.TIME)
+ .to(SqlTypeName.BIGINT)
+ .convert(new Function<BeamSqlPrimitive, BeamSqlPrimitive>() {
+ @Override
+ public BeamSqlPrimitive apply(@Nonnull BeamSqlPrimitive beamSqlPrimitive) {
+ GregorianCalendar date = (GregorianCalendar) beamSqlPrimitive.getValue();
+ return BeamSqlPrimitive.of(SqlTypeName.BIGINT, date.getTimeInMillis());
+ }
+ }).build();
+
+ public static final ReinterpretConversion DATE_TYPES_TO_BIGINT =
+ ReinterpretConversion.builder()
+ .from(SqlTypeName.DATE, SqlTypeName.TIMESTAMP)
+ .to(SqlTypeName.BIGINT)
+ .convert(new Function<BeamSqlPrimitive, BeamSqlPrimitive>() {
+ @Override
+ public BeamSqlPrimitive apply(@Nonnull BeamSqlPrimitive beamSqlPrimitive) {
+ Date date = (Date) beamSqlPrimitive.getValue();
+ return BeamSqlPrimitive.of(SqlTypeName.BIGINT, date.getTime());
+ }
+ }).build();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversions.java
new file mode 100644
index 0000000..9539909
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversions.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import com.google.common.base.Function;
+
+import javax.annotation.Nonnull;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utility class to contain implementations of SQL integer type conversions.
+ */
+public abstract class IntegerReinterpretConversions {
+
+ public static final ReinterpretConversion INTEGER_TYPES_TO_BIGINT =
+ ReinterpretConversion.builder()
+ .from(SqlTypeName.INT_TYPES)
+ .to(SqlTypeName.BIGINT)
+ .convert(new Function<BeamSqlPrimitive, BeamSqlPrimitive>() {
+ @Override
+ public BeamSqlPrimitive apply(@Nonnull BeamSqlPrimitive beamSqlPrimitive) {
+ Long value = ((Number) beamSqlPrimitive.getValue()).longValue();
+ return BeamSqlPrimitive.of(SqlTypeName.BIGINT, value);
+ }
+ }).build();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversion.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversion.java
new file mode 100644
index 0000000..df29962
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversion.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Defines conversion between 2 SQL types.
+ */
+public class ReinterpretConversion {
+
+ /**
+ * Builder for {@link ReinterpretConversion}.
+ */
+ public static class Builder {
+
+ private Set<SqlTypeName> from = new HashSet<>();
+ private SqlTypeName to;
+ private Function<BeamSqlPrimitive, BeamSqlPrimitive> convert;
+
+ public Builder from(SqlTypeName from) {
+ this.from.add(from);
+ return this;
+ }
+
+ public Builder from(Collection<SqlTypeName> from) {
+ this.from.addAll(from);
+ return this;
+ }
+
+ public Builder from(SqlTypeName ... from) {
+ return from(Arrays.asList(from));
+ }
+
+ public Builder to(SqlTypeName to) {
+ this.to = to;
+ return this;
+ }
+
+ public Builder convert(Function<BeamSqlPrimitive, BeamSqlPrimitive> convert) {
+ this.convert = convert;
+ return this;
+ }
+
+ public ReinterpretConversion build() {
+ if (from.isEmpty() || to == null || convert == null) {
+ throw new IllegalArgumentException("All arguments to ReinterpretConversion.Builder"
+ + " are mandatory.");
+ }
+ return new ReinterpretConversion(this);
+ }
+ }
+
+ private Set<SqlTypeName> from;
+ private SqlTypeName to;
+ private Function<BeamSqlPrimitive, BeamSqlPrimitive> convertFunction;
+
+ private ReinterpretConversion(Builder builder) {
+ this.from = ImmutableSet.copyOf(builder.from);
+ this.to = builder.to;
+ this.convertFunction = builder.convert;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public BeamSqlPrimitive convert(BeamSqlPrimitive input) {
+ if (!from.contains(input.getOutputType())) {
+ throw new IllegalArgumentException("Unable to convert from " + input.getOutputType().name()
+ + " to " + to.name() + ". This conversion only supports " + toString());
+ }
+
+ return convertFunction.apply(input);
+ }
+
+ public SqlTypeName to() {
+ return to;
+ }
+
+ public Set<SqlTypeName> from() {
+ return from;
+ }
+
+ @Override
+ public String toString() {
+ return from.toString() + "->" + to.name();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/Reinterpreter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/Reinterpreter.java
new file mode 100644
index 0000000..29d4ea4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/Reinterpreter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import com.google.common.base.Optional;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Class that tracks conversions between SQL types.
+ */
+public class Reinterpreter {
+
+ /**
+ * Builder for Reinterpreter.
+ */
+ public static class Builder {
+
+ private Map<SqlTypeName, Map<SqlTypeName, ReinterpretConversion>> conversions = new HashMap<>();
+
+ public Builder withConversion(ReinterpretConversion conversion) {
+ Set<SqlTypeName> fromTypes = conversion.from();
+ SqlTypeName toType = conversion.to();
+
+ for (SqlTypeName fromType : fromTypes) {
+ if (!conversions.containsKey(fromType)) {
+ conversions.put(fromType, new HashMap<SqlTypeName, ReinterpretConversion>());
+ }
+
+ conversions.get(fromType).put(toType, conversion);
+ }
+
+ return this;
+ }
+
+ public Reinterpreter build() {
+ if (conversions.isEmpty()) {
+ throw new IllegalArgumentException("Conversions should not be empty");
+ }
+
+ return new Reinterpreter(this);
+ }
+ }
+
+ private Map<SqlTypeName, Map<SqlTypeName, ReinterpretConversion>> conversions;
+
+ private Reinterpreter(Builder builder) {
+ this.conversions = builder.conversions;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public boolean canConvert(SqlTypeName from, SqlTypeName to) {
+ return getConversion(from, to).isPresent();
+ }
+
+ public BeamSqlPrimitive convert(SqlTypeName to, BeamSqlPrimitive value) {
+ Optional<ReinterpretConversion> conversion = getConversion(value.getOutputType(), to);
+ if (!conversion.isPresent()) {
+ throw new UnsupportedOperationException("Unsupported conversion: "
+ + value.getOutputType().name() + "->" + to.name());
+ }
+
+ return conversion.get().convert(value);
+ }
+
+ private Optional<ReinterpretConversion> getConversion(SqlTypeName from, SqlTypeName to) {
+ if (!conversions.containsKey(from)) {
+ return Optional.absent();
+ }
+
+ Map<SqlTypeName, ReinterpretConversion> allConversionsFrom = conversions.get(from);
+
+ ReinterpretConversion conversionTo = allConversionsFrom.get(to);
+
+ return Optional.fromNullable(conversionTo);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/package-info.java
new file mode 100644
index 0000000..8694937
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation for Reinterpret type conversions.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpression.java
new file mode 100644
index 0000000..5146b14
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'CHAR_LENGTH' operator.
+ */
+public class BeamSqlCharLengthExpression extends BeamSqlStringUnaryExpression {
+ public BeamSqlCharLengthExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.INTEGER);
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+ return BeamSqlPrimitive.of(SqlTypeName.INTEGER, str.length());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpression.java
new file mode 100644
index 0000000..c2f317f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpression.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * String concat operator.
+ */
+public class BeamSqlConcatExpression extends BeamSqlExpression {
+
+ protected BeamSqlConcatExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ public BeamSqlConcatExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() != 2) {
+ return false;
+ }
+
+ for (BeamSqlExpression exp : getOperands()) {
+ if (!SqlTypeName.CHAR_TYPES.contains(exp.getOutputType())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String left = opValueEvaluated(0, inputRow, window);
+ String right = opValueEvaluated(1, inputRow, window);
+
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+ new StringBuilder(left.length() + right.length())
+ .append(left).append(right).toString());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpression.java
new file mode 100644
index 0000000..bf0b8f5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpression.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'INITCAP' operator.
+ */
+public class BeamSqlInitCapExpression extends BeamSqlStringUnaryExpression {
+ public BeamSqlInitCapExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+
+ StringBuilder ret = new StringBuilder(str);
+ boolean isInit = true;
+ for (int i = 0; i < str.length(); i++) {
+ if (Character.isWhitespace(str.charAt(i))) {
+ isInit = true;
+ continue;
+ }
+
+ if (isInit) {
+ ret.setCharAt(i, Character.toUpperCase(str.charAt(i)));
+ isInit = false;
+ } else {
+ ret.setCharAt(i, Character.toLowerCase(str.charAt(i)));
+ }
+ }
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, ret.toString());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpression.java
new file mode 100644
index 0000000..55f8d6d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'LOWER' operator.
+ */
+public class BeamSqlLowerExpression extends BeamSqlStringUnaryExpression {
+ public BeamSqlLowerExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.toLowerCase());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpression.java
new file mode 100644
index 0000000..62d5a64
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpression.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'OVERLAY' operator.
+ *
+ * <p>
+ * OVERLAY(string1 PLACING string2 FROM integer [ FOR integer2 ])
+ * </p>
+ */
+public class BeamSqlOverlayExpression extends BeamSqlExpression {
+ public BeamSqlOverlayExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() < 3 || operands.size() > 4) {
+ return false;
+ }
+
+ if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+ || !SqlTypeName.CHAR_TYPES.contains(opType(1))
+ || !SqlTypeName.INT_TYPES.contains(opType(2))) {
+ return false;
+ }
+
+ if (operands.size() == 4 && !SqlTypeName.INT_TYPES.contains(opType(3))) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+ String replaceStr = opValueEvaluated(1, inputRow, window);
+ int idx = opValueEvaluated(2, inputRow, window);
+ // the index is 1 based.
+ idx -= 1;
+ int length = replaceStr.length();
+ if (operands.size() == 4) {
+ length = opValueEvaluated(3, inputRow, window);
+ }
+
+ StringBuilder result = new StringBuilder(
+ str.length() + replaceStr.length() - length);
+ result.append(str.substring(0, idx))
+ .append(replaceStr)
+ .append(str.substring(idx + length));
+
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, result.toString());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpression.java
new file mode 100644
index 0000000..f97547e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpression.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * String position operator.
+ *
+ * <p>
+ * example:
+ * POSITION(string1 IN string2)
+ * POSITION(string1 IN string2 FROM integer)
+ * </p>
+ */
+public class BeamSqlPositionExpression extends BeamSqlExpression {
+ public BeamSqlPositionExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.INTEGER);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() < 2 || operands.size() > 3) {
+ return false;
+ }
+
+ if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+ || !SqlTypeName.CHAR_TYPES.contains(opType(1))) {
+ return false;
+ }
+
+ if (operands.size() == 3
+ && !SqlTypeName.INT_TYPES.contains(opType(2))) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String targetStr = opValueEvaluated(0, inputRow, window);
+ String containingStr = opValueEvaluated(1, inputRow, window);
+ int from = -1;
+ if (operands.size() == 3) {
+ Number tmp = opValueEvaluated(2, inputRow, window);
+ from = tmp.intValue();
+ }
+
+ int idx = containingStr.indexOf(targetStr, from);
+
+ return BeamSqlPrimitive.of(SqlTypeName.INTEGER, idx);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpression.java
new file mode 100644
index 0000000..1e1b553
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpression.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for all string unary operators.
+ */
+public abstract class BeamSqlStringUnaryExpression extends BeamSqlExpression {
+ public BeamSqlStringUnaryExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+ super(operands, outputType);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() != 1) {
+ return false;
+ }
+
+ if (!SqlTypeName.CHAR_TYPES.contains(opType(0))) {
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpression.java
new file mode 100644
index 0000000..a521ef0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpression.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'SUBSTRING' operator.
+ *
+ * <p>
+ * SUBSTRING(string FROM integer)
+ * SUBSTRING(string FROM integer FOR integer)
+ * </p>
+ */
+public class BeamSqlSubstringExpression extends BeamSqlExpression {
+ public BeamSqlSubstringExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() < 2 || operands.size() > 3) {
+ return false;
+ }
+
+ if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+ || !SqlTypeName.INT_TYPES.contains(opType(1))) {
+ return false;
+ }
+
+ if (operands.size() == 3 && !SqlTypeName.INT_TYPES.contains(opType(2))) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+ int idx = opValueEvaluated(1, inputRow, window);
+ int startIdx = idx;
+ if (startIdx > 0) {
+ // NOTE: SQL substring is 1 based(rather than 0 based)
+ startIdx -= 1;
+ } else if (startIdx < 0) {
+ // NOTE: SQL also support negative index...
+ startIdx += str.length();
+ } else {
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "");
+ }
+
+ if (operands.size() == 3) {
+ int length = opValueEvaluated(2, inputRow, window);
+ if (length < 0) {
+ length = 0;
+ }
+ int endIdx = Math.min(startIdx + length, str.length());
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.substring(startIdx, endIdx));
+ } else {
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.substring(startIdx));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpression.java
new file mode 100644
index 0000000..3c3083c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpression.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.fun.SqlTrimFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Trim operator.
+ *
+ * <p>
+ * TRIM( { BOTH | LEADING | TRAILING } string1 FROM string2)
+ * </p>
+ */
+public class BeamSqlTrimExpression extends BeamSqlExpression {
+ public BeamSqlTrimExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public boolean accept() {
+ if (operands.size() != 1 && operands.size() != 3) {
+ return false;
+ }
+
+ if (operands.size() == 1 && !SqlTypeName.CHAR_TYPES.contains(opType(0))) {
+ return false;
+ }
+
+ if (operands.size() == 3
+ && (
+ SqlTypeName.SYMBOL != opType(0)
+ || !SqlTypeName.CHAR_TYPES.contains(opType(1))
+ || !SqlTypeName.CHAR_TYPES.contains(opType(2)))
+ ) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ if (operands.size() == 1) {
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+ opValueEvaluated(0, inputRow, window).toString().trim());
+ } else {
+ SqlTrimFunction.Flag type = opValueEvaluated(0, inputRow, window);
+ String targetStr = opValueEvaluated(1, inputRow, window);
+ String containingStr = opValueEvaluated(2, inputRow, window);
+
+ switch (type) {
+ case LEADING:
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, leadingTrim(containingStr, targetStr));
+ case TRAILING:
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, trailingTrim(containingStr, targetStr));
+ case BOTH:
+ default:
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+ trailingTrim(leadingTrim(containingStr, targetStr), targetStr));
+ }
+ }
+ }
+
+ static String leadingTrim(String containingStr, String targetStr) {
+ int idx = 0;
+ while (containingStr.startsWith(targetStr, idx)) {
+ idx += targetStr.length();
+ }
+
+ return containingStr.substring(idx);
+ }
+
+ static String trailingTrim(String containingStr, String targetStr) {
+ int idx = containingStr.length() - targetStr.length();
+ while (containingStr.startsWith(targetStr, idx)) {
+ idx -= targetStr.length();
+ }
+
+ idx += targetStr.length();
+ return containingStr.substring(0, idx);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpression.java
new file mode 100644
index 0000000..bc29ec8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'UPPER' operator.
+ */
+public class BeamSqlUpperExpression extends BeamSqlStringUnaryExpression {
+ public BeamSqlUpperExpression(List<BeamSqlExpression> operands) {
+ super(operands, SqlTypeName.VARCHAR);
+ }
+
+ @Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
+ String str = opValueEvaluated(0, inputRow, window);
+ return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.toUpperCase());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/package-info.java
new file mode 100644
index 0000000..f8fc4be
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * String operators.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/package-info.java
new file mode 100644
index 0000000..3e58a08
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * interpreter generate runnable 'code' to execute SQL relational expressions.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/package-info.java
new file mode 100644
index 0000000..de237d6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation classes of BeamSql.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParser.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParser.java
new file mode 100644
index 0000000..bfa4d47
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParser.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.StringReader;
+import org.apache.beam.sdk.extensions.sql.impl.parser.impl.BeamSqlParserImpl;
+import org.apache.calcite.config.Lex;
+
+/**
+ * SQL Parser which handles DDL for Beam.
+ */
+public class BeamSqlParser {
+ public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 128;
+ private final BeamSqlParserImpl impl;
+
+ public BeamSqlParser(String s) {
+ this.impl = new BeamSqlParserImpl(new StringReader(s));
+ this.impl.setTabSize(1);
+ this.impl.setQuotedCasing(Lex.ORACLE.quotedCasing);
+ this.impl.setUnquotedCasing(Lex.ORACLE.unquotedCasing);
+ this.impl.setIdentifierMaxLength(DEFAULT_IDENTIFIER_MAX_LENGTH);
+ /*
+ * By default parser uses [ ] for quoting identifiers. Switching to
+ * DQID (double quoted identifiers) is needed for array and map access
+ * (m['x'] = 1 or arr[2] = 10 etc) to work.
+ */
+ this.impl.switchTo("DQID");
+ }
+
+ @VisibleForTesting
+ public BeamSqlParserImpl impl() {
+ return impl;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnConstraint.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnConstraint.java
new file mode 100644
index 0000000..965daa2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnConstraint.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Column constraint such as primary key.
+ */
+public class ColumnConstraint extends SqlLiteral {
+ private ColumnConstraint(
+ Object value, SqlTypeName typeName, SqlParserPos pos) {
+ super(value, typeName, pos);
+ }
+
+ /**
+ * A primary key constraint.
+ */
+ public static class PrimaryKey extends ColumnConstraint {
+ public PrimaryKey(SqlParserPos pos) {
+ super(SqlDDLKeywords.PRIMARY, SqlTypeName.SYMBOL, pos);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnDefinition.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnDefinition.java
new file mode 100644
index 0000000..fce8d2c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnDefinition.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import java.util.Arrays;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.util.NlsString;
+
+/**
+ * Column definition used during sql parsing(mainly DDL which not supported by default).
+ */
+public class ColumnDefinition extends SqlNodeList {
+ public ColumnDefinition(
+ SqlIdentifier name, SqlDataTypeSpec type, ColumnConstraint constraint,
+ SqlNode comment, SqlParserPos pos) {
+ super(Arrays.asList(name, type, constraint, comment), pos);
+ }
+
+ public String name() {
+ return get(0).toString();
+ }
+
+ public SqlDataTypeSpec type() {
+ return (SqlDataTypeSpec) get(1);
+ }
+
+ public ColumnConstraint constraint() {
+ SqlNode constraintNode = get(2);
+ return constraintNode == null ? null : (ColumnConstraint) constraintNode;
+ }
+
+ public String comment() {
+ SqlNode commentNode = get(3);
+ return commentNode == null ? null : ((NlsString) SqlLiteral.value(commentNode)).getValue();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ParserUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ParserUtils.java
new file mode 100644
index 0000000..dae82a6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ParserUtils.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * Util method for parser.
+ */
+public class ParserUtils {
+
+ /**
+ * Convert a create table statement to a {@code Table} object.
+ * @param stmt
+ * @return the table
+ */
+ public static Table convertCreateTableStmtToTable(SqlCreateTable stmt) {
+ List<Column> columns = new ArrayList<>(stmt.fieldList().size());
+ for (ColumnDefinition columnDef : stmt.fieldList()) {
+ Column column = Column.builder()
+ .name(columnDef.name().toLowerCase())
+ .type(
+ CalciteUtils.toJavaType(
+ columnDef.type().deriveType(BeamQueryPlanner.TYPE_FACTORY).getSqlTypeName()
+ )
+ )
+ .comment(columnDef.comment())
+ .primaryKey(columnDef.constraint() instanceof ColumnConstraint.PrimaryKey)
+ .build();
+ columns.add(column);
+ }
+
+ Table table = Table.builder()
+ .type(stmt.type().toLowerCase())
+ .name(stmt.tableName().toLowerCase())
+ .columns(columns)
+ .comment(stmt.comment())
+ .location(stmt.location())
+ .properties(stmt.properties())
+ .build();
+
+ return table;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateTable.java
new file mode 100644
index 0000000..794e3e6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateTable.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.base.Strings;
+import java.net.URI;
+import java.util.List;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlSpecialOperator;
+import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.util.ImmutableNullableList;
+import org.apache.calcite.util.NlsString;
+
+/**
+ * A Calcite {@code SqlCall} which represents a create table statement.
+ */
+public class SqlCreateTable extends SqlCall {
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator(
+ "CREATE_TABLE", SqlKind.OTHER) {
+ @Override
+ public SqlCall createCall(
+ SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... o) {
+ assert functionQualifier == null;
+ return new SqlCreateTable(pos, (SqlIdentifier) o[0], (SqlNodeList) o[1],
+ o[2], o[3], o[4], o[5], o[6]);
+ }
+
+ @Override
+ public void unparse(
+ SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
+ SqlCreateTable t = (SqlCreateTable) call;
+ UnparseUtil u = new UnparseUtil(writer, leftPrec, rightPrec);
+ u.keyword("CREATE", "TABLE").node(t.tblName).nodeList(
+ t.fieldList);
+ u.keyword("TYPE").node(t.type);
+ u.keyword("COMMENT").node(t.comment);
+ u.keyword("LOCATION").node(t.location);
+ if (t.properties != null) {
+ u.keyword("TBLPROPERTIES").node(t.properties);
+ }
+ if (t.query != null) {
+ u.keyword("AS").node(t.query);
+ }
+ }
+ };
+
+ private final SqlIdentifier tblName;
+ private final SqlNodeList fieldList;
+ private final SqlNode type;
+ private final SqlNode comment;
+ private final SqlNode location;
+ private final SqlNode properties;
+ private final SqlNode query;
+
+ public SqlCreateTable(
+ SqlParserPos pos, SqlIdentifier tblName, SqlNodeList fieldList, SqlNode type,
+ SqlNode comment, SqlNode location, SqlNode properties, SqlNode query) {
+ super(pos);
+ this.tblName = tblName;
+ this.fieldList = fieldList;
+ this.type = type;
+ this.comment = comment;
+ this.location = location;
+ this.properties = properties;
+ this.query = query;
+ }
+
+ @Override
+ public SqlOperator getOperator() {
+ return OPERATOR;
+ }
+
+ @Override
+ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
+ getOperator().unparse(writer, this, leftPrec, rightPrec);
+ }
+
+ @Override
+ public List<SqlNode> getOperandList() {
+ return ImmutableNullableList.of(tblName, fieldList, location, properties,
+ query);
+ }
+
+ public String tableName() {
+ return tblName.toString();
+ }
+
+ public URI location() {
+ return location == null ? null : URI.create(getString(location));
+ }
+
+ public String type() {
+ return type == null ? null : getString(type);
+ }
+
+ public String comment() {
+ return comment == null ? null : getString(comment);
+ }
+
+ public JSONObject properties() {
+ String propertiesStr = getString(properties);
+ if (Strings.isNullOrEmpty(propertiesStr)) {
+ return new JSONObject();
+ } else {
+ return JSON.parseObject(propertiesStr);
+ }
+ }
+
+ private String getString(SqlNode n) {
+ return n == null ? null : ((NlsString) SqlLiteral.value(n)).getValue();
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<ColumnDefinition> fieldList() {
+ return (List<ColumnDefinition>) ((List<? extends SqlNode>) fieldList.getList());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDDLKeywords.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDDLKeywords.java
new file mode 100644
index 0000000..14a1b12
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDDLKeywords.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import org.apache.calcite.sql.SqlLiteral;
+
+/**
+ * Define the keywords that can occur in a CREATE TABLE statement.
+ */
+public enum SqlDDLKeywords implements SqlLiteral.SqlSymbol {
+ PRIMARY
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/UnparseUtil.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/UnparseUtil.java
new file mode 100644
index 0000000..30e06b5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/UnparseUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlWriter;
+
+class UnparseUtil {
+ private final SqlWriter writer;
+ private final int leftPrec;
+ private final int rightPrec;
+
+ UnparseUtil(SqlWriter writer, int leftPrec, int rightPrec) {
+ this.writer = writer;
+ this.leftPrec = leftPrec;
+ this.rightPrec = rightPrec;
+ }
+
+ UnparseUtil keyword(String... keywords) {
+ for (String k : keywords) {
+ writer.keyword(k);
+ }
+ return this;
+ }
+
+ UnparseUtil node(SqlNode n) {
+ n.unparse(writer, leftPrec, rightPrec);
+ return this;
+ }
+
+ UnparseUtil nodeList(SqlNodeList l) {
+ writer.keyword("(");
+ if (l.size() > 0) {
+ l.get(0).unparse(writer, leftPrec, rightPrec);
+ for (int i = 1; i < l.size(); ++i) {
+ writer.keyword(",");
+ l.get(i).unparse(writer, leftPrec, rightPrec);
+ }
+ }
+ writer.keyword(")");
+ return this;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/package-info.java
new file mode 100644
index 0000000..73ae2c5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Created by xumingmingv on 16/06/2017.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
new file mode 100644
index 0000000..ce46e2d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.planner;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The core component to handle through a SQL statement, from explain execution plan,
+ * to generate a Beam pipeline.
+ *
+ */
+public class BeamQueryPlanner {
+ private static final Logger LOG = LoggerFactory.getLogger(BeamQueryPlanner.class);
+
+ protected final Planner planner;
+ private Map<String, BeamSqlTable> sourceTables = new HashMap<>();
+
+ public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+ RelDataTypeSystem.DEFAULT);
+
+ public BeamQueryPlanner(SchemaPlus schema) {
+ final List<RelTraitDef> traitDefs = new ArrayList<>();
+ traitDefs.add(ConventionTraitDef.INSTANCE);
+ traitDefs.add(RelCollationTraitDef.INSTANCE);
+
+ List<SqlOperatorTable> sqlOperatorTables = new ArrayList<>();
+ sqlOperatorTables.add(SqlStdOperatorTable.instance());
+ sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), false,
+ Collections.<String>emptyList(), TYPE_FACTORY));
+
+ FrameworkConfig config = Frameworks.newConfigBuilder()
+ .parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+ .traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+ .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM)
+ .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
+ .build();
+ this.planner = Frameworks.getPlanner(config);
+
+ for (String t : schema.getTableNames()) {
+ sourceTables.put(t, (BaseBeamTable) schema.getTable(t));
+ }
+ }
+
+ /**
+ * Parse input SQL query, and return a {@link SqlNode} as grammar tree.
+ */
+ public SqlNode parseQuery(String sqlQuery) throws SqlParseException{
+ return planner.parse(sqlQuery);
+ }
+
+ /**
+ * {@code compileBeamPipeline} translate a SQL statement to executed as Beam data flow,
+ * which is linked with the given {@code pipeline}. The final output stream is returned as
+ * {@code PCollection} so more operations can be applied.
+ */
+ public PCollection<BeamRecord> compileBeamPipeline(String sqlStatement, Pipeline basePipeline
+ , BeamSqlEnv sqlEnv) throws Exception {
+ BeamRelNode relNode = convertToBeamRel(sqlStatement);
+
+ // the input PCollectionTuple is empty, and be rebuilt in BeamIOSourceRel.
+ return relNode.buildBeamPipeline(PCollectionTuple.empty(basePipeline), sqlEnv);
+ }
+
+ /**
+ * It parses and validate the input query, then convert into a
+ * {@link BeamRelNode} tree.
+ *
+ */
+ public BeamRelNode convertToBeamRel(String sqlStatement)
+ throws ValidationException, RelConversionException, SqlParseException {
+ BeamRelNode beamRelNode;
+ try {
+ beamRelNode = (BeamRelNode) validateAndConvert(planner.parse(sqlStatement));
+ } finally {
+ planner.close();
+ }
+ return beamRelNode;
+ }
+
+ private RelNode validateAndConvert(SqlNode sqlNode)
+ throws ValidationException, RelConversionException {
+ SqlNode validated = validateNode(sqlNode);
+ LOG.info("SQL:\n" + validated);
+ RelNode relNode = convertToRelNode(validated);
+ return convertToBeamRel(relNode);
+ }
+
+ private RelNode convertToBeamRel(RelNode relNode) throws RelConversionException {
+ RelTraitSet traitSet = relNode.getTraitSet();
+
+ LOG.info("SQLPlan>\n" + RelOptUtil.toString(relNode));
+
+ // PlannerImpl.transform() optimizes RelNode with ruleset
+ return planner.transform(0, traitSet.plus(BeamLogicalConvention.INSTANCE), relNode);
+ }
+
+ private RelNode convertToRelNode(SqlNode sqlNode) throws RelConversionException {
+ return planner.rel(sqlNode).rel;
+ }
+
+ private SqlNode validateNode(SqlNode sqlNode) throws ValidationException {
+ return planner.validate(sqlNode);
+ }
+
+ public Map<String, BeamSqlTable> getSourceTables() {
+ return sourceTables;
+ }
+
+ public Planner getPlanner() {
+ return planner;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRelDataTypeSystem.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRelDataTypeSystem.java
new file mode 100644
index 0000000..5734653
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRelDataTypeSystem.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.planner;
+
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
+
+/**
+ * customized data type in Beam.
+ *
+ */
+public class BeamRelDataTypeSystem extends RelDataTypeSystemImpl {
+ public static final RelDataTypeSystem BEAM_REL_DATATYPE_SYSTEM = new BeamRelDataTypeSystem();
+
+ @Override
+ public int getMaxNumericScale() {
+ return 38;
+ }
+
+ @Override
+ public int getMaxNumericPrecision() {
+ return 38;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
new file mode 100644
index 0000000..d3c9871
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.planner;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.util.Iterator;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamAggregationRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamFilterRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamIOSinkRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamIOSourceRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamIntersectRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamJoinRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamMinusRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamProjectRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamSortRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamUnionRule;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamValuesRule;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.RuleSet;
+
+/**
+ * {@link RuleSet} used in {@link BeamQueryPlanner}. It translates a standard
+ * Calcite {@link RelNode} tree, to represent with {@link BeamRelNode}
+ *
+ */
+public class BeamRuleSets {
+ private static final ImmutableSet<RelOptRule> calciteToBeamConversionRules = ImmutableSet
+ .<RelOptRule>builder().add(BeamIOSourceRule.INSTANCE, BeamProjectRule.INSTANCE,
+ BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE,
+ BeamAggregationRule.INSTANCE, BeamSortRule.INSTANCE, BeamValuesRule.INSTANCE,
+ BeamIntersectRule.INSTANCE, BeamMinusRule.INSTANCE, BeamUnionRule.INSTANCE,
+ BeamJoinRule.INSTANCE)
+ .build();
+
+ public static RuleSet[] getRuleSets() {
+ return new RuleSet[] { new BeamRuleSet(
+ ImmutableSet.<RelOptRule>builder().addAll(calciteToBeamConversionRules).build()) };
+ }
+
+ private static class BeamRuleSet implements RuleSet {
+ final ImmutableSet<RelOptRule> rules;
+
+ public BeamRuleSet(ImmutableSet<RelOptRule> rules) {
+ this.rules = rules;
+ }
+
+ public BeamRuleSet(ImmutableList<RelOptRule> rules) {
+ this.rules = ImmutableSet.<RelOptRule>builder().addAll(rules).build();
+ }
+
+ @Override
+ public Iterator<RelOptRule> iterator() {
+ return rules.iterator();
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/package-info.java
new file mode 100644
index 0000000..a5552b3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner} is the main interface.
+ * It defines data sources, validate a SQL statement, and convert it as a Beam
+ * pipeline.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.planner;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java
new file mode 100644
index 0000000..e49e79c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamAggregationTransforms;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.WithTimestamps;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Util;
+import org.joda.time.Duration;
+
+/**
+ * {@link BeamRelNode} to replace a {@link Aggregate} node.
+ *
+ */
+public class BeamAggregationRel extends Aggregate implements BeamRelNode {
+ private int windowFieldIdx = -1;
+ private WindowFn<BeamRecord, BoundedWindow> windowFn;
+ private Trigger trigger;
+ private Duration allowedLatence = Duration.ZERO;
+
+ public BeamAggregationRel(RelOptCluster cluster, RelTraitSet traits
+ , RelNode child, boolean indicator,
+ ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls
+ , WindowFn windowFn, Trigger trigger, int windowFieldIdx, Duration allowedLatence) {
+ super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls);
+ this.windowFn = windowFn;
+ this.trigger = trigger;
+ this.windowFieldIdx = windowFieldIdx;
+ this.allowedLatence = allowedLatence;
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ RelNode input = getInput();
+ String stageName = BeamSqlRelUtils.getStageName(this) + "_";
+
+ PCollection<BeamRecord> upstream =
+ BeamSqlRelUtils.getBeamRelInput(input).buildBeamPipeline(inputPCollections, sqlEnv);
+ if (windowFieldIdx != -1) {
+ upstream = upstream.apply(stageName + "assignEventTimestamp", WithTimestamps
+ .of(new BeamAggregationTransforms.WindowTimestampFn(windowFieldIdx)))
+ .setCoder(upstream.getCoder());
+ }
+
+ PCollection<BeamRecord> windowStream = upstream.apply(stageName + "window",
+ Window.into(windowFn)
+ .triggering(trigger)
+ .withAllowedLateness(allowedLatence)
+ .accumulatingFiredPanes());
+
+ BeamRecordCoder keyCoder = exKeyFieldsSchema(input.getRowType()).getRecordCoder();
+ PCollection<KV<BeamRecord, BeamRecord>> exCombineByStream = windowStream.apply(
+ stageName + "exCombineBy",
+ WithKeys
+ .of(new BeamAggregationTransforms.AggregationGroupByKeyFn(
+ windowFieldIdx, groupSet)))
+ .setCoder(KvCoder.of(keyCoder, upstream.getCoder()));
+
+
+ BeamRecordCoder aggCoder = exAggFieldsSchema().getRecordCoder();
+
+ PCollection<KV<BeamRecord, BeamRecord>> aggregatedStream = exCombineByStream.apply(
+ stageName + "combineBy",
+ Combine.<BeamRecord, BeamRecord, BeamRecord>perKey(
+ new BeamAggregationTransforms.AggregationAdaptor(getAggCallList(),
+ CalciteUtils.toBeamRowType(input.getRowType()))))
+ .setCoder(KvCoder.of(keyCoder, aggCoder));
+
+ PCollection<BeamRecord> mergedStream = aggregatedStream.apply(stageName + "mergeRecord",
+ ParDo.of(new BeamAggregationTransforms.MergeAggregationRecord(
+ CalciteUtils.toBeamRowType(getRowType()), getAggCallList(), windowFieldIdx)));
+ mergedStream.setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+
+ return mergedStream;
+ }
+
+ /**
+ * Type of sub-rowrecord used as Group-By keys.
+ */
+ private BeamRecordSqlType exKeyFieldsSchema(RelDataType relDataType) {
+ BeamRecordSqlType inputRowType = CalciteUtils.toBeamRowType(relDataType);
+ List<String> fieldNames = new ArrayList<>();
+ List<Integer> fieldTypes = new ArrayList<>();
+ for (int i : groupSet.asList()) {
+ if (i != windowFieldIdx) {
+ fieldNames.add(inputRowType.getFieldNameByIndex(i));
+ fieldTypes.add(inputRowType.getFieldTypeByIndex(i));
+ }
+ }
+ return BeamRecordSqlType.create(fieldNames, fieldTypes);
+ }
+
+ /**
+ * Type of sub-rowrecord, that represents the list of aggregation fields.
+ */
+ private BeamRecordSqlType exAggFieldsSchema() {
+ List<String> fieldNames = new ArrayList<>();
+ List<Integer> fieldTypes = new ArrayList<>();
+ for (AggregateCall ac : getAggCallList()) {
+ fieldNames.add(ac.name);
+ fieldTypes.add(CalciteUtils.toJavaType(ac.type.getSqlTypeName()));
+ }
+
+ return BeamRecordSqlType.create(fieldNames, fieldTypes);
+ }
+
+ @Override
+ public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator
+ , ImmutableBitSet groupSet,
+ List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ return new BeamAggregationRel(getCluster(), traitSet, input, indicator
+ , groupSet, groupSets, aggCalls, windowFn, trigger, windowFieldIdx, allowedLatence);
+ }
+
+ public void setWindowFn(WindowFn windowFn) {
+ this.windowFn = windowFn;
+ }
+
+ public void setTrigger(Trigger trigger) {
+ this.trigger = trigger;
+ }
+
+ public RelWriter explainTerms(RelWriter pw) {
+ // We skip the "groups" element if it is a singleton of "group".
+ pw.item("group", groupSet)
+ .itemIf("window", windowFn, windowFn != null)
+ .itemIf("trigger", trigger, trigger != null)
+ .itemIf("event_time", windowFieldIdx, windowFieldIdx != -1)
+ .itemIf("groups", groupSets, getGroupType() != Group.SIMPLE)
+ .itemIf("indicator", indicator, indicator)
+ .itemIf("aggs", aggCalls, pw.nest());
+ if (!pw.nest()) {
+ for (Ord<AggregateCall> ord : Ord.zip(aggCalls)) {
+ pw.item(Util.first(ord.e.name, "agg#" + ord.i), ord.e);
+ }
+ }
+ return pw;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamFilterRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamFilterRel.java
new file mode 100644
index 0000000..9d36a47
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamFilterRel.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlExpressionExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSqlFilterFn;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rex.RexNode;
+
+/**
+ * BeamRelNode to replace a {@code Filter} node.
+ *
+ */
+public class BeamFilterRel extends Filter implements BeamRelNode {
+
+ public BeamFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child,
+ RexNode condition) {
+ super(cluster, traits, child, condition);
+ }
+
+ @Override
+ public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
+ return new BeamFilterRel(getCluster(), traitSet, input, condition);
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ RelNode input = getInput();
+ String stageName = BeamSqlRelUtils.getStageName(this);
+
+ PCollection<BeamRecord> upstream =
+ BeamSqlRelUtils.getBeamRelInput(input).buildBeamPipeline(inputPCollections, sqlEnv);
+
+ BeamSqlExpressionExecutor executor = new BeamSqlFnExecutor(this);
+
+ PCollection<BeamRecord> filterStream = upstream.apply(stageName,
+ ParDo.of(new BeamSqlFilterFn(getRelTypeName(), executor)));
+ filterStream.setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+
+ return filterStream;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java
new file mode 100644
index 0000000..d3de0fb
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import com.google.common.base.Joiner;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rex.RexNode;
+
+/**
+ * BeamRelNode to replace a {@code TableModify} node.
+ *
+ */
+public class BeamIOSinkRel extends TableModify implements BeamRelNode {
+ public BeamIOSinkRel(RelOptCluster cluster, RelTraitSet traits, RelOptTable table,
+ Prepare.CatalogReader catalogReader, RelNode child, Operation operation,
+ List<String> updateColumnList, List<RexNode> sourceExpressionList, boolean flattened) {
+ super(cluster, traits, table, catalogReader, child, operation, updateColumnList,
+ sourceExpressionList, flattened);
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new BeamIOSinkRel(getCluster(), traitSet, getTable(), getCatalogReader(), sole(inputs),
+ getOperation(), getUpdateColumnList(), getSourceExpressionList(), isFlattened());
+ }
+
+ /**
+ * Note that {@code BeamIOSinkRel} returns the input PCollection,
+ * which is the persisted PCollection.
+ */
+ @Override
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ RelNode input = getInput();
+ String stageName = BeamSqlRelUtils.getStageName(this);
+
+ PCollection<BeamRecord> upstream =
+ BeamSqlRelUtils.getBeamRelInput(input).buildBeamPipeline(inputPCollections, sqlEnv);
+
+ String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
+
+ BeamSqlTable targetTable = sqlEnv.findTable(sourceName);
+
+ upstream.apply(stageName, targetTable.buildIOWriter());
+
+ return upstream;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java
new file mode 100644
index 0000000..2d6e46f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import com.google.common.base.Joiner;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.core.TableScan;
+
+/**
+ * BeamRelNode to replace a {@code TableScan} node.
+ *
+ */
+public class BeamIOSourceRel extends TableScan implements BeamRelNode {
+
+ public BeamIOSourceRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) {
+ super(cluster, traitSet, table);
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
+
+ TupleTag<BeamRecord> sourceTupleTag = new TupleTag<>(sourceName);
+ if (inputPCollections.has(sourceTupleTag)) {
+ //choose PCollection from input PCollectionTuple if exists there.
+ PCollection<BeamRecord> sourceStream = inputPCollections
+ .get(new TupleTag<BeamRecord>(sourceName));
+ return sourceStream;
+ } else {
+ //If not, the source PColection is provided with BaseBeamTable.buildIOReader().
+ BeamSqlTable sourceTable = sqlEnv.findTable(sourceName);
+ return sourceTable.buildIOReader(inputPCollections.getPipeline())
+ .setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRel.java
new file mode 100644
index 0000000..1ffb636
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRel.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Intersect;
+import org.apache.calcite.rel.core.SetOp;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Intersect} node.
+ *
+ * <p>This is used to combine two SELECT statements, but returns rows only from the
+ * first SELECT statement that are identical to a row in the second SELECT statement.
+ */
+public class BeamIntersectRel extends Intersect implements BeamRelNode {
+ private BeamSetOperatorRelBase delegate;
+ public BeamIntersectRel(
+ RelOptCluster cluster,
+ RelTraitSet traits,
+ List<RelNode> inputs,
+ boolean all) {
+ super(cluster, traits, inputs, all);
+ delegate = new BeamSetOperatorRelBase(this,
+ BeamSetOperatorRelBase.OpType.INTERSECT, inputs, all);
+ }
+
+ @Override public SetOp copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
+ return new BeamIntersectRel(getCluster(), traitSet, inputs, all);
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ return delegate.buildBeamPipeline(inputPCollections, sqlEnv);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
new file mode 100644
index 0000000..cc26aa6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamJoinTransforms;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.Pair;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Join} node.
+ *
+ * <p>Support for join can be categorized into 3 cases:
+ * <ul>
+ * <li>BoundedTable JOIN BoundedTable</li>
+ * <li>UnboundedTable JOIN UnboundedTable</li>
+ * <li>BoundedTable JOIN UnboundedTable</li>
+ * </ul>
+ *
+ * <p>For the first two cases, a standard join is utilized as long as the windowFn of the both
+ * sides match.
+ *
+ * <p>For the third case, {@code sideInput} is utilized to implement the join, so there are some
+ * constraints:
+ *
+ * <ul>
+ * <li>{@code FULL OUTER JOIN} is not supported.</li>
+ * <li>If it's a {@code LEFT OUTER JOIN}, the unbounded table should on the left side.</li>
+ * <li>If it's a {@code RIGHT OUTER JOIN}, the unbounded table should on the right side.</li>
+ * </ul>
+ *
+ *
+ * <p>There are also some general constraints:
+ *
+ * <ul>
+ * <li>Only equi-join is supported.</li>
+ * <li>CROSS JOIN is not supported.</li>
+ * </ul>
+ */
+public class BeamJoinRel extends Join implements BeamRelNode {
+ public BeamJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right,
+ RexNode condition, Set<CorrelationId> variablesSet, JoinRelType joinType) {
+ super(cluster, traits, left, right, condition, variablesSet, joinType);
+ }
+
+ @Override public Join copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left,
+ RelNode right, JoinRelType joinType, boolean semiJoinDone) {
+ return new BeamJoinRel(getCluster(), traitSet, left, right, conditionExpr, variablesSet,
+ joinType);
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections,
+ BeamSqlEnv sqlEnv)
+ throws Exception {
+ BeamRelNode leftRelNode = BeamSqlRelUtils.getBeamRelInput(left);
+ BeamRecordSqlType leftRowType = CalciteUtils.toBeamRowType(left.getRowType());
+ PCollection<BeamRecord> leftRows = leftRelNode.buildBeamPipeline(inputPCollections, sqlEnv);
+
+ final BeamRelNode rightRelNode = BeamSqlRelUtils.getBeamRelInput(right);
+ PCollection<BeamRecord> rightRows = rightRelNode.buildBeamPipeline(inputPCollections, sqlEnv);
+
+ String stageName = BeamSqlRelUtils.getStageName(this);
+ WindowFn leftWinFn = leftRows.getWindowingStrategy().getWindowFn();
+ WindowFn rightWinFn = rightRows.getWindowingStrategy().getWindowFn();
+
+ // extract the join fields
+ List<Pair<Integer, Integer>> pairs = extractJoinColumns(
+ leftRelNode.getRowType().getFieldCount());
+
+ // build the extract key type
+ // the name of the join field is not important
+ List<String> names = new ArrayList<>(pairs.size());
+ List<Integer> types = new ArrayList<>(pairs.size());
+ for (int i = 0; i < pairs.size(); i++) {
+ names.add("c" + i);
+ types.add(leftRowType.getFieldTypeByIndex(pairs.get(i).getKey()));
+ }
+ BeamRecordSqlType extractKeyRowType = BeamRecordSqlType.create(names, types);
+
+ Coder extractKeyRowCoder = extractKeyRowType.getRecordCoder();
+
+ // BeamSqlRow -> KV<BeamSqlRow, BeamSqlRow>
+ PCollection<KV<BeamRecord, BeamRecord>> extractedLeftRows = leftRows
+ .apply(stageName + "_left_ExtractJoinFields",
+ MapElements.via(new BeamJoinTransforms.ExtractJoinFields(true, pairs)))
+ .setCoder(KvCoder.of(extractKeyRowCoder, leftRows.getCoder()));
+
+ PCollection<KV<BeamRecord, BeamRecord>> extractedRightRows = rightRows
+ .apply(stageName + "_right_ExtractJoinFields",
+ MapElements.via(new BeamJoinTransforms.ExtractJoinFields(false, pairs)))
+ .setCoder(KvCoder.of(extractKeyRowCoder, rightRows.getCoder()));
+
+ // prepare the NullRows
+ BeamRecord leftNullRow = buildNullRow(leftRelNode);
+ BeamRecord rightNullRow = buildNullRow(rightRelNode);
+
+ // a regular join
+ if ((leftRows.isBounded() == PCollection.IsBounded.BOUNDED
+ && rightRows.isBounded() == PCollection.IsBounded.BOUNDED)
+ || (leftRows.isBounded() == PCollection.IsBounded.UNBOUNDED
+ && rightRows.isBounded() == PCollection.IsBounded.UNBOUNDED)) {
+ try {
+ leftWinFn.verifyCompatibility(rightWinFn);
+ } catch (IncompatibleWindowException e) {
+ throw new IllegalArgumentException(
+ "WindowFns must match for a bounded-vs-bounded/unbounded-vs-unbounded join.", e);
+ }
+
+ return standardJoin(extractedLeftRows, extractedRightRows,
+ leftNullRow, rightNullRow, stageName);
+ } else if (
+ (leftRows.isBounded() == PCollection.IsBounded.BOUNDED
+ && rightRows.isBounded() == PCollection.IsBounded.UNBOUNDED)
+ || (leftRows.isBounded() == PCollection.IsBounded.UNBOUNDED
+ && rightRows.isBounded() == PCollection.IsBounded.BOUNDED)
+ ) {
+ // if one of the sides is Bounded & the other is Unbounded
+ // then do a sideInput join
+ // when doing a sideInput join, the windowFn does not need to match
+ // Only support INNER JOIN & LEFT OUTER JOIN where left side of the join must be
+ // the unbounded
+ if (joinType == JoinRelType.FULL) {
+ throw new UnsupportedOperationException("FULL OUTER JOIN is not supported when join "
+ + "a bounded table with an unbounded table.");
+ }
+
+ if ((joinType == JoinRelType.LEFT
+ && leftRows.isBounded() == PCollection.IsBounded.BOUNDED)
+ || (joinType == JoinRelType.RIGHT
+ && rightRows.isBounded() == PCollection.IsBounded.BOUNDED)) {
+ throw new UnsupportedOperationException(
+ "LEFT side of an OUTER JOIN must be Unbounded table.");
+ }
+
+ return sideInputJoin(extractedLeftRows, extractedRightRows,
+ leftNullRow, rightNullRow);
+ } else {
+ throw new UnsupportedOperationException(
+ "The inputs to the JOIN have un-joinnable windowFns: " + leftWinFn + ", " + rightWinFn);
+ }
+ }
+
+ private PCollection<BeamRecord> standardJoin(
+ PCollection<KV<BeamRecord, BeamRecord>> extractedLeftRows,
+ PCollection<KV<BeamRecord, BeamRecord>> extractedRightRows,
+ BeamRecord leftNullRow, BeamRecord rightNullRow, String stageName) {
+ PCollection<KV<BeamRecord, KV<BeamRecord, BeamRecord>>> joinedRows = null;
+ switch (joinType) {
+ case LEFT:
+ joinedRows = org.apache.beam.sdk.extensions.joinlibrary.Join
+ .leftOuterJoin(extractedLeftRows, extractedRightRows, rightNullRow);
+ break;
+ case RIGHT:
+ joinedRows = org.apache.beam.sdk.extensions.joinlibrary.Join
+ .rightOuterJoin(extractedLeftRows, extractedRightRows, leftNullRow);
+ break;
+ case FULL:
+ joinedRows = org.apache.beam.sdk.extensions.joinlibrary.Join
+ .fullOuterJoin(extractedLeftRows, extractedRightRows, leftNullRow,
+ rightNullRow);
+ break;
+ case INNER:
+ default:
+ joinedRows = org.apache.beam.sdk.extensions.joinlibrary.Join
+ .innerJoin(extractedLeftRows, extractedRightRows);
+ break;
+ }
+
+ PCollection<BeamRecord> ret = joinedRows
+ .apply(stageName + "_JoinParts2WholeRow",
+ MapElements.via(new BeamJoinTransforms.JoinParts2WholeRow()))
+ .setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+ return ret;
+ }
+
+ public PCollection<BeamRecord> sideInputJoin(
+ PCollection<KV<BeamRecord, BeamRecord>> extractedLeftRows,
+ PCollection<KV<BeamRecord, BeamRecord>> extractedRightRows,
+ BeamRecord leftNullRow, BeamRecord rightNullRow) {
+ // we always make the Unbounded table on the left to do the sideInput join
+ // (will convert the result accordingly before return)
+ boolean swapped = (extractedLeftRows.isBounded() == PCollection.IsBounded.BOUNDED);
+ JoinRelType realJoinType =
+ (swapped && joinType != JoinRelType.INNER) ? JoinRelType.LEFT : joinType;
+
+ PCollection<KV<BeamRecord, BeamRecord>> realLeftRows =
+ swapped ? extractedRightRows : extractedLeftRows;
+ PCollection<KV<BeamRecord, BeamRecord>> realRightRows =
+ swapped ? extractedLeftRows : extractedRightRows;
+ BeamRecord realRightNullRow = swapped ? leftNullRow : rightNullRow;
+
+ // swapped still need to pass down because, we need to swap the result back.
+ return sideInputJoinHelper(realJoinType, realLeftRows, realRightRows,
+ realRightNullRow, swapped);
+ }
+
+ private PCollection<BeamRecord> sideInputJoinHelper(
+ JoinRelType joinType,
+ PCollection<KV<BeamRecord, BeamRecord>> leftRows,
+ PCollection<KV<BeamRecord, BeamRecord>> rightRows,
+ BeamRecord rightNullRow, boolean swapped) {
+ final PCollectionView<Map<BeamRecord, Iterable<BeamRecord>>> rowsView = rightRows
+ .apply(View.<BeamRecord, BeamRecord>asMultimap());
+
+ PCollection<BeamRecord> ret = leftRows
+ .apply(ParDo.of(new BeamJoinTransforms.SideInputJoinDoFn(
+ joinType, rightNullRow, rowsView, swapped)).withSideInputs(rowsView))
+ .setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+
+ return ret;
+ }
+
+ private BeamRecord buildNullRow(BeamRelNode relNode) {
+ BeamRecordSqlType leftType = CalciteUtils.toBeamRowType(relNode.getRowType());
+ return new BeamRecord(leftType, Collections.nCopies(leftType.getFieldCount(), null));
+ }
+
+ private List<Pair<Integer, Integer>> extractJoinColumns(int leftRowColumnCount) {
+ // it's a CROSS JOIN because: condition == true
+ if (condition instanceof RexLiteral && (Boolean) ((RexLiteral) condition).getValue()) {
+ throw new UnsupportedOperationException("CROSS JOIN is not supported!");
+ }
+
+ RexCall call = (RexCall) condition;
+ List<Pair<Integer, Integer>> pairs = new ArrayList<>();
+ if ("AND".equals(call.getOperator().getName())) {
+ List<RexNode> operands = call.getOperands();
+ for (RexNode rexNode : operands) {
+ Pair<Integer, Integer> pair = extractOneJoinColumn((RexCall) rexNode, leftRowColumnCount);
+ pairs.add(pair);
+ }
+ } else if ("=".equals(call.getOperator().getName())) {
+ pairs.add(extractOneJoinColumn(call, leftRowColumnCount));
+ } else {
+ throw new UnsupportedOperationException(
+ "Operator " + call.getOperator().getName() + " is not supported in join condition");
+ }
+
+ return pairs;
+ }
+
+ private Pair<Integer, Integer> extractOneJoinColumn(RexCall oneCondition,
+ int leftRowColumnCount) {
+ List<RexNode> operands = oneCondition.getOperands();
+ final int leftIndex = Math.min(((RexInputRef) operands.get(0)).getIndex(),
+ ((RexInputRef) operands.get(1)).getIndex());
+
+ final int rightIndex1 = Math.max(((RexInputRef) operands.get(0)).getIndex(),
+ ((RexInputRef) operands.get(1)).getIndex());
+ final int rightIndex = rightIndex1 - leftRowColumnCount;
+
+ return new Pair<>(leftIndex, rightIndex);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamLogicalConvention.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamLogicalConvention.java
new file mode 100644
index 0000000..11e4f5e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamLogicalConvention.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+
+/**
+ * Convertion for Beam SQL.
+ *
+ */
+public enum BeamLogicalConvention implements Convention {
+ INSTANCE;
+
+ @Override
+ public Class getInterface() {
+ return BeamRelNode.class;
+ }
+
+ @Override
+ public String getName() {
+ return "BEAM_LOGICAL";
+ }
+
+ @Override
+ public RelTraitDef getTraitDef() {
+ return ConventionTraitDef.INSTANCE;
+ }
+
+ @Override
+ public boolean satisfies(RelTrait trait) {
+ return this == trait;
+ }
+
+ @Override
+ public void register(RelOptPlanner planner) {
+ }
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ @Override
+ public boolean canConvertConvention(Convention toConvention) {
+ return false;
+ }
+
+ @Override
+ public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits, RelTraitSet toTraits) {
+ return false;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRel.java
new file mode 100644
index 0000000..6f5dff2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRel.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Minus;
+import org.apache.calcite.rel.core.SetOp;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Minus} node.
+ *
+ * <p>Corresponds to the SQL {@code EXCEPT} operator.
+ */
+public class BeamMinusRel extends Minus implements BeamRelNode {
+
+ private BeamSetOperatorRelBase delegate;
+
+ public BeamMinusRel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs,
+ boolean all) {
+ super(cluster, traits, inputs, all);
+ delegate = new BeamSetOperatorRelBase(this,
+ BeamSetOperatorRelBase.OpType.MINUS, inputs, all);
+ }
+
+ @Override public SetOp copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
+ return new BeamMinusRel(getCluster(), traitSet, inputs, all);
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ return delegate.buildBeamPipeline(inputPCollections, sqlEnv);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamProjectRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamProjectRel.java
new file mode 100644
index 0000000..501feb3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamProjectRel.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlExpressionExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSqlProjectFn;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+
+/**
+ * BeamRelNode to replace a {@code Project} node.
+ *
+ */
+public class BeamProjectRel extends Project implements BeamRelNode {
+
+ /**
+ * projects: {@link RexLiteral}, {@link RexInputRef}, {@link RexCall}.
+ *
+ */
+ public BeamProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode input,
+ List<? extends RexNode> projects, RelDataType rowType) {
+ super(cluster, traits, input, projects, rowType);
+ }
+
+ @Override
+ public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> projects,
+ RelDataType rowType) {
+ return new BeamProjectRel(getCluster(), traitSet, input, projects, rowType);
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ RelNode input = getInput();
+ String stageName = BeamSqlRelUtils.getStageName(this);
+
+ PCollection<BeamRecord> upstream =
+ BeamSqlRelUtils.getBeamRelInput(input).buildBeamPipeline(inputPCollections, sqlEnv);
+
+ BeamSqlExpressionExecutor executor = new BeamSqlFnExecutor(this);
+
+ PCollection<BeamRecord> projectStream = upstream.apply(stageName, ParDo
+ .of(new BeamSqlProjectFn(getRelTypeName(), executor,
+ CalciteUtils.toBeamRowType(rowType))));
+ projectStream.setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+
+ return projectStream;
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamRelNode.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamRelNode.java
new file mode 100644
index 0000000..9e8d46d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamRelNode.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.rel.RelNode;
+
+/**
+ * A new method {@link #buildBeamPipeline(PCollectionTuple, BeamSqlEnv)} is added.
+ */
+public interface BeamRelNode extends RelNode {
+
+ /**
+ * A {@link BeamRelNode} is a recursive structure, the
+ * {@code BeamQueryPlanner} visits it with a DFS(Depth-First-Search)
+ * algorithm.
+ */
+ PCollection<BeamRecord> buildBeamPipeline(
+ PCollectionTuple inputPCollections, BeamSqlEnv sqlEnv)
+ throws Exception;
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java
new file mode 100644
index 0000000..a1f3e2b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSetOperatorsTransforms;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.calcite.rel.RelNode;
+
+/**
+ * Delegate for Set operators: {@code BeamUnionRel}, {@code BeamIntersectRel}
+ * and {@code BeamMinusRel}.
+ */
+public class BeamSetOperatorRelBase {
+ /**
+ * Set operator type.
+ */
+ public enum OpType implements Serializable {
+ UNION,
+ INTERSECT,
+ MINUS
+ }
+
+ private BeamRelNode beamRelNode;
+ private List<RelNode> inputs;
+ private boolean all;
+ private OpType opType;
+
+ public BeamSetOperatorRelBase(BeamRelNode beamRelNode, OpType opType,
+ List<RelNode> inputs, boolean all) {
+ this.beamRelNode = beamRelNode;
+ this.opType = opType;
+ this.inputs = inputs;
+ this.all = all;
+ }
+
+ public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ PCollection<BeamRecord> leftRows = BeamSqlRelUtils.getBeamRelInput(inputs.get(0))
+ .buildBeamPipeline(inputPCollections, sqlEnv);
+ PCollection<BeamRecord> rightRows = BeamSqlRelUtils.getBeamRelInput(inputs.get(1))
+ .buildBeamPipeline(inputPCollections, sqlEnv);
+
+ WindowFn leftWindow = leftRows.getWindowingStrategy().getWindowFn();
+ WindowFn rightWindow = rightRows.getWindowingStrategy().getWindowFn();
+ if (!leftWindow.isCompatible(rightWindow)) {
+ throw new IllegalArgumentException(
+ "inputs of " + opType + " have different window strategy: "
+ + leftWindow + " VS " + rightWindow);
+ }
+
+ final TupleTag<BeamRecord> leftTag = new TupleTag<>();
+ final TupleTag<BeamRecord> rightTag = new TupleTag<>();
+
+ // co-group
+ String stageName = BeamSqlRelUtils.getStageName(beamRelNode);
+ PCollection<KV<BeamRecord, CoGbkResult>> coGbkResultCollection = KeyedPCollectionTuple
+ .of(leftTag, leftRows.apply(
+ stageName + "_CreateLeftIndex", MapElements.via(
+ new BeamSetOperatorsTransforms.BeamSqlRow2KvFn())))
+ .and(rightTag, rightRows.apply(
+ stageName + "_CreateRightIndex", MapElements.via(
+ new BeamSetOperatorsTransforms.BeamSqlRow2KvFn())))
+ .apply(CoGroupByKey.<BeamRecord>create());
+ PCollection<BeamRecord> ret = coGbkResultCollection
+ .apply(ParDo.of(new BeamSetOperatorsTransforms.SetOperatorFilteringDoFn(leftTag, rightTag,
+ opType, all)));
+ return ret;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java
new file mode 100644
index 0000000..99626aa
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Top;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Sort} node.
+ *
+ * <p>Since Beam does not fully supported global sort we are using {@link Top} to implement
+ * the {@code Sort} algebra. The following types of ORDER BY are supported:
+
+ * <pre>{@code
+ * select * from t order by id desc limit 10;
+ * select * from t order by id desc limit 10 offset 5;
+ * }</pre>
+ *
+ * <p>but Order BY without a limit is NOT supported:
+ *
+ * <pre>{@code
+ * select * from t order by id desc
+ * }</pre>
+ *
+ * <h3>Constraints</h3>
+ * <ul>
+ * <li>Due to the constraints of {@link Top}, the result of a `ORDER BY LIMIT`
+ * must fit into the memory of a single machine.</li>
+ * <li>Since `WINDOW`(HOP, TUMBLE, SESSION etc) is always associated with `GroupBy`,
+ * it does not make much sense to use `ORDER BY` with `WINDOW`.
+ * </li>
+ * </ul>
+ */
+public class BeamSortRel extends Sort implements BeamRelNode {
+ private List<Integer> fieldIndices = new ArrayList<>();
+ private List<Boolean> orientation = new ArrayList<>();
+ private List<Boolean> nullsFirst = new ArrayList<>();
+
+ private int startIndex = 0;
+ private int count;
+
+ public BeamSortRel(
+ RelOptCluster cluster,
+ RelTraitSet traits,
+ RelNode child,
+ RelCollation collation,
+ RexNode offset,
+ RexNode fetch) {
+ super(cluster, traits, child, collation, offset, fetch);
+
+ List<RexNode> fieldExps = getChildExps();
+ RelCollationImpl collationImpl = (RelCollationImpl) collation;
+ List<RelFieldCollation> collations = collationImpl.getFieldCollations();
+ for (int i = 0; i < fieldExps.size(); i++) {
+ RexNode fieldExp = fieldExps.get(i);
+ RexInputRef inputRef = (RexInputRef) fieldExp;
+ fieldIndices.add(inputRef.getIndex());
+ orientation.add(collations.get(i).getDirection() == RelFieldCollation.Direction.ASCENDING);
+
+ RelFieldCollation.NullDirection rawNullDirection = collations.get(i).nullDirection;
+ if (rawNullDirection == RelFieldCollation.NullDirection.UNSPECIFIED) {
+ rawNullDirection = collations.get(i).getDirection().defaultNullDirection();
+ }
+ nullsFirst.add(rawNullDirection == RelFieldCollation.NullDirection.FIRST);
+ }
+
+ if (fetch == null) {
+ throw new UnsupportedOperationException("ORDER BY without a LIMIT is not supported!");
+ }
+
+ RexLiteral fetchLiteral = (RexLiteral) fetch;
+ count = ((BigDecimal) fetchLiteral.getValue()).intValue();
+
+ if (offset != null) {
+ RexLiteral offsetLiteral = (RexLiteral) offset;
+ startIndex = ((BigDecimal) offsetLiteral.getValue()).intValue();
+ }
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ RelNode input = getInput();
+ PCollection<BeamRecord> upstream = BeamSqlRelUtils.getBeamRelInput(input)
+ .buildBeamPipeline(inputPCollections, sqlEnv);
+ Type windowType = upstream.getWindowingStrategy().getWindowFn()
+ .getWindowTypeDescriptor().getType();
+ if (!windowType.equals(GlobalWindow.class)) {
+ throw new UnsupportedOperationException(
+ "`ORDER BY` is only supported for GlobalWindow, actual window: " + windowType);
+ }
+
+ BeamSqlRowComparator comparator = new BeamSqlRowComparator(fieldIndices, orientation,
+ nullsFirst);
+ // first find the top (offset + count)
+ PCollection<List<BeamRecord>> rawStream =
+ upstream.apply("extractTopOffsetAndFetch",
+ Top.of(startIndex + count, comparator).withoutDefaults())
+ .setCoder(ListCoder.<BeamRecord>of(upstream.getCoder()));
+
+ // strip the `leading offset`
+ if (startIndex > 0) {
+ rawStream = rawStream.apply("stripLeadingOffset", ParDo.of(
+ new SubListFn<BeamRecord>(startIndex, startIndex + count)))
+ .setCoder(ListCoder.<BeamRecord>of(upstream.getCoder()));
+ }
+
+ PCollection<BeamRecord> orderedStream = rawStream.apply(
+ "flatten", Flatten.<BeamRecord>iterables());
+ orderedStream.setCoder(CalciteUtils.toBeamRowType(getRowType()).getRecordCoder());
+
+ return orderedStream;
+ }
+
+ private static class SubListFn<T> extends DoFn<List<T>, List<T>> {
+ private int startIndex;
+ private int endIndex;
+
+ public SubListFn(int startIndex, int endIndex) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext ctx) {
+ ctx.output(ctx.element().subList(startIndex, endIndex));
+ }
+ }
+
+ @Override public Sort copy(RelTraitSet traitSet, RelNode newInput, RelCollation newCollation,
+ RexNode offset, RexNode fetch) {
+ return new BeamSortRel(getCluster(), traitSet, newInput, newCollation, offset, fetch);
+ }
+
+ private static class BeamSqlRowComparator implements Comparator<BeamRecord>, Serializable {
+ private List<Integer> fieldsIndices;
+ private List<Boolean> orientation;
+ private List<Boolean> nullsFirst;
+
+ public BeamSqlRowComparator(List<Integer> fieldsIndices,
+ List<Boolean> orientation,
+ List<Boolean> nullsFirst) {
+ this.fieldsIndices = fieldsIndices;
+ this.orientation = orientation;
+ this.nullsFirst = nullsFirst;
+ }
+
+ @Override public int compare(BeamRecord row1, BeamRecord row2) {
+ for (int i = 0; i < fieldsIndices.size(); i++) {
+ int fieldIndex = fieldsIndices.get(i);
+ int fieldRet = 0;
+ SqlTypeName fieldType = CalciteUtils.getFieldType(
+ BeamSqlRecordHelper.getSqlRecordType(row1), fieldIndex);
+ // whether NULL should be ordered first or last(compared to non-null values) depends on
+ // what user specified in SQL(NULLS FIRST/NULLS LAST)
+ boolean isValue1Null = (row1.getFieldValue(fieldIndex) == null);
+ boolean isValue2Null = (row2.getFieldValue(fieldIndex) == null);
+ if (isValue1Null && isValue2Null) {
+ continue;
+ } else if (isValue1Null && !isValue2Null) {
+ fieldRet = -1 * (nullsFirst.get(i) ? -1 : 1);
+ } else if (!isValue1Null && isValue2Null) {
+ fieldRet = 1 * (nullsFirst.get(i) ? -1 : 1);
+ } else {
+ switch (fieldType) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ case VARCHAR:
+ case DATE:
+ case TIMESTAMP:
+ Comparable v1 = (Comparable) row1.getFieldValue(fieldIndex);
+ Comparable v2 = (Comparable) row2.getFieldValue(fieldIndex);
+ fieldRet = v1.compareTo(v2);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ "Data type: " + fieldType + " not supported yet!");
+ }
+ }
+
+ fieldRet *= (orientation.get(i) ? -1 : 1);
+ if (fieldRet != 0) {
+ return fieldRet;
+ }
+ }
+ return 0;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSqlRelUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSqlRelUtils.java
new file mode 100644
index 0000000..6467d9f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSqlRelUtils.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utilities for {@code BeamRelNode}.
+ */
+class BeamSqlRelUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRelUtils.class);
+
+ private static final AtomicInteger sequence = new AtomicInteger(0);
+ private static final AtomicInteger classSequence = new AtomicInteger(0);
+
+ public static String getStageName(BeamRelNode relNode) {
+ return relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() + "_"
+ + sequence.getAndIncrement();
+ }
+
+ public static String getClassName(BeamRelNode relNode) {
+ return "Generated_" + relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId()
+ + "_" + classSequence.getAndIncrement();
+ }
+
+ public static BeamRelNode getBeamRelInput(RelNode input) {
+ if (input instanceof RelSubset) {
+ // go with known best input
+ input = ((RelSubset) input).getBest();
+ }
+ return (BeamRelNode) input;
+ }
+
+ public static String explain(final RelNode rel) {
+ return explain(rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES);
+ }
+
+ public static String explain(final RelNode rel, SqlExplainLevel detailLevel) {
+ String explain = "";
+ try {
+ explain = RelOptUtil.toString(rel);
+ } catch (StackOverflowError e) {
+ LOG.error("StackOverflowError occurred while extracting plan. "
+ + "Please report it to the dev@ mailing list.");
+ LOG.error("RelNode " + rel + " ExplainLevel " + detailLevel, e);
+ LOG.error("Forcing plan to empty string and continue... "
+ + "SQL Runner may not working properly after.");
+ }
+ return explain;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRel.java
new file mode 100644
index 0000000..85d676e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRel.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelInput;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.SetOp;
+import org.apache.calcite.rel.core.Union;
+
+/**
+ * {@link BeamRelNode} to replace a {@link Union}.
+ *
+ * <p>{@code BeamUnionRel} needs the input of it have the same {@link WindowFn}. From the SQL
+ * perspective, two cases are supported:
+ *
+ * <p>1) Do not use {@code grouped window function}:
+ *
+ * <pre>{@code
+ * select * from person UNION select * from person
+ * }</pre>
+ *
+ * <p>2) Use the same {@code grouped window function}, with the same param:
+ * <pre>{@code
+ * select id, count(*) from person
+ * group by id, TUMBLE(order_time, INTERVAL '1' HOUR)
+ * UNION
+ * select * from person
+ * group by id, TUMBLE(order_time, INTERVAL '1' HOUR)
+ * }</pre>
+ *
+ * <p>Inputs with different group functions are NOT supported:
+ * <pre>{@code
+ * select id, count(*) from person
+ * group by id, TUMBLE(order_time, INTERVAL '1' HOUR)
+ * UNION
+ * select * from person
+ * group by id, TUMBLE(order_time, INTERVAL '2' HOUR)
+ * }</pre>
+ */
+public class BeamUnionRel extends Union implements BeamRelNode {
+ private BeamSetOperatorRelBase delegate;
+ public BeamUnionRel(RelOptCluster cluster,
+ RelTraitSet traits,
+ List<RelNode> inputs,
+ boolean all) {
+ super(cluster, traits, inputs, all);
+ this.delegate = new BeamSetOperatorRelBase(this,
+ BeamSetOperatorRelBase.OpType.UNION,
+ inputs, all);
+ }
+
+ public BeamUnionRel(RelInput input) {
+ super(input);
+ }
+
+ @Override public SetOp copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
+ return new BeamUnionRel(getCluster(), traitSet, inputs, all);
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ return delegate.buildBeamPipeline(inputPCollections, sqlEnv);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java
new file mode 100644
index 0000000..d684294
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexLiteral;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Values} node.
+ *
+ * <p>{@code BeamValuesRel} will be used in the following SQLs:
+ * <ul>
+ * <li>{@code insert into t (name, desc) values ('hello', 'world')}</li>
+ * <li>{@code select 1, '1', LOCALTIME}</li>
+ * </ul>
+ */
+public class BeamValuesRel extends Values implements BeamRelNode {
+
+ public BeamValuesRel(
+ RelOptCluster cluster,
+ RelDataType rowType,
+ ImmutableList<ImmutableList<RexLiteral>> tuples,
+ RelTraitSet traits) {
+ super(cluster, rowType, tuples, traits);
+
+ }
+
+ @Override public PCollection<BeamRecord> buildBeamPipeline(PCollectionTuple inputPCollections
+ , BeamSqlEnv sqlEnv) throws Exception {
+ List<BeamRecord> rows = new ArrayList<>(tuples.size());
+ String stageName = BeamSqlRelUtils.getStageName(this);
+ if (tuples.isEmpty()) {
+ throw new IllegalStateException("Values with empty tuples!");
+ }
+
+ BeamRecordSqlType beamSQLRowType = CalciteUtils.toBeamRowType(this.getRowType());
+ for (ImmutableList<RexLiteral> tuple : tuples) {
+ List<Object> fieldsValue = new ArrayList<>(beamSQLRowType.getFieldCount());
+ for (int i = 0; i < tuple.size(); i++) {
+ fieldsValue.add(BeamTableUtils.autoCastField(
+ beamSQLRowType.getFieldTypeByIndex(i), tuple.get(i).getValue()));
+ }
+ rows.add(new BeamRecord(beamSQLRowType, fieldsValue));
+ }
+
+ return inputPCollections.getPipeline().apply(stageName, Create.of(rows))
+ .setCoder(beamSQLRowType.getRecordCoder());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
new file mode 100644
index 0000000..76b335d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL specified nodes, to replace {@link org.apache.calcite.rel.RelNode}.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
new file mode 100644
index 0000000..cdf6712
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import com.google.common.collect.ImmutableList;
+import java.util.GregorianCalendar;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamAggregationRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Duration;
+
+/**
+ * Rule to detect the window/trigger settings.
+ *
+ */
+public class BeamAggregationRule extends RelOptRule {
+ public static final BeamAggregationRule INSTANCE =
+ new BeamAggregationRule(Aggregate.class, Project.class, RelFactories.LOGICAL_BUILDER);
+
+ public BeamAggregationRule(
+ Class<? extends Aggregate> aggregateClass,
+ Class<? extends Project> projectClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(
+ operand(aggregateClass,
+ operand(projectClass, any())),
+ relBuilderFactory, null);
+ }
+
+ public BeamAggregationRule(RelOptRuleOperand operand, String description) {
+ super(operand, description);
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final Aggregate aggregate = call.rel(0);
+ final Project project = call.rel(1);
+ updateWindowTrigger(call, aggregate, project);
+ }
+
+ private void updateWindowTrigger(RelOptRuleCall call, Aggregate aggregate,
+ Project project) {
+ ImmutableBitSet groupByFields = aggregate.getGroupSet();
+ List<RexNode> projectMapping = project.getProjects();
+
+ WindowFn windowFn = new GlobalWindows();
+ Trigger triggerFn = Repeatedly.forever(AfterWatermark.pastEndOfWindow());
+ int windowFieldIdx = -1;
+ Duration allowedLatence = Duration.ZERO;
+
+ for (int groupField : groupByFields.asList()) {
+ RexNode projNode = projectMapping.get(groupField);
+ if (projNode instanceof RexCall) {
+ SqlOperator op = ((RexCall) projNode).op;
+ ImmutableList<RexNode> parameters = ((RexCall) projNode).operands;
+ String functionName = op.getName();
+ switch (functionName) {
+ case "TUMBLE":
+ windowFieldIdx = groupField;
+ windowFn = FixedWindows
+ .of(Duration.millis(getWindowParameterAsMillis(parameters.get(1))));
+ if (parameters.size() == 3) {
+ GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(2))
+ .getValue();
+ triggerFn = createTriggerWithDelay(delayTime);
+ allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+ }
+ break;
+ case "HOP":
+ windowFieldIdx = groupField;
+ windowFn = SlidingWindows
+ .of(Duration.millis(getWindowParameterAsMillis(parameters.get(1))))
+ .every(Duration.millis(getWindowParameterAsMillis(parameters.get(2))));
+ if (parameters.size() == 4) {
+ GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(3))
+ .getValue();
+ triggerFn = createTriggerWithDelay(delayTime);
+ allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+ }
+ break;
+ case "SESSION":
+ windowFieldIdx = groupField;
+ windowFn = Sessions
+ .withGapDuration(Duration.millis(getWindowParameterAsMillis(parameters.get(1))));
+ if (parameters.size() == 3) {
+ GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(2))
+ .getValue();
+ triggerFn = createTriggerWithDelay(delayTime);
+ allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ BeamAggregationRel newAggregator = new BeamAggregationRel(aggregate.getCluster(),
+ aggregate.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convert(aggregate.getInput(),
+ aggregate.getInput().getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ aggregate.indicator,
+ aggregate.getGroupSet(),
+ aggregate.getGroupSets(),
+ aggregate.getAggCallList(),
+ windowFn,
+ triggerFn,
+ windowFieldIdx,
+ allowedLatence);
+ call.transformTo(newAggregator);
+ }
+
+ private Trigger createTriggerWithDelay(GregorianCalendar delayTime) {
+ return Repeatedly.forever(AfterWatermark.pastEndOfWindow().withLateFirings(AfterProcessingTime
+ .pastFirstElementInPane().plusDelayOf(Duration.millis(delayTime.getTimeInMillis()))));
+ }
+
+ private long getWindowParameterAsMillis(RexNode parameterNode) {
+ if (parameterNode instanceof RexLiteral) {
+ return RexLiteral.intValue(parameterNode);
+ } else {
+ throw new IllegalArgumentException(String.format("[%s] is not valid.", parameterNode));
+ }
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamFilterRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamFilterRule.java
new file mode 100644
index 0000000..bc25085
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamFilterRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamFilterRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.logical.LogicalFilter;
+
+/**
+ * A {@code ConverterRule} to replace {@link Filter} with {@link BeamFilterRel}.
+ *
+ */
+public class BeamFilterRule extends ConverterRule {
+ public static final BeamFilterRule INSTANCE = new BeamFilterRule();
+
+ private BeamFilterRule() {
+ super(LogicalFilter.class, Convention.NONE, BeamLogicalConvention.INSTANCE, "BeamFilterRule");
+ }
+
+ @Override
+ public RelNode convert(RelNode rel) {
+ final Filter filter = (Filter) rel;
+ final RelNode input = filter.getInput();
+
+ return new BeamFilterRel(filter.getCluster(),
+ filter.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ filter.getCondition());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSinkRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSinkRule.java
new file mode 100644
index 0000000..77f4bdd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSinkRule.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSinkRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.Table;
+
+/**
+ * A {@code ConverterRule} to replace {@link TableModify} with
+ * {@link BeamIOSinkRel}.
+ *
+ */
+public class BeamIOSinkRule extends ConverterRule {
+ public static final BeamIOSinkRule INSTANCE = new BeamIOSinkRule();
+
+ private BeamIOSinkRule() {
+ super(LogicalTableModify.class, Convention.NONE, BeamLogicalConvention.INSTANCE,
+ "BeamIOSinkRule");
+ }
+
+ @Override
+ public RelNode convert(RelNode rel) {
+ final TableModify tableModify = (TableModify) rel;
+ final RelNode input = tableModify.getInput();
+
+ final RelOptCluster cluster = tableModify.getCluster();
+ final RelTraitSet traitSet = tableModify.getTraitSet().replace(BeamLogicalConvention.INSTANCE);
+ final RelOptTable relOptTable = tableModify.getTable();
+ final Prepare.CatalogReader catalogReader = tableModify.getCatalogReader();
+ final RelNode convertedInput = convert(input,
+ input.getTraitSet().replace(BeamLogicalConvention.INSTANCE));
+ final TableModify.Operation operation = tableModify.getOperation();
+ final List<String> updateColumnList = tableModify.getUpdateColumnList();
+ final List<RexNode> sourceExpressionList = tableModify.getSourceExpressionList();
+ final boolean flattened = tableModify.isFlattened();
+
+ final Table table = tableModify.getTable().unwrap(Table.class);
+
+ switch (table.getJdbcTableType()) {
+ case TABLE:
+ case STREAM:
+ if (operation != TableModify.Operation.INSERT) {
+ throw new UnsupportedOperationException(
+ String.format("Streams doesn't support %s modify operation", operation));
+ }
+ return new BeamIOSinkRel(cluster, traitSet,
+ relOptTable, catalogReader, convertedInput, operation, updateColumnList,
+ sourceExpressionList, flattened);
+ default:
+ throw new IllegalArgumentException(
+ String.format("Unsupported table type: %s", table.getJdbcTableType()));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSourceRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSourceRule.java
new file mode 100644
index 0000000..a257d3d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSourceRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.logical.LogicalTableScan;
+
+/**
+ * A {@code ConverterRule} to replace {@link TableScan} with
+ * {@link BeamIOSourceRel}.
+ *
+ */
+public class BeamIOSourceRule extends ConverterRule {
+ public static final BeamIOSourceRule INSTANCE = new BeamIOSourceRule();
+
+ private BeamIOSourceRule() {
+ super(LogicalTableScan.class, Convention.NONE, BeamLogicalConvention.INSTANCE,
+ "BeamIOSourceRule");
+ }
+
+ @Override
+ public RelNode convert(RelNode rel) {
+ final TableScan scan = (TableScan) rel;
+
+ return new BeamIOSourceRel(scan.getCluster(),
+ scan.getTraitSet().replace(BeamLogicalConvention.INSTANCE), scan.getTable());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIntersectRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIntersectRule.java
new file mode 100644
index 0000000..03d7129
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIntersectRule.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIntersectRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Intersect;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+
+/**
+ * {@code ConverterRule} to replace {@code Intersect} with {@code BeamIntersectRel}.
+ */
+public class BeamIntersectRule extends ConverterRule {
+ public static final BeamIntersectRule INSTANCE = new BeamIntersectRule();
+ private BeamIntersectRule() {
+ super(LogicalIntersect.class, Convention.NONE,
+ BeamLogicalConvention.INSTANCE, "BeamIntersectRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Intersect intersect = (Intersect) rel;
+ final List<RelNode> inputs = intersect.getInputs();
+ return new BeamIntersectRel(
+ intersect.getCluster(),
+ intersect.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convertList(inputs, BeamLogicalConvention.INSTANCE),
+ intersect.all
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamJoinRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamJoinRule.java
new file mode 100644
index 0000000..4d9dd20
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamJoinRule.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamJoinRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.logical.LogicalJoin;
+
+/**
+ * {@code ConverterRule} to replace {@code Join} with {@code BeamJoinRel}.
+ */
+public class BeamJoinRule extends ConverterRule {
+ public static final BeamJoinRule INSTANCE = new BeamJoinRule();
+ private BeamJoinRule() {
+ super(LogicalJoin.class, Convention.NONE,
+ BeamLogicalConvention.INSTANCE, "BeamJoinRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Join join = (Join) rel;
+ return new BeamJoinRel(
+ join.getCluster(),
+ join.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convert(join.getLeft(),
+ join.getLeft().getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ convert(join.getRight(),
+ join.getRight().getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ join.getCondition(),
+ join.getVariablesSet(),
+ join.getJoinType()
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamMinusRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamMinusRule.java
new file mode 100644
index 0000000..9efdf70
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamMinusRule.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamMinusRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Minus;
+import org.apache.calcite.rel.logical.LogicalMinus;
+
+/**
+ * {@code ConverterRule} to replace {@code Minus} with {@code BeamMinusRel}.
+ */
+public class BeamMinusRule extends ConverterRule {
+ public static final BeamMinusRule INSTANCE = new BeamMinusRule();
+ private BeamMinusRule() {
+ super(LogicalMinus.class, Convention.NONE,
+ BeamLogicalConvention.INSTANCE, "BeamMinusRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Minus minus = (Minus) rel;
+ final List<RelNode> inputs = minus.getInputs();
+ return new BeamMinusRel(
+ minus.getCluster(),
+ minus.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convertList(inputs, BeamLogicalConvention.INSTANCE),
+ minus.all
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamProjectRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamProjectRule.java
new file mode 100644
index 0000000..d19a01d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamProjectRule.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamProjectRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.logical.LogicalProject;
+
+/**
+ * A {@code ConverterRule} to replace {@link Project} with
+ * {@link BeamProjectRel}.
+ *
+ */
+public class BeamProjectRule extends ConverterRule {
+ public static final BeamProjectRule INSTANCE = new BeamProjectRule();
+
+ private BeamProjectRule() {
+ super(LogicalProject.class, Convention.NONE, BeamLogicalConvention.INSTANCE, "BeamProjectRule");
+ }
+
+ @Override
+ public RelNode convert(RelNode rel) {
+ final Project project = (Project) rel;
+ final RelNode input = project.getInput();
+
+ return new BeamProjectRel(project.getCluster(),
+ project.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ project.getProjects(), project.getRowType());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamSortRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamSortRule.java
new file mode 100644
index 0000000..36a7c1b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamSortRule.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSortRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.logical.LogicalSort;
+
+/**
+ * {@code ConverterRule} to replace {@code Sort} with {@code BeamSortRel}.
+ */
+public class BeamSortRule extends ConverterRule {
+ public static final BeamSortRule INSTANCE = new BeamSortRule();
+ private BeamSortRule() {
+ super(LogicalSort.class, Convention.NONE,
+ BeamLogicalConvention.INSTANCE, "BeamSortRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Sort sort = (Sort) rel;
+ final RelNode input = sort.getInput();
+ return new BeamSortRel(
+ sort.getCluster(),
+ sort.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+ sort.getCollation(),
+ sort.offset,
+ sort.fetch
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnionRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnionRule.java
new file mode 100644
index 0000000..6065b72
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnionRule.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamUnionRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Union;
+import org.apache.calcite.rel.logical.LogicalUnion;
+
+/**
+ * A {@code ConverterRule} to replace {@link org.apache.calcite.rel.core.Union} with
+ * {@link BeamUnionRule}.
+ */
+public class BeamUnionRule extends ConverterRule {
+ public static final BeamUnionRule INSTANCE = new BeamUnionRule();
+ private BeamUnionRule() {
+ super(LogicalUnion.class, Convention.NONE, BeamLogicalConvention.INSTANCE,
+ "BeamUnionRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Union union = (Union) rel;
+
+ return new BeamUnionRel(
+ union.getCluster(),
+ union.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+ convertList(union.getInputs(), BeamLogicalConvention.INSTANCE),
+ union.all
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamValuesRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamValuesRule.java
new file mode 100644
index 0000000..b5dc30c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamValuesRule.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamValuesRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.logical.LogicalValues;
+
+/**
+ * {@code ConverterRule} to replace {@code Values} with {@code BeamValuesRel}.
+ */
+public class BeamValuesRule extends ConverterRule {
+ public static final BeamValuesRule INSTANCE = new BeamValuesRule();
+ private BeamValuesRule() {
+ super(LogicalValues.class, Convention.NONE,
+ BeamLogicalConvention.INSTANCE, "BeamValuesRule");
+ }
+
+ @Override public RelNode convert(RelNode rel) {
+ Values values = (Values) rel;
+ return new BeamValuesRel(
+ values.getCluster(),
+ values.getRowType(),
+ values.getTuples(),
+ values.getTraitSet().replace(BeamLogicalConvention.INSTANCE)
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/package-info.java
new file mode 100644
index 0000000..fa32b44
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.calcite.plan.RelOptRule} to generate
+ * {@link org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode}.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BaseBeamTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BaseBeamTable.java
new file mode 100644
index 0000000..7f99e12
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BaseBeamTable.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+
+/**
+ * Each IO in Beam has one table schema, by extending {@link BaseBeamTable}.
+ */
+public abstract class BaseBeamTable implements BeamSqlTable, Serializable {
+ protected BeamRecordSqlType beamRecordSqlType;
+ public BaseBeamTable(BeamRecordSqlType beamRecordSqlType) {
+ this.beamRecordSqlType = beamRecordSqlType;
+ }
+
+ @Override public BeamRecordSqlType getRowType() {
+ return beamRecordSqlType;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamIOType.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamIOType.java
new file mode 100644
index 0000000..5ced467
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamIOType.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema;
+
+import java.io.Serializable;
+
+/**
+ * Type as a source IO, determined whether it's a STREAMING process, or batch
+ * process.
+ */
+public enum BeamIOType implements Serializable {
+ BOUNDED, UNBOUNDED;
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamPCollectionTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamPCollectionTable.java
new file mode 100644
index 0000000..31e60e0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamPCollectionTable.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+ * {@code BeamPCollectionTable} converts a {@code PCollection<BeamSqlRow>} as a virtual table,
+ * then a downstream query can query directly.
+ */
+public class BeamPCollectionTable extends BaseBeamTable {
+ private BeamIOType ioType;
+ private transient PCollection<BeamRecord> upstream;
+
+ protected BeamPCollectionTable(BeamRecordSqlType beamSqlRowType) {
+ super(beamSqlRowType);
+ }
+
+ public BeamPCollectionTable(PCollection<BeamRecord> upstream,
+ BeamRecordSqlType beamSqlRowType){
+ this(beamSqlRowType);
+ ioType = upstream.isBounded().equals(IsBounded.BOUNDED)
+ ? BeamIOType.BOUNDED : BeamIOType.UNBOUNDED;
+ this.upstream = upstream;
+ }
+
+ @Override
+ public BeamIOType getSourceType() {
+ return ioType;
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildIOReader(Pipeline pipeline) {
+ return upstream;
+ }
+
+ @Override
+ public PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter() {
+ throw new IllegalArgumentException("cannot use [BeamPCollectionTable] as target");
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamTableUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamTableUtils.java
new file mode 100644
index 0000000..e9f3c76
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamTableUtils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.schema;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+
+/**
+ * Utility methods for working with {@code BeamTable}.
+ */
+public final class BeamTableUtils {
+ public static BeamRecord csvLine2BeamRecord(
+ CSVFormat csvFormat,
+ String line,
+ BeamRecordSqlType beamRecordSqlType) {
+ List<Object> fieldsValue = new ArrayList<>(beamRecordSqlType.getFieldCount());
+ try (StringReader reader = new StringReader(line)) {
+ CSVParser parser = csvFormat.parse(reader);
+ CSVRecord rawRecord = parser.getRecords().get(0);
+
+ if (rawRecord.size() != beamRecordSqlType.getFieldCount()) {
+ throw new IllegalArgumentException(String.format(
+ "Expect %d fields, but actually %d",
+ beamRecordSqlType.getFieldCount(), rawRecord.size()
+ ));
+ } else {
+ for (int idx = 0; idx < beamRecordSqlType.getFieldCount(); idx++) {
+ String raw = rawRecord.get(idx);
+ fieldsValue.add(autoCastField(beamRecordSqlType.getFieldTypeByIndex(idx), raw));
+ }
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("decodeRecord failed!", e);
+ }
+ return new BeamRecord(beamRecordSqlType, fieldsValue);
+ }
+
+ public static String beamRecord2CsvLine(BeamRecord row, CSVFormat csvFormat) {
+ StringWriter writer = new StringWriter();
+ try (CSVPrinter printer = csvFormat.print(writer)) {
+ for (int i = 0; i < row.getFieldCount(); i++) {
+ printer.print(row.getFieldValue(i).toString());
+ }
+ printer.println();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("encodeRecord failed!", e);
+ }
+ return writer.toString();
+ }
+
+ public static Object autoCastField(int fieldType, Object rawObj) {
+ if (rawObj == null) {
+ return null;
+ }
+
+ SqlTypeName columnType = CalciteUtils.toCalciteType(fieldType);
+ // auto-casting for numberics
+ if ((rawObj instanceof String && SqlTypeName.NUMERIC_TYPES.contains(columnType))
+ || (rawObj instanceof BigDecimal && columnType != SqlTypeName.DECIMAL)) {
+ String raw = rawObj.toString();
+ switch (columnType) {
+ case TINYINT:
+ return Byte.valueOf(raw);
+ case SMALLINT:
+ return Short.valueOf(raw);
+ case INTEGER:
+ return Integer.valueOf(raw);
+ case BIGINT:
+ return Long.valueOf(raw);
+ case FLOAT:
+ return Float.valueOf(raw);
+ case DOUBLE:
+ return Double.valueOf(raw);
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Column type %s is not supported yet!", columnType));
+ }
+ } else if (SqlTypeName.CHAR_TYPES.contains(columnType)) {
+ // convert NlsString to String
+ if (rawObj instanceof NlsString) {
+ return ((NlsString) rawObj).getValue();
+ } else {
+ return rawObj;
+ }
+ } else {
+ return rawObj;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/package-info.java
new file mode 100644
index 0000000..86e7d06
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * define table schema, to map with Beam IO components.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamAggregationTransforms.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamAggregationTransforms.java
new file mode 100644
index 0000000..f8c4c6f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamAggregationTransforms.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.coders.BigDecimalCoder;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.UdafImpl;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Instant;
+
+/**
+ * Collections of {@code PTransform} and {@code DoFn} used to perform GROUP-BY operation.
+ */
+public class BeamAggregationTransforms implements Serializable{
+ /**
+ * Merge KV to single record.
+ */
+ public static class MergeAggregationRecord extends DoFn<KV<BeamRecord, BeamRecord>, BeamRecord> {
+ private BeamRecordSqlType outRowType;
+ private List<String> aggFieldNames;
+ private int windowStartFieldIdx;
+
+ public MergeAggregationRecord(BeamRecordSqlType outRowType, List<AggregateCall> aggList
+ , int windowStartFieldIdx) {
+ this.outRowType = outRowType;
+ this.aggFieldNames = new ArrayList<>();
+ for (AggregateCall ac : aggList) {
+ aggFieldNames.add(ac.getName());
+ }
+ this.windowStartFieldIdx = windowStartFieldIdx;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window) {
+ KV<BeamRecord, BeamRecord> kvRecord = c.element();
+ List<Object> fieldValues = new ArrayList<>();
+ fieldValues.addAll(kvRecord.getKey().getDataValues());
+ fieldValues.addAll(kvRecord.getValue().getDataValues());
+
+ if (windowStartFieldIdx != -1) {
+ fieldValues.add(windowStartFieldIdx, ((IntervalWindow) window).start().toDate());
+ }
+
+ BeamRecord outRecord = new BeamRecord(outRowType, fieldValues);
+ c.output(outRecord);
+ }
+ }
+
+ /**
+ * extract group-by fields.
+ */
+ public static class AggregationGroupByKeyFn
+ implements SerializableFunction<BeamRecord, BeamRecord> {
+ private List<Integer> groupByKeys;
+
+ public AggregationGroupByKeyFn(int windowFieldIdx, ImmutableBitSet groupSet) {
+ this.groupByKeys = new ArrayList<>();
+ for (int i : groupSet.asList()) {
+ if (i != windowFieldIdx) {
+ groupByKeys.add(i);
+ }
+ }
+ }
+
+ @Override
+ public BeamRecord apply(BeamRecord input) {
+ BeamRecordSqlType typeOfKey = exTypeOfKeyRecord(BeamSqlRecordHelper.getSqlRecordType(input));
+
+ List<Object> fieldValues = new ArrayList<>(groupByKeys.size());
+ for (int idx = 0; idx < groupByKeys.size(); ++idx) {
+ fieldValues.add(input.getFieldValue(groupByKeys.get(idx)));
+ }
+
+ BeamRecord keyOfRecord = new BeamRecord(typeOfKey, fieldValues);
+ return keyOfRecord;
+ }
+
+ private BeamRecordSqlType exTypeOfKeyRecord(BeamRecordSqlType dataType) {
+ List<String> fieldNames = new ArrayList<>();
+ List<Integer> fieldTypes = new ArrayList<>();
+ for (int idx : groupByKeys) {
+ fieldNames.add(dataType.getFieldNameByIndex(idx));
+ fieldTypes.add(dataType.getFieldTypeByIndex(idx));
+ }
+ return BeamRecordSqlType.create(fieldNames, fieldTypes);
+ }
+ }
+
+ /**
+ * Assign event timestamp.
+ */
+ public static class WindowTimestampFn implements SerializableFunction<BeamRecord, Instant> {
+ private int windowFieldIdx = -1;
+
+ public WindowTimestampFn(int windowFieldIdx) {
+ super();
+ this.windowFieldIdx = windowFieldIdx;
+ }
+
+ @Override
+ public Instant apply(BeamRecord input) {
+ return new Instant(input.getDate(windowFieldIdx).getTime());
+ }
+ }
+
+ /**
+ * An adaptor class to invoke Calcite UDAF instances in Beam {@code CombineFn}.
+ */
+ public static class AggregationAdaptor
+ extends CombineFn<BeamRecord, AggregationAccumulator, BeamRecord> {
+ private List<CombineFn> aggregators;
+ private List<BeamSqlInputRefExpression> sourceFieldExps;
+ private BeamRecordSqlType finalRowType;
+
+ public AggregationAdaptor(List<AggregateCall> aggregationCalls,
+ BeamRecordSqlType sourceRowType) {
+ aggregators = new ArrayList<>();
+ sourceFieldExps = new ArrayList<>();
+ List<String> outFieldsName = new ArrayList<>();
+ List<Integer> outFieldsType = new ArrayList<>();
+ for (AggregateCall call : aggregationCalls) {
+ int refIndex = call.getArgList().size() > 0 ? call.getArgList().get(0) : 0;
+ BeamSqlInputRefExpression sourceExp = new BeamSqlInputRefExpression(
+ CalciteUtils.getFieldType(sourceRowType, refIndex), refIndex);
+ sourceFieldExps.add(sourceExp);
+
+ outFieldsName.add(call.name);
+ int outFieldType = CalciteUtils.toJavaType(call.type.getSqlTypeName());
+ outFieldsType.add(outFieldType);
+
+ switch (call.getAggregation().getName()) {
+ case "COUNT":
+ aggregators.add(Count.combineFn());
+ break;
+ case "MAX":
+ aggregators.add(BeamBuiltinAggregations.createMax(call.type.getSqlTypeName()));
+ break;
+ case "MIN":
+ aggregators.add(BeamBuiltinAggregations.createMin(call.type.getSqlTypeName()));
+ break;
+ case "SUM":
+ aggregators.add(BeamBuiltinAggregations.createSum(call.type.getSqlTypeName()));
+ break;
+ case "AVG":
+ aggregators.add(BeamBuiltinAggregations.createAvg(call.type.getSqlTypeName()));
+ break;
+ case "VAR_POP":
+ aggregators.add(BeamBuiltinAggregations.createVar(call.type.getSqlTypeName(),
+ false));
+ break;
+ case "VAR_SAMP":
+ aggregators.add(BeamBuiltinAggregations.createVar(call.type.getSqlTypeName(),
+ true));
+ break;
+ default:
+ if (call.getAggregation() instanceof SqlUserDefinedAggFunction) {
+ // handle UDAF.
+ SqlUserDefinedAggFunction udaf = (SqlUserDefinedAggFunction) call.getAggregation();
+ UdafImpl fn = (UdafImpl) udaf.function;
+ try {
+ aggregators.add(fn.getCombineFn());
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ String.format("Aggregator [%s] is not supported",
+ call.getAggregation().getName()));
+ }
+ break;
+ }
+ }
+ finalRowType = BeamRecordSqlType.create(outFieldsName, outFieldsType);
+ }
+ @Override
+ public AggregationAccumulator createAccumulator() {
+ AggregationAccumulator initialAccu = new AggregationAccumulator();
+ for (CombineFn agg : aggregators) {
+ initialAccu.accumulatorElements.add(agg.createAccumulator());
+ }
+ return initialAccu;
+ }
+ @Override
+ public AggregationAccumulator addInput(AggregationAccumulator accumulator, BeamRecord input) {
+ AggregationAccumulator deltaAcc = new AggregationAccumulator();
+ for (int idx = 0; idx < aggregators.size(); ++idx) {
+ deltaAcc.accumulatorElements.add(
+ aggregators.get(idx).addInput(accumulator.accumulatorElements.get(idx),
+ sourceFieldExps.get(idx).evaluate(input, null).getValue()));
+ }
+ return deltaAcc;
+ }
+ @Override
+ public AggregationAccumulator mergeAccumulators(Iterable<AggregationAccumulator> accumulators) {
+ AggregationAccumulator deltaAcc = new AggregationAccumulator();
+ for (int idx = 0; idx < aggregators.size(); ++idx) {
+ List accs = new ArrayList<>();
+ Iterator<AggregationAccumulator> ite = accumulators.iterator();
+ while (ite.hasNext()) {
+ accs.add(ite.next().accumulatorElements.get(idx));
+ }
+ deltaAcc.accumulatorElements.add(aggregators.get(idx).mergeAccumulators(accs));
+ }
+ return deltaAcc;
+ }
+ @Override
+ public BeamRecord extractOutput(AggregationAccumulator accumulator) {
+ List<Object> fieldValues = new ArrayList<>(aggregators.size());
+ for (int idx = 0; idx < aggregators.size(); ++idx) {
+ fieldValues
+ .add(aggregators.get(idx).extractOutput(accumulator.accumulatorElements.get(idx)));
+ }
+ return new BeamRecord(finalRowType, fieldValues);
+ }
+ @Override
+ public Coder<AggregationAccumulator> getAccumulatorCoder(
+ CoderRegistry registry, Coder<BeamRecord> inputCoder)
+ throws CannotProvideCoderException {
+ BeamRecordCoder beamRecordCoder = (BeamRecordCoder) inputCoder;
+ registry.registerCoderForClass(BigDecimal.class, BigDecimalCoder.of());
+ List<Coder> aggAccuCoderList = new ArrayList<>();
+ for (int idx = 0; idx < aggregators.size(); ++idx) {
+ int srcFieldIndex = sourceFieldExps.get(idx).getInputRef();
+ Coder srcFieldCoder = beamRecordCoder.getCoders().get(srcFieldIndex);
+ aggAccuCoderList.add(aggregators.get(idx).getAccumulatorCoder(registry, srcFieldCoder));
+ }
+ return new AggregationAccumulatorCoder(aggAccuCoderList);
+ }
+ }
+
+ /**
+ * A class to holder varied accumulator objects.
+ */
+ public static class AggregationAccumulator{
+ private List accumulatorElements = new ArrayList<>();
+ }
+
+ /**
+ * Coder for {@link AggregationAccumulator}.
+ */
+ public static class AggregationAccumulatorCoder extends CustomCoder<AggregationAccumulator>{
+ private VarIntCoder sizeCoder = VarIntCoder.of();
+ private List<Coder> elementCoders;
+
+ public AggregationAccumulatorCoder(List<Coder> elementCoders) {
+ this.elementCoders = elementCoders;
+ }
+
+ @Override
+ public void encode(AggregationAccumulator value, OutputStream outStream)
+ throws CoderException, IOException {
+ sizeCoder.encode(value.accumulatorElements.size(), outStream);
+ for (int idx = 0; idx < value.accumulatorElements.size(); ++idx) {
+ elementCoders.get(idx).encode(value.accumulatorElements.get(idx), outStream);
+ }
+ }
+
+ @Override
+ public AggregationAccumulator decode(InputStream inStream) throws CoderException, IOException {
+ AggregationAccumulator accu = new AggregationAccumulator();
+ int size = sizeCoder.decode(inStream);
+ for (int idx = 0; idx < size; ++idx) {
+ accu.accumulatorElements.add(elementCoders.get(idx).decode(inStream));
+ }
+ return accu;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java
new file mode 100644
index 0000000..ad15f98
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
+import java.util.Date;
+import java.util.Iterator;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.BigDecimalCoder;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Max;
+import org.apache.beam.sdk.transforms.Min;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Built-in aggregations functions for COUNT/MAX/MIN/SUM/AVG/VAR_POP/VAR_SAMP.
+ */
+class BeamBuiltinAggregations {
+ private static MathContext mc = new MathContext(10, RoundingMode.HALF_UP);
+
+ /**
+ * {@link CombineFn} for MAX based on {@link Max} and {@link Combine.BinaryCombineFn}.
+ */
+ public static CombineFn createMax(SqlTypeName fieldType) {
+ switch (fieldType) {
+ case INTEGER:
+ return Max.ofIntegers();
+ case SMALLINT:
+ return new CustMax<Short>();
+ case TINYINT:
+ return new CustMax<Byte>();
+ case BIGINT:
+ return Max.ofLongs();
+ case FLOAT:
+ return new CustMax<Float>();
+ case DOUBLE:
+ return Max.ofDoubles();
+ case TIMESTAMP:
+ return new CustMax<Date>();
+ case DECIMAL:
+ return new CustMax<BigDecimal>();
+ default:
+ throw new UnsupportedOperationException(
+ String.format("[%s] is not support in MAX", fieldType));
+ }
+ }
+
+ /**
+ * {@link CombineFn} for MAX based on {@link Min} and {@link Combine.BinaryCombineFn}.
+ */
+ public static CombineFn createMin(SqlTypeName fieldType) {
+ switch (fieldType) {
+ case INTEGER:
+ return Min.ofIntegers();
+ case SMALLINT:
+ return new CustMin<Short>();
+ case TINYINT:
+ return new CustMin<Byte>();
+ case BIGINT:
+ return Min.ofLongs();
+ case FLOAT:
+ return new CustMin<Float>();
+ case DOUBLE:
+ return Min.ofDoubles();
+ case TIMESTAMP:
+ return new CustMin<Date>();
+ case DECIMAL:
+ return new CustMin<BigDecimal>();
+ default:
+ throw new UnsupportedOperationException(
+ String.format("[%s] is not support in MIN", fieldType));
+ }
+ }
+
+ /**
+ * {@link CombineFn} for MAX based on {@link Sum} and {@link Combine.BinaryCombineFn}.
+ */
+ public static CombineFn createSum(SqlTypeName fieldType) {
+ switch (fieldType) {
+ case INTEGER:
+ return Sum.ofIntegers();
+ case SMALLINT:
+ return new ShortSum();
+ case TINYINT:
+ return new ByteSum();
+ case BIGINT:
+ return Sum.ofLongs();
+ case FLOAT:
+ return new FloatSum();
+ case DOUBLE:
+ return Sum.ofDoubles();
+ case DECIMAL:
+ return new BigDecimalSum();
+ default:
+ throw new UnsupportedOperationException(
+ String.format("[%s] is not support in SUM", fieldType));
+ }
+ }
+
+ /**
+ * {@link CombineFn} for AVG.
+ */
+ public static CombineFn createAvg(SqlTypeName fieldType) {
+ switch (fieldType) {
+ case INTEGER:
+ return new IntegerAvg();
+ case SMALLINT:
+ return new ShortAvg();
+ case TINYINT:
+ return new ByteAvg();
+ case BIGINT:
+ return new LongAvg();
+ case FLOAT:
+ return new FloatAvg();
+ case DOUBLE:
+ return new DoubleAvg();
+ case DECIMAL:
+ return new BigDecimalAvg();
+ default:
+ throw new UnsupportedOperationException(
+ String.format("[%s] is not support in AVG", fieldType));
+ }
+ }
+
+ /**
+ * {@link CombineFn} for VAR_POP and VAR_SAMP.
+ */
+ public static CombineFn createVar(SqlTypeName fieldType, boolean isSamp) {
+ switch (fieldType) {
+ case INTEGER:
+ return new IntegerVar(isSamp);
+ case SMALLINT:
+ return new ShortVar(isSamp);
+ case TINYINT:
+ return new ByteVar(isSamp);
+ case BIGINT:
+ return new LongVar(isSamp);
+ case FLOAT:
+ return new FloatVar(isSamp);
+ case DOUBLE:
+ return new DoubleVar(isSamp);
+ case DECIMAL:
+ return new BigDecimalVar(isSamp);
+ default:
+ throw new UnsupportedOperationException(
+ String.format("[%s] is not support in AVG", fieldType));
+ }
+ }
+
+ static class CustMax<T extends Comparable<T>> extends Combine.BinaryCombineFn<T> {
+ public T apply(T left, T right) {
+ return (right == null || right.compareTo(left) < 0) ? left : right;
+ }
+ }
+
+ static class CustMin<T extends Comparable<T>> extends Combine.BinaryCombineFn<T> {
+ public T apply(T left, T right) {
+ return (left == null || left.compareTo(right) < 0) ? left : right;
+ }
+ }
+
+ static class ShortSum extends Combine.BinaryCombineFn<Short> {
+ public Short apply(Short left, Short right) {
+ return (short) (left + right);
+ }
+ }
+
+ static class ByteSum extends Combine.BinaryCombineFn<Byte> {
+ public Byte apply(Byte left, Byte right) {
+ return (byte) (left + right);
+ }
+ }
+
+ static class FloatSum extends Combine.BinaryCombineFn<Float> {
+ public Float apply(Float left, Float right) {
+ return left + right;
+ }
+ }
+
+ static class BigDecimalSum extends Combine.BinaryCombineFn<BigDecimal> {
+ public BigDecimal apply(BigDecimal left, BigDecimal right) {
+ return left.add(right);
+ }
+ }
+
+ /**
+ * {@link CombineFn} for <em>AVG</em> on {@link Number} types.
+ */
+ abstract static class Avg<T extends Number>
+ extends CombineFn<T, KV<Integer, BigDecimal>, T> {
+ @Override
+ public KV<Integer, BigDecimal> createAccumulator() {
+ return KV.of(0, new BigDecimal(0));
+ }
+
+ @Override
+ public KV<Integer, BigDecimal> addInput(KV<Integer, BigDecimal> accumulator, T input) {
+ return KV.of(accumulator.getKey() + 1, accumulator.getValue().add(toBigDecimal(input)));
+ }
+
+ @Override
+ public KV<Integer, BigDecimal> mergeAccumulators(
+ Iterable<KV<Integer, BigDecimal>> accumulators) {
+ int size = 0;
+ BigDecimal acc = new BigDecimal(0);
+ Iterator<KV<Integer, BigDecimal>> ite = accumulators.iterator();
+ while (ite.hasNext()) {
+ KV<Integer, BigDecimal> ele = ite.next();
+ size += ele.getKey();
+ acc = acc.add(ele.getValue());
+ }
+ return KV.of(size, acc);
+ }
+
+ @Override
+ public Coder<KV<Integer, BigDecimal>> getAccumulatorCoder(CoderRegistry registry,
+ Coder<T> inputCoder) throws CannotProvideCoderException {
+ return KvCoder.of(BigEndianIntegerCoder.of(), BigDecimalCoder.of());
+ }
+
+ protected BigDecimal prepareOutput(KV<Integer, BigDecimal> accumulator){
+ return accumulator.getValue().divide(new BigDecimal(accumulator.getKey()), mc);
+ }
+
+ public abstract T extractOutput(KV<Integer, BigDecimal> accumulator);
+ public abstract BigDecimal toBigDecimal(T record);
+ }
+
+ static class IntegerAvg extends Avg<Integer>{
+ @Nullable
+ public Integer extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).intValue();
+ }
+
+ public BigDecimal toBigDecimal(Integer record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class LongAvg extends Avg<Long>{
+ @Nullable
+ public Long extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).longValue();
+ }
+
+ public BigDecimal toBigDecimal(Long record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class ShortAvg extends Avg<Short>{
+ @Nullable
+ public Short extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).shortValue();
+ }
+
+ public BigDecimal toBigDecimal(Short record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class ByteAvg extends Avg<Byte> {
+ @Nullable
+ public Byte extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).byteValue();
+ }
+
+ public BigDecimal toBigDecimal(Byte record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class FloatAvg extends Avg<Float>{
+ @Nullable
+ public Float extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).floatValue();
+ }
+
+ public BigDecimal toBigDecimal(Float record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class DoubleAvg extends Avg<Double>{
+ @Nullable
+ public Double extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator).doubleValue();
+ }
+
+ public BigDecimal toBigDecimal(Double record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class BigDecimalAvg extends Avg<BigDecimal>{
+ @Nullable
+ public BigDecimal extractOutput(KV<Integer, BigDecimal> accumulator) {
+ return accumulator.getKey() == 0 ? null : prepareOutput(accumulator);
+ }
+
+ public BigDecimal toBigDecimal(BigDecimal record) {
+ return record;
+ }
+ }
+
+ static class VarAgg implements Serializable {
+ long count; // number of elements
+ BigDecimal sum; // sum of elements
+
+ public VarAgg(long count, BigDecimal sum) {
+ this.count = count;
+ this.sum = sum;
+ }
+ }
+
+ /**
+ * {@link CombineFn} for <em>Var</em> on {@link Number} types.
+ * Variance Pop and Variance Sample
+ * <p>Evaluate the variance using the algorithm described by Chan, Golub, and LeVeque in
+ * "Algorithms for computing the sample variance: analysis and recommendations"
+ * The American Statistician, 37 (1983) pp. 242--247.</p>
+ * <p>variance = variance1 + variance2 + n/(m*(m+n)) * pow(((m/n)*t1 - t2),2)</p>
+ * <p>where: - variance is sum[x-avg^2] (this is actually n times the variance)
+ * and is updated at every step. - n is the count of elements in chunk1 - m is
+ * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 =
+ * sum of elements in chunk2.</p>
+ */
+ abstract static class Var<T extends Number>
+ extends CombineFn<T, KV<BigDecimal, VarAgg>, T> {
+ boolean isSamp; // flag to determine return value should be Variance Pop or Variance Sample
+
+ public Var(boolean isSamp){
+ this.isSamp = isSamp;
+ }
+
+ @Override
+ public KV<BigDecimal, VarAgg> createAccumulator() {
+ VarAgg varagg = new VarAgg(0L, new BigDecimal(0));
+ return KV.of(new BigDecimal(0), varagg);
+ }
+
+ @Override
+ public KV<BigDecimal, VarAgg> addInput(KV<BigDecimal, VarAgg> accumulator, T input) {
+ BigDecimal v;
+ if (input == null) {
+ return accumulator;
+ } else {
+ v = new BigDecimal(input.toString());
+ accumulator.getValue().count++;
+ accumulator.getValue().sum = accumulator.getValue().sum
+ .add(new BigDecimal(input.toString()));
+ BigDecimal variance;
+ if (accumulator.getValue().count > 1) {
+
+// pseudo code for the formula
+// t = count * v - sum;
+// variance = (t^2) / (count * (count - 1));
+ BigDecimal t = v.multiply(new BigDecimal(accumulator.getValue().count))
+ .subtract(accumulator.getValue().sum);
+ variance = t.pow(2)
+ .divide(new BigDecimal(accumulator.getValue().count)
+ .multiply(new BigDecimal(accumulator.getValue().count)
+ .subtract(BigDecimal.ONE)), mc);
+ } else {
+ variance = BigDecimal.ZERO;
+ }
+ return KV.of(accumulator.getKey().add(variance), accumulator.getValue());
+ }
+ }
+
+ @Override
+ public KV<BigDecimal, VarAgg> mergeAccumulators(
+ Iterable<KV<BigDecimal, VarAgg>> accumulators) {
+ BigDecimal variance = new BigDecimal(0);
+ long count = 0;
+ BigDecimal sum = new BigDecimal(0);
+
+ Iterator<KV<BigDecimal, VarAgg>> ite = accumulators.iterator();
+ while (ite.hasNext()) {
+ KV<BigDecimal, VarAgg> r = ite.next();
+
+ BigDecimal b = r.getValue().sum;
+
+ count += r.getValue().count;
+ sum = sum.add(b);
+
+// t = ( r.count / count ) * sum - b;
+// d = t^2 * ( ( count / r.count ) / ( count + r.count ) );
+ BigDecimal t = new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
+ .multiply(sum).subtract(b);
+ BigDecimal d = t.pow(2)
+ .multiply(new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
+ .divide(new BigDecimal(count)
+ .add(new BigDecimal(r.getValue().count))), mc);
+ variance = variance.add(r.getKey().add(d));
+ }
+
+ return KV.of(variance, new VarAgg(count, sum));
+ }
+
+ @Override
+ public Coder<KV<BigDecimal, VarAgg>> getAccumulatorCoder(CoderRegistry registry,
+ Coder<T> inputCoder) throws CannotProvideCoderException {
+ return KvCoder.of(BigDecimalCoder.of(), SerializableCoder.of(VarAgg.class));
+ }
+
+ protected BigDecimal prepareOutput(KV<BigDecimal, VarAgg> accumulator){
+ BigDecimal decimalVar;
+ if (accumulator.getValue().count > 1) {
+ BigDecimal a = accumulator.getKey();
+ BigDecimal b = new BigDecimal(accumulator.getValue().count)
+ .subtract(this.isSamp ? BigDecimal.ONE : BigDecimal.ZERO);
+
+ decimalVar = a.divide(b, mc);
+ } else {
+ decimalVar = BigDecimal.ZERO;
+ }
+ return decimalVar;
+ }
+
+ public abstract T extractOutput(KV<BigDecimal, VarAgg> accumulator);
+
+ public abstract BigDecimal toBigDecimal(T record);
+ }
+
+ static class IntegerVar extends Var<Integer> {
+ public IntegerVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Integer extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).intValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Integer record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class ShortVar extends Var<Short> {
+ public ShortVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Short extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).shortValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Short record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class ByteVar extends Var<Byte> {
+ public ByteVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Byte extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).byteValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Byte record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class LongVar extends Var<Long> {
+ public LongVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Long extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).longValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Long record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class FloatVar extends Var<Float> {
+ public FloatVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Float extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).floatValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Float record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class DoubleVar extends Var<Double> {
+ public DoubleVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public Double extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator).doubleValue();
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(Double record) {
+ return new BigDecimal(record);
+ }
+ }
+
+ static class BigDecimalVar extends Var<BigDecimal> {
+ public BigDecimalVar(boolean isSamp) {
+ super(isSamp);
+ }
+
+ public BigDecimal extractOutput(KV<BigDecimal, VarAgg> accumulator) {
+ return prepareOutput(accumulator);
+ }
+
+ @Override
+ public BigDecimal toBigDecimal(BigDecimal record) {
+ return record;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java
new file mode 100644
index 0000000..3c6b20f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlRecordHelper;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.util.Pair;
+
+/**
+ * Collections of {@code PTransform} and {@code DoFn} used to perform JOIN operation.
+ */
+public class BeamJoinTransforms {
+
+ /**
+ * A {@code SimpleFunction} to extract join fields from the specified row.
+ */
+ public static class ExtractJoinFields
+ extends SimpleFunction<BeamRecord, KV<BeamRecord, BeamRecord>> {
+ private final boolean isLeft;
+ private final List<Pair<Integer, Integer>> joinColumns;
+
+ public ExtractJoinFields(boolean isLeft, List<Pair<Integer, Integer>> joinColumns) {
+ this.isLeft = isLeft;
+ this.joinColumns = joinColumns;
+ }
+
+ @Override public KV<BeamRecord, BeamRecord> apply(BeamRecord input) {
+ // build the type
+ // the name of the join field is not important
+ List<String> names = new ArrayList<>(joinColumns.size());
+ List<Integer> types = new ArrayList<>(joinColumns.size());
+ for (int i = 0; i < joinColumns.size(); i++) {
+ names.add("c" + i);
+ types.add(isLeft
+ ? BeamSqlRecordHelper.getSqlRecordType(input).getFieldTypeByIndex(
+ joinColumns.get(i).getKey())
+ : BeamSqlRecordHelper.getSqlRecordType(input).getFieldTypeByIndex(
+ joinColumns.get(i).getValue()));
+ }
+ BeamRecordSqlType type = BeamRecordSqlType.create(names, types);
+
+ // build the row
+ List<Object> fieldValues = new ArrayList<>(joinColumns.size());
+ for (int i = 0; i < joinColumns.size(); i++) {
+ fieldValues.add(input
+ .getFieldValue(isLeft ? joinColumns.get(i).getKey() : joinColumns.get(i).getValue()));
+ }
+ return KV.of(new BeamRecord(type, fieldValues), input);
+ }
+ }
+
+
+ /**
+ * A {@code DoFn} which implement the sideInput-JOIN.
+ */
+ public static class SideInputJoinDoFn extends DoFn<KV<BeamRecord, BeamRecord>, BeamRecord> {
+ private final PCollectionView<Map<BeamRecord, Iterable<BeamRecord>>> sideInputView;
+ private final JoinRelType joinType;
+ private final BeamRecord rightNullRow;
+ private final boolean swap;
+
+ public SideInputJoinDoFn(JoinRelType joinType, BeamRecord rightNullRow,
+ PCollectionView<Map<BeamRecord, Iterable<BeamRecord>>> sideInputView,
+ boolean swap) {
+ this.joinType = joinType;
+ this.rightNullRow = rightNullRow;
+ this.sideInputView = sideInputView;
+ this.swap = swap;
+ }
+
+ @ProcessElement public void processElement(ProcessContext context) {
+ BeamRecord key = context.element().getKey();
+ BeamRecord leftRow = context.element().getValue();
+ Map<BeamRecord, Iterable<BeamRecord>> key2Rows = context.sideInput(sideInputView);
+ Iterable<BeamRecord> rightRowsIterable = key2Rows.get(key);
+
+ if (rightRowsIterable != null && rightRowsIterable.iterator().hasNext()) {
+ Iterator<BeamRecord> it = rightRowsIterable.iterator();
+ while (it.hasNext()) {
+ context.output(combineTwoRowsIntoOne(leftRow, it.next(), swap));
+ }
+ } else {
+ if (joinType == JoinRelType.LEFT) {
+ context.output(combineTwoRowsIntoOne(leftRow, rightNullRow, swap));
+ }
+ }
+ }
+ }
+
+
+ /**
+ * A {@code SimpleFunction} to combine two rows into one.
+ */
+ public static class JoinParts2WholeRow
+ extends SimpleFunction<KV<BeamRecord, KV<BeamRecord, BeamRecord>>, BeamRecord> {
+ @Override public BeamRecord apply(KV<BeamRecord, KV<BeamRecord, BeamRecord>> input) {
+ KV<BeamRecord, BeamRecord> parts = input.getValue();
+ BeamRecord leftRow = parts.getKey();
+ BeamRecord rightRow = parts.getValue();
+ return combineTwoRowsIntoOne(leftRow, rightRow, false);
+ }
+ }
+
+ /**
+ * As the method name suggests: combine two rows into one wide row.
+ */
+ private static BeamRecord combineTwoRowsIntoOne(BeamRecord leftRow,
+ BeamRecord rightRow, boolean swap) {
+ if (swap) {
+ return combineTwoRowsIntoOneHelper(rightRow, leftRow);
+ } else {
+ return combineTwoRowsIntoOneHelper(leftRow, rightRow);
+ }
+ }
+
+ /**
+ * As the method name suggests: combine two rows into one wide row.
+ */
+ private static BeamRecord combineTwoRowsIntoOneHelper(BeamRecord leftRow,
+ BeamRecord rightRow) {
+ // build the type
+ List<String> names = new ArrayList<>(leftRow.getFieldCount() + rightRow.getFieldCount());
+ names.addAll(leftRow.getDataType().getFieldNames());
+ names.addAll(rightRow.getDataType().getFieldNames());
+
+ List<Integer> types = new ArrayList<>(leftRow.getFieldCount() + rightRow.getFieldCount());
+ types.addAll(BeamSqlRecordHelper.getSqlRecordType(leftRow).getFieldTypes());
+ types.addAll(BeamSqlRecordHelper.getSqlRecordType(rightRow).getFieldTypes());
+ BeamRecordSqlType type = BeamRecordSqlType.create(names, types);
+
+ List<Object> fieldValues = new ArrayList<>(leftRow.getDataValues());
+ fieldValues.addAll(rightRow.getDataValues());
+ return new BeamRecord(type, fieldValues);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java
new file mode 100644
index 0000000..33ac807
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.util.Iterator;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSetOperatorRelBase;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Collections of {@code PTransform} and {@code DoFn} used to perform Set operations.
+ */
+public abstract class BeamSetOperatorsTransforms {
+ /**
+ * Transform a {@code BeamSqlRow} to a {@code KV<BeamSqlRow, BeamSqlRow>}.
+ */
+ public static class BeamSqlRow2KvFn extends
+ SimpleFunction<BeamRecord, KV<BeamRecord, BeamRecord>> {
+ @Override public KV<BeamRecord, BeamRecord> apply(BeamRecord input) {
+ return KV.of(input, input);
+ }
+ }
+
+ /**
+ * Filter function used for Set operators.
+ */
+ public static class SetOperatorFilteringDoFn extends
+ DoFn<KV<BeamRecord, CoGbkResult>, BeamRecord> {
+ private TupleTag<BeamRecord> leftTag;
+ private TupleTag<BeamRecord> rightTag;
+ private BeamSetOperatorRelBase.OpType opType;
+ // ALL?
+ private boolean all;
+
+ public SetOperatorFilteringDoFn(TupleTag<BeamRecord> leftTag, TupleTag<BeamRecord> rightTag,
+ BeamSetOperatorRelBase.OpType opType, boolean all) {
+ this.leftTag = leftTag;
+ this.rightTag = rightTag;
+ this.opType = opType;
+ this.all = all;
+ }
+
+ @ProcessElement public void processElement(ProcessContext ctx) {
+ CoGbkResult coGbkResult = ctx.element().getValue();
+ Iterable<BeamRecord> leftRows = coGbkResult.getAll(leftTag);
+ Iterable<BeamRecord> rightRows = coGbkResult.getAll(rightTag);
+ switch (opType) {
+ case UNION:
+ if (all) {
+ // output both left & right
+ Iterator<BeamRecord> iter = leftRows.iterator();
+ while (iter.hasNext()) {
+ ctx.output(iter.next());
+ }
+ iter = rightRows.iterator();
+ while (iter.hasNext()) {
+ ctx.output(iter.next());
+ }
+ } else {
+ // only output the key
+ ctx.output(ctx.element().getKey());
+ }
+ break;
+ case INTERSECT:
+ if (leftRows.iterator().hasNext() && rightRows.iterator().hasNext()) {
+ if (all) {
+ for (BeamRecord leftRow : leftRows) {
+ ctx.output(leftRow);
+ }
+ } else {
+ ctx.output(ctx.element().getKey());
+ }
+ }
+ break;
+ case MINUS:
+ if (leftRows.iterator().hasNext() && !rightRows.iterator().hasNext()) {
+ Iterator<BeamRecord> iter = leftRows.iterator();
+ if (all) {
+ // output all
+ while (iter.hasNext()) {
+ ctx.output(iter.next());
+ }
+ } else {
+ // only output one
+ ctx.output(iter.next());
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlFilterFn.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlFilterFn.java
new file mode 100644
index 0000000..d3a3f7b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlFilterFn.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlExpressionExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamFilterRel;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ * {@code BeamSqlFilterFn} is the executor for a {@link BeamFilterRel} step.
+ *
+ */
+public class BeamSqlFilterFn extends DoFn<BeamRecord, BeamRecord> {
+
+ private String stepName;
+ private BeamSqlExpressionExecutor executor;
+
+ public BeamSqlFilterFn(String stepName, BeamSqlExpressionExecutor executor) {
+ super();
+ this.stepName = stepName;
+ this.executor = executor;
+ }
+
+ @Setup
+ public void setup() {
+ executor.prepare();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window) {
+ BeamRecord in = c.element();
+
+ List<Object> result = executor.execute(in, window);
+
+ if ((Boolean) result.get(0)) {
+ c.output(in);
+ }
+ }
+
+ @Teardown
+ public void close() {
+ executor.close();
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlOutputToConsoleFn.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlOutputToConsoleFn.java
new file mode 100644
index 0000000..f97a90a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlOutputToConsoleFn.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ * A test PTransform to display output in console.
+ *
+ */
+public class BeamSqlOutputToConsoleFn extends DoFn<BeamRecord, Void> {
+
+ private String stepName;
+
+ public BeamSqlOutputToConsoleFn(String stepName) {
+ super();
+ this.stepName = stepName;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ System.out.println("Output: " + c.element().getDataValues());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
new file mode 100644
index 0000000..719fbf3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlExpressionExecutor;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamProjectRel;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ *
+ * {@code BeamSqlProjectFn} is the executor for a {@link BeamProjectRel} step.
+ *
+ */
+public class BeamSqlProjectFn extends DoFn<BeamRecord, BeamRecord> {
+ private String stepName;
+ private BeamSqlExpressionExecutor executor;
+ private BeamRecordSqlType outputRowType;
+
+ public BeamSqlProjectFn(String stepName, BeamSqlExpressionExecutor executor,
+ BeamRecordSqlType outputRowType) {
+ super();
+ this.stepName = stepName;
+ this.executor = executor;
+ this.outputRowType = outputRowType;
+ }
+
+ @Setup
+ public void setup() {
+ executor.prepare();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window) {
+ BeamRecord inputRow = c.element();
+ List<Object> results = executor.execute(inputRow, window);
+ List<Object> fieldsValue = new ArrayList<>(results.size());
+ for (int idx = 0; idx < results.size(); ++idx) {
+ fieldsValue.add(
+ BeamTableUtils.autoCastField(outputRowType.getFieldTypeByIndex(idx), results.get(idx)));
+ }
+ BeamRecord outRow = new BeamRecord(outputRowType, fieldsValue);
+
+ c.output(outRow);
+ }
+
+ @Teardown
+ public void close() {
+ executor.close();
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/package-info.java
new file mode 100644
index 0000000..bc90e5b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.beam.sdk.transforms.PTransform} used in a BeamSql pipeline.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.transform;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
new file mode 100644
index 0000000..8c44780
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.utils;
+
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utility methods for Calcite related operations.
+ */
+public class CalciteUtils {
+ private static final Map<Integer, SqlTypeName> JAVA_TO_CALCITE_MAPPING = new HashMap<>();
+ private static final Map<SqlTypeName, Integer> CALCITE_TO_JAVA_MAPPING = new HashMap<>();
+ static {
+ JAVA_TO_CALCITE_MAPPING.put(Types.TINYINT, SqlTypeName.TINYINT);
+ JAVA_TO_CALCITE_MAPPING.put(Types.SMALLINT, SqlTypeName.SMALLINT);
+ JAVA_TO_CALCITE_MAPPING.put(Types.INTEGER, SqlTypeName.INTEGER);
+ JAVA_TO_CALCITE_MAPPING.put(Types.BIGINT, SqlTypeName.BIGINT);
+
+ JAVA_TO_CALCITE_MAPPING.put(Types.FLOAT, SqlTypeName.FLOAT);
+ JAVA_TO_CALCITE_MAPPING.put(Types.DOUBLE, SqlTypeName.DOUBLE);
+
+ JAVA_TO_CALCITE_MAPPING.put(Types.DECIMAL, SqlTypeName.DECIMAL);
+
+ JAVA_TO_CALCITE_MAPPING.put(Types.CHAR, SqlTypeName.CHAR);
+ JAVA_TO_CALCITE_MAPPING.put(Types.VARCHAR, SqlTypeName.VARCHAR);
+
+ JAVA_TO_CALCITE_MAPPING.put(Types.DATE, SqlTypeName.DATE);
+ JAVA_TO_CALCITE_MAPPING.put(Types.TIME, SqlTypeName.TIME);
+ JAVA_TO_CALCITE_MAPPING.put(Types.TIMESTAMP, SqlTypeName.TIMESTAMP);
+
+ JAVA_TO_CALCITE_MAPPING.put(Types.BOOLEAN, SqlTypeName.BOOLEAN);
+
+ for (Map.Entry<Integer, SqlTypeName> pair : JAVA_TO_CALCITE_MAPPING.entrySet()) {
+ CALCITE_TO_JAVA_MAPPING.put(pair.getValue(), pair.getKey());
+ }
+ }
+
+ /**
+ * Get the corresponding {@code SqlTypeName} for an integer sql type.
+ */
+ public static SqlTypeName toCalciteType(int type) {
+ return JAVA_TO_CALCITE_MAPPING.get(type);
+ }
+
+ /**
+ * Get the integer sql type from Calcite {@code SqlTypeName}.
+ */
+ public static Integer toJavaType(SqlTypeName typeName) {
+ return CALCITE_TO_JAVA_MAPPING.get(typeName);
+ }
+
+ /**
+ * Get the {@code SqlTypeName} for the specified column of a table.
+ */
+ public static SqlTypeName getFieldType(BeamRecordSqlType schema, int index) {
+ return toCalciteType(schema.getFieldTypeByIndex(index));
+ }
+
+ /**
+ * Generate {@code BeamSqlRowType} from {@code RelDataType} which is used to create table.
+ */
+ public static BeamRecordSqlType toBeamRowType(RelDataType tableInfo) {
+ List<String> fieldNames = new ArrayList<>();
+ List<Integer> fieldTypes = new ArrayList<>();
+ for (RelDataTypeField f : tableInfo.getFieldList()) {
+ fieldNames.add(f.getName());
+ fieldTypes.add(toJavaType(f.getType().getSqlTypeName()));
+ }
+ return BeamRecordSqlType.create(fieldNames, fieldTypes);
+ }
+
+ /**
+ * Create an instance of {@code RelDataType} so it can be used to create a table.
+ */
+ public static RelProtoDataType toCalciteRowType(final BeamRecordSqlType that) {
+ return new RelProtoDataType() {
+ @Override
+ public RelDataType apply(RelDataTypeFactory a) {
+ RelDataTypeFactory.FieldInfoBuilder builder = a.builder();
+ for (int idx = 0; idx < that.getFieldNames().size(); ++idx) {
+ builder.add(that.getFieldNameByIndex(idx), toCalciteType(that.getFieldTypeByIndex(idx)));
+ }
+ return builder.build();
+ }
+ };
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtils.java
new file mode 100644
index 0000000..9658bab
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtils.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.utils;
+
+import com.google.common.base.Optional;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utils to help with SqlTypes.
+ */
+public class SqlTypeUtils {
+ /**
+ * Finds an operand with provided type.
+ * Returns Optional.absent() if no operand found with matching type
+ */
+ public static Optional<BeamSqlExpression> findExpressionOfType(
+ List<BeamSqlExpression> operands, SqlTypeName type) {
+
+ for (BeamSqlExpression operand : operands) {
+ if (type.equals(operand.getOutputType())) {
+ return Optional.of(operand);
+ }
+ }
+
+ return Optional.absent();
+ }
+
+ /**
+ * Finds an operand with the type in typesToFind.
+ * Returns Optional.absent() if no operand found with matching type
+ */
+ public static Optional<BeamSqlExpression> findExpressionOfType(
+ List<BeamSqlExpression> operands, Collection<SqlTypeName> typesToFind) {
+
+ for (BeamSqlExpression operand : operands) {
+ if (typesToFind.contains(operand.getOutputType())) {
+ return Optional.of(operand);
+ }
+ }
+
+ return Optional.absent();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/package-info.java
new file mode 100644
index 0000000..b00ed0c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utility classes.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.utils;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Column.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Column.java
new file mode 100644
index 0000000..9bcc16a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Column.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+
+/**
+ * Metadata class for a {@code BeamSqlTable} column.
+ */
+@AutoValue
+public abstract class Column implements Serializable {
+ public abstract String getName();
+ public abstract Integer getType();
+ @Nullable
+ public abstract String getComment();
+ public abstract boolean isPrimaryKey();
+
+ public static Builder builder() {
+ return new org.apache.beam.sdk.extensions.sql.meta.AutoValue_Column.Builder();
+ }
+
+ /**
+ * Builder class for {@link Column}.
+ */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder name(String name);
+ public abstract Builder type(Integer type);
+ public abstract Builder comment(String comment);
+ public abstract Builder primaryKey(boolean isPrimaryKey);
+ public abstract Column build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java
new file mode 100644
index 0000000..4af82a0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Represents the metadata of a {@code BeamSqlTable}.
+ */
+@AutoValue
+public abstract class Table implements Serializable {
+ /** type of the table. */
+ public abstract String getType();
+ public abstract String getName();
+ public abstract List<Column> getColumns();
+ @Nullable
+ public abstract String getComment();
+ @Nullable
+ public abstract URI getLocation();
+ @Nullable
+ public abstract JSONObject getProperties();
+
+ public static Builder builder() {
+ return new org.apache.beam.sdk.extensions.sql.meta.AutoValue_Table.Builder();
+ }
+
+ public String getLocationAsString() {
+ if (getLocation() == null) {
+ return null;
+ }
+
+ return "/" + getLocation().getHost() + getLocation().getPath();
+ }
+
+ /**
+ * Builder class for {@link Table}.
+ */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder type(String type);
+ public abstract Builder name(String name);
+ public abstract Builder columns(List<Column> columns);
+ public abstract Builder comment(String name);
+ public abstract Builder location(URI location);
+ public abstract Builder properties(JSONObject properties);
+ public abstract Table build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/package-info.java
new file mode 100644
index 0000000..a50ce5f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Metadata related classes.
+ */
+package org.apache.beam.sdk.extensions.sql.meta;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/MetaUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/MetaUtils.java
new file mode 100644
index 0000000..35ecdce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/MetaUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * Utility methods for metadata.
+ */
+public class MetaUtils {
+ public static BeamRecordSqlType getBeamSqlRecordTypeFromTable(Table table) {
+ List<String> columnNames = new ArrayList<>(table.getColumns().size());
+ List<Integer> columnTypes = new ArrayList<>(table.getColumns().size());
+ for (Column column : table.getColumns()) {
+ columnNames.add(column.getName());
+ columnTypes.add(column.getType());
+ }
+ return BeamRecordSqlType.create(columnNames, columnTypes);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
new file mode 100644
index 0000000..d57f703
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * A {@code TableProvider} handles the metadata CRUD of a specified kind of tables.
+ *
+ * <p>So there will be a provider to handle textfile(CSV) based tables, there is a provider to
+ * handle MySQL based tables, a provider to handle Casandra based tables etc.
+ */
+public interface TableProvider {
+ /**
+ * Init the provider.
+ */
+ void init();
+
+ /**
+ * Gets the table type this provider handles.
+ */
+ String getTableType();
+
+ /**
+ * Creates a table.
+ */
+ void createTable(Table table);
+
+ /**
+ * List all tables from this provider.
+ */
+ List<Table> listTables();
+
+ /**
+ * Build a {@link BeamSqlTable} using the given table meta info.
+ */
+ BeamSqlTable buildBeamSqlTable(Table table);
+
+ /**
+ * Close the provider.
+ */
+ void close();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java
new file mode 100644
index 0000000..a8c8a30
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.beamRecord2CsvLine;
+import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.csvLine2BeamRecord;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.commons.csv.CSVFormat;
+
+/**
+ * A Kafka topic that saves records as CSV format.
+ *
+ */
+public class BeamKafkaCSVTable extends BeamKafkaTable {
+ private CSVFormat csvFormat;
+ public BeamKafkaCSVTable(BeamRecordSqlType beamSqlRowType, String bootstrapServers,
+ List<String> topics) {
+ this(beamSqlRowType, bootstrapServers, topics, CSVFormat.DEFAULT);
+ }
+
+ public BeamKafkaCSVTable(BeamRecordSqlType beamSqlRowType, String bootstrapServers,
+ List<String> topics, CSVFormat format) {
+ super(beamSqlRowType, bootstrapServers, topics);
+ this.csvFormat = format;
+ }
+
+ @Override
+ public PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamRecord>>
+ getPTransformForInput() {
+ return new CsvRecorderDecoder(beamRecordSqlType, csvFormat);
+ }
+
+ @Override
+ public PTransform<PCollection<BeamRecord>, PCollection<KV<byte[], byte[]>>>
+ getPTransformForOutput() {
+ return new CsvRecorderEncoder(beamRecordSqlType, csvFormat);
+ }
+
+ /**
+ * A PTransform to convert {@code KV<byte[], byte[]>} to {@link BeamRecord}.
+ *
+ */
+ public static class CsvRecorderDecoder
+ extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamRecord>> {
+ private BeamRecordSqlType rowType;
+ private CSVFormat format;
+ public CsvRecorderDecoder(BeamRecordSqlType rowType, CSVFormat format) {
+ this.rowType = rowType;
+ this.format = format;
+ }
+
+ @Override
+ public PCollection<BeamRecord> expand(PCollection<KV<byte[], byte[]>> input) {
+ return input.apply("decodeRecord", ParDo.of(new DoFn<KV<byte[], byte[]>, BeamRecord>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ String rowInString = new String(c.element().getValue());
+ c.output(csvLine2BeamRecord(format, rowInString, rowType));
+ }
+ }));
+ }
+ }
+
+ /**
+ * A PTransform to convert {@link BeamRecord} to {@code KV<byte[], byte[]>}.
+ *
+ */
+ public static class CsvRecorderEncoder
+ extends PTransform<PCollection<BeamRecord>, PCollection<KV<byte[], byte[]>>> {
+ private BeamRecordSqlType rowType;
+ private CSVFormat format;
+ public CsvRecorderEncoder(BeamRecordSqlType rowType, CSVFormat format) {
+ this.rowType = rowType;
+ this.format = format;
+ }
+
+ @Override
+ public PCollection<KV<byte[], byte[]>> expand(PCollection<BeamRecord> input) {
+ return input.apply("encodeRecord", ParDo.of(new DoFn<BeamRecord, KV<byte[], byte[]>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ BeamRecord in = c.element();
+ c.output(KV.of(new byte[] {}, beamRecord2CsvLine(in, format).getBytes()));
+ }
+ }));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java
new file mode 100644
index 0000000..50f7496
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
+import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+
+/**
+ * {@code BeamKafkaTable} represent a Kafka topic, as source or target. Need to
+ * extend to convert between {@code BeamSqlRow} and {@code KV<byte[], byte[]>}.
+ *
+ */
+public abstract class BeamKafkaTable extends BaseBeamTable implements Serializable {
+ private String bootstrapServers;
+ private List<String> topics;
+ private Map<String, Object> configUpdates;
+
+ protected BeamKafkaTable(BeamRecordSqlType beamSqlRowType) {
+ super(beamSqlRowType);
+ }
+
+ public BeamKafkaTable(BeamRecordSqlType beamSqlRowType, String bootstrapServers,
+ List<String> topics) {
+ super(beamSqlRowType);
+ this.bootstrapServers = bootstrapServers;
+ this.topics = topics;
+ }
+
+ public BeamKafkaTable updateConsumerProperties(Map<String, Object> configUpdates) {
+ this.configUpdates = configUpdates;
+ return this;
+ }
+
+ @Override
+ public BeamIOType getSourceType() {
+ return BeamIOType.UNBOUNDED;
+ }
+
+ public abstract PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamRecord>>
+ getPTransformForInput();
+
+ public abstract PTransform<PCollection<BeamRecord>, PCollection<KV<byte[], byte[]>>>
+ getPTransformForOutput();
+
+ @Override
+ public PCollection<BeamRecord> buildIOReader(Pipeline pipeline) {
+ return PBegin.in(pipeline).apply("read",
+ KafkaIO.<byte[], byte[]>read()
+ .withBootstrapServers(bootstrapServers)
+ .withTopics(topics)
+ .updateConsumerProperties(configUpdates)
+ .withKeyDeserializerAndCoder(ByteArrayDeserializer.class, ByteArrayCoder.of())
+ .withValueDeserializerAndCoder(ByteArrayDeserializer.class, ByteArrayCoder.of())
+ .withoutMetadata())
+ .apply("in_format", getPTransformForInput());
+ }
+
+ @Override
+ public PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter() {
+ checkArgument(topics != null && topics.size() == 1,
+ "Only one topic can be acceptable as output.");
+
+ return new PTransform<PCollection<BeamRecord>, PDone>() {
+ @Override
+ public PDone expand(PCollection<BeamRecord> input) {
+ return input.apply("out_reformat", getPTransformForOutput()).apply("persistent",
+ KafkaIO.<byte[], byte[]>write()
+ .withBootstrapServers(bootstrapServers)
+ .withTopic(topics.get(0))
+ .withKeySerializer(ByteArraySerializer.class)
+ .withValueSerializer(ByteArraySerializer.class));
+ }
+ };
+ }
+
+ public String getBootstrapServers() {
+ return bootstrapServers;
+ }
+
+ public List<String> getTopics() {
+ return topics;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
new file mode 100644
index 0000000..8c37d46
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static org.apache.beam.sdk.extensions.sql.meta.provider.MetaUtils.getBeamSqlRecordTypeFromTable;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+
+/**
+ * Kafka table provider.
+ *
+ * <p>A sample of text table is:
+ *
+ * <pre>{@code
+ * CREATE TABLE ORDERS(
+ * ID INT PRIMARY KEY COMMENT 'this is the primary key',
+ * NAME VARCHAR(127) COMMENT 'this is the name'
+ * )
+ * COMMENT 'this is the table orders'
+ * LOCATION 'kafka://localhost:2181/brokers?topic=test'
+ * TBLPROPERTIES '{"bootstrap.servers":"localhost:9092", "topics": ["topic1", "topic2"]}'
+ * }</pre>
+ */
+public class KafkaTableProvider implements TableProvider {
+ @Override public BeamSqlTable buildBeamSqlTable(Table table) {
+ BeamRecordSqlType recordType = getBeamSqlRecordTypeFromTable(table);
+
+ JSONObject properties = table.getProperties();
+ String bootstrapServers = properties.getString("bootstrap.servers");
+ JSONArray topicsArr = properties.getJSONArray("topics");
+ List<String> topics = new ArrayList<>(topicsArr.size());
+ for (Object topic : topicsArr) {
+ topics.add(topic.toString());
+ }
+ BeamKafkaCSVTable txtTable = new BeamKafkaCSVTable(recordType, bootstrapServers, topics);
+ return txtTable;
+ }
+
+ @Override public String getTableType() {
+ return "kafka";
+ }
+
+ @Override public void createTable(Table table) {
+ // empty
+ }
+
+ @Override public List<Table> listTables() {
+ return Collections.emptyList();
+ }
+
+ @Override public void init() {
+ // empty
+ }
+
+ @Override public void close() {
+ // empty
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
new file mode 100644
index 0000000..4101da7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * table schema for KafkaIO.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/package-info.java
new file mode 100644
index 0000000..c271261
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Table providers.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTable.java
new file mode 100644
index 0000000..78cec74
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTable.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.commons.csv.CSVFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@code BeamTextCSVTable} is a {@code BeamTextTable} which formatted in CSV.
+ *
+ * <p>
+ * {@link CSVFormat} itself has many dialects, check its javadoc for more info.
+ * </p>
+ */
+public class BeamTextCSVTable extends BeamTextTable {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BeamTextCSVTable.class);
+
+ private String filePattern;
+ private CSVFormat csvFormat;
+
+ /**
+ * CSV table with {@link CSVFormat#DEFAULT DEFAULT} format.
+ */
+ public BeamTextCSVTable(BeamRecordSqlType beamSqlRowType, String filePattern) {
+ this(beamSqlRowType, filePattern, CSVFormat.DEFAULT);
+ }
+
+ public BeamTextCSVTable(BeamRecordSqlType beamRecordSqlType, String filePattern,
+ CSVFormat csvFormat) {
+ super(beamRecordSqlType, filePattern);
+ this.filePattern = filePattern;
+ this.csvFormat = csvFormat;
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildIOReader(Pipeline pipeline) {
+ return PBegin.in(pipeline).apply("decodeRecord", TextIO.read().from(filePattern))
+ .apply("parseCSVLine",
+ new BeamTextCSVTableIOReader(beamRecordSqlType, filePattern, csvFormat));
+ }
+
+ @Override
+ public PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter() {
+ return new BeamTextCSVTableIOWriter(beamRecordSqlType, filePattern, csvFormat);
+ }
+
+ public CSVFormat getCsvFormat() {
+ return csvFormat;
+ }
+
+ public String getFilePattern() {
+ return filePattern;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOReader.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOReader.java
new file mode 100644
index 0000000..953ac03
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOReader.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.csvLine2BeamRecord;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.commons.csv.CSVFormat;
+
+/**
+ * IOReader for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableIOReader
+ extends PTransform<PCollection<String>, PCollection<BeamRecord>>
+ implements Serializable {
+ private String filePattern;
+ protected BeamRecordSqlType beamRecordSqlType;
+ protected CSVFormat csvFormat;
+
+ public BeamTextCSVTableIOReader(BeamRecordSqlType beamRecordSqlType, String filePattern,
+ CSVFormat csvFormat) {
+ this.filePattern = filePattern;
+ this.beamRecordSqlType = beamRecordSqlType;
+ this.csvFormat = csvFormat;
+ }
+
+ @Override
+ public PCollection<BeamRecord> expand(PCollection<String> input) {
+ return input.apply(ParDo.of(new DoFn<String, BeamRecord>() {
+ @ProcessElement
+ public void processElement(ProcessContext ctx) {
+ String str = ctx.element();
+ ctx.output(csvLine2BeamRecord(csvFormat, str, beamRecordSqlType));
+ }
+ }));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOWriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOWriter.java
new file mode 100644
index 0000000..80481d2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOWriter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.beamRecord2CsvLine;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.commons.csv.CSVFormat;
+
+/**
+ * IOWriter for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableIOWriter extends PTransform<PCollection<BeamRecord>, PDone>
+ implements Serializable {
+ private String filePattern;
+ protected BeamRecordSqlType beamRecordSqlType;
+ protected CSVFormat csvFormat;
+
+ public BeamTextCSVTableIOWriter(BeamRecordSqlType beamRecordSqlType, String filePattern,
+ CSVFormat csvFormat) {
+ this.filePattern = filePattern;
+ this.beamRecordSqlType = beamRecordSqlType;
+ this.csvFormat = csvFormat;
+ }
+
+ @Override public PDone expand(PCollection<BeamRecord> input) {
+ return input.apply("encodeRecord", ParDo.of(new DoFn<BeamRecord, String>() {
+
+ @ProcessElement public void processElement(ProcessContext ctx) {
+ BeamRecord row = ctx.element();
+ ctx.output(beamRecord2CsvLine(row, csvFormat));
+ }
+ })).apply(TextIO.write().to(filePattern));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextTable.java
new file mode 100644
index 0000000..76616ef
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextTable.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
+
+/**
+ * {@code BeamTextTable} represents a text file/directory(backed by {@code TextIO}).
+ */
+public abstract class BeamTextTable extends BaseBeamTable implements Serializable {
+ protected String filePattern;
+
+ protected BeamTextTable(BeamRecordSqlType beamRecordSqlType, String filePattern) {
+ super(beamRecordSqlType);
+ this.filePattern = filePattern;
+ }
+
+ @Override
+ public BeamIOType getSourceType() {
+ return BeamIOType.BOUNDED;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java
new file mode 100644
index 0000000..bc9f03f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import static org.apache.beam.sdk.extensions.sql.meta.provider.MetaUtils.getBeamSqlRecordTypeFromTable;
+
+import com.alibaba.fastjson.JSONObject;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.commons.csv.CSVFormat;
+
+/**
+ * Text table provider.
+ *
+ * <p>A sample of text table is:
+ * <pre>{@code
+ * CREATE TABLE ORDERS(
+ * ID INT PRIMARY KEY COMMENT 'this is the primary key',
+ * NAME VARCHAR(127) COMMENT 'this is the name'
+ * )
+ * TYPE 'text'
+ * COMMENT 'this is the table orders'
+ * LOCATION 'text://home/admin/orders'
+ * TBLPROPERTIES '{"format": "Excel"}' -- format of each text line(csv format)
+ * }</pre>
+ */
+public class TextTableProvider implements TableProvider {
+
+ @Override public String getTableType() {
+ return "text";
+ }
+
+ @Override public BeamSqlTable buildBeamSqlTable(Table table) {
+ BeamRecordSqlType recordType = getBeamSqlRecordTypeFromTable(table);
+
+ String filePattern = table.getLocationAsString();
+ CSVFormat format = CSVFormat.DEFAULT;
+ JSONObject properties = table.getProperties();
+ String csvFormatStr = properties.getString("format");
+ if (csvFormatStr != null && !csvFormatStr.isEmpty()) {
+ format = CSVFormat.valueOf(csvFormatStr);
+ }
+
+ BeamTextCSVTable txtTable = new BeamTextCSVTable(recordType, filePattern, format);
+ return txtTable;
+ }
+
+ @Override public void createTable(Table table) {
+ // empty
+ }
+
+ @Override public List<Table> listTables() {
+ return Collections.emptyList();
+ }
+
+ @Override public void init() {
+ // empty
+ }
+
+ @Override public void close() {
+ // empty
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/package-info.java
new file mode 100644
index 0000000..2dd9e6e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Table schema for text files.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java
new file mode 100644
index 0000000..bacfbff
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.store;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+
+/**
+ * A {@link MetaStore} which stores the meta info in memory.
+ *
+ * <p>NOTE, because this implementation is memory based, the metadata is NOT persistent.
+ * for tables which created, you need to create again every time you launch the
+ * {@link org.apache.beam.sdk.extensions.sql.BeamSqlCli}.
+ */
+public class InMemoryMetaStore implements MetaStore {
+ private Map<String, Table> tables = new HashMap<>();
+ private Map<String, TableProvider> providers = new HashMap<>();
+
+ public InMemoryMetaStore() {
+ }
+
+ @Override public void createTable(Table table) {
+ validateTableType(table);
+
+ // first assert the table name is unique
+ if (tables.containsKey(table.getName())) {
+ throw new IllegalArgumentException("Duplicate table name: " + table.getName());
+ }
+
+ // invoke the provider's create
+ providers.get(table.getType()).createTable(table);
+
+ // store to the global metastore
+ tables.put(table.getName(), table);
+ }
+
+ @Override public Table getTable(String tableName) {
+ if (tableName == null) {
+ return null;
+ }
+ return tables.get(tableName.toLowerCase());
+ }
+
+ @Override public List<Table> listTables() {
+ return new ArrayList<>(tables.values());
+ }
+
+ @Override public BeamSqlTable buildBeamSqlTable(String tableName) {
+ Table table = getTable(tableName);
+
+ if (table == null) {
+ throw new IllegalArgumentException("The specified table: " + tableName + " does not exists!");
+ }
+
+ TableProvider provider = providers.get(table.getType());
+
+ return provider.buildBeamSqlTable(table);
+ }
+
+ private void validateTableType(Table table) {
+ if (!providers.containsKey(table.getType())) {
+ throw new IllegalArgumentException(
+ "Table type: " + table.getType() + " not supported!");
+ }
+ }
+
+ public void registerProvider(TableProvider provider) {
+ if (providers.containsKey(provider.getTableType())) {
+ throw new IllegalArgumentException("Provider is already registered for table type: "
+ + provider.getTableType());
+ }
+
+ this.providers.put(provider.getTableType(), provider);
+ initTablesFromProvider(provider);
+ }
+
+ private void initTablesFromProvider(TableProvider provider) {
+ List<Table> tables = provider.listTables();
+ for (Table table : tables) {
+ if (this.tables.containsKey(table.getName())) {
+ throw new IllegalStateException(
+ "Duplicate table: " + table.getName() + " from provider: " + provider);
+ }
+
+ this.tables.put(table.getName(), table);
+ }
+ }
+
+ Map<String, TableProvider> getProviders() {
+ return providers;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/MetaStore.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/MetaStore.java
new file mode 100644
index 0000000..2f395f0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/MetaStore.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.store;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+
+/**
+ * The interface to handle CRUD of {@code BeamSql} table metadata.
+ */
+public interface MetaStore {
+
+ /**
+ * create a table.
+ */
+ void createTable(Table table);
+
+ /**
+ * Get table with the specified name.
+ */
+ Table getTable(String tableName);
+
+ /**
+ * List all the tables.
+ */
+ List<Table> listTables();
+
+ /**
+ * Build the {@code BeamSqlTable} for the specified table.
+ */
+ BeamSqlTable buildBeamSqlTable(String tableName);
+
+ /**
+ * Register a table provider.
+ * @param provider
+ */
+ void registerProvider(TableProvider provider);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/package-info.java
new file mode 100644
index 0000000..39f1385
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Meta stores.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.store;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/package-info.java
new file mode 100644
index 0000000..bae08b3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL provides a new interface to run a SQL statement with Beam.
+ */
+package org.apache.beam.sdk.extensions.sql;
diff --git a/sdks/java/extensions/sql/src/main/resources/log4j.properties b/sdks/java/extensions/sql/src/main/resources/log4j.properties
new file mode 100644
index 0000000..709484b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/resources/log4j.properties
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=ERROR,console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
\ No newline at end of file
diff --git a/sdks/java/extensions/sql/src/main/resources/org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.commons.compiler.properties b/sdks/java/extensions/sql/src/main/resources/org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.commons.compiler.properties
new file mode 100644
index 0000000..72a4eec
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/resources/org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.commons.compiler.properties
@@ -0,0 +1,18 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+compilerFactory=org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.janino.CompilerFactory
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
new file mode 100644
index 0000000..156d7ff
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql;
+
+import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.apache.beam.sdk.util.ApiSurface;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Surface test for BeamSql api.
+ */
+@RunWith(JUnit4.class)
+public class BeamSqlApiSurfaceTest {
+ @Test
+ public void testSdkApiSurface() throws Exception {
+
+ @SuppressWarnings("unchecked")
+ final Set<String> allowed =
+ ImmutableSet.of(
+ "org.apache.beam",
+ "org.joda.time",
+ "com.alibaba.fastjson",
+ // exposed by fastjson
+ "sun.reflect"
+ );
+
+ ApiSurface surface = ApiSurface
+ .ofClass(BeamSql.class)
+ .includingClass(BeamSqlCli.class)
+ .includingClass(BeamSqlUdf.class)
+ .includingClass(BeamRecordSqlType.class)
+ .includingClass(BeamSqlRecordHelper.class)
+ .pruningPrefix("java")
+ .pruningPattern("org[.]apache[.]beam[.]sdk[.]extensions[.]sql[.].*Test")
+ .pruningPattern("org[.]apache[.]beam[.]sdk[.]extensions[.]sql[.].*TestBase");
+
+ assertThat(surface, containsOnlyPackages(allowed));
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
new file mode 100644
index 0000000..62d6933
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.text.TextTableProvider;
+import org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore;
+import org.junit.Test;
+
+/**
+ * UnitTest for {@link BeamSqlCli}.
+ */
+public class BeamSqlCliTest {
+ @Test
+ public void testExecute_createTextTable() throws Exception {
+ InMemoryMetaStore metaStore = new InMemoryMetaStore();
+ metaStore.registerProvider(new TextTableProvider());
+
+ BeamSqlCli cli = new BeamSqlCli()
+ .metaStore(metaStore);
+ cli.execute(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name', \n"
+ + "age int COMMENT 'age') \n"
+ + "TYPE 'text' \n"
+ + "COMMENT '' LOCATION 'text://home/admin/orders'"
+ );
+ Table table = metaStore.getTable("person");
+ assertNotNull(table);
+ }
+
+ @Test
+ public void testExplainQuery() throws Exception {
+ InMemoryMetaStore metaStore = new InMemoryMetaStore();
+ metaStore.registerProvider(new TextTableProvider());
+
+ BeamSqlCli cli = new BeamSqlCli()
+ .metaStore(metaStore);
+
+ cli.execute(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name', \n"
+ + "age int COMMENT 'age') \n"
+ + "TYPE 'text' \n"
+ + "COMMENT '' LOCATION 'text://home/admin/orders'"
+ );
+
+ String plan = cli.explainQuery("select * from person");
+ assertEquals(
+ "BeamProjectRel(id=[$0], name=[$1], age=[$2])\n"
+ + " BeamIOSourceRel(table=[[person]])\n",
+ plan
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
new file mode 100644
index 0000000..76d2313
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for GROUP-BY/aggregation, with global_window/fix_time_window/sliding_window/session_window
+ * with BOUNDED PCollection.
+ */
+public class BeamSqlDslAggregationTest extends BeamSqlDslBase {
+ public PCollection<BeamRecord> boundedInput3;
+
+ @Before
+ public void setUp(){
+ BeamRecordSqlType rowTypeInTableB = BeamRecordSqlType.create(
+ Arrays.asList("f_int", "f_double", "f_int2", "f_decimal"),
+ Arrays.asList(Types.INTEGER, Types.DOUBLE, Types.INTEGER, Types.DECIMAL));
+
+ List<BeamRecord> recordsInTableB = new ArrayList<>();
+ BeamRecord row1 = new BeamRecord(rowTypeInTableB
+ , 1, 1.0, 0, new BigDecimal(1));
+ recordsInTableB.add(row1);
+
+ BeamRecord row2 = new BeamRecord(rowTypeInTableB
+ , 4, 4.0, 0, new BigDecimal(4));
+ recordsInTableB.add(row2);
+
+ BeamRecord row3 = new BeamRecord(rowTypeInTableB
+ , 7, 7.0, 0, new BigDecimal(7));
+ recordsInTableB.add(row3);
+
+ BeamRecord row4 = new BeamRecord(rowTypeInTableB
+ , 13, 13.0, 0, new BigDecimal(13));
+ recordsInTableB.add(row4);
+
+ BeamRecord row5 = new BeamRecord(rowTypeInTableB
+ , 5, 5.0, 0, new BigDecimal(5));
+ recordsInTableB.add(row5);
+
+ BeamRecord row6 = new BeamRecord(rowTypeInTableB
+ , 10, 10.0, 0, new BigDecimal(10));
+ recordsInTableB.add(row6);
+
+ BeamRecord row7 = new BeamRecord(rowTypeInTableB
+ , 17, 17.0, 0, new BigDecimal(17));
+ recordsInTableB.add(row7);
+
+ boundedInput3 = PBegin.in(pipeline).apply("boundedInput3",
+ Create.of(recordsInTableB).withCoder(rowTypeInTableB.getRecordCoder()));
+ }
+
+ /**
+ * GROUP-BY with single aggregation function with bounded PCollection.
+ */
+ @Test
+ public void testAggregationWithoutWindowWithBounded() throws Exception {
+ runAggregationWithoutWindow(boundedInput1);
+ }
+
+ /**
+ * GROUP-BY with single aggregation function with unbounded PCollection.
+ */
+ @Test
+ public void testAggregationWithoutWindowWithUnbounded() throws Exception {
+ runAggregationWithoutWindow(unboundedInput1);
+ }
+
+ private void runAggregationWithoutWindow(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int2, COUNT(*) AS `getFieldCount` FROM PCOLLECTION GROUP BY f_int2";
+
+ PCollection<BeamRecord> result =
+ input.apply("testAggregationWithoutWindow", BeamSql.query(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int2", "size"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT));
+
+
+ BeamRecord record = new BeamRecord(resultType, 0, 4L);
+
+ PAssert.that(result).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * GROUP-BY with multiple aggregation functions with bounded PCollection.
+ */
+ @Test
+ public void testAggregationFunctionsWithBounded() throws Exception{
+ runAggregationFunctions(boundedInput1);
+ }
+
+ /**
+ * GROUP-BY with multiple aggregation functions with unbounded PCollection.
+ */
+ @Test
+ public void testAggregationFunctionsWithUnbounded() throws Exception{
+ runAggregationFunctions(unboundedInput1);
+ }
+
+ private void runAggregationFunctions(PCollection<BeamRecord> input) throws Exception{
+ String sql = "select f_int2, count(*) as getFieldCount, "
+ + "sum(f_long) as sum1, avg(f_long) as avg1, max(f_long) as max1, min(f_long) as min1, "
+ + "sum(f_short) as sum2, avg(f_short) as avg2, max(f_short) as max2, min(f_short) as min2, "
+ + "sum(f_byte) as sum3, avg(f_byte) as avg3, max(f_byte) as max3, min(f_byte) as min3, "
+ + "sum(f_float) as sum4, avg(f_float) as avg4, max(f_float) as max4, min(f_float) as min4, "
+ + "sum(f_double) as sum5, avg(f_double) as avg5, "
+ + "max(f_double) as max5, min(f_double) as min5, "
+ + "max(f_timestamp) as max6, min(f_timestamp) as min6, "
+ + "var_pop(f_double) as varpop1, var_samp(f_double) as varsamp1, "
+ + "var_pop(f_int) as varpop2, var_samp(f_int) as varsamp2 "
+ + "FROM TABLE_A group by f_int2";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testAggregationFunctions", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(
+ Arrays.asList("f_int2", "size", "sum1", "avg1", "max1", "min1", "sum2", "avg2", "max2",
+ "min2", "sum3", "avg3", "max3", "min3", "sum4", "avg4", "max4", "min4", "sum5", "avg5",
+ "max5", "min5", "max6", "min6",
+ "varpop1", "varsamp1", "varpop2", "varsamp2"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT, Types.BIGINT, Types.BIGINT, Types.BIGINT,
+ Types.BIGINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT,
+ Types.TINYINT, Types.TINYINT, Types.TINYINT, Types.TINYINT, Types.FLOAT, Types.FLOAT,
+ Types.FLOAT, Types.FLOAT, Types.DOUBLE, Types.DOUBLE, Types.DOUBLE, Types.DOUBLE,
+ Types.TIMESTAMP, Types.TIMESTAMP,
+ Types.DOUBLE, Types.DOUBLE, Types.INTEGER, Types.INTEGER));
+
+ BeamRecord record = new BeamRecord(resultType
+ , 0, 4L
+ , 10000L, 2500L, 4000L, 1000L
+ , (short) 10, (short) 2, (short) 4, (short) 1
+ , (byte) 10, (byte) 2, (byte) 4, (byte) 1
+ , 10.0F, 2.5F, 4.0F, 1.0F
+ , 10.0, 2.5, 4.0, 1.0
+ , FORMAT.parse("2017-01-01 02:04:03"), FORMAT.parse("2017-01-01 01:01:03")
+ , 1.25, 1.666666667, 1, 1);
+
+ PAssert.that(result).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ private static class CheckerBigDecimalDivide
+ implements SerializableFunction<Iterable<BeamRecord>, Void> {
+ @Override public Void apply(Iterable<BeamRecord> input) {
+ Iterator<BeamRecord> iter = input.iterator();
+ assertTrue(iter.hasNext());
+ BeamRecord row = iter.next();
+ assertEquals(row.getDouble("avg1"), 8.142857143, 1e-7);
+ assertTrue(row.getInteger("avg2") == 8);
+ assertEquals(row.getDouble("varpop1"), 26.40816326, 1e-7);
+ assertTrue(row.getInteger("varpop2") == 26);
+ assertEquals(row.getDouble("varsamp1"), 30.80952381, 1e-7);
+ assertTrue(row.getInteger("varsamp2") == 30);
+ assertFalse(iter.hasNext());
+ return null;
+ }
+ }
+
+ /**
+ * GROUP-BY with aggregation functions with BigDeciaml Calculation (Avg, Var_Pop, etc).
+ */
+ @Test
+ public void testAggregationFunctionsWithBoundedOnBigDecimalDivide() throws Exception {
+ String sql = "SELECT AVG(f_double) as avg1, AVG(f_int) as avg2, "
+ + "VAR_POP(f_double) as varpop1, VAR_POP(f_int) as varpop2, "
+ + "VAR_SAMP(f_double) as varsamp1, VAR_SAMP(f_int) as varsamp2 "
+ + "FROM PCOLLECTION GROUP BY f_int2";
+
+ PCollection<BeamRecord> result =
+ boundedInput3.apply("testAggregationWithDecimalValue", BeamSql.query(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(
+ Arrays.asList("avg1", "avg2", "avg3",
+ "varpop1", "varpop2",
+ "varsamp1", "varsamp2"),
+ Arrays.asList(Types.DOUBLE, Types.INTEGER, Types.DECIMAL,
+ Types.DOUBLE, Types.INTEGER,
+ Types.DOUBLE, Types.INTEGER));
+
+ PAssert.that(result).satisfies(new CheckerBigDecimalDivide());
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * Implicit GROUP-BY with DISTINCT with bounded PCollection.
+ */
+ @Test
+ public void testDistinctWithBounded() throws Exception {
+ runDistinct(boundedInput1);
+ }
+
+ /**
+ * Implicit GROUP-BY with DISTINCT with unbounded PCollection.
+ */
+ @Test
+ public void testDistinctWithUnbounded() throws Exception {
+ runDistinct(unboundedInput1);
+ }
+
+ private void runDistinct(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT distinct f_int, f_long FROM PCOLLECTION ";
+
+ PCollection<BeamRecord> result =
+ input.apply("testDistinct", BeamSql.query(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int", "f_long"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT));
+
+ BeamRecord record1 = new BeamRecord(resultType, 1, 1000L);
+ BeamRecord record2 = new BeamRecord(resultType, 2, 2000L);
+ BeamRecord record3 = new BeamRecord(resultType, 3, 3000L);
+ BeamRecord record4 = new BeamRecord(resultType, 4, 4000L);
+
+ PAssert.that(result).containsInAnyOrder(record1, record2, record3, record4);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * GROUP-BY with TUMBLE window(aka fix_time_window) with bounded PCollection.
+ */
+ @Test
+ public void testTumbleWindowWithBounded() throws Exception {
+ runTumbleWindow(boundedInput1);
+ }
+
+ /**
+ * GROUP-BY with TUMBLE window(aka fix_time_window) with unbounded PCollection.
+ */
+ @Test
+ public void testTumbleWindowWithUnbounded() throws Exception {
+ runTumbleWindow(unboundedInput1);
+ }
+
+ private void runTumbleWindow(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int2, COUNT(*) AS `getFieldCount`,"
+ + " TUMBLE_START(f_timestamp, INTERVAL '1' HOUR) AS `window_start`"
+ + " FROM TABLE_A"
+ + " GROUP BY f_int2, TUMBLE(f_timestamp, INTERVAL '1' HOUR)";
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testTumbleWindow", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(
+ Arrays.asList("f_int2", "size", "window_start"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT, Types.TIMESTAMP));
+
+ BeamRecord record1 = new BeamRecord(resultType, 0, 3L, FORMAT.parse("2017-01-01 01:00:00"));
+ BeamRecord record2 = new BeamRecord(resultType, 0, 1L, FORMAT.parse("2017-01-01 02:00:00"));
+
+ PAssert.that(result).containsInAnyOrder(record1, record2);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * GROUP-BY with HOP window(aka sliding_window) with bounded PCollection.
+ */
+ @Test
+ public void testHopWindowWithBounded() throws Exception {
+ runHopWindow(boundedInput1);
+ }
+
+ /**
+ * GROUP-BY with HOP window(aka sliding_window) with unbounded PCollection.
+ */
+ @Test
+ public void testHopWindowWithUnbounded() throws Exception {
+ runHopWindow(unboundedInput1);
+ }
+
+ private void runHopWindow(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int2, COUNT(*) AS `getFieldCount`,"
+ + " HOP_START(f_timestamp, INTERVAL '1' HOUR, INTERVAL '30' MINUTE) AS `window_start`"
+ + " FROM PCOLLECTION"
+ + " GROUP BY f_int2, HOP(f_timestamp, INTERVAL '1' HOUR, INTERVAL '30' MINUTE)";
+ PCollection<BeamRecord> result =
+ input.apply("testHopWindow", BeamSql.query(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(
+ Arrays.asList("f_int2", "size", "window_start"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT, Types.TIMESTAMP));
+
+ BeamRecord record1 = new BeamRecord(resultType, 0, 3L, FORMAT.parse("2017-01-01 00:30:00"));
+ BeamRecord record2 = new BeamRecord(resultType, 0, 3L, FORMAT.parse("2017-01-01 01:00:00"));
+ BeamRecord record3 = new BeamRecord(resultType, 0, 1L, FORMAT.parse("2017-01-01 01:30:00"));
+ BeamRecord record4 = new BeamRecord(resultType, 0, 1L, FORMAT.parse("2017-01-01 02:00:00"));
+
+ PAssert.that(result).containsInAnyOrder(record1, record2, record3, record4);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * GROUP-BY with SESSION window with bounded PCollection.
+ */
+ @Test
+ public void testSessionWindowWithBounded() throws Exception {
+ runSessionWindow(boundedInput1);
+ }
+
+ /**
+ * GROUP-BY with SESSION window with unbounded PCollection.
+ */
+ @Test
+ public void testSessionWindowWithUnbounded() throws Exception {
+ runSessionWindow(unboundedInput1);
+ }
+
+ private void runSessionWindow(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int2, COUNT(*) AS `getFieldCount`,"
+ + " SESSION_START(f_timestamp, INTERVAL '5' MINUTE) AS `window_start`"
+ + " FROM TABLE_A"
+ + " GROUP BY f_int2, SESSION(f_timestamp, INTERVAL '5' MINUTE)";
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testSessionWindow", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(
+ Arrays.asList("f_int2", "size", "window_start"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT, Types.TIMESTAMP));
+
+ BeamRecord record1 = new BeamRecord(resultType, 0, 3L, FORMAT.parse("2017-01-01 01:01:03"));
+ BeamRecord record2 = new BeamRecord(resultType, 0, 1L, FORMAT.parse("2017-01-01 02:04:03"));
+
+ PAssert.that(result).containsInAnyOrder(record1, record2);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testWindowOnNonTimestampField() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage(
+ "Cannot apply 'TUMBLE' to arguments of type 'TUMBLE(<BIGINT>, <INTERVAL HOUR>)'");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT f_int2, COUNT(*) AS `getFieldCount` FROM TABLE_A "
+ + "GROUP BY f_int2, TUMBLE(f_long, INTERVAL '1' HOUR)";
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), boundedInput1)
+ .apply("testWindowOnNonTimestampField", BeamSql.queryMulti(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testUnsupportedDistinct() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage("Encountered \"*\"");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT f_int2, COUNT(DISTINCT *) AS `size` "
+ + "FROM PCOLLECTION GROUP BY f_int2";
+
+ PCollection<BeamRecord> result =
+ boundedInput1.apply("testUnsupportedDistinct", BeamSql.query(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
new file mode 100644
index 0000000..b27435c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+/**
+ * prepare input records to test {@link BeamSql}.
+ *
+ * <p>Note that, any change in these records would impact tests in this package.
+ *
+ */
+public class BeamSqlDslBase {
+ public static final DateFormat FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ @Rule
+ public ExpectedException exceptions = ExpectedException.none();
+
+ public static BeamRecordSqlType rowTypeInTableA;
+ public static List<BeamRecord> recordsInTableA;
+
+ //bounded PCollections
+ public PCollection<BeamRecord> boundedInput1;
+ public PCollection<BeamRecord> boundedInput2;
+
+ //unbounded PCollections
+ public PCollection<BeamRecord> unboundedInput1;
+ public PCollection<BeamRecord> unboundedInput2;
+
+ @BeforeClass
+ public static void prepareClass() throws ParseException {
+ rowTypeInTableA = BeamRecordSqlType.create(
+ Arrays.asList("f_int", "f_long", "f_short", "f_byte", "f_float", "f_double", "f_string",
+ "f_timestamp", "f_int2", "f_decimal"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT, Types.SMALLINT, Types.TINYINT, Types.FLOAT,
+ Types.DOUBLE, Types.VARCHAR, Types.TIMESTAMP, Types.INTEGER, Types.DECIMAL));
+
+ recordsInTableA = prepareInputRowsInTableA();
+ }
+
+ @Before
+ public void preparePCollections(){
+ boundedInput1 = PBegin.in(pipeline).apply("boundedInput1",
+ Create.of(recordsInTableA).withCoder(rowTypeInTableA.getRecordCoder()));
+
+ boundedInput2 = PBegin.in(pipeline).apply("boundedInput2",
+ Create.of(recordsInTableA.get(0)).withCoder(rowTypeInTableA.getRecordCoder()));
+
+ unboundedInput1 = prepareUnboundedPCollection1();
+ unboundedInput2 = prepareUnboundedPCollection2();
+ }
+
+ private PCollection<BeamRecord> prepareUnboundedPCollection1() {
+ TestStream.Builder<BeamRecord> values = TestStream
+ .create(rowTypeInTableA.getRecordCoder());
+
+ for (BeamRecord row : recordsInTableA) {
+ values = values.advanceWatermarkTo(new Instant(row.getDate("f_timestamp")));
+ values = values.addElements(row);
+ }
+
+ return PBegin.in(pipeline).apply("unboundedInput1", values.advanceWatermarkToInfinity());
+ }
+
+ private PCollection<BeamRecord> prepareUnboundedPCollection2() {
+ TestStream.Builder<BeamRecord> values = TestStream
+ .create(rowTypeInTableA.getRecordCoder());
+
+ BeamRecord row = recordsInTableA.get(0);
+ values = values.advanceWatermarkTo(new Instant(row.getDate("f_timestamp")));
+ values = values.addElements(row);
+
+ return PBegin.in(pipeline).apply("unboundedInput2", values.advanceWatermarkToInfinity());
+ }
+
+ private static List<BeamRecord> prepareInputRowsInTableA() throws ParseException{
+ List<BeamRecord> rows = new ArrayList<>();
+
+ BeamRecord row1 = new BeamRecord(rowTypeInTableA
+ , 1, 1000L, Short.valueOf("1"), Byte.valueOf("1"), 1.0f, 1.0, "string_row1"
+ , FORMAT.parse("2017-01-01 01:01:03"), 0, new BigDecimal(1));
+ rows.add(row1);
+
+ BeamRecord row2 = new BeamRecord(rowTypeInTableA
+ , 2, 2000L, Short.valueOf("2"), Byte.valueOf("2"), 2.0f, 2.0, "string_row2"
+ , FORMAT.parse("2017-01-01 01:02:03"), 0, new BigDecimal(2));
+ rows.add(row2);
+
+ BeamRecord row3 = new BeamRecord(rowTypeInTableA
+ , 3, 3000L, Short.valueOf("3"), Byte.valueOf("3"), 3.0f, 3.0, "string_row3"
+ , FORMAT.parse("2017-01-01 01:06:03"), 0, new BigDecimal(3));
+ rows.add(row3);
+
+ BeamRecord row4 = new BeamRecord(rowTypeInTableA
+ , 4, 4000L, Short.valueOf("4"), Byte.valueOf("4"), 4.0f, 4.0, "string_row4"
+ , FORMAT.parse("2017-01-01 02:04:03"), 0, new BigDecimal(4));
+ rows.add(row4);
+
+ return rows;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslFilterTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslFilterTest.java
new file mode 100644
index 0000000..bd430e5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslFilterTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Test;
+
+/**
+ * Tests for WHERE queries with BOUNDED PCollection.
+ */
+public class BeamSqlDslFilterTest extends BeamSqlDslBase {
+ /**
+ * single filter with bounded PCollection.
+ */
+ @Test
+ public void testSingleFilterWithBounded() throws Exception {
+ runSingleFilter(boundedInput1);
+ }
+
+ /**
+ * single filter with unbounded PCollection.
+ */
+ @Test
+ public void testSingleFilterWithUnbounded() throws Exception {
+ runSingleFilter(unboundedInput1);
+ }
+
+ private void runSingleFilter(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT * FROM PCOLLECTION WHERE f_int = 1";
+
+ PCollection<BeamRecord> result =
+ input.apply("testSingleFilter", BeamSql.query(sql));
+
+ PAssert.that(result).containsInAnyOrder(recordsInTableA.get(0));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * composite filters with bounded PCollection.
+ */
+ @Test
+ public void testCompositeFilterWithBounded() throws Exception {
+ runCompositeFilter(boundedInput1);
+ }
+
+ /**
+ * composite filters with unbounded PCollection.
+ */
+ @Test
+ public void testCompositeFilterWithUnbounded() throws Exception {
+ runCompositeFilter(unboundedInput1);
+ }
+
+ private void runCompositeFilter(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT * FROM TABLE_A"
+ + " WHERE f_int > 1 AND (f_long < 3000 OR f_string = 'string_row3')";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testCompositeFilter", BeamSql.queryMulti(sql));
+
+ PAssert.that(result).containsInAnyOrder(recordsInTableA.get(1), recordsInTableA.get(2));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * nothing return with filters in bounded PCollection.
+ */
+ @Test
+ public void testNoReturnFilterWithBounded() throws Exception {
+ runNoReturnFilter(boundedInput1);
+ }
+
+ /**
+ * nothing return with filters in unbounded PCollection.
+ */
+ @Test
+ public void testNoReturnFilterWithUnbounded() throws Exception {
+ runNoReturnFilter(unboundedInput1);
+ }
+
+ private void runNoReturnFilter(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT * FROM TABLE_A WHERE f_int < 1";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testNoReturnFilter", BeamSql.queryMulti(sql));
+
+ PAssert.that(result).empty();
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testFromInvalidTableName1() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage("Object 'TABLE_B' not found");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT * FROM TABLE_B WHERE f_int < 1";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), boundedInput1)
+ .apply("testFromInvalidTableName1", BeamSql.queryMulti(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testFromInvalidTableName2() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage("Use fixed table name PCOLLECTION");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT * FROM PCOLLECTION_NA";
+
+ PCollection<BeamRecord> result = boundedInput1.apply(BeamSql.query(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testInvalidFilter() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage("Column 'f_int_na' not found in any table");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT * FROM PCOLLECTION WHERE f_int_na = 0";
+
+ PCollection<BeamRecord> result = boundedInput1.apply(BeamSql.query(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
new file mode 100644
index 0000000..bbfa3d3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql;
+
+import static org.apache.beam.sdk.extensions.sql.impl.rel.BeamJoinRelBoundedVsBoundedTest.ORDER_DETAILS1;
+import static org.apache.beam.sdk.extensions.sql.impl.rel.BeamJoinRelBoundedVsBoundedTest.ORDER_DETAILS2;
+
+import java.sql.Types;
+import java.util.Arrays;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Tests for joins in queries.
+ */
+public class BeamSqlDslJoinTest {
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ private static final BeamRecordSqlType SOURCE_RECORD_TYPE =
+ BeamRecordSqlType.create(
+ Arrays.asList(
+ "order_id", "site_id", "price"
+ ),
+ Arrays.asList(
+ Types.INTEGER, Types.INTEGER, Types.INTEGER
+ )
+ );
+
+ private static final BeamRecordCoder SOURCE_CODER = SOURCE_RECORD_TYPE.getRecordCoder();
+
+ private static final BeamRecordSqlType RESULT_RECORD_TYPE =
+ BeamRecordSqlType.create(
+ Arrays.asList(
+ "order_id", "site_id", "price", "order_id0", "site_id0", "price0"
+ ),
+ Arrays.asList(
+ Types.INTEGER, Types.INTEGER, Types.INTEGER, Types.INTEGER
+ , Types.INTEGER, Types.INTEGER
+ )
+ );
+
+ private static final BeamRecordCoder RESULT_CODER = RESULT_RECORD_TYPE.getRecordCoder();
+
+ @Test
+ public void testInnerJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PAssert.that(queryFromOrderTables(sql)).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ RESULT_RECORD_TYPE
+ ).addRows(
+ 2, 3, 3, 1, 2, 3
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testLeftOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " LEFT OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PAssert.that(queryFromOrderTables(sql)).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ RESULT_RECORD_TYPE
+ ).addRows(
+ 1, 2, 3, null, null, null,
+ 2, 3, 3, 1, 2, 3,
+ 3, 4, 5, null, null, null
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testRightOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " RIGHT OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PAssert.that(queryFromOrderTables(sql)).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ RESULT_RECORD_TYPE
+ ).addRows(
+ 2, 3, 3, 1, 2, 3,
+ null, null, null, 2, 3, 3,
+ null, null, null, 3, 4, 5
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testFullOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " FULL OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PAssert.that(queryFromOrderTables(sql)).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ RESULT_RECORD_TYPE
+ ).addRows(
+ 2, 3, 3, 1, 2, 3,
+ 1, 2, 3, null, null, null,
+ 3, 4, 5, null, null, null,
+ null, null, null, 2, 3, 3,
+ null, null, null, 3, 4, 5
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testException_nonEqualJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id>o2.site_id"
+ ;
+
+ pipeline.enableAbandonedNodeEnforcement(false);
+ queryFromOrderTables(sql);
+ pipeline.run();
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testException_crossJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1, ORDER_DETAILS2 o2";
+
+ pipeline.enableAbandonedNodeEnforcement(false);
+ queryFromOrderTables(sql);
+ pipeline.run();
+ }
+
+ private PCollection<BeamRecord> queryFromOrderTables(String sql) {
+ return PCollectionTuple
+ .of(
+ new TupleTag<BeamRecord>("ORDER_DETAILS1"),
+ ORDER_DETAILS1.buildIOReader(pipeline).setCoder(SOURCE_CODER)
+ )
+ .and(new TupleTag<BeamRecord>("ORDER_DETAILS2"),
+ ORDER_DETAILS2.buildIOReader(pipeline).setCoder(SOURCE_CODER)
+ ).apply("join", BeamSql.queryMulti(sql)).setCoder(RESULT_CODER);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslProjectTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslProjectTest.java
new file mode 100644
index 0000000..b288270
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslProjectTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.sql.Types;
+import java.util.Arrays;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Test;
+
+/**
+ * Tests for field-project in queries with BOUNDED PCollection.
+ */
+public class BeamSqlDslProjectTest extends BeamSqlDslBase {
+ /**
+ * select all fields with bounded PCollection.
+ */
+ @Test
+ public void testSelectAllWithBounded() throws Exception {
+ runSelectAll(boundedInput2);
+ }
+
+ /**
+ * select all fields with unbounded PCollection.
+ */
+ @Test
+ public void testSelectAllWithUnbounded() throws Exception {
+ runSelectAll(unboundedInput2);
+ }
+
+ private void runSelectAll(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT * FROM PCOLLECTION";
+
+ PCollection<BeamRecord> result =
+ input.apply("testSelectAll", BeamSql.query(sql));
+
+ PAssert.that(result).containsInAnyOrder(recordsInTableA.get(0));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * select partial fields with bounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsWithBounded() throws Exception {
+ runPartialFields(boundedInput2);
+ }
+
+ /**
+ * select partial fields with unbounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsWithUnbounded() throws Exception {
+ runPartialFields(unboundedInput2);
+ }
+
+ private void runPartialFields(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int, f_long FROM TABLE_A";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testPartialFields", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int", "f_long"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT));
+
+ BeamRecord record = new BeamRecord(resultType
+ , recordsInTableA.get(0).getFieldValue(0), recordsInTableA.get(0).getFieldValue(1));
+
+ PAssert.that(result).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * select partial fields for multiple rows with bounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsInMultipleRowWithBounded() throws Exception {
+ runPartialFieldsInMultipleRow(boundedInput1);
+ }
+
+ /**
+ * select partial fields for multiple rows with unbounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsInMultipleRowWithUnbounded() throws Exception {
+ runPartialFieldsInMultipleRow(unboundedInput1);
+ }
+
+ private void runPartialFieldsInMultipleRow(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int, f_long FROM TABLE_A";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testPartialFieldsInMultipleRow", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int", "f_long"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT));
+
+ BeamRecord record1 = new BeamRecord(resultType
+ , recordsInTableA.get(0).getFieldValue(0), recordsInTableA.get(0).getFieldValue(1));
+
+ BeamRecord record2 = new BeamRecord(resultType
+ , recordsInTableA.get(1).getFieldValue(0), recordsInTableA.get(1).getFieldValue(1));
+
+ BeamRecord record3 = new BeamRecord(resultType
+ , recordsInTableA.get(2).getFieldValue(0), recordsInTableA.get(2).getFieldValue(1));
+
+ BeamRecord record4 = new BeamRecord(resultType
+ , recordsInTableA.get(3).getFieldValue(0), recordsInTableA.get(3).getFieldValue(1));
+
+ PAssert.that(result).containsInAnyOrder(record1, record2, record3, record4);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * select partial fields with bounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsInRowsWithBounded() throws Exception {
+ runPartialFieldsInRows(boundedInput1);
+ }
+
+ /**
+ * select partial fields with unbounded PCollection.
+ */
+ @Test
+ public void testPartialFieldsInRowsWithUnbounded() throws Exception {
+ runPartialFieldsInRows(unboundedInput1);
+ }
+
+ private void runPartialFieldsInRows(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT f_int, f_long FROM TABLE_A";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testPartialFieldsInRows", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int", "f_long"),
+ Arrays.asList(Types.INTEGER, Types.BIGINT));
+
+ BeamRecord record1 = new BeamRecord(resultType
+ , recordsInTableA.get(0).getFieldValue(0), recordsInTableA.get(0).getFieldValue(1));
+
+ BeamRecord record2 = new BeamRecord(resultType
+ , recordsInTableA.get(1).getFieldValue(0), recordsInTableA.get(1).getFieldValue(1));
+
+ BeamRecord record3 = new BeamRecord(resultType
+ , recordsInTableA.get(2).getFieldValue(0), recordsInTableA.get(2).getFieldValue(1));
+
+ BeamRecord record4 = new BeamRecord(resultType
+ , recordsInTableA.get(3).getFieldValue(0), recordsInTableA.get(3).getFieldValue(1));
+
+ PAssert.that(result).containsInAnyOrder(record1, record2, record3, record4);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * select literal field with bounded PCollection.
+ */
+ @Test
+ public void testLiteralFieldWithBounded() throws Exception {
+ runLiteralField(boundedInput2);
+ }
+
+ /**
+ * select literal field with unbounded PCollection.
+ */
+ @Test
+ public void testLiteralFieldWithUnbounded() throws Exception {
+ runLiteralField(unboundedInput2);
+ }
+
+ public void runLiteralField(PCollection<BeamRecord> input) throws Exception {
+ String sql = "SELECT 1 as literal_field FROM TABLE_A";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), input)
+ .apply("testLiteralField", BeamSql.queryMulti(sql));
+
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("literal_field"),
+ Arrays.asList(Types.INTEGER));
+
+ BeamRecord record = new BeamRecord(resultType, 1);
+
+ PAssert.that(result).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testProjectUnknownField() throws Exception {
+ exceptions.expect(IllegalStateException.class);
+ exceptions.expectMessage("Column 'f_int_na' not found in any table");
+ pipeline.enableAbandonedNodeEnforcement(false);
+
+ String sql = "SELECT f_int_na FROM TABLE_A";
+
+ PCollection<BeamRecord> result =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("TABLE_A"), boundedInput1)
+ .apply("testProjectUnknownField", BeamSql.queryMulti(sql));
+
+ pipeline.run().waitUntilFinish();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java
new file mode 100644
index 0000000..0d8bc12
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import java.sql.Types;
+import java.util.Arrays;
+import java.util.Iterator;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.junit.Test;
+
+/**
+ * Tests for UDF/UDAF.
+ */
+public class BeamSqlDslUdfUdafTest extends BeamSqlDslBase {
+ /**
+ * GROUP-BY with UDAF.
+ */
+ @Test
+ public void testUdaf() throws Exception {
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int2", "squaresum"),
+ Arrays.asList(Types.INTEGER, Types.INTEGER));
+
+ BeamRecord record = new BeamRecord(resultType, 0, 30);
+
+ String sql1 = "SELECT f_int2, squaresum1(f_int) AS `squaresum`"
+ + " FROM PCOLLECTION GROUP BY f_int2";
+ PCollection<BeamRecord> result1 =
+ boundedInput1.apply("testUdaf1",
+ BeamSql.query(sql1).withUdaf("squaresum1", new SquareSum()));
+ PAssert.that(result1).containsInAnyOrder(record);
+
+ String sql2 = "SELECT f_int2, squaresum2(f_int) AS `squaresum`"
+ + " FROM PCOLLECTION GROUP BY f_int2";
+ PCollection<BeamRecord> result2 =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("PCOLLECTION"), boundedInput1)
+ .apply("testUdaf2",
+ BeamSql.queryMulti(sql2).withUdaf("squaresum2", new SquareSum()));
+ PAssert.that(result2).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * test UDF.
+ */
+ @Test
+ public void testUdf() throws Exception{
+ BeamRecordSqlType resultType = BeamRecordSqlType.create(Arrays.asList("f_int", "cubicvalue"),
+ Arrays.asList(Types.INTEGER, Types.INTEGER));
+
+ BeamRecord record = new BeamRecord(resultType, 2, 8);
+
+ String sql1 = "SELECT f_int, cubic1(f_int) as cubicvalue FROM PCOLLECTION WHERE f_int = 2";
+ PCollection<BeamRecord> result1 =
+ boundedInput1.apply("testUdf1",
+ BeamSql.query(sql1).withUdf("cubic1", CubicInteger.class));
+ PAssert.that(result1).containsInAnyOrder(record);
+
+ String sql2 = "SELECT f_int, cubic2(f_int) as cubicvalue FROM PCOLLECTION WHERE f_int = 2";
+ PCollection<BeamRecord> result2 =
+ PCollectionTuple.of(new TupleTag<BeamRecord>("PCOLLECTION"), boundedInput1)
+ .apply("testUdf2",
+ BeamSql.queryMulti(sql2).withUdf("cubic2", new CubicIntegerFn()));
+ PAssert.that(result2).containsInAnyOrder(record);
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * UDAF(CombineFn) for test, which returns the sum of square.
+ */
+ public static class SquareSum extends CombineFn<Integer, Integer, Integer> {
+ @Override
+ public Integer createAccumulator() {
+ return 0;
+ }
+
+ @Override
+ public Integer addInput(Integer accumulator, Integer input) {
+ return accumulator + input * input;
+ }
+
+ @Override
+ public Integer mergeAccumulators(Iterable<Integer> accumulators) {
+ int v = 0;
+ Iterator<Integer> ite = accumulators.iterator();
+ while (ite.hasNext()) {
+ v += ite.next();
+ }
+ return v;
+ }
+
+ @Override
+ public Integer extractOutput(Integer accumulator) {
+ return accumulator;
+ }
+
+ }
+
+ /**
+ * A example UDF for test.
+ */
+ public static class CubicInteger implements BeamSqlUdf {
+ public static Integer eval(Integer input){
+ return input * input * input;
+ }
+ }
+
+ /**
+ * A example UDF with {@link SerializableFunction}.
+ */
+ public static class CubicIntegerFn implements SerializableFunction<Integer, Integer> {
+ @Override
+ public Integer apply(Integer input) {
+ return input * input * input;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java
new file mode 100644
index 0000000..d4cc53a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.BeamRecord;
+
+/**
+ * Test utilities.
+ */
+public class TestUtils {
+ /**
+ * A {@code DoFn} to convert a {@code BeamSqlRow} to a comparable {@code String}.
+ */
+ public static class BeamSqlRow2StringDoFn extends DoFn<BeamRecord, String> {
+ @ProcessElement
+ public void processElement(ProcessContext ctx) {
+ ctx.output(ctx.element().toString());
+ }
+ }
+
+ /**
+ * Convert list of {@code BeamSqlRow} to list of {@code String}.
+ */
+ public static List<String> beamSqlRows2Strings(List<BeamRecord> rows) {
+ List<String> strs = new ArrayList<>();
+ for (BeamRecord row : rows) {
+ strs.add(row.toString());
+ }
+
+ return strs;
+ }
+
+ /**
+ * Convenient way to build a list of {@code BeamSqlRow}s.
+ *
+ * <p>You can use it like this:
+ *
+ * <pre>{@code
+ * TestUtils.RowsBuilder.of(
+ * Types.INTEGER, "order_id",
+ * Types.INTEGER, "sum_site_id",
+ * Types.VARCHAR, "buyer"
+ * ).addRows(
+ * 1, 3, "james",
+ * 2, 5, "bond"
+ * ).getStringRows()
+ * }</pre>
+ * {@code}
+ */
+ public static class RowsBuilder {
+ private BeamRecordSqlType type;
+ private List<BeamRecord> rows = new ArrayList<>();
+
+ /**
+ * Create a RowsBuilder with the specified row type info.
+ *
+ * <p>For example:
+ * <pre>{@code
+ * TestUtils.RowsBuilder.of(
+ * Types.INTEGER, "order_id",
+ * Types.INTEGER, "sum_site_id",
+ * Types.VARCHAR, "buyer"
+ * )}</pre>
+ *
+ * @args pairs of column type and column names.
+ */
+ public static RowsBuilder of(final Object... args) {
+ BeamRecordSqlType beamSQLRowType = buildBeamSqlRowType(args);
+ RowsBuilder builder = new RowsBuilder();
+ builder.type = beamSQLRowType;
+
+ return builder;
+ }
+
+ /**
+ * Create a RowsBuilder with the specified row type info.
+ *
+ * <p>For example:
+ * <pre>{@code
+ * TestUtils.RowsBuilder.of(
+ * beamRecordSqlType
+ * )}</pre>
+ * @beamSQLRowType the record type.
+ */
+ public static RowsBuilder of(final BeamRecordSqlType beamSQLRowType) {
+ RowsBuilder builder = new RowsBuilder();
+ builder.type = beamSQLRowType;
+
+ return builder;
+ }
+
+ /**
+ * Add rows to the builder.
+ *
+ * <p>Note: check the class javadoc for for detailed example.
+ */
+ public RowsBuilder addRows(final Object... args) {
+ this.rows.addAll(buildRows(type, Arrays.asList(args)));
+ return this;
+ }
+
+ /**
+ * Add rows to the builder.
+ *
+ * <p>Note: check the class javadoc for for detailed example.
+ */
+ public RowsBuilder addRows(final List args) {
+ this.rows.addAll(buildRows(type, args));
+ return this;
+ }
+
+ public List<BeamRecord> getRows() {
+ return rows;
+ }
+
+ public List<String> getStringRows() {
+ return beamSqlRows2Strings(rows);
+ }
+ }
+
+ /**
+ * Convenient way to build a {@code BeamSqlRowType}.
+ *
+ * <p>e.g.
+ *
+ * <pre>{@code
+ * buildBeamSqlRowType(
+ * Types.BIGINT, "order_id",
+ * Types.INTEGER, "site_id",
+ * Types.DOUBLE, "price",
+ * Types.TIMESTAMP, "order_time"
+ * )
+ * }</pre>
+ */
+ public static BeamRecordSqlType buildBeamSqlRowType(Object... args) {
+ List<Integer> types = new ArrayList<>();
+ List<String> names = new ArrayList<>();
+
+ for (int i = 0; i < args.length - 1; i += 2) {
+ types.add((int) args[i]);
+ names.add((String) args[i + 1]);
+ }
+
+ return BeamRecordSqlType.create(names, types);
+ }
+
+ /**
+ * Convenient way to build a {@code BeamSqlRow}s.
+ *
+ * <p>e.g.
+ *
+ * <pre>{@code
+ * buildRows(
+ * rowType,
+ * 1, 1, 1, // the first row
+ * 2, 2, 2, // the second row
+ * ...
+ * )
+ * }</pre>
+ */
+ public static List<BeamRecord> buildRows(BeamRecordSqlType type, List args) {
+ List<BeamRecord> rows = new ArrayList<>();
+ int fieldCount = type.getFieldCount();
+
+ for (int i = 0; i < args.size(); i += fieldCount) {
+ rows.add(new BeamRecord(type, args.subList(i, i + fieldCount)));
+ }
+ return rows;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
new file mode 100644
index 0000000..382404e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
+
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlCaseExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlDivideExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMinusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlModExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlPlusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentDateExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimeExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimestampExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateCeilExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateFloorExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimeMinusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimePlusExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlExtractExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlIntervalMultiplyExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlAndExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlNotExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlOrExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlCharLengthExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlConcatExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlInitCapExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlLowerExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlOverlayExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlPositionExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlSubstringExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlTrimExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlUpperExpression;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamFilterRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamProjectRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.fun.SqlTrimFunction;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit test cases for {@link BeamSqlFnExecutor}.
+ */
+public class BeamSqlFnExecutorTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testBeamFilterRel() {
+ RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.AND,
+ Arrays.asList(
+ rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
+ Arrays.asList(rexBuilder.makeInputRef(relDataType, 0),
+ rexBuilder.makeBigintLiteral(new BigDecimal(1000L)))),
+ rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ Arrays.asList(rexBuilder.makeInputRef(relDataType, 1),
+ rexBuilder.makeExactLiteral(new BigDecimal(0))))));
+
+ BeamFilterRel beamFilterRel = new BeamFilterRel(cluster, RelTraitSet.createEmpty(), null,
+ condition);
+
+ BeamSqlFnExecutor executor = new BeamSqlFnExecutor(beamFilterRel);
+ executor.prepare();
+
+ Assert.assertEquals(1, executor.exps.size());
+
+ BeamSqlExpression l1Exp = executor.exps.get(0);
+ assertTrue(l1Exp instanceof BeamSqlAndExpression);
+ Assert.assertEquals(SqlTypeName.BOOLEAN, l1Exp.getOutputType());
+
+ Assert.assertEquals(2, l1Exp.getOperands().size());
+ BeamSqlExpression l1Left = (BeamSqlExpression) l1Exp.getOperands().get(0);
+ BeamSqlExpression l1Right = (BeamSqlExpression) l1Exp.getOperands().get(1);
+
+ assertTrue(l1Left instanceof BeamSqlLessThanOrEqualsExpression);
+ assertTrue(l1Right instanceof BeamSqlEqualsExpression);
+
+ Assert.assertEquals(2, l1Left.getOperands().size());
+ BeamSqlExpression l1LeftLeft = (BeamSqlExpression) l1Left.getOperands().get(0);
+ BeamSqlExpression l1LeftRight = (BeamSqlExpression) l1Left.getOperands().get(1);
+ assertTrue(l1LeftLeft instanceof BeamSqlInputRefExpression);
+ assertTrue(l1LeftRight instanceof BeamSqlPrimitive);
+
+ Assert.assertEquals(2, l1Right.getOperands().size());
+ BeamSqlExpression l1RightLeft = (BeamSqlExpression) l1Right.getOperands().get(0);
+ BeamSqlExpression l1RightRight = (BeamSqlExpression) l1Right.getOperands().get(1);
+ assertTrue(l1RightLeft instanceof BeamSqlInputRefExpression);
+ assertTrue(l1RightRight instanceof BeamSqlPrimitive);
+ }
+
+ @Test
+ public void testBeamProjectRel() {
+ BeamRelNode relNode = new BeamProjectRel(cluster, RelTraitSet.createEmpty(),
+ relBuilder.values(relDataType, 1234567L, 0, 8.9, null).build(),
+ rexBuilder.identityProjects(relDataType), relDataType);
+ BeamSqlFnExecutor executor = new BeamSqlFnExecutor(relNode);
+
+ executor.prepare();
+ Assert.assertEquals(4, executor.exps.size());
+ assertTrue(executor.exps.get(0) instanceof BeamSqlInputRefExpression);
+ assertTrue(executor.exps.get(1) instanceof BeamSqlInputRefExpression);
+ assertTrue(executor.exps.get(2) instanceof BeamSqlInputRefExpression);
+ assertTrue(executor.exps.get(3) instanceof BeamSqlInputRefExpression);
+ }
+
+
+ @Test
+ public void testBuildExpression_logical() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true),
+ rexBuilder.makeLiteral(false)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlAndExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.OR,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true),
+ rexBuilder.makeLiteral(false)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlOrExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlNotExpression);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testBuildExpression_logical_andOr_invalidOperand() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true),
+ rexBuilder.makeLiteral("hello")
+ )
+ );
+ BeamSqlFnExecutor.buildExpression(rexNode);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testBuildExpression_logical_not_invalidOperand() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello")
+ )
+ );
+ BeamSqlFnExecutor.buildExpression(rexNode);
+ }
+
+
+ @Test(expected = IllegalStateException.class)
+ public void testBuildExpression_logical_not_invalidOperandCount() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true),
+ rexBuilder.makeLiteral(true)
+ )
+ );
+ BeamSqlFnExecutor.buildExpression(rexNode);
+ }
+
+ @Test
+ public void testBuildExpression_arithmetic() {
+ testBuildArithmeticExpression(SqlStdOperatorTable.PLUS, BeamSqlPlusExpression.class);
+ testBuildArithmeticExpression(SqlStdOperatorTable.MINUS, BeamSqlMinusExpression.class);
+ testBuildArithmeticExpression(SqlStdOperatorTable.MULTIPLY, BeamSqlMultiplyExpression.class);
+ testBuildArithmeticExpression(SqlStdOperatorTable.DIVIDE, BeamSqlDivideExpression.class);
+ testBuildArithmeticExpression(SqlStdOperatorTable.MOD, BeamSqlModExpression.class);
+ }
+
+ private void testBuildArithmeticExpression(SqlOperator fn,
+ Class<? extends BeamSqlExpression> clazz) {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(fn, Arrays.asList(
+ rexBuilder.makeBigintLiteral(new BigDecimal(1L)),
+ rexBuilder.makeBigintLiteral(new BigDecimal(1L))
+ ));
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+
+ assertTrue(exp.getClass().equals(clazz));
+ }
+
+ @Test
+ public void testBuildExpression_string() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CONCAT,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello "),
+ rexBuilder.makeLiteral("world")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlConcatExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.POSITION,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello"),
+ rexBuilder.makeLiteral("worldhello")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlPositionExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.POSITION,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello"),
+ rexBuilder.makeLiteral("worldhello"),
+ rexBuilder.makeCast(BeamQueryPlanner.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER),
+ rexBuilder.makeBigintLiteral(BigDecimal.ONE))
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlPositionExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CHAR_LENGTH,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlCharLengthExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.UPPER,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlUpperExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.LOWER,
+ Arrays.asList(
+ rexBuilder.makeLiteral("HELLO")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlLowerExpression);
+
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.INITCAP,
+ Arrays.asList(
+ rexBuilder.makeLiteral("hello")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlInitCapExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.TRIM,
+ Arrays.asList(
+ rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH),
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeLiteral("HELLO")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlTrimExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING,
+ Arrays.asList(
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlSubstringExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING,
+ Arrays.asList(
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlSubstringExpression);
+
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.OVERLAY,
+ Arrays.asList(
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlOverlayExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.OVERLAY,
+ Arrays.asList(
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlOverlayExpression);
+
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CASE,
+ Arrays.asList(
+ rexBuilder.makeLiteral(true),
+ rexBuilder.makeLiteral("HELLO"),
+ rexBuilder.makeLiteral("HELLO")
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlCaseExpression);
+ }
+
+ @Test
+ public void testBuildExpression_date() {
+ RexNode rexNode;
+ BeamSqlExpression exp;
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeZone(TimeZone.getTimeZone("GMT"));
+ calendar.setTime(new Date());
+
+ // CEIL
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CEIL,
+ Arrays.asList(
+ rexBuilder.makeDateLiteral(calendar),
+ rexBuilder.makeFlag(TimeUnitRange.MONTH)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlDateCeilExpression);
+
+ // FLOOR
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.FLOOR,
+ Arrays.asList(
+ rexBuilder.makeDateLiteral(calendar),
+ rexBuilder.makeFlag(TimeUnitRange.MONTH)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlDateFloorExpression);
+
+ // EXTRACT == EXTRACT_DATE?
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.EXTRACT,
+ Arrays.asList(
+ rexBuilder.makeFlag(TimeUnitRange.MONTH),
+ rexBuilder.makeDateLiteral(calendar)
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlExtractExpression);
+
+ // CURRENT_DATE
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CURRENT_DATE,
+ Arrays.<RexNode>asList(
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlCurrentDateExpression);
+
+ // LOCALTIME
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.LOCALTIME,
+ Arrays.<RexNode>asList(
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlCurrentTimeExpression);
+
+ // LOCALTIMESTAMP
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.LOCALTIMESTAMP,
+ Arrays.<RexNode>asList(
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlCurrentTimestampExpression);
+
+ // DATETIME_PLUS
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.DATETIME_PLUS,
+ Arrays.<RexNode>asList(
+ rexBuilder.makeDateLiteral(calendar),
+ rexBuilder.makeIntervalLiteral(
+ new BigDecimal(10),
+ new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.DAY, SqlParserPos.ZERO))
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlDatetimePlusExpression);
+
+ // * for intervals
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY,
+ Arrays.<RexNode>asList(
+ rexBuilder.makeExactLiteral(new BigDecimal(1)),
+ rexBuilder.makeIntervalLiteral(
+ new BigDecimal(10),
+ new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.DAY, SqlParserPos.ZERO))
+ )
+ );
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlIntervalMultiplyExpression);
+
+ // minus for dates
+ rexNode = rexBuilder.makeCall(
+ TYPE_FACTORY.createSqlType(SqlTypeName.INTERVAL_DAY),
+ SqlStdOperatorTable.MINUS,
+ Arrays.<RexNode>asList(
+ rexBuilder.makeTimestampLiteral(Calendar.getInstance(), 1000),
+ rexBuilder.makeTimestampLiteral(Calendar.getInstance(), 1000)
+ )
+ );
+
+ exp = BeamSqlFnExecutor.buildExpression(rexNode);
+ assertTrue(exp instanceof BeamSqlDatetimeMinusExpression);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTestBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTestBase.java
new file mode 100644
index 0000000..9d12126
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTestBase.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelDataTypeSystem;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRuleSets;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelBuilder;
+import org.junit.BeforeClass;
+
+/**
+ * base class to test {@link BeamSqlFnExecutor} and subclasses of {@link BeamSqlExpression}.
+ */
+public class BeamSqlFnExecutorTestBase {
+ public static RexBuilder rexBuilder = new RexBuilder(BeamQueryPlanner.TYPE_FACTORY);
+ public static RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder);
+
+ public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+ RelDataTypeSystem.DEFAULT);
+ public static RelDataType relDataType;
+
+ public static BeamRecordSqlType beamRowType;
+ public static BeamRecord record;
+
+ public static RelBuilder relBuilder;
+
+ @BeforeClass
+ public static void prepare() {
+ relDataType = TYPE_FACTORY.builder()
+ .add("order_id", SqlTypeName.BIGINT)
+ .add("site_id", SqlTypeName.INTEGER)
+ .add("price", SqlTypeName.DOUBLE)
+ .add("order_time", SqlTypeName.BIGINT).build();
+
+ beamRowType = CalciteUtils.toBeamRowType(relDataType);
+ record = new BeamRecord(beamRowType
+ , 1234567L, 0, 8.9, 1234567L);
+
+ SchemaPlus schema = Frameworks.createRootSchema(true);
+ final List<RelTraitDef> traitDefs = new ArrayList<>();
+ traitDefs.add(ConventionTraitDef.INSTANCE);
+ traitDefs.add(RelCollationTraitDef.INSTANCE);
+ FrameworkConfig config = Frameworks.newConfigBuilder()
+ .parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+ .traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+ .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM).build();
+
+ relBuilder = RelBuilder.create(config);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamNullExperssionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamNullExperssionTest.java
new file mode 100644
index 0000000..1bcda2c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamNullExperssionTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlIsNotNullExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlIsNullExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlIsNullExpression} and
+ * {@link BeamSqlIsNotNullExpression}.
+ */
+public class BeamNullExperssionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testIsNull() {
+ BeamSqlIsNullExpression exp1 = new BeamSqlIsNullExpression(
+ new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0));
+ Assert.assertEquals(false, exp1.evaluate(record, null).getValue());
+
+ BeamSqlIsNullExpression exp2 = new BeamSqlIsNullExpression(
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, null));
+ Assert.assertEquals(true, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testIsNotNull() {
+ BeamSqlIsNotNullExpression exp1 = new BeamSqlIsNotNullExpression(
+ new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0));
+ Assert.assertEquals(true, exp1.evaluate(record, null).getValue());
+
+ BeamSqlIsNotNullExpression exp2 = new BeamSqlIsNotNullExpression(
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, null));
+ Assert.assertEquals(false, exp2.evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlAndOrExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlAndOrExpressionTest.java
new file mode 100644
index 0000000..51a170d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlAndOrExpressionTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlAndExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlOrExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlAndExpression}, {@link BeamSqlOrExpression}.
+ */
+public class BeamSqlAndOrExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testAnd() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+
+ Assert.assertTrue(new BeamSqlAndExpression(operands).evaluate(record, null).getValue());
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+
+ Assert.assertFalse(new BeamSqlAndExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testOr() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+
+ Assert.assertFalse(new BeamSqlOrExpression(operands).evaluate(record, null).getValue());
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+
+ Assert.assertTrue(new BeamSqlOrExpression(operands).evaluate(record, null).getValue());
+
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpressionTest.java
new file mode 100644
index 0000000..e02554f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpressionTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlCaseExpression.
+ */
+public class BeamSqlCaseExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertTrue(new BeamSqlCaseExpression(operands).accept());
+
+ // even param count
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+ // `when` type error
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "error"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+ // `then` type mixing
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 10));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertEquals("hello", new BeamSqlCaseExpression(operands)
+ .evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertEquals("world", new BeamSqlCaseExpression(operands)
+ .evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello1"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertEquals("hello1", new BeamSqlCaseExpression(operands)
+ .evaluate(record, null).getValue());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpressionTest.java
new file mode 100644
index 0000000..f4e3cf9
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpressionTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test for {@link BeamSqlCastExpression}.
+ */
+public class BeamSqlCastExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ private List<BeamSqlExpression> operands;
+
+ @Before
+ public void setup() {
+ operands = new ArrayList<>();
+ }
+
+ @Test
+ public void testForOperands() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "aaa"));
+ Assert.assertFalse(new BeamSqlCastExpression(operands, SqlTypeName.BIGINT).accept());
+ }
+
+ @Test
+ public void testForIntegerToBigintTypeCasting() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+ Assert.assertEquals(5L,
+ new BeamSqlCastExpression(operands, SqlTypeName.BIGINT).evaluate(record, null).getLong());
+ }
+
+ @Test
+ public void testForDoubleToBigIntCasting() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 5.45));
+ Assert.assertEquals(5L,
+ new BeamSqlCastExpression(operands, SqlTypeName.BIGINT).evaluate(record, null).getLong());
+ }
+
+ @Test
+ public void testForIntegerToDateCast() {
+ // test for yyyyMMdd format
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 20170521));
+ Assert.assertEquals(Date.valueOf("2017-05-21"),
+ new BeamSqlCastExpression(operands, SqlTypeName.DATE).evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testyyyyMMddDateFormat() {
+ //test for yyyy-MM-dd format
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2017-05-21"));
+ Assert.assertEquals(Date.valueOf("2017-05-21"),
+ new BeamSqlCastExpression(operands, SqlTypeName.DATE).evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testyyMMddDateFormat() {
+ // test for yy.MM.dd format
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "17.05.21"));
+ Assert.assertEquals(Date.valueOf("2017-05-21"),
+ new BeamSqlCastExpression(operands, SqlTypeName.DATE).evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testForTimestampCastExpression() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "17-05-21 23:59:59.989"));
+ Assert.assertEquals(SqlTypeName.TIMESTAMP,
+ new BeamSqlCastExpression(operands, SqlTypeName.TIMESTAMP).evaluate(record, null)
+ .getOutputType());
+ }
+
+ @Test
+ public void testDateTimeFormatWithMillis() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2017-05-21 23:59:59.989"));
+ Assert.assertEquals(Timestamp.valueOf("2017-05-22 00:00:00.0"),
+ new BeamSqlCastExpression(operands, SqlTypeName.TIMESTAMP)
+ .evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testDateTimeFormatWithTimezone() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2017-05-21 23:59:59.89079 PST"));
+ Assert.assertEquals(Timestamp.valueOf("2017-05-22 00:00:00.0"),
+ new BeamSqlCastExpression(operands, SqlTypeName.TIMESTAMP)
+ .evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testDateTimeFormat() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2017-05-21 23:59:59"));
+ Assert.assertEquals(Timestamp.valueOf("2017-05-21 23:59:59"),
+ new BeamSqlCastExpression(operands, SqlTypeName.TIMESTAMP)
+ .evaluate(record, null).getValue());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testForCastTypeNotSupported() {
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.TIME, Calendar.getInstance().getTime()));
+ Assert.assertEquals(Timestamp.valueOf("2017-05-22 00:00:00.0"),
+ new BeamSqlCastExpression(operands, SqlTypeName.TIMESTAMP)
+ .evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCompareExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCompareExpressionTest.java
new file mode 100644
index 0000000..8aad6b3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCompareExpressionTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.Arrays;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlCompareExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlGreaterThanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlGreaterThanOrEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlNotEqualsExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for the collections of {@link BeamSqlCompareExpression}.
+ */
+public class BeamSqlCompareExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testEqual() {
+ BeamSqlEqualsExpression exp1 = new BeamSqlEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 100L)));
+ Assert.assertEquals(false, exp1.evaluate(record, null).getValue());
+
+ BeamSqlEqualsExpression exp2 = new BeamSqlEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+ Assert.assertEquals(true, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testLargerThan(){
+ BeamSqlGreaterThanExpression exp1 = new BeamSqlGreaterThanExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+ Assert.assertEquals(false, exp1.evaluate(record, null).getValue());
+
+ BeamSqlGreaterThanExpression exp2 = new BeamSqlGreaterThanExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234566L)));
+ Assert.assertEquals(true, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testLargerThanEqual(){
+ BeamSqlGreaterThanOrEqualsExpression exp1 = new BeamSqlGreaterThanOrEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+ Assert.assertEquals(true, exp1.evaluate(record, null).getValue());
+
+ BeamSqlGreaterThanOrEqualsExpression exp2 = new BeamSqlGreaterThanOrEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234568L)));
+ Assert.assertEquals(false, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testLessThan(){
+ BeamSqlLessThanExpression exp1 = new BeamSqlLessThanExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1)));
+ Assert.assertEquals(true, exp1.evaluate(record, null).getValue());
+
+ BeamSqlLessThanExpression exp2 = new BeamSqlLessThanExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1)));
+ Assert.assertEquals(false, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testLessThanEqual(){
+ BeamSqlLessThanOrEqualsExpression exp1 = new BeamSqlLessThanOrEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2),
+ BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 8.9)));
+ Assert.assertEquals(true, exp1.evaluate(record, null).getValue());
+
+ BeamSqlLessThanOrEqualsExpression exp2 = new BeamSqlLessThanOrEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2),
+ BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 8.0)));
+ Assert.assertEquals(false, exp2.evaluate(record, null).getValue());
+ }
+
+ @Test
+ public void testNotEqual(){
+ BeamSqlNotEqualsExpression exp1 = new BeamSqlNotEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+ Assert.assertEquals(false, exp1.evaluate(record, null).getValue());
+
+ BeamSqlNotEqualsExpression exp2 = new BeamSqlNotEqualsExpression(
+ Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3),
+ BeamSqlPrimitive.of(SqlTypeName.BIGINT, 0L)));
+ Assert.assertEquals(true, exp2.evaluate(record, null).getValue());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpressionTest.java
new file mode 100644
index 0000000..e543d4f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpressionTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlInputRefExpression}.
+ */
+public class BeamSqlInputRefExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testRefInRange() {
+ BeamSqlInputRefExpression ref0 = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0);
+ Assert.assertEquals(record.getLong(0), ref0.evaluate(record, null).getValue());
+
+ BeamSqlInputRefExpression ref1 = new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1);
+ Assert.assertEquals(record.getInteger(1), ref1.evaluate(record, null).getValue());
+
+ BeamSqlInputRefExpression ref2 = new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2);
+ Assert.assertEquals(record.getDouble(2), ref2.evaluate(record, null).getValue());
+
+ BeamSqlInputRefExpression ref3 = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3);
+ Assert.assertEquals(record.getLong(3), ref3.evaluate(record, null).getValue());
+ }
+
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testRefOutOfRange(){
+ BeamSqlInputRefExpression ref = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 4);
+ ref.evaluate(record, null).getValue();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testTypeUnMatch(){
+ BeamSqlInputRefExpression ref = new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 0);
+ ref.evaluate(record, null).getValue();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitiveTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitiveTest.java
new file mode 100644
index 0000000..81f9ce0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitiveTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlPrimitive}.
+ *
+ */
+public class BeamSqlPrimitiveTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testPrimitiveInt(){
+ BeamSqlPrimitive<Integer> expInt = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100);
+ Assert.assertEquals(expInt.getValue(), expInt.evaluate(record, null).getValue());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrimitiveTypeUnMatch1(){
+ BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100L);
+ Assert.assertEquals(expInt.getValue(), expInt.evaluate(record, null).getValue());
+ }
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrimitiveTypeUnMatch2(){
+ BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.DECIMAL, 100L);
+ Assert.assertEquals(expInt.getValue(), expInt.evaluate(record, null).getValue());
+ }
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrimitiveTypeUnMatch3(){
+ BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.FLOAT, 100L);
+ Assert.assertEquals(expInt.getValue(), expInt.evaluate(record, null).getValue());
+ }
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrimitiveTypeUnMatch4(){
+ BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 100L);
+ Assert.assertEquals(expInt.getValue(), expInt.evaluate(record, null).getValue());
+ }
+
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlReinterpretExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlReinterpretExpressionTest.java
new file mode 100644
index 0000000..3d7b8ad
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlReinterpretExpressionTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.GregorianCalendar;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret.BeamSqlReinterpretExpression;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlReinterpretExpression}.
+ */
+public class BeamSqlReinterpretExpressionTest extends BeamSqlFnExecutorTestBase {
+ private static final long DATE_LONG = 1000L;
+ private static final Date DATE = new Date(DATE_LONG);
+ private static final GregorianCalendar CALENDAR = new GregorianCalendar(2017, 8, 9);
+
+ private static final BeamRecord NULL_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+
+ private static final BeamSqlExpression DATE_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.DATE, DATE);
+
+ private static final BeamSqlExpression TIME_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.TIME, CALENDAR);
+
+ private static final BeamSqlExpression TIMESTAMP_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.TIMESTAMP, DATE);
+
+ private static final BeamSqlExpression TINYINT_PRIMITIVE_5 = BeamSqlPrimitive.of(
+ SqlTypeName.TINYINT, (byte) 5);
+
+ private static final BeamSqlExpression SMALLINT_PRIMITIVE_6 = BeamSqlPrimitive.of(
+ SqlTypeName.SMALLINT, (short) 6);
+
+ private static final BeamSqlExpression INTEGER_PRIMITIVE_8 = BeamSqlPrimitive.of(
+ SqlTypeName.INTEGER, 8);
+
+ private static final BeamSqlExpression BIGINT_PRIMITIVE_15 = BeamSqlPrimitive.of(
+ SqlTypeName.BIGINT, 15L);
+
+ private static final BeamSqlExpression VARCHAR_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.VARCHAR, "hello");
+
+ @Test
+ public void testAcceptsDateTypes() throws Exception {
+ assertTrue(reinterpretExpression(DATE_PRIMITIVE).accept());
+ assertTrue(reinterpretExpression(TIMESTAMP_PRIMITIVE).accept());
+ }
+
+ @Test
+ public void testAcceptsTime() {
+ assertTrue(reinterpretExpression(TIME_PRIMITIVE).accept());
+ }
+
+ @Test
+ public void testAcceptsIntTypes() {
+ assertTrue(reinterpretExpression(TINYINT_PRIMITIVE_5).accept());
+ assertTrue(reinterpretExpression(SMALLINT_PRIMITIVE_6).accept());
+ assertTrue(reinterpretExpression(INTEGER_PRIMITIVE_8).accept());
+ assertTrue(reinterpretExpression(BIGINT_PRIMITIVE_15).accept());
+ }
+
+ @Test
+ public void testDoesNotAcceptUnsupportedType() {
+ assertFalse(reinterpretExpression(VARCHAR_PRIMITIVE).accept());
+ }
+
+ @Test
+ public void testHasCorrectOutputType() {
+ BeamSqlReinterpretExpression reinterpretExpression1 =
+ new BeamSqlReinterpretExpression(Arrays.asList(DATE_PRIMITIVE), SqlTypeName.BIGINT);
+ assertEquals(SqlTypeName.BIGINT, reinterpretExpression1.getOutputType());
+
+ BeamSqlReinterpretExpression reinterpretExpression2 =
+ new BeamSqlReinterpretExpression(Arrays.asList(DATE_PRIMITIVE), SqlTypeName.INTERVAL_YEAR);
+ assertEquals(SqlTypeName.INTERVAL_YEAR, reinterpretExpression2.getOutputType());
+ }
+
+ @Test
+ public void evaluateDate() {
+ assertEquals(DATE_LONG, evaluateReinterpretExpression(DATE_PRIMITIVE));
+ assertEquals(DATE_LONG, evaluateReinterpretExpression(TIMESTAMP_PRIMITIVE));
+ }
+
+ @Test
+ public void evaluateTime() {
+ assertEquals(CALENDAR.getTimeInMillis(), evaluateReinterpretExpression(TIME_PRIMITIVE));
+ }
+
+ @Test
+ public void evaluateInts() {
+ assertEquals(5L, evaluateReinterpretExpression(TINYINT_PRIMITIVE_5));
+ assertEquals(6L, evaluateReinterpretExpression(SMALLINT_PRIMITIVE_6));
+ assertEquals(8L, evaluateReinterpretExpression(INTEGER_PRIMITIVE_8));
+ assertEquals(15L, evaluateReinterpretExpression(BIGINT_PRIMITIVE_15));
+ }
+
+ private static long evaluateReinterpretExpression(BeamSqlExpression operand) {
+ return reinterpretExpression(operand).evaluate(NULL_ROW, NULL_WINDOW).getLong();
+ }
+
+ private static BeamSqlReinterpretExpression reinterpretExpression(
+ BeamSqlExpression... operands) {
+ return new BeamSqlReinterpretExpression(Arrays.asList(operands), SqlTypeName.BIGINT);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpressionTest.java
new file mode 100644
index 0000000..19098a6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpressionTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlUdfExpression.
+ */
+public class BeamSqlUdfExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test
+ public void testUdf() throws NoSuchMethodException, SecurityException {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 10));
+
+ BeamSqlUdfExpression exp = new BeamSqlUdfExpression(
+ UdfFn.class.getMethod("negative", Integer.class), operands, SqlTypeName.INTEGER);
+
+ Assert.assertEquals(-10, exp.evaluate(record, null).getValue());
+ }
+
+ /**
+ * UDF example.
+ */
+ public static final class UdfFn {
+ public static int negative(Integer number) {
+ return number == null ? 0 : 0 - number;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
new file mode 100644
index 0000000..a8d5e43
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamSqlArithmeticExpression}.
+ */
+public class BeamSqlArithmeticExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void testAccept_normal() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // byte, short
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+ assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+ // integer, long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+ // float, double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+ assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+ // varchar
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "1"));
+ assertFalse(new BeamSqlPlusExpression(operands).accept());
+ }
+
+ @Test public void testAccept_exception() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // more than 2 operands
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+ assertFalse(new BeamSqlPlusExpression(operands).accept());
+
+ // boolean
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ assertFalse(new BeamSqlPlusExpression(operands).accept());
+ }
+
+ @Test public void testPlus() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // integer + integer => integer
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(2, new BeamSqlPlusExpression(operands).evaluate(record, null).getValue());
+
+ // integer + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlPlusExpression(operands).evaluate(record, null).getValue());
+
+ // long + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlPlusExpression(operands).evaluate(record, null).getValue());
+
+ // float + long => float
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(Float.valueOf(1.1F + 1),
+ new BeamSqlPlusExpression(operands).evaluate(record, null).getValue());
+
+ // double + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2.1, new BeamSqlPlusExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testMinus() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // integer + integer => long
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(1, new BeamSqlMinusExpression(operands).evaluate(record, null).getValue());
+
+ // integer + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(1L, new BeamSqlMinusExpression(operands).evaluate(record, null).getValue());
+
+ // long + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(1L, new BeamSqlMinusExpression(operands).evaluate(record, null).getValue());
+
+ // float + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2.1F - 1L,
+ new BeamSqlMinusExpression(operands).evaluate(record, null).getValue().floatValue(), 0.1);
+
+ // double + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(1.1, new BeamSqlMinusExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testMultiply() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // integer + integer => integer
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(2, new BeamSqlMultiplyExpression(operands).evaluate(record, null).getValue());
+
+ // integer + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlMultiplyExpression(operands).evaluate(record, null).getValue());
+
+ // long + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlMultiplyExpression(operands).evaluate(record, null).getValue());
+
+ // float + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(Float.valueOf(2.1F * 1L),
+ new BeamSqlMultiplyExpression(operands).evaluate(record, null).getValue());
+
+ // double + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2.1, new BeamSqlMultiplyExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testDivide() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // integer + integer => integer
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(2, new BeamSqlDivideExpression(operands).evaluate(record, null).getValue());
+
+ // integer + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlDivideExpression(operands).evaluate(record, null).getValue());
+
+ // long + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2L, new BeamSqlDivideExpression(operands).evaluate(record, null).getValue());
+
+ // float + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2.1F / 1,
+ new BeamSqlDivideExpression(operands).evaluate(record, null).getValue());
+
+ // double + long => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ assertEquals(2.1, new BeamSqlDivideExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testMod() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // integer + integer => long
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ assertEquals(1, new BeamSqlModExpression(operands).evaluate(record, null).getValue());
+
+ // integer + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ assertEquals(1L, new BeamSqlModExpression(operands).evaluate(record, null).getValue());
+
+ // long + long => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 3L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ assertEquals(1L, new BeamSqlModExpression(operands).evaluate(record, null).getValue());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpressionTest.java
new file mode 100644
index 0000000..bfca720
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpressionTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlCurrentDateExpression.
+ */
+public class BeamSqlCurrentDateExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test
+ public void test() {
+ Assert.assertEquals(
+ SqlTypeName.DATE,
+ new BeamSqlCurrentDateExpression()
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getOutputType()
+ );
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
new file mode 100644
index 0000000..af3cacd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlLocalTimeExpression.
+ */
+public class BeamSqlCurrentTimeExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test
+ public void test() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ assertEquals(SqlTypeName.TIME,
+ new BeamSqlCurrentTimeExpression(operands).evaluate(record, null).getOutputType());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
new file mode 100644
index 0000000..c171e40
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlLocalTimestampExpression.
+ */
+public class BeamSqlCurrentTimestampExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test
+ public void test() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ assertEquals(SqlTypeName.TIMESTAMP,
+ new BeamSqlCurrentTimestampExpression(operands).evaluate(record, null).getOutputType());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java
new file mode 100644
index 0000000..141bbf5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlDateCeilExpression}.
+ */
+public class BeamSqlDateCeilExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DATE,
+ str2DateTime("2017-05-22 09:10:11")));
+ // YEAR
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.YEAR));
+ Assert.assertEquals(str2DateTime("2018-01-01 00:00:00"),
+ new BeamSqlDateCeilExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getDate());
+
+ operands.set(1, BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.MONTH));
+ Assert.assertEquals(str2DateTime("2017-06-01 00:00:00"),
+ new BeamSqlDateCeilExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getDate());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateExpressionTestBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateExpressionTestBase.java
new file mode 100644
index 0000000..cb0b6ec
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateExpressionTestBase.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+
+/**
+ * Base class for all date related expression test.
+ */
+public class BeamSqlDateExpressionTestBase extends BeamSqlFnExecutorTestBase {
+ static long str2LongTime(String dateStr) {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ try {
+ Date date = format.parse(dateStr);
+ return date.getTime();
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static Date str2DateTime(String dateStr) {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ try {
+ format.setTimeZone(TimeZone.getTimeZone("GMT"));
+ Date date = format.parse(dateStr);
+ return date;
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpressionTest.java
new file mode 100644
index 0000000..ede12ce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpressionTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlDateFloorExpression}.
+ */
+public class BeamSqlDateFloorExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DATE,
+ str2DateTime("2017-05-22 09:10:11")));
+ // YEAR
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.YEAR));
+ assertEquals(str2DateTime("2017-01-01 00:00:00"),
+ new BeamSqlDateFloorExpression(operands).evaluate(record, null).getDate());
+ // MONTH
+ operands.set(1, BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.MONTH));
+ assertEquals(str2DateTime("2017-05-01 00:00:00"),
+ new BeamSqlDateFloorExpression(operands).evaluate(record, null).getDate());
+
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpressionTest.java
new file mode 100644
index 0000000..ef837ca
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpressionTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Date;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link BeamSqlDatetimeMinusExpression}.
+ */
+public class BeamSqlDatetimeMinusExpressionTest {
+
+ private static final BeamRecord NULL_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+
+ private static final Date DATE = new Date(329281L);
+ private static final Date DATE_MINUS_2_SEC = new DateTime(DATE).minusSeconds(2).toDate();
+
+ private static final BeamSqlPrimitive TIMESTAMP = BeamSqlPrimitive.of(
+ SqlTypeName.TIMESTAMP, DATE);
+
+ private static final BeamSqlPrimitive TIMESTAMP_MINUS_2_SEC = BeamSqlPrimitive.of(
+ SqlTypeName.TIMESTAMP, DATE_MINUS_2_SEC);
+
+ private static final BeamSqlPrimitive INTERVAL_2_SEC = BeamSqlPrimitive.of(
+ SqlTypeName.INTERVAL_SECOND, TimeUnit.SECOND.multiplier.multiply(new BigDecimal(2)));
+
+ private static final BeamSqlPrimitive STRING = BeamSqlPrimitive.of(
+ SqlTypeName.VARCHAR, "hello");
+
+ private static final BeamSqlPrimitive INTERVAL_3_MONTHS = BeamSqlPrimitive.of(
+ SqlTypeName.INTERVAL_MONTH, TimeUnit.MONTH.multiplier.multiply(new BigDecimal(3)));
+
+ @Test public void testOutputType() {
+ BeamSqlDatetimeMinusExpression minusExpression1 =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC);
+ BeamSqlDatetimeMinusExpression minusExpression2 =
+ minusExpression(SqlTypeName.BIGINT, TIMESTAMP, TIMESTAMP_MINUS_2_SEC);
+
+ assertEquals(SqlTypeName.TIMESTAMP, minusExpression1.getOutputType());
+ assertEquals(SqlTypeName.BIGINT, minusExpression2.getOutputType());
+ }
+
+ @Test public void testAcceptsTimestampMinusTimestamp() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.INTERVAL_SECOND, TIMESTAMP, TIMESTAMP_MINUS_2_SEC);
+
+ assertTrue(minusExpression.accept());
+ }
+
+ @Test public void testAcceptsTimestampMinusInteval() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC);
+
+ assertTrue(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptUnsupportedReturnType() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.BIGINT, TIMESTAMP, INTERVAL_2_SEC);
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptUnsupportedFirstOperand() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, STRING, INTERVAL_2_SEC);
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptUnsupportedSecondOperand() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, STRING);
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testEvaluateTimestampMinusTimestamp() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.INTERVAL_SECOND, TIMESTAMP, TIMESTAMP_MINUS_2_SEC);
+
+ BeamSqlPrimitive subtractionResult = minusExpression.evaluate(NULL_ROW, NULL_WINDOW);
+
+ assertEquals(SqlTypeName.BIGINT, subtractionResult.getOutputType());
+ assertEquals(2L * TimeUnit.SECOND.multiplier.longValue(), subtractionResult.getLong());
+ }
+
+ @Test public void testEvaluateTimestampMinusInteval() {
+ BeamSqlDatetimeMinusExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC);
+
+ BeamSqlPrimitive subtractionResult = minusExpression.evaluate(NULL_ROW, NULL_WINDOW);
+
+ assertEquals(SqlTypeName.TIMESTAMP, subtractionResult.getOutputType());
+ assertEquals(DATE_MINUS_2_SEC, subtractionResult.getDate());
+ }
+
+ private static BeamSqlDatetimeMinusExpression minusExpression(
+ SqlTypeName outputType, BeamSqlExpression ... operands) {
+ return new BeamSqlDatetimeMinusExpression(Arrays.asList(operands), outputType);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpressionTest.java
new file mode 100644
index 0000000..57e709f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpressionTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Date;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Test for {@link BeamSqlDatetimePlusExpression}.
+ */
+public class BeamSqlDatetimePlusExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ private static final BeamRecord NULL_INPUT_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+ private static final Date DATE = str2DateTime("1984-04-19 01:02:03");
+
+ private static final Date DATE_PLUS_15_SECONDS = new DateTime(DATE).plusSeconds(15).toDate();
+ private static final Date DATE_PLUS_10_MINUTES = new DateTime(DATE).plusMinutes(10).toDate();
+ private static final Date DATE_PLUS_7_HOURS = new DateTime(DATE).plusHours(7).toDate();
+ private static final Date DATE_PLUS_3_DAYS = new DateTime(DATE).plusDays(3).toDate();
+ private static final Date DATE_PLUS_2_MONTHS = new DateTime(DATE).plusMonths(2).toDate();
+ private static final Date DATE_PLUS_11_YEARS = new DateTime(DATE).plusYears(11).toDate();
+
+ private static final BeamSqlExpression SQL_INTERVAL_15_SECONDS =
+ interval(SqlTypeName.INTERVAL_SECOND, 15);
+ private static final BeamSqlExpression SQL_INTERVAL_10_MINUTES =
+ interval(SqlTypeName.INTERVAL_MINUTE, 10);
+ private static final BeamSqlExpression SQL_INTERVAL_7_HOURS =
+ interval(SqlTypeName.INTERVAL_HOUR, 7);
+ private static final BeamSqlExpression SQL_INTERVAL_3_DAYS =
+ interval(SqlTypeName.INTERVAL_DAY, 3);
+ private static final BeamSqlExpression SQL_INTERVAL_2_MONTHS =
+ interval(SqlTypeName.INTERVAL_MONTH, 2);
+ private static final BeamSqlExpression SQL_INTERVAL_4_MONTHS =
+ interval(SqlTypeName.INTERVAL_MONTH, 4);
+ private static final BeamSqlExpression SQL_INTERVAL_11_YEARS =
+ interval(SqlTypeName.INTERVAL_YEAR, 11);
+
+ private static final BeamSqlExpression SQL_TIMESTAMP =
+ BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP, DATE);
+
+ @Test public void testHappyPath_outputTypeAndAccept() {
+ BeamSqlExpression plusExpression = dateTimePlus(SQL_TIMESTAMP, SQL_INTERVAL_3_DAYS);
+
+ assertEquals(SqlTypeName.TIMESTAMP, plusExpression.getOutputType());
+ assertTrue(plusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptTreeOperands() {
+ BeamSqlDatetimePlusExpression plusExpression =
+ dateTimePlus(SQL_TIMESTAMP, SQL_INTERVAL_3_DAYS, SQL_INTERVAL_4_MONTHS);
+
+ assertEquals(SqlTypeName.TIMESTAMP, plusExpression.getOutputType());
+ assertFalse(plusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptWithoutTimestampOperand() {
+ BeamSqlDatetimePlusExpression plusExpression =
+ dateTimePlus(SQL_INTERVAL_3_DAYS, SQL_INTERVAL_4_MONTHS);
+
+ assertEquals(SqlTypeName.TIMESTAMP, plusExpression.getOutputType());
+ assertFalse(plusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptWithoutIntervalOperand() {
+ BeamSqlDatetimePlusExpression plusExpression =
+ dateTimePlus(SQL_TIMESTAMP, SQL_TIMESTAMP);
+
+ assertEquals(SqlTypeName.TIMESTAMP, plusExpression.getOutputType());
+ assertFalse(plusExpression.accept());
+ }
+
+ @Test public void testEvaluate() {
+ assertEquals(DATE_PLUS_15_SECONDS, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_15_SECONDS));
+ assertEquals(DATE_PLUS_10_MINUTES, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_10_MINUTES));
+ assertEquals(DATE_PLUS_7_HOURS, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_7_HOURS));
+ assertEquals(DATE_PLUS_3_DAYS, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_3_DAYS));
+ assertEquals(DATE_PLUS_2_MONTHS, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_2_MONTHS));
+ assertEquals(DATE_PLUS_11_YEARS, evalDatetimePlus(SQL_TIMESTAMP, SQL_INTERVAL_11_YEARS));
+ }
+
+ @Test public void testEvaluateThrowsForUnsupportedIntervalType() {
+ thrown.expect(UnsupportedOperationException.class);
+
+ BeamSqlPrimitive unsupportedInterval = BeamSqlPrimitive.of(SqlTypeName.INTERVAL_YEAR_MONTH, 3);
+ evalDatetimePlus(SQL_TIMESTAMP, unsupportedInterval);
+ }
+
+ private static Date evalDatetimePlus(BeamSqlExpression date, BeamSqlExpression interval) {
+ return dateTimePlus(date, interval).evaluate(NULL_INPUT_ROW, NULL_WINDOW).getDate();
+ }
+
+ private static BeamSqlDatetimePlusExpression dateTimePlus(BeamSqlExpression ... operands) {
+ return new BeamSqlDatetimePlusExpression(Arrays.asList(operands));
+ }
+
+ private static BeamSqlExpression interval(SqlTypeName type, int multiplier) {
+ return BeamSqlPrimitive.of(type,
+ timeUnitInternalMultiplier(type)
+ .multiply(new BigDecimal(multiplier)));
+ }
+
+ private static BigDecimal timeUnitInternalMultiplier(final SqlTypeName sqlIntervalType) {
+ switch (sqlIntervalType) {
+ case INTERVAL_SECOND:
+ return TimeUnit.SECOND.multiplier;
+ case INTERVAL_MINUTE:
+ return TimeUnit.MINUTE.multiplier;
+ case INTERVAL_HOUR:
+ return TimeUnit.HOUR.multiplier;
+ case INTERVAL_DAY:
+ return TimeUnit.DAY.multiplier;
+ case INTERVAL_MONTH:
+ return TimeUnit.MONTH.multiplier;
+ case INTERVAL_YEAR:
+ return TimeUnit.YEAR.multiplier;
+ default:
+ throw new IllegalArgumentException("Interval " + sqlIntervalType
+ + " cannot be converted to TimeUnit");
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
new file mode 100644
index 0000000..b03827a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlExtractExpression}.
+ */
+public class BeamSqlExtractExpressionTest extends BeamSqlDateExpressionTestBase {
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ long time = str2LongTime("2017-05-22 16:17:18");
+
+ // YEAR
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.YEAR));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(2017L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // MONTH
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.MONTH));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(5L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // DAY
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DAY));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(22L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // DAY_OF_WEEK
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DOW));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(2L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // DAY_OF_YEAR
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DOY));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(142L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // WEEK
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.WEEK));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(21L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ // QUARTER
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.QUARTER));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+ time));
+ assertEquals(2L,
+ new BeamSqlExtractExpression(operands)
+ .evaluate(BeamSqlFnExecutorTestBase.record, null).getValue());
+
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpressionTest.java
new file mode 100644
index 0000000..0c91f40
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpressionTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.TimeUnitUtils.timeUnitInternalMultiplier;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlIntervalMultiplyExpression.
+ */
+public class BeamSqlIntervalMultiplyExpressionTest {
+ private static final BeamRecord NULL_INPUT_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+ private static final BigDecimal DECIMAL_THREE = new BigDecimal(3);
+ private static final BigDecimal DECIMAL_FOUR = new BigDecimal(4);
+
+ private static final BeamSqlExpression SQL_INTERVAL_DAY =
+ BeamSqlPrimitive.of(SqlTypeName.INTERVAL_DAY, DECIMAL_THREE);
+
+ private static final BeamSqlExpression SQL_INTERVAL_MONTH =
+ BeamSqlPrimitive.of(SqlTypeName.INTERVAL_MONTH, DECIMAL_FOUR);
+
+ private static final BeamSqlExpression SQL_INTEGER_FOUR =
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4);
+
+ private static final BeamSqlExpression SQL_INTEGER_FIVE =
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5);
+
+ @Test public void testHappyPath_outputTypeAndAccept() {
+ BeamSqlExpression multiplyExpression =
+ newMultiplyExpression(SQL_INTERVAL_DAY, SQL_INTEGER_FOUR);
+
+ assertEquals(SqlTypeName.INTERVAL_DAY, multiplyExpression.getOutputType());
+ assertTrue(multiplyExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptTreeOperands() {
+ BeamSqlIntervalMultiplyExpression multiplyExpression =
+ newMultiplyExpression(SQL_INTERVAL_DAY, SQL_INTEGER_FIVE, SQL_INTEGER_FOUR);
+
+ assertEquals(SqlTypeName.INTERVAL_DAY, multiplyExpression.getOutputType());
+ assertFalse(multiplyExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptWithoutIntervalOperand() {
+ BeamSqlIntervalMultiplyExpression multiplyExpression =
+ newMultiplyExpression(SQL_INTEGER_FOUR, SQL_INTEGER_FIVE);
+
+ assertNull(multiplyExpression.getOutputType());
+ assertFalse(multiplyExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptWithoutIntegerOperand() {
+ BeamSqlIntervalMultiplyExpression multiplyExpression =
+ newMultiplyExpression(SQL_INTERVAL_DAY, SQL_INTERVAL_MONTH);
+
+ assertEquals(SqlTypeName.INTERVAL_DAY, multiplyExpression.getOutputType());
+ assertFalse(multiplyExpression.accept());
+ }
+
+ @Test public void testEvaluate_integerOperand() {
+ BeamSqlIntervalMultiplyExpression multiplyExpression =
+ newMultiplyExpression(SQL_INTERVAL_DAY, SQL_INTEGER_FOUR);
+
+ BeamSqlPrimitive multiplicationResult =
+ multiplyExpression.evaluate(NULL_INPUT_ROW, NULL_WINDOW);
+
+ BigDecimal expectedResult =
+ DECIMAL_FOUR.multiply(timeUnitInternalMultiplier(SqlTypeName.INTERVAL_DAY));
+
+ assertEquals(expectedResult, multiplicationResult.getDecimal());
+ assertEquals(SqlTypeName.INTERVAL_DAY, multiplicationResult.getOutputType());
+ }
+
+ private BeamSqlIntervalMultiplyExpression newMultiplyExpression(BeamSqlExpression ... operands) {
+ return new BeamSqlIntervalMultiplyExpression(Arrays.asList(operands));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpressionTest.java
new file mode 100644
index 0000000..5232487
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpressionTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDatetimeMinusExpression.INTERVALS_DURATIONS_TYPES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Unit tests for {@link BeamSqlTimestampMinusIntervalExpression}.
+ */
+public class BeamSqlTimestampMinusIntervalExpressionTest {
+ private static final BeamRecord NULL_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+
+ private static final Date DATE = new Date(329281L);
+ private static final Date DATE_MINUS_2_SEC = new DateTime(DATE).minusSeconds(2).toDate();
+
+ private static final BeamSqlPrimitive TIMESTAMP = BeamSqlPrimitive.of(
+ SqlTypeName.TIMESTAMP, DATE);
+
+ private static final BeamSqlPrimitive INTERVAL_2_SEC = BeamSqlPrimitive.of(
+ SqlTypeName.INTERVAL_SECOND, TimeUnit.SECOND.multiplier.multiply(new BigDecimal(2)));
+
+ private static final BeamSqlPrimitive INTERVAL_3_MONTHS = BeamSqlPrimitive.of(
+ SqlTypeName.INTERVAL_MONTH, TimeUnit.MONTH.multiplier.multiply(new BigDecimal(3)));
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test public void testBasicProperties() {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.INTERVAL_DAY_MINUTE, TIMESTAMP, INTERVAL_3_MONTHS);
+
+ assertEquals(SqlTypeName.INTERVAL_DAY_MINUTE, minusExpression.getOutputType());
+ assertEquals(Arrays.asList(TIMESTAMP, INTERVAL_3_MONTHS), minusExpression.getOperands());
+ }
+
+ @Test public void testAcceptsHappyPath() {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC);
+
+ assertTrue(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptOneOperand() {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP);
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptThreeOperands() {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC, INTERVAL_3_MONTHS);
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptWrongOutputType() {
+ Set<SqlTypeName> unsupportedTypes = new HashSet<>(SqlTypeName.ALL_TYPES);
+ unsupportedTypes.remove(SqlTypeName.TIMESTAMP);
+
+ for (SqlTypeName unsupportedType : unsupportedTypes) {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(unsupportedType, TIMESTAMP, INTERVAL_2_SEC);
+
+ assertFalse(minusExpression.accept());
+ }
+ }
+
+ @Test public void testDoesNotAcceptWrongFirstOperand() {
+ Set<SqlTypeName> unsupportedTypes = new HashSet<>(SqlTypeName.ALL_TYPES);
+ unsupportedTypes.remove(SqlTypeName.TIMESTAMP);
+
+ for (SqlTypeName unsupportedType : unsupportedTypes) {
+ BeamSqlPrimitive unsupportedOperand = mock(BeamSqlPrimitive.class);
+ doReturn(unsupportedType).when(unsupportedOperand).getOutputType();
+
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, unsupportedOperand, INTERVAL_2_SEC);
+
+ assertFalse(minusExpression.accept());
+ }
+ }
+
+ @Test public void testDoesNotAcceptWrongSecondOperand() {
+ Set<SqlTypeName> unsupportedTypes = new HashSet<>(SqlTypeName.ALL_TYPES);
+ unsupportedTypes.removeAll(INTERVALS_DURATIONS_TYPES.keySet());
+
+ for (SqlTypeName unsupportedType : unsupportedTypes) {
+ BeamSqlPrimitive unsupportedOperand = mock(BeamSqlPrimitive.class);
+ doReturn(unsupportedType).when(unsupportedOperand).getOutputType();
+
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, unsupportedOperand);
+
+ assertFalse(minusExpression.accept());
+ }
+ }
+
+ @Test public void testAcceptsAllSupportedIntervalTypes() {
+ for (SqlTypeName unsupportedType : INTERVALS_DURATIONS_TYPES.keySet()) {
+ BeamSqlPrimitive unsupportedOperand = mock(BeamSqlPrimitive.class);
+ doReturn(unsupportedType).when(unsupportedOperand).getOutputType();
+
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, unsupportedOperand);
+
+ assertTrue(minusExpression.accept());
+ }
+ }
+
+ @Test public void testEvaluateHappyPath() {
+ BeamSqlTimestampMinusIntervalExpression minusExpression =
+ minusExpression(SqlTypeName.TIMESTAMP, TIMESTAMP, INTERVAL_2_SEC);
+
+ BeamSqlPrimitive subtractionResult = minusExpression.evaluate(NULL_ROW, NULL_WINDOW);
+
+ assertEquals(SqlTypeName.TIMESTAMP, subtractionResult.getOutputType());
+ assertEquals(DATE_MINUS_2_SEC, subtractionResult.getDate());
+ }
+
+ private static BeamSqlTimestampMinusIntervalExpression minusExpression(
+ SqlTypeName intervalsToCount, BeamSqlExpression... operands) {
+ return new BeamSqlTimestampMinusIntervalExpression(Arrays.asList(operands), intervalsToCount);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpressionTest.java
new file mode 100644
index 0000000..54bf52d
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpressionTest.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.Date;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.DateTime;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Unit tests for {@link BeamSqlTimestampMinusTimestampExpression}.
+ */
+public class BeamSqlTimestampMinusTimestampExpressionTest {
+
+ private static final BeamRecord NULL_ROW = null;
+ private static final BoundedWindow NULL_WINDOW = null;
+
+ private static final Date DATE = new Date(2017, 3, 4, 3, 2, 1);
+ private static final Date DATE_MINUS_2_SEC = new DateTime(DATE).minusSeconds(2).toDate();
+ private static final Date DATE_MINUS_3_MIN = new DateTime(DATE).minusMinutes(3).toDate();
+ private static final Date DATE_MINUS_4_HOURS = new DateTime(DATE).minusHours(4).toDate();
+ private static final Date DATE_MINUS_7_DAYS = new DateTime(DATE).minusDays(7).toDate();
+ private static final Date DATE_MINUS_2_MONTHS = new DateTime(DATE).minusMonths(2).toDate();
+ private static final Date DATE_MINUS_1_YEAR = new DateTime(DATE).minusYears(1).toDate();
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test public void testOutputTypeIsBigint() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE));
+
+ assertEquals(SqlTypeName.BIGINT, minusExpression.getOutputType());
+ }
+
+ @Test public void testAccepts2Timestamps() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE));
+
+ assertTrue(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAccept3Timestamps() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE_MINUS_1_YEAR),
+ timestamp(DATE));
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAccept1Timestamp() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE));
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptUnsupportedIntervalToCount() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY_MINUTE,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE));
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptNotTimestampAsOperandOne() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3),
+ timestamp(DATE));
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testDoesNotAcceptNotTimestampAsOperandTwo() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE),
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+
+ assertFalse(minusExpression.accept());
+ }
+
+ @Test public void testEvaluateDiffSeconds() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_SECOND,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_2_SEC));
+
+ long expectedResult = applyMultiplier(2L, TimeUnit.SECOND);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateDiffMinutes() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_MINUTE,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_3_MIN));
+
+ long expectedResult = applyMultiplier(3L, TimeUnit.MINUTE);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateDiffHours() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_HOUR,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_4_HOURS));
+
+ long expectedResult = applyMultiplier(4L, TimeUnit.HOUR);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateDiffDays() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_7_DAYS));
+
+ long expectedResult = applyMultiplier(7L, TimeUnit.DAY);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateDiffMonths() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_MONTH,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_2_MONTHS));
+
+ long expectedResult = applyMultiplier(2L, TimeUnit.MONTH);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateDiffYears() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_YEAR,
+ timestamp(DATE),
+ timestamp(DATE_MINUS_1_YEAR));
+
+ long expectedResult = applyMultiplier(1L, TimeUnit.YEAR);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateNegativeDiffSeconds() {
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_SECOND,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE));
+
+ long expectedResult = applyMultiplier(-2L, TimeUnit.SECOND);
+ assertEquals(expectedResult, eval(minusExpression));
+ }
+
+ @Test public void testEvaluateThrowsForUnsupportedIntervalType() {
+
+ thrown.expect(IllegalArgumentException.class);
+
+ BeamSqlTimestampMinusTimestampExpression minusExpression =
+ minusExpression(
+ SqlTypeName.INTERVAL_DAY_MINUTE,
+ timestamp(DATE_MINUS_2_SEC),
+ timestamp(DATE));
+
+ eval(minusExpression);
+ }
+
+ private static BeamSqlTimestampMinusTimestampExpression minusExpression(
+ SqlTypeName intervalsToCount, BeamSqlExpression... operands) {
+ return new BeamSqlTimestampMinusTimestampExpression(Arrays.asList(operands), intervalsToCount);
+ }
+
+ private BeamSqlExpression timestamp(Date date) {
+ return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP, date);
+ }
+
+ private long eval(BeamSqlTimestampMinusTimestampExpression minusExpression) {
+ return minusExpression.evaluate(NULL_ROW, NULL_WINDOW).getLong();
+ }
+
+ private long applyMultiplier(long value, TimeUnit timeUnit) {
+ return value * timeUnit.multiplier.longValue();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtilsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtilsTest.java
new file mode 100644
index 0000000..91552ae
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtilsTest.java
@@ -0,0 +1,54 @@
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit tests for {@link TimeUnitUtils}.
+ */
+public class TimeUnitUtilsTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test public void testReturnsInternalTimeUnitMultipliers() {
+ assertEquals(TimeUnit.SECOND.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_SECOND));
+ assertEquals(TimeUnit.MINUTE.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_MINUTE));
+ assertEquals(TimeUnit.HOUR.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_HOUR));
+ assertEquals(TimeUnit.DAY.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_DAY));
+ assertEquals(TimeUnit.MONTH.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_MONTH));
+ assertEquals(TimeUnit.YEAR.multiplier,
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_YEAR));
+ }
+
+ @Test public void testThrowsForUnsupportedIntervalType() {
+ thrown.expect(IllegalArgumentException.class);
+ TimeUnitUtils.timeUnitInternalMultiplier(SqlTypeName.INTERVAL_DAY_MINUTE);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpressionTest.java
new file mode 100644
index 0000000..c98ce23
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpressionTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlNotExpression}.
+ */
+public class BeamSqlNotExpressionTest extends BeamSqlFnExecutorTestBase {
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+ Assert.assertTrue(new BeamSqlNotExpression(operands).evaluate(record, null).getBoolean());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+ Assert.assertFalse(new BeamSqlNotExpression(operands).evaluate(record, null).getBoolean());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, null));
+ Assert.assertNull(new BeamSqlNotExpression(operands).evaluate(record, null).getValue());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpressionTest.java
new file mode 100644
index 0000000..6665253
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpressionTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for {@link BeamSqlMathBinaryExpression}.
+ */
+public class BeamSqlMathBinaryExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void testForGreaterThanTwoOperands() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // operands more than 2 not allowed
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+ Assert.assertFalse(new BeamSqlRoundExpression(operands).accept());
+ }
+
+ @Test public void testForOneOperand() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // only one operand allowed in round function
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ }
+
+ @Test public void testForOperandsType() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // varchar operand not allowed
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+ Assert.assertFalse(new BeamSqlRoundExpression(operands).accept());
+ }
+
+ @Test public void testRoundFunction() {
+ // test round functions with operands of type bigint, int,
+ // tinyint, smallint, double, decimal
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ // round(double, double) => double
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.0));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 4.0));
+ Assert.assertEquals(2.0,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+ // round(integer,integer) => integer
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ Assert.assertEquals(2, new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // round(long,long) => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 5L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 3L));
+ Assert.assertEquals(5L, new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // round(short) => short
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, new Short("4")));
+ Assert.assertEquals(SqlFunctions.toShort(4),
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // round(long,long) => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ Assert.assertEquals(2L, new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // round(double, long) => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ Assert.assertEquals(1.1,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.368768));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ Assert.assertEquals(2.37,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 3.78683686458));
+ Assert.assertEquals(4.0,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 378.683686458));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -2));
+ Assert.assertEquals(400.0,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 378.683686458));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1));
+ Assert.assertEquals(380.0,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // round(integer, double) => integer
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.2));
+ Assert.assertEquals(2, new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+
+ // operand with a BeamSqlInputRefExpression
+ // to select a column value from row of a record
+ operands.clear();
+ BeamSqlInputRefExpression ref0 = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0);
+ operands.add(ref0);
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+
+ Assert.assertEquals(1234567L,
+ new BeamSqlRoundExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testPowerFunction() {
+ // test power functions with operands of type bigint, int,
+ // tinyint, smallint, double, decimal
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.0));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 4.0));
+ Assert.assertEquals(16.0,
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+ // power(integer,integer) => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ Assert.assertEquals(4.0,
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+ // power(integer,long) => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 3L));
+ Assert.assertEquals(8.0
+ , new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+
+ // power(long,long) => long
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+ Assert.assertEquals(4.0,
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+
+ // power(double, int) => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ Assert.assertEquals(1.1,
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+
+ // power(double, long) => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+ Assert.assertEquals(1.1,
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+
+ // power(integer, double) => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.2));
+ Assert.assertEquals(Math.pow(2, 2.2),
+ new BeamSqlPowerExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForTruncate() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.0));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 4.0));
+ Assert.assertEquals(2.0,
+ new BeamSqlTruncateExpression(operands).evaluate(record, null).getValue());
+ // truncate(double, integer) => double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.80685));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+ Assert.assertEquals(2.8068,
+ new BeamSqlTruncateExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForAtan2() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.875));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.56));
+ Assert.assertEquals(Math.atan2(0.875, 0.56),
+ new BeamSqlAtan2Expression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpressionTest.java
new file mode 100644
index 0000000..d80a670
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpressionTest.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for {@link BeamSqlMathUnaryExpression}.
+ */
+public class BeamSqlMathUnaryExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void testForGreaterThanOneOperands() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // operands more than 1 not allowed
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+ Assert.assertFalse(new BeamSqlAbsExpression(operands).accept());
+ }
+
+ @Test public void testForOperandsType() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // varchar operand not allowed
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "2"));
+ Assert.assertFalse(new BeamSqlAbsExpression(operands).accept());
+ }
+
+ @Test public void testForUnaryExpressions() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for sqrt function
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+
+ // test for abs function
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, -28965734597L));
+ Assert.assertEquals(28965734597L,
+ new BeamSqlAbsExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForLnExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for LN function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.log(2),
+ new BeamSqlLnExpression(operands).evaluate(record, null).getValue());
+
+ // test for LN function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert
+ .assertEquals(Math.log(2.4),
+ new BeamSqlLnExpression(operands).evaluate(record, null).getValue());
+ // test for LN function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.log(2.56),
+ new BeamSqlLnExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForLog10Expression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for log10 function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.log10(2),
+ new BeamSqlLogExpression(operands).evaluate(record, null).getValue());
+ // test for log10 function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.log10(2.4),
+ new BeamSqlLogExpression(operands).evaluate(record, null).getValue());
+ // test for log10 function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.log10(2.56),
+ new BeamSqlLogExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForExpExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.exp(2),
+ new BeamSqlExpExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.exp(2.4),
+ new BeamSqlExpExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.exp(2.56),
+ new BeamSqlExpExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForAcosExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Double.NaN,
+ new BeamSqlAcosExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.45));
+ Assert.assertEquals(Math.acos(0.45),
+ new BeamSqlAcosExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(-0.367)));
+ Assert.assertEquals(Math.acos(-0.367),
+ new BeamSqlAcosExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForAsinExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type double
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.45));
+ Assert.assertEquals(Math.asin(0.45),
+ new BeamSqlAsinExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(-0.367)));
+ Assert.assertEquals(Math.asin(-0.367),
+ new BeamSqlAsinExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForAtanExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type double
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.45));
+ Assert.assertEquals(Math.atan(0.45),
+ new BeamSqlAtanExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(-0.367)));
+ Assert.assertEquals(Math.atan(-0.367),
+ new BeamSqlAtanExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForCosExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type double
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 0.45));
+ Assert.assertEquals(Math.cos(0.45),
+ new BeamSqlCosExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(-0.367)));
+ Assert.assertEquals(Math.cos(-0.367),
+ new BeamSqlCosExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForCotExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type double
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, .45));
+ Assert.assertEquals(1.0d / Math.tan(0.45),
+ new BeamSqlCotExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(-.367)));
+ Assert.assertEquals(1.0d / Math.tan(-0.367),
+ new BeamSqlCotExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForDegreesExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.toDegrees(2),
+ new BeamSqlDegreesExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.toDegrees(2.4),
+ new BeamSqlDegreesExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.toDegrees(2.56),
+ new BeamSqlDegreesExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForRadiansExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.toRadians(2),
+ new BeamSqlRadiansExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.toRadians(2.4),
+ new BeamSqlRadiansExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.toRadians(2.56),
+ new BeamSqlRadiansExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForSinExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.sin(2),
+ new BeamSqlSinExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.sin(2.4),
+ new BeamSqlSinExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.sin(2.56),
+ new BeamSqlSinExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForTanExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals(Math.tan(2),
+ new BeamSqlTanExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(Math.tan(2.4),
+ new BeamSqlTanExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(Math.tan(2.56),
+ new BeamSqlTanExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForSignExpression() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ // test for exp function with operand type smallint
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.valueOf("2")));
+ Assert.assertEquals((short) 1
+ , new BeamSqlSignExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type double
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.4));
+ Assert.assertEquals(1.0, new BeamSqlSignExpression(operands).evaluate(record, null).getValue());
+ // test for exp function with operand type decimal
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DECIMAL, BigDecimal.valueOf(2.56)));
+ Assert.assertEquals(BigDecimal.ONE,
+ new BeamSqlSignExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForPi() {
+ Assert.assertEquals(Math.PI, new BeamSqlPiExpression().evaluate(record, null).getValue());
+ }
+
+ @Test public void testForCeil() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.68687979));
+ Assert.assertEquals(Math.ceil(2.68687979),
+ new BeamSqlCeilExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void testForFloor() {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.68687979));
+ Assert.assertEquals(Math.floor(2.68687979),
+ new BeamSqlFloorExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversionsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversionsTest.java
new file mode 100644
index 0000000..894d094
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversionsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.GregorianCalendar;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Unit test for {@link DatetimeReinterpretConversions}.
+ */
+public class DatetimeReinterpretConversionsTest {
+ private static final long DATE_LONG = 1000L;
+ private static final Date DATE = new Date(DATE_LONG);
+ private static final GregorianCalendar CALENDAR = new GregorianCalendar(2017, 8, 9);
+
+ private static final BeamSqlPrimitive DATE_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.DATE, DATE);
+
+ private static final BeamSqlPrimitive TIME_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.TIME, CALENDAR);
+
+ private static final BeamSqlPrimitive TIMESTAMP_PRIMITIVE = BeamSqlPrimitive.of(
+ SqlTypeName.TIMESTAMP, DATE);
+
+ @Test public void testTimeToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ DatetimeReinterpretConversions.TIME_TO_BIGINT
+ .convert(TIME_PRIMITIVE);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(CALENDAR.getTimeInMillis(), conversionResultPrimitive.getLong());
+ }
+
+ @Test public void testDateToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ DatetimeReinterpretConversions.DATE_TYPES_TO_BIGINT
+ .convert(DATE_PRIMITIVE);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(DATE_LONG, conversionResultPrimitive.getLong());
+ }
+
+ @Test public void testTimestampToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ DatetimeReinterpretConversions.DATE_TYPES_TO_BIGINT
+ .convert(TIMESTAMP_PRIMITIVE);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(DATE_LONG, conversionResultPrimitive.getLong());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversionsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversionsTest.java
new file mode 100644
index 0000000..2bf14f6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversionsTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+
+/**
+ * Unit tests for {@link IntegerReinterpretConversions}.
+ */
+
+public class IntegerReinterpretConversionsTest {
+
+ private static final BeamSqlPrimitive TINYINT_PRIMITIVE_5 = BeamSqlPrimitive.of(
+ SqlTypeName.TINYINT, (byte) 5);
+
+ private static final BeamSqlPrimitive SMALLINT_PRIMITIVE_6 = BeamSqlPrimitive.of(
+ SqlTypeName.SMALLINT, (short) 6);
+
+ private static final BeamSqlPrimitive INTEGER_PRIMITIVE_8 = BeamSqlPrimitive.of(
+ SqlTypeName.INTEGER, 8);
+
+ private static final BeamSqlPrimitive BIGINT_PRIMITIVE_15 = BeamSqlPrimitive.of(
+ SqlTypeName.BIGINT, 15L);
+
+ @Test public void testTinyIntToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ IntegerReinterpretConversions.INTEGER_TYPES_TO_BIGINT
+ .convert(TINYINT_PRIMITIVE_5);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(5L, conversionResultPrimitive.getLong());
+ }
+
+ @Test public void testSmallIntToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ IntegerReinterpretConversions.INTEGER_TYPES_TO_BIGINT
+ .convert(SMALLINT_PRIMITIVE_6);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(6L, conversionResultPrimitive.getLong());
+ }
+
+ @Test public void testIntegerToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ IntegerReinterpretConversions.INTEGER_TYPES_TO_BIGINT
+ .convert(INTEGER_PRIMITIVE_8);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(8L, conversionResultPrimitive.getLong());
+ }
+
+ @Test public void testBigintToBigint() {
+ BeamSqlPrimitive conversionResultPrimitive =
+ IntegerReinterpretConversions.INTEGER_TYPES_TO_BIGINT
+ .convert(BIGINT_PRIMITIVE_15);
+
+ assertEquals(SqlTypeName.BIGINT, conversionResultPrimitive.getOutputType());
+ assertEquals(15L, conversionResultPrimitive.getLong());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversionTest.java
new file mode 100644
index 0000000..31cdab8
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversionTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+/**
+ * Unit test for {@link ReinterpretConversion}.
+ */
+public class ReinterpretConversionTest {
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test public void testNewInstanceProperties() {
+ Set<SqlTypeName> from = ImmutableSet.of(SqlTypeName.FLOAT, SqlTypeName.TIME);
+ SqlTypeName to = SqlTypeName.BOOLEAN;
+ Function<BeamSqlPrimitive, BeamSqlPrimitive> mockConversionFunction = mock(Function.class);
+
+ ReinterpretConversion conversion = ReinterpretConversion.builder()
+ .from(from)
+ .to(to)
+ .convert(mockConversionFunction)
+ .build();
+
+ assertEquals(from, conversion.from());
+ assertEquals(to, conversion.to());
+ }
+
+ @Test public void testConvert() {
+ BeamSqlPrimitive integerPrimitive = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3);
+ BeamSqlPrimitive booleanPrimitive = BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true);
+
+ Function<BeamSqlPrimitive, BeamSqlPrimitive> mockConversionFunction = mock(Function.class);
+ doReturn(booleanPrimitive).when(mockConversionFunction).apply(same(integerPrimitive));
+
+ ReinterpretConversion conversion = ReinterpretConversion.builder()
+ .from(SqlTypeName.INTEGER)
+ .to(SqlTypeName.BOOLEAN)
+ .convert(mockConversionFunction)
+ .build();
+
+ BeamSqlPrimitive conversionResult = conversion.convert(integerPrimitive);
+
+ assertSame(booleanPrimitive, conversionResult);
+ verify(mockConversionFunction).apply(same(integerPrimitive));
+ }
+
+ @Test public void testBuilderThrowsWithoutFrom() {
+ thrown.expect(IllegalArgumentException.class);
+ ReinterpretConversion.builder()
+ .to(SqlTypeName.BOOLEAN)
+ .convert(mock(Function.class))
+ .build();
+ }
+
+ @Test public void testBuilderThrowsWihtoutTo() {
+ thrown.expect(IllegalArgumentException.class);
+ ReinterpretConversion.builder()
+ .from(SqlTypeName.BOOLEAN)
+ .convert(mock(Function.class))
+ .build();
+ }
+
+ @Test public void testBuilderThrowsWihtoutConversionFunction() {
+ thrown.expect(IllegalArgumentException.class);
+ ReinterpretConversion.builder()
+ .from(SqlTypeName.BOOLEAN)
+ .to(SqlTypeName.SMALLINT)
+ .build();
+ }
+
+ @Test public void testConvertThrowsForUnsupportedInput() {
+ thrown.expect(IllegalArgumentException.class);
+
+ ReinterpretConversion conversion = ReinterpretConversion.builder()
+ .from(SqlTypeName.DATE)
+ .to(SqlTypeName.BOOLEAN)
+ .convert(mock(Function.class))
+ .build();
+
+ conversion.convert(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpreterTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpreterTest.java
new file mode 100644
index 0000000..6406831
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpreterTest.java
@@ -0,0 +1,155 @@
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.reinterpret;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.internal.util.collections.Sets;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit tests for {@link Reinterpreter}.
+ */
+public class ReinterpreterTest {
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test public void testBuilderCreatesInstance() {
+ Reinterpreter reinterpreter = newReinterpreter();
+ assertNotNull(reinterpreter);
+ }
+
+ @Test public void testBuilderThrowsWithoutConverters() {
+ thrown.expect(IllegalArgumentException.class);
+ Reinterpreter.builder().build();
+ }
+
+ @Test public void testCanConvertBetweenSupportedTypes() {
+ Reinterpreter reinterpreter = Reinterpreter.builder()
+ .withConversion(mockConversion(SqlTypeName.SYMBOL, SqlTypeName.SMALLINT, SqlTypeName.DATE))
+ .withConversion(mockConversion(SqlTypeName.INTEGER, SqlTypeName.FLOAT))
+ .build();
+
+ assertTrue(reinterpreter.canConvert(SqlTypeName.SMALLINT, SqlTypeName.SYMBOL));
+ assertTrue(reinterpreter.canConvert(SqlTypeName.DATE, SqlTypeName.SYMBOL));
+ assertTrue(reinterpreter.canConvert(SqlTypeName.FLOAT, SqlTypeName.INTEGER));
+ }
+
+ @Test public void testCannotConvertFromUnsupportedTypes() {
+ Reinterpreter reinterpreter = Reinterpreter.builder()
+ .withConversion(mockConversion(SqlTypeName.SYMBOL, SqlTypeName.SMALLINT, SqlTypeName.DATE))
+ .withConversion(mockConversion(SqlTypeName.INTEGER, SqlTypeName.FLOAT))
+ .build();
+
+ Set<SqlTypeName> unsupportedTypes = new HashSet<>(SqlTypeName.ALL_TYPES);
+ unsupportedTypes.removeAll(
+ Sets.newSet(SqlTypeName.DATE, SqlTypeName.SMALLINT, SqlTypeName.FLOAT));
+
+ for (SqlTypeName unsupportedType : unsupportedTypes) {
+ assertFalse(reinterpreter.canConvert(unsupportedType, SqlTypeName.DATE));
+ assertFalse(reinterpreter.canConvert(unsupportedType, SqlTypeName.INTEGER));
+ }
+ }
+
+ @Test public void testCannotConvertToUnsupportedTypes() {
+ Reinterpreter reinterpreter = Reinterpreter.builder()
+ .withConversion(mockConversion(SqlTypeName.SYMBOL, SqlTypeName.SMALLINT, SqlTypeName.DATE))
+ .withConversion(mockConversion(SqlTypeName.INTEGER, SqlTypeName.FLOAT))
+ .build();
+
+ Set<SqlTypeName> unsupportedTypes = new HashSet<>(SqlTypeName.ALL_TYPES);
+ unsupportedTypes.removeAll(Sets.newSet(SqlTypeName.SYMBOL, SqlTypeName.INTEGER));
+
+ for (SqlTypeName unsupportedType : unsupportedTypes) {
+ assertFalse(reinterpreter.canConvert(SqlTypeName.SMALLINT, unsupportedType));
+ assertFalse(reinterpreter.canConvert(SqlTypeName.DATE, unsupportedType));
+ assertFalse(reinterpreter.canConvert(SqlTypeName.FLOAT, unsupportedType));
+ }
+ }
+
+ @Test public void testConvert() {
+ Date date = new Date(12345L);
+ BeamSqlPrimitive stringPrimitive = BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello");
+ BeamSqlPrimitive datePrimitive = BeamSqlPrimitive.of(SqlTypeName.DATE, date);
+
+ ReinterpretConversion mockConversion = mock(ReinterpretConversion.class);
+ doReturn(Sets.newSet(SqlTypeName.VARCHAR)).when(mockConversion).from();
+ doReturn(SqlTypeName.DATE).when(mockConversion).to();
+ doReturn(datePrimitive).when(mockConversion).convert(same(stringPrimitive));
+
+ Reinterpreter reinterpreter = Reinterpreter.builder().withConversion(mockConversion).build();
+ BeamSqlPrimitive converted = reinterpreter.convert(SqlTypeName.DATE, stringPrimitive);
+
+ assertSame(datePrimitive, converted);
+ verify(mockConversion).convert(same(stringPrimitive));
+ }
+
+ @Test public void testConvertThrowsForUnsupportedFromType() {
+ thrown.expect(UnsupportedOperationException.class);
+
+ BeamSqlPrimitive intervalPrimitive = BeamSqlPrimitive
+ .of(SqlTypeName.INTERVAL_DAY, new BigDecimal(2));
+
+ Reinterpreter reinterpreter = newReinterpreter();
+ reinterpreter.convert(SqlTypeName.DATE, intervalPrimitive);
+ }
+
+ @Test public void testConvertThrowsForUnsupportedToType() {
+ thrown.expect(UnsupportedOperationException.class);
+
+ BeamSqlPrimitive stringPrimitive = BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello");
+
+ Reinterpreter reinterpreter = newReinterpreter();
+ reinterpreter.convert(SqlTypeName.INTERVAL_DAY, stringPrimitive);
+ }
+
+ private Reinterpreter newReinterpreter() {
+ return Reinterpreter.builder()
+ .withConversion(
+ mockConversion(
+ SqlTypeName.DATE,
+ SqlTypeName.SMALLINT, SqlTypeName.VARCHAR))
+ .build();
+ }
+
+ private ReinterpretConversion mockConversion(SqlTypeName convertTo, SqlTypeName ... convertFrom) {
+ ReinterpretConversion conversion = mock(ReinterpretConversion.class);
+
+ doReturn(Sets.newSet(convertFrom)).when(conversion).from();
+ doReturn(convertTo).when(conversion).to();
+
+ return conversion;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
new file mode 100644
index 0000000..d6c3565
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlCharLengthExpression.
+ */
+public class BeamSqlCharLengthExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertEquals(5,
+ new BeamSqlCharLengthExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpressionTest.java
new file mode 100644
index 0000000..c350fe2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpressionTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlConcatExpression.
+ */
+public class BeamSqlConcatExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertTrue(new BeamSqlConcatExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ assertFalse(new BeamSqlConcatExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertFalse(new BeamSqlConcatExpression(operands).accept());
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " world"));
+ Assert.assertEquals("hello world",
+ new BeamSqlConcatExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
new file mode 100644
index 0000000..7ea83d1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlInitCapExpression.
+ */
+public class BeamSqlInitCapExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello world"));
+ assertEquals("Hello World",
+ new BeamSqlInitCapExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hEllO wOrld"));
+ assertEquals("Hello World",
+ new BeamSqlInitCapExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello world"));
+ assertEquals("Hello World",
+ new BeamSqlInitCapExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpressionTest.java
new file mode 100644
index 0000000..393680c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpressionTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlLowerExpression.
+ */
+public class BeamSqlLowerExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "HELLO"));
+ assertEquals("hello",
+ new BeamSqlLowerExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
new file mode 100644
index 0000000..2b4c0ea
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlOverlayExpression.
+ */
+public class BeamSqlOverlayExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertTrue(new BeamSqlOverlayExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ assertTrue(new BeamSqlOverlayExpression(operands).accept());
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ Assert.assertEquals("w3resou3rce",
+ new BeamSqlOverlayExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+ Assert.assertEquals("w3resou33rce",
+ new BeamSqlOverlayExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+ Assert.assertEquals("w3resou3rce",
+ new BeamSqlOverlayExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 7));
+ Assert.assertEquals("w3resouce",
+ new BeamSqlOverlayExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
new file mode 100644
index 0000000..3b477cc
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlPositionExpression.
+ */
+public class BeamSqlPositionExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ assertTrue(new BeamSqlPositionExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertTrue(new BeamSqlPositionExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ assertFalse(new BeamSqlPositionExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertFalse(new BeamSqlPositionExpression(operands).accept());
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals(-1, new BeamSqlPositionExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
new file mode 100644
index 0000000..b999ca1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlStringUnaryExpression.
+ */
+public class BeamSqlStringUnaryExpressionTest {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertTrue(new BeamSqlCharLengthExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertFalse(new BeamSqlCharLengthExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertFalse(new BeamSqlCharLengthExpression(operands).accept());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
new file mode 100644
index 0000000..b48a8be
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlSubstringExpression.
+ */
+public class BeamSqlSubstringExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertTrue(new BeamSqlSubstringExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ assertTrue(new BeamSqlSubstringExpression(operands).accept());
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ assertEquals("hello",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+ assertEquals("he",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+ assertEquals("hello",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100));
+ assertEquals("hello",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 0));
+ assertEquals("",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1));
+ assertEquals("",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1));
+ assertEquals("o",
+ new BeamSqlSubstringExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpressionTest.java
new file mode 100644
index 0000000..3645082
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpressionTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.fun.SqlTrimFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlTrimExpression.
+ */
+public class BeamSqlTrimExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void accept() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+ assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, SqlTrimFunction.Flag.BOTH));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+ assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+ assertFalse(new BeamSqlTrimExpression(operands).accept());
+ }
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, SqlTrimFunction.Flag.LEADING));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+ Assert.assertEquals("__hehe",
+ new BeamSqlTrimExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, SqlTrimFunction.Flag.TRAILING));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+ Assert.assertEquals("hehe__",
+ new BeamSqlTrimExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, SqlTrimFunction.Flag.BOTH));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "__"));
+ Assert.assertEquals("__",
+ new BeamSqlTrimExpression(operands).evaluate(record, null).getValue());
+
+ operands.clear();
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+ Assert.assertEquals("hello",
+ new BeamSqlTrimExpression(operands).evaluate(record, null).getValue());
+ }
+
+ @Test public void leadingTrim() throws Exception {
+ assertEquals("__hehe",
+ BeamSqlTrimExpression.leadingTrim("hehe__hehe", "he"));
+ }
+
+ @Test public void trailingTrim() throws Exception {
+ assertEquals("hehe__",
+ BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"));
+ }
+
+ @Test public void trim() throws Exception {
+ assertEquals("__",
+ BeamSqlTrimExpression.leadingTrim(
+ BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"), "he"
+ ));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpressionTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpressionTest.java
new file mode 100644
index 0000000..41e5a28
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpressionTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlUpperExpression.
+ */
+public class BeamSqlUpperExpressionTest extends BeamSqlFnExecutorTestBase {
+
+ @Test public void evaluate() throws Exception {
+ List<BeamSqlExpression> operands = new ArrayList<>();
+
+ operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+ assertEquals("HELLO",
+ new BeamSqlUpperExpression(operands).evaluate(record, null).getValue());
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParserTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParserTest.java
new file mode 100644
index 0000000..c7c8bf4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParserTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.parser;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableList;
+import java.net.URI;
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.calcite.sql.SqlNode;
+import org.junit.Test;
+
+/**
+ * UnitTest for {@link BeamSqlParser}.
+ */
+public class BeamSqlParserTest {
+ @Test
+ public void testParseCreateTable_full() throws Exception {
+ JSONObject properties = new JSONObject();
+ JSONArray hello = new JSONArray();
+ hello.add("james");
+ hello.add("bond");
+ properties.put("hello", hello);
+
+ Table table = parseTable(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name') \n"
+ + "TYPE 'text' \n"
+ + "COMMENT 'person table' \n"
+ + "LOCATION 'text://home/admin/person'\n"
+ + "TBLPROPERTIES '{\"hello\": [\"james\", \"bond\"]}'"
+ );
+ assertEquals(
+ mockTable("person", "text", "person table", properties),
+ table
+ );
+ }
+
+ @Test(expected = org.apache.beam.sdk.extensions.sql.impl.parser.impl.ParseException.class)
+ public void testParseCreateTable_withoutType() throws Exception {
+ parseTable(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name') \n"
+ + "COMMENT 'person table' \n"
+ + "LOCATION 'text://home/admin/person'\n"
+ + "TBLPROPERTIES '{\"hello\": [\"james\", \"bond\"]}'"
+ );
+ }
+
+ @Test
+ public void testParseCreateTable_withoutTableComment() throws Exception {
+ JSONObject properties = new JSONObject();
+ JSONArray hello = new JSONArray();
+ hello.add("james");
+ hello.add("bond");
+ properties.put("hello", hello);
+
+ Table table = parseTable(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name') \n"
+ + "TYPE 'text' \n"
+ + "LOCATION 'text://home/admin/person'\n"
+ + "TBLPROPERTIES '{\"hello\": [\"james\", \"bond\"]}'"
+ );
+ assertEquals(mockTable("person", "text", null, properties), table);
+ }
+
+ @Test
+ public void testParseCreateTable_withoutTblProperties() throws Exception {
+ Table table = parseTable(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name') \n"
+ + "TYPE 'text' \n"
+ + "COMMENT 'person table' \n"
+ + "LOCATION 'text://home/admin/person'\n"
+ );
+ assertEquals(
+ mockTable("person", "text", "person table", new JSONObject()),
+ table
+ );
+ }
+
+ @Test
+ public void testParseCreateTable_withoutLocation() throws Exception {
+ Table table = parseTable(
+ "create table person (\n"
+ + "id int COMMENT 'id', \n"
+ + "name varchar(31) COMMENT 'name') \n"
+ + "TYPE 'text' \n"
+ + "COMMENT 'person table' \n"
+ );
+
+ assertEquals(
+ mockTable("person", "text", "person table", new JSONObject(), null),
+ table
+ );
+ }
+
+ private Table parseTable(String sql) throws Exception {
+ BeamSqlParser parser = new BeamSqlParser(sql);
+ SqlNode sqlNode = parser.impl().parseSqlStmtEof();
+
+ assertNotNull(sqlNode);
+ assertTrue(sqlNode instanceof SqlCreateTable);
+ SqlCreateTable stmt = (SqlCreateTable) sqlNode;
+ return ParserUtils.convertCreateTableStmtToTable(stmt);
+ }
+
+ private static Table mockTable(String name, String type, String comment, JSONObject properties) {
+ return mockTable(name, type, comment, properties, "text://home/admin/" + name);
+ }
+
+ private static Table mockTable(String name, String type, String comment, JSONObject properties,
+ String location) {
+ URI locationURI = null;
+ if (location != null) {
+ locationURI = URI.create(location);
+ }
+
+ return Table.builder()
+ .name(name)
+ .type(type)
+ .comment(comment)
+ .location(locationURI)
+ .columns(ImmutableList.of(
+ Column.builder()
+ .name("id")
+ .type(Types.INTEGER)
+ .primaryKey(false)
+ .comment("id")
+ .build(),
+ Column.builder()
+ .name("name")
+ .type(Types.VARCHAR)
+ .primaryKey(false)
+ .comment("name")
+ .build()
+ ))
+ .properties(properties)
+ .build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BaseRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BaseRelTest.java
new file mode 100644
index 0000000..906ccfd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BaseRelTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Base class for rel test.
+ */
+public class BaseRelTest {
+ public PCollection<BeamRecord> compilePipeline (
+ String sql, Pipeline pipeline, BeamSqlEnv sqlEnv) throws Exception {
+ return sqlEnv.getPlanner().compileBeamPipeline(sql, pipeline, sqlEnv);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRelTest.java
new file mode 100644
index 0000000..8e41d0a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRelTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamIntersectRel}.
+ */
+public class BeamIntersectRelTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void prepare() {
+ sqlEnv.registerTable("ORDER_DETAILS1",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 1L, 1, 1.0,
+ 2L, 2, 2.0,
+ 4L, 4, 4.0
+ )
+ );
+
+ sqlEnv.registerTable("ORDER_DETAILS2",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 2L, 2, 2.0,
+ 3L, 3, 3.0
+ )
+ );
+ }
+
+ @Test
+ public void testIntersect() throws Exception {
+ String sql = "";
+ sql += "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS1 "
+ + " INTERSECT "
+ + "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS2 ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 2L, 2, 2.0
+ ).getRows());
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testIntersectAll() throws Exception {
+ String sql = "";
+ sql += "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS1 "
+ + " INTERSECT ALL "
+ + "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS2 ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).satisfies(new CheckSize(3));
+
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 1L, 1, 1.0,
+ 2L, 2, 2.0
+ ).getRows());
+
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelBoundedVsBoundedTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelBoundedVsBoundedTest.java
new file mode 100644
index 0000000..e0d691b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelBoundedVsBoundedTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Bounded + Bounded Test for {@code BeamJoinRel}.
+ */
+public class BeamJoinRelBoundedVsBoundedTest extends BaseRelTest {
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ private static final BeamSqlEnv BEAM_SQL_ENV = new BeamSqlEnv();
+
+ public static final MockedBoundedTable ORDER_DETAILS1 =
+ MockedBoundedTable.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price"
+ ).addRows(
+ 1, 2, 3,
+ 2, 3, 3,
+ 3, 4, 5
+ );
+
+ public static final MockedBoundedTable ORDER_DETAILS2 =
+ MockedBoundedTable.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price"
+ ).addRows(
+ 1, 2, 3,
+ 2, 3, 3,
+ 3, 4, 5
+ );
+
+ @BeforeClass
+ public static void prepare() {
+ BEAM_SQL_ENV.registerTable("ORDER_DETAILS1", ORDER_DETAILS1);
+ BEAM_SQL_ENV.registerTable("ORDER_DETAILS2", ORDER_DETAILS2);
+ }
+
+ @Test
+ public void testInnerJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "site_id0",
+ Types.INTEGER, "price0"
+ ).addRows(
+ 2, 3, 3, 1, 2, 3
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testLeftOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " LEFT OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.enableAbandonedNodeEnforcement(false);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "site_id0",
+ Types.INTEGER, "price0"
+ ).addRows(
+ 1, 2, 3, null, null, null,
+ 2, 3, 3, 1, 2, 3,
+ 3, 4, 5, null, null, null
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testRightOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " RIGHT OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "site_id0",
+ Types.INTEGER, "price0"
+ ).addRows(
+ 2, 3, 3, 1, 2, 3,
+ null, null, null, 2, 3, 3,
+ null, null, null, 3, 4, 5
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test
+ public void testFullOuterJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " FULL OUTER JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id=o2.site_id AND o2.price=o1.site_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "site_id0",
+ Types.INTEGER, "price0"
+ ).addRows(
+ 2, 3, 3, 1, 2, 3,
+ 1, 2, 3, null, null, null,
+ 3, 4, 5, null, null, null,
+ null, null, null, 2, 3, 3,
+ null, null, null, 3, 4, 5
+ ).getRows());
+ pipeline.run();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testException_nonEqualJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1"
+ + " JOIN ORDER_DETAILS2 o2"
+ + " on "
+ + " o1.order_id>o2.site_id"
+ ;
+
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testException_crossJoin() throws Exception {
+ String sql =
+ "SELECT * "
+ + "FROM ORDER_DETAILS1 o1, ORDER_DETAILS2 o2";
+
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
new file mode 100644
index 0000000..c5145ec
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import java.util.Date;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSqlOutputToConsoleFn;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.extensions.sql.mock.MockedUnboundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Unbounded + Unbounded Test for {@code BeamJoinRel}.
+ */
+public class BeamJoinRelUnboundedVsBoundedTest extends BaseRelTest {
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ private static final BeamSqlEnv BEAM_SQL_ENV = new BeamSqlEnv();
+ public static final Date FIRST_DATE = new Date(1);
+ public static final Date SECOND_DATE = new Date(1 + 3600 * 1000);
+ public static final Date THIRD_DATE = new Date(1 + 3600 * 1000 + 3600 * 1000 + 1);
+ private static final Duration WINDOW_SIZE = Duration.standardHours(1);
+
+ @BeforeClass
+ public static void prepare() {
+ BEAM_SQL_ENV.registerTable("ORDER_DETAILS", MockedUnboundedTable
+ .of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.TIMESTAMP, "order_time"
+ )
+ .timestampColumnIndex(3)
+ .addRows(
+ Duration.ZERO,
+ 1, 1, 1, FIRST_DATE,
+ 1, 2, 2, FIRST_DATE
+ )
+ .addRows(
+ WINDOW_SIZE.plus(Duration.standardSeconds(1)),
+ 2, 2, 3, SECOND_DATE,
+ 2, 3, 3, SECOND_DATE,
+ // this late data is omitted
+ 1, 2, 3, FIRST_DATE
+ )
+ .addRows(
+ WINDOW_SIZE.plus(WINDOW_SIZE).plus(Duration.standardSeconds(1)),
+ 3, 3, 3, THIRD_DATE,
+ // this late data is omitted
+ 2, 2, 3, SECOND_DATE
+ )
+ );
+
+ BEAM_SQL_ENV.registerTable("ORDER_DETAILS1", MockedBoundedTable
+ .of(Types.INTEGER, "order_id",
+ Types.VARCHAR, "buyer"
+ ).addRows(
+ 1, "james",
+ 2, "bond"
+ ));
+ }
+
+ @Test
+ public void testInnerJoin_unboundedTableOnTheLeftSide() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " JOIN "
+ + " ORDER_DETAILS1 o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.VARCHAR, "buyer"
+ ).addRows(
+ 1, 3, "james",
+ 2, 5, "bond"
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testInnerJoin_boundedTableOnTheLeftSide() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + " ORDER_DETAILS1 o2 "
+ + " JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.VARCHAR, "buyer"
+ ).addRows(
+ 1, 3, "james",
+ 2, 5, "bond"
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testLeftOuterJoin() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " LEFT OUTER JOIN "
+ + " ORDER_DETAILS1 o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ rows.apply(ParDo.of(new BeamSqlOutputToConsoleFn("helloworld")));
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.VARCHAR, "buyer"
+ ).addRows(
+ 1, 3, "james",
+ 2, 5, "bond",
+ 3, 3, null
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testLeftOuterJoinError() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + " ORDER_DETAILS1 o2 "
+ + " LEFT OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+
+ @Test
+ public void testRightOuterJoin() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + " ORDER_DETAILS1 o2 "
+ + " RIGHT OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.VARCHAR, "buyer"
+ ).addRows(
+ 1, 3, "james",
+ 2, 5, "bond",
+ 3, 3, null
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testRightOuterJoinError() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " RIGHT OUTER JOIN "
+ + " ORDER_DETAILS1 o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testFullOuterJoinError() throws Exception {
+ String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
+ + " ORDER_DETAILS1 o2 "
+ + " FULL OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsUnboundedTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsUnboundedTest.java
new file mode 100644
index 0000000..e5470ca
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsUnboundedTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import java.util.Date;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSqlOutputToConsoleFn;
+import org.apache.beam.sdk.extensions.sql.mock.MockedUnboundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Unbounded + Unbounded Test for {@code BeamJoinRel}.
+ */
+public class BeamJoinRelUnboundedVsUnboundedTest extends BaseRelTest {
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ private static final BeamSqlEnv BEAM_SQL_ENV = new BeamSqlEnv();
+ public static final Date FIRST_DATE = new Date(1);
+ public static final Date SECOND_DATE = new Date(1 + 3600 * 1000);
+
+ private static final Duration WINDOW_SIZE = Duration.standardHours(1);
+
+ @BeforeClass
+ public static void prepare() {
+ BEAM_SQL_ENV.registerTable("ORDER_DETAILS", MockedUnboundedTable
+ .of(Types.INTEGER, "order_id",
+ Types.INTEGER, "site_id",
+ Types.INTEGER, "price",
+ Types.TIMESTAMP, "order_time"
+ )
+ .timestampColumnIndex(3)
+ .addRows(
+ Duration.ZERO,
+ 1, 1, 1, FIRST_DATE,
+ 1, 2, 6, FIRST_DATE
+ )
+ .addRows(
+ WINDOW_SIZE.plus(Duration.standardMinutes(1)),
+ 2, 2, 7, SECOND_DATE,
+ 2, 3, 8, SECOND_DATE,
+ // this late record is omitted(First window)
+ 1, 3, 3, FIRST_DATE
+ )
+ .addRows(
+ // this late record is omitted(Second window)
+ WINDOW_SIZE.plus(WINDOW_SIZE).plus(Duration.standardMinutes(1)),
+ 2, 3, 3, SECOND_DATE
+ )
+ );
+ }
+
+ @Test
+ public void testInnerJoin() throws Exception {
+ String sql = "SELECT * FROM "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "sum_site_id0").addRows(
+ 1, 3, 1, 3,
+ 2, 5, 2, 5
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testLeftOuterJoin() throws Exception {
+ String sql = "SELECT * FROM "
+ + "(select site_id as order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY site_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " LEFT OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ // 1, 1 | 1, 3
+ // 2, 2 | NULL, NULL
+ // ---- | -----
+ // 2, 2 | 2, 5
+ // 3, 3 | NULL, NULL
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "sum_site_id0"
+ ).addRows(
+ 1, 1, 1, 3,
+ 2, 2, null, null,
+ 2, 2, 2, 5,
+ 3, 3, null, null
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testRightOuterJoin() throws Exception {
+ String sql = "SELECT * FROM "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " RIGHT OUTER JOIN "
+ + "(select site_id as order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY site_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id",
+ Types.INTEGER, "order_id0",
+ Types.INTEGER, "sum_site_id0"
+ ).addRows(
+ 1, 3, 1, 1,
+ null, null, 2, 2,
+ 2, 5, 2, 2,
+ null, null, 3, 3
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testFullOuterJoin() throws Exception {
+ String sql = "SELECT * FROM "
+ + "(select price as order_id1, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY price, TUMBLE(order_time, INTERVAL '1' HOUR)) o1 "
+ + " FULL OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id , TUMBLE(order_time, INTERVAL '1' HOUR)) o2 "
+ + " on "
+ + " o1.order_id1=o2.order_id"
+ ;
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ rows.apply(ParDo.of(new BeamSqlOutputToConsoleFn("hello")));
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "order_id1",
+ Types.INTEGER, "sum_site_id",
+ Types.INTEGER, "order_id",
+ Types.INTEGER, "sum_site_id0"
+ ).addRows(
+ 1, 1, 1, 3,
+ 6, 2, null, null,
+ 7, 2, null, null,
+ 8, 3, null, null,
+ null, null, 2, 5
+ ).getStringRows()
+ );
+ pipeline.run();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testWindowsMismatch() throws Exception {
+ String sql = "SELECT * FROM "
+ + "(select site_id as order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY site_id, TUMBLE(order_time, INTERVAL '2' HOUR)) o1 "
+ + " LEFT OUTER JOIN "
+ + "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
+ + " GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) o2 "
+ + " on "
+ + " o1.order_id=o2.order_id"
+ ;
+ pipeline.enableAbandonedNodeEnforcement(false);
+ compilePipeline(sql, pipeline, BEAM_SQL_ENV);
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRelTest.java
new file mode 100644
index 0000000..5c4ae2c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRelTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamMinusRel}.
+ */
+public class BeamMinusRelTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void prepare() {
+ sqlEnv.registerTable("ORDER_DETAILS1",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 1L, 1, 1.0,
+ 2L, 2, 2.0,
+ 4L, 4, 4.0,
+ 4L, 4, 4.0
+ )
+ );
+
+ sqlEnv.registerTable("ORDER_DETAILS2",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 2L, 2, 2.0,
+ 3L, 3, 3.0
+ )
+ );
+ }
+
+ @Test
+ public void testExcept() throws Exception {
+ String sql = "";
+ sql += "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS1 "
+ + " EXCEPT "
+ + "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS2 ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 4L, 4, 4.0
+ ).getRows());
+
+ pipeline.run();
+ }
+
+ @Test
+ public void testExceptAll() throws Exception {
+ String sql = "";
+ sql += "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS1 "
+ + " EXCEPT ALL "
+ + "SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS2 ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).satisfies(new CheckSize(2));
+
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 4L, 4, 4.0,
+ 4L, 4, 4.0
+ ).getRows());
+
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBaseTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBaseTest.java
new file mode 100644
index 0000000..cd0297a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBaseTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import java.util.Date;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSetOperatorRelBase}.
+ */
+public class BeamSetOperatorRelBaseTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+ public static final Date THE_DATE = new Date(100000);
+
+ @BeforeClass
+ public static void prepare() {
+ sqlEnv.registerTable("ORDER_DETAILS",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price",
+ Types.TIMESTAMP, "order_time"
+ ).addRows(
+ 1L, 1, 1.0, THE_DATE,
+ 2L, 2, 2.0, THE_DATE
+ )
+ );
+ }
+
+ @Test
+ public void testSameWindow() throws Exception {
+ String sql = "SELECT "
+ + " order_id, site_id, count(*) as cnt "
+ + "FROM ORDER_DETAILS GROUP BY order_id, site_id"
+ + ", TUMBLE(order_time, INTERVAL '1' HOUR) "
+ + " UNION SELECT "
+ + " order_id, site_id, count(*) as cnt "
+ + "FROM ORDER_DETAILS GROUP BY order_id, site_id"
+ + ", TUMBLE(order_time, INTERVAL '1' HOUR) ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ // compare valueInString to ignore the windowStart & windowEnd
+ PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn())))
+ .containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.BIGINT, "cnt"
+ ).addRows(
+ 1L, 1, 1L,
+ 2L, 2, 1L
+ ).getStringRows());
+ pipeline.run();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDifferentWindows() throws Exception {
+ String sql = "SELECT "
+ + " order_id, site_id, count(*) as cnt "
+ + "FROM ORDER_DETAILS GROUP BY order_id, site_id"
+ + ", TUMBLE(order_time, INTERVAL '1' HOUR) "
+ + " UNION SELECT "
+ + " order_id, site_id, count(*) as cnt "
+ + "FROM ORDER_DETAILS GROUP BY order_id, site_id"
+ + ", TUMBLE(order_time, INTERVAL '2' HOUR) ";
+
+ // use a real pipeline rather than the TestPipeline because we are
+ // testing exceptions, the pipeline will not actually run.
+ Pipeline pipeline1 = Pipeline.create(PipelineOptionsFactory.create());
+ compilePipeline(sql, pipeline1, sqlEnv);
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRelTest.java
new file mode 100644
index 0000000..bab5296
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRelTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import java.util.Date;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSortRel}.
+ */
+public class BeamSortRelTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ @Before
+ public void prepare() {
+ sqlEnv.registerTable("ORDER_DETAILS",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price",
+ Types.TIMESTAMP, "order_time"
+ ).addRows(
+ 1L, 2, 1.0, new Date(0),
+ 1L, 1, 2.0, new Date(1),
+ 2L, 4, 3.0, new Date(2),
+ 2L, 1, 4.0, new Date(3),
+ 5L, 5, 5.0, new Date(4),
+ 6L, 6, 6.0, new Date(5),
+ 7L, 7, 7.0, new Date(6),
+ 8L, 8888, 8.0, new Date(7),
+ 8L, 999, 9.0, new Date(8),
+ 10L, 100, 10.0, new Date(9)
+ )
+ );
+ sqlEnv.registerTable("SUB_ORDER_RAM",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ )
+ );
+ }
+
+ @Test
+ public void testOrderBy_basic() throws Exception {
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_id asc, site_id desc limit 4";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 2, 1.0,
+ 1L, 1, 2.0,
+ 2L, 4, 3.0,
+ 2L, 1, 4.0
+ ).getRows());
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testOrderBy_timestamp() throws Exception {
+ String sql = "SELECT order_id, site_id, price, order_time "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_time desc limit 4";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price",
+ Types.TIMESTAMP, "order_time"
+ ).addRows(
+ 7L, 7, 7.0, new Date(6),
+ 8L, 8888, 8.0, new Date(7),
+ 8L, 999, 9.0, new Date(8),
+ 10L, 100, 10.0, new Date(9)
+ ).getRows());
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testOrderBy_nullsFirst() throws Exception {
+ sqlEnv.registerTable("ORDER_DETAILS",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 2, 1.0,
+ 1L, null, 2.0,
+ 2L, 1, 3.0,
+ 2L, null, 4.0,
+ 5L, 5, 5.0
+ )
+ );
+ sqlEnv.registerTable("SUB_ORDER_RAM", MockedBoundedTable
+ .of(Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"));
+
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_id asc, site_id desc NULLS FIRST limit 4";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, null, 2.0,
+ 1L, 2, 1.0,
+ 2L, null, 4.0,
+ 2L, 1, 3.0
+ ).getRows()
+ );
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testOrderBy_nullsLast() throws Exception {
+ sqlEnv.registerTable("ORDER_DETAILS", MockedBoundedTable
+ .of(Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 2, 1.0,
+ 1L, null, 2.0,
+ 2L, 1, 3.0,
+ 2L, null, 4.0,
+ 5L, 5, 5.0));
+ sqlEnv.registerTable("SUB_ORDER_RAM", MockedBoundedTable
+ .of(Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"));
+
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_id asc, site_id desc NULLS LAST limit 4";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 2, 1.0,
+ 1L, null, 2.0,
+ 2L, 1, 3.0,
+ 2L, null, 4.0
+ ).getRows()
+ );
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testOrderBy_with_offset() throws Exception {
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_id asc, site_id desc limit 4 offset 4";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 5L, 5, 5.0,
+ 6L, 6, 6.0,
+ 7L, 7, 7.0,
+ 8L, 8888, 8.0
+ ).getRows()
+ );
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testOrderBy_bigFetch() throws Exception {
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + "ORDER BY order_id asc, site_id desc limit 11";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 2, 1.0,
+ 1L, 1, 2.0,
+ 2L, 4, 3.0,
+ 2L, 1, 4.0,
+ 5L, 5, 5.0,
+ 6L, 6, 6.0,
+ 7L, 7, 7.0,
+ 8L, 8888, 8.0,
+ 8L, 999, 9.0,
+ 10L, 100, 10.0
+ ).getRows()
+ );
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testOrderBy_exception() throws Exception {
+ String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id) SELECT "
+ + " order_id, COUNT(*) "
+ + "FROM ORDER_DETAILS "
+ + "GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)"
+ + "ORDER BY order_id asc limit 11";
+
+ TestPipeline pipeline = TestPipeline.create();
+ compilePipeline(sql, pipeline, sqlEnv);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRelTest.java
new file mode 100644
index 0000000..d79a54e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRelTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamUnionRel}.
+ */
+public class BeamUnionRelTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void prepare() {
+ sqlEnv.registerTable("ORDER_DETAILS",
+ MockedBoundedTable.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 2L, 2, 2.0
+ )
+ );
+ }
+
+ @Test
+ public void testUnion() throws Exception {
+ String sql = "SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS "
+ + " UNION SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS ";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 2L, 2, 2.0
+ ).getRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testUnionAll() throws Exception {
+ String sql = "SELECT "
+ + " order_id, site_id, price "
+ + "FROM ORDER_DETAILS"
+ + " UNION ALL "
+ + " SELECT order_id, site_id, price "
+ + "FROM ORDER_DETAILS";
+
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.BIGINT, "order_id",
+ Types.INTEGER, "site_id",
+ Types.DOUBLE, "price"
+ ).addRows(
+ 1L, 1, 1.0,
+ 1L, 1, 1.0,
+ 2L, 2, 2.0,
+ 2L, 2, 2.0
+ ).getRows()
+ );
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRelTest.java
new file mode 100644
index 0000000..5604e32
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRelTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamValuesRel}.
+ */
+public class BeamValuesRelTest extends BaseRelTest {
+ static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void prepare() {
+ sqlEnv.registerTable("string_table",
+ MockedBoundedTable.of(
+ Types.VARCHAR, "name",
+ Types.VARCHAR, "description"
+ )
+ );
+ sqlEnv.registerTable("int_table",
+ MockedBoundedTable.of(
+ Types.INTEGER, "c0",
+ Types.INTEGER, "c1"
+ )
+ );
+ }
+
+ @Test
+ public void testValues() throws Exception {
+ String sql = "insert into string_table(name, description) values "
+ + "('hello', 'world'), ('james', 'bond')";
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.VARCHAR, "name",
+ Types.VARCHAR, "description"
+ ).addRows(
+ "hello", "world",
+ "james", "bond"
+ ).getRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testValues_castInt() throws Exception {
+ String sql = "insert into int_table (c0, c1) values(cast(1 as int), cast(2 as int))";
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "c0",
+ Types.INTEGER, "c1"
+ ).addRows(
+ 1, 2
+ ).getRows()
+ );
+ pipeline.run();
+ }
+
+ @Test
+ public void testValues_onlySelect() throws Exception {
+ String sql = "select 1, '1'";
+ PCollection<BeamRecord> rows = compilePipeline(sql, pipeline, sqlEnv);
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder.of(
+ Types.INTEGER, "EXPR$0",
+ Types.CHAR, "EXPR$1"
+ ).addRows(
+ 1, "1"
+ ).getRows()
+ );
+ pipeline.run();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/CheckSize.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/CheckSize.java
new file mode 100644
index 0000000..7407a76
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/CheckSize.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.junit.Assert;
+
+/**
+ * Utility class to check size of BeamSQLRow iterable.
+ */
+public class CheckSize implements SerializableFunction<Iterable<BeamRecord>, Void> {
+ private int size;
+ public CheckSize(int size) {
+ this.size = size;
+ }
+ @Override public Void apply(Iterable<BeamRecord> input) {
+ int count = 0;
+ for (BeamRecord row : input) {
+ count++;
+ }
+ Assert.assertEquals(size, count);
+ return null;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamSqlRowCoderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamSqlRowCoderTest.java
new file mode 100644
index 0000000..0a320db
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamSqlRowCoderTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.schema;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.testing.CoderProperties;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for BeamSqlRowCoder.
+ */
+public class BeamSqlRowCoderTest {
+
+ @Test
+ public void encodeAndDecode() throws Exception {
+ final RelProtoDataType protoRowType = new RelProtoDataType() {
+ @Override
+ public RelDataType apply(RelDataTypeFactory a0) {
+ return a0.builder()
+ .add("col_tinyint", SqlTypeName.TINYINT)
+ .add("col_smallint", SqlTypeName.SMALLINT)
+ .add("col_integer", SqlTypeName.INTEGER)
+ .add("col_bigint", SqlTypeName.BIGINT)
+ .add("col_float", SqlTypeName.FLOAT)
+ .add("col_double", SqlTypeName.DOUBLE)
+ .add("col_decimal", SqlTypeName.DECIMAL)
+ .add("col_string_varchar", SqlTypeName.VARCHAR)
+ .add("col_time", SqlTypeName.TIME)
+ .add("col_timestamp", SqlTypeName.TIMESTAMP)
+ .add("col_boolean", SqlTypeName.BOOLEAN)
+ .build();
+ }
+ };
+
+ BeamRecordSqlType beamSQLRowType = CalciteUtils.toBeamRowType(
+ protoRowType.apply(new JavaTypeFactoryImpl(
+ RelDataTypeSystem.DEFAULT)));
+
+ GregorianCalendar calendar = new GregorianCalendar();
+ calendar.setTime(new Date());
+ BeamRecord row = new BeamRecord(beamSQLRowType
+ , Byte.valueOf("1"), Short.valueOf("1"), 1, 1L, 1.1F, 1.1
+ , BigDecimal.ZERO, "hello", calendar, new Date(), true);
+
+
+ BeamRecordCoder coder = beamSQLRowType.getRecordCoder();
+ CoderProperties.coderDecodeEncodeEqual(coder, row);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamAggregationTransformTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamAggregationTransformTest.java
new file mode 100644
index 0000000..948e86c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamAggregationTransformTest.java
@@ -0,0 +1,453 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema.transform;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.transform.BeamAggregationTransforms;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlAvgAggFunction;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
+import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
+import org.apache.calcite.sql.fun.SqlSumAggFunction;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link BeamAggregationTransforms}.
+ *
+ */
+public class BeamAggregationTransformTest extends BeamTransformBaseTest{
+
+ @Rule
+ public TestPipeline p = TestPipeline.create();
+
+ private List<AggregateCall> aggCalls;
+
+ private BeamRecordSqlType keyType;
+ private BeamRecordSqlType aggPartType;
+ private BeamRecordSqlType outputType;
+
+ private BeamRecordCoder inRecordCoder;
+ private BeamRecordCoder keyCoder;
+ private BeamRecordCoder aggCoder;
+ private BeamRecordCoder outRecordCoder;
+
+ /**
+ * This step equals to below query.
+ * <pre>
+ * SELECT `f_int`
+ * , COUNT(*) AS `size`
+ * , SUM(`f_long`) AS `sum1`, AVG(`f_long`) AS `avg1`
+ * , MAX(`f_long`) AS `max1`, MIN(`f_long`) AS `min1`
+ * , SUM(`f_short`) AS `sum2`, AVG(`f_short`) AS `avg2`
+ * , MAX(`f_short`) AS `max2`, MIN(`f_short`) AS `min2`
+ * , SUM(`f_byte`) AS `sum3`, AVG(`f_byte`) AS `avg3`
+ * , MAX(`f_byte`) AS `max3`, MIN(`f_byte`) AS `min3`
+ * , SUM(`f_float`) AS `sum4`, AVG(`f_float`) AS `avg4`
+ * , MAX(`f_float`) AS `max4`, MIN(`f_float`) AS `min4`
+ * , SUM(`f_double`) AS `sum5`, AVG(`f_double`) AS `avg5`
+ * , MAX(`f_double`) AS `max5`, MIN(`f_double`) AS `min5`
+ * , MAX(`f_timestamp`) AS `max7`, MIN(`f_timestamp`) AS `min7`
+ * ,SUM(`f_int2`) AS `sum8`, AVG(`f_int2`) AS `avg8`
+ * , MAX(`f_int2`) AS `max8`, MIN(`f_int2`) AS `min8`
+ * FROM TABLE_NAME
+ * GROUP BY `f_int`
+ * </pre>
+ * @throws ParseException
+ */
+ @Test
+ public void testCountPerElementBasic() throws ParseException {
+ setupEnvironment();
+
+ PCollection<BeamRecord> input = p.apply(Create.of(inputRows));
+
+ //1. extract fields in group-by key part
+ PCollection<KV<BeamRecord, BeamRecord>> exGroupByStream = input.apply("exGroupBy",
+ WithKeys
+ .of(new BeamAggregationTransforms.AggregationGroupByKeyFn(-1, ImmutableBitSet.of(0))))
+ .setCoder(KvCoder.<BeamRecord, BeamRecord>of(keyCoder, inRecordCoder));
+
+ //2. apply a GroupByKey.
+ PCollection<KV<BeamRecord, Iterable<BeamRecord>>> groupedStream = exGroupByStream
+ .apply("groupBy", GroupByKey.<BeamRecord, BeamRecord>create())
+ .setCoder(KvCoder.<BeamRecord, Iterable<BeamRecord>>of(keyCoder,
+ IterableCoder.<BeamRecord>of(inRecordCoder)));
+
+ //3. run aggregation functions
+ PCollection<KV<BeamRecord, BeamRecord>> aggregatedStream = groupedStream.apply("aggregation",
+ Combine.<BeamRecord, BeamRecord, BeamRecord>groupedValues(
+ new BeamAggregationTransforms.AggregationAdaptor(aggCalls, inputRowType)))
+ .setCoder(KvCoder.<BeamRecord, BeamRecord>of(keyCoder, aggCoder));
+
+ //4. flat KV to a single record
+ PCollection<BeamRecord> mergedStream = aggregatedStream.apply("mergeRecord",
+ ParDo.of(new BeamAggregationTransforms.MergeAggregationRecord(outputType, aggCalls, -1)));
+ mergedStream.setCoder(outRecordCoder);
+
+ //assert function BeamAggregationTransform.AggregationGroupByKeyFn
+ PAssert.that(exGroupByStream).containsInAnyOrder(prepareResultOfAggregationGroupByKeyFn());
+
+ //assert BeamAggregationTransform.AggregationCombineFn
+ PAssert.that(aggregatedStream).containsInAnyOrder(prepareResultOfAggregationCombineFn());
+
+ //assert BeamAggregationTransform.MergeAggregationRecord
+ PAssert.that(mergedStream).containsInAnyOrder(prepareResultOfMergeAggregationRecord());
+
+ p.run();
+}
+
+ private void setupEnvironment() {
+ prepareAggregationCalls();
+ prepareTypeAndCoder();
+ }
+
+ /**
+ * create list of all {@link AggregateCall}.
+ */
+ @SuppressWarnings("deprecation")
+ private void prepareAggregationCalls() {
+ //aggregations for all data type
+ aggCalls = new ArrayList<>();
+ aggCalls.add(
+ new AggregateCall(new SqlCountAggFunction(), false,
+ Arrays.<Integer>asList(),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+ "count")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlSumAggFunction(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT)), false,
+ Arrays.<Integer>asList(1),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+ "sum1")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(1),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+ "avg1")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(1),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+ "max1")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(1),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+ "min1")
+ );
+
+ aggCalls.add(
+ new AggregateCall(new SqlSumAggFunction(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT)), false,
+ Arrays.<Integer>asList(2),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+ "sum2")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(2),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+ "avg2")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(2),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+ "max2")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(2),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+ "min2")
+ );
+
+ aggCalls.add(
+ new AggregateCall(
+ new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT)),
+ false,
+ Arrays.<Integer>asList(3),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+ "sum3")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(3),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+ "avg3")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(3),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+ "max3")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(3),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+ "min3")
+ );
+
+ aggCalls.add(
+ new AggregateCall(
+ new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT)),
+ false,
+ Arrays.<Integer>asList(4),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+ "sum4")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(4),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+ "avg4")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(4),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+ "max4")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(4),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+ "min4")
+ );
+
+ aggCalls.add(
+ new AggregateCall(
+ new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE)),
+ false,
+ Arrays.<Integer>asList(5),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+ "sum5")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(5),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+ "avg5")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(5),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+ "max5")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(5),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+ "min5")
+ );
+
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(7),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TIMESTAMP),
+ "max7")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(7),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TIMESTAMP),
+ "min7")
+ );
+
+ aggCalls.add(
+ new AggregateCall(
+ new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER)),
+ false,
+ Arrays.<Integer>asList(8),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+ "sum8")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+ Arrays.<Integer>asList(8),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+ "avg8")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+ Arrays.<Integer>asList(8),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+ "max8")
+ );
+ aggCalls.add(
+ new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+ Arrays.<Integer>asList(8),
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+ "min8")
+ );
+ }
+
+ /**
+ * Coders used in aggregation steps.
+ */
+ private void prepareTypeAndCoder() {
+ inRecordCoder = inputRowType.getRecordCoder();
+
+ keyType = initTypeOfSqlRow(Arrays.asList(KV.of("f_int", SqlTypeName.INTEGER)));
+ keyCoder = keyType.getRecordCoder();
+
+ aggPartType = initTypeOfSqlRow(
+ Arrays.asList(KV.of("count", SqlTypeName.BIGINT),
+
+ KV.of("sum1", SqlTypeName.BIGINT), KV.of("avg1", SqlTypeName.BIGINT),
+ KV.of("max1", SqlTypeName.BIGINT), KV.of("min1", SqlTypeName.BIGINT),
+
+ KV.of("sum2", SqlTypeName.SMALLINT), KV.of("avg2", SqlTypeName.SMALLINT),
+ KV.of("max2", SqlTypeName.SMALLINT), KV.of("min2", SqlTypeName.SMALLINT),
+
+ KV.of("sum3", SqlTypeName.TINYINT), KV.of("avg3", SqlTypeName.TINYINT),
+ KV.of("max3", SqlTypeName.TINYINT), KV.of("min3", SqlTypeName.TINYINT),
+
+ KV.of("sum4", SqlTypeName.FLOAT), KV.of("avg4", SqlTypeName.FLOAT),
+ KV.of("max4", SqlTypeName.FLOAT), KV.of("min4", SqlTypeName.FLOAT),
+
+ KV.of("sum5", SqlTypeName.DOUBLE), KV.of("avg5", SqlTypeName.DOUBLE),
+ KV.of("max5", SqlTypeName.DOUBLE), KV.of("min5", SqlTypeName.DOUBLE),
+
+ KV.of("max7", SqlTypeName.TIMESTAMP), KV.of("min7", SqlTypeName.TIMESTAMP),
+
+ KV.of("sum8", SqlTypeName.INTEGER), KV.of("avg8", SqlTypeName.INTEGER),
+ KV.of("max8", SqlTypeName.INTEGER), KV.of("min8", SqlTypeName.INTEGER)
+ ));
+ aggCoder = aggPartType.getRecordCoder();
+
+ outputType = prepareFinalRowType();
+ outRecordCoder = outputType.getRecordCoder();
+ }
+
+ /**
+ * expected results after {@link BeamAggregationTransforms.AggregationGroupByKeyFn}.
+ */
+ private List<KV<BeamRecord, BeamRecord>> prepareResultOfAggregationGroupByKeyFn() {
+ return Arrays.asList(
+ KV.of(new BeamRecord(keyType, Arrays.<Object>asList(inputRows.get(0).getInteger(0))),
+ inputRows.get(0)),
+ KV.of(new BeamRecord(keyType, Arrays.<Object>asList(inputRows.get(1).getInteger(0))),
+ inputRows.get(1)),
+ KV.of(new BeamRecord(keyType, Arrays.<Object>asList(inputRows.get(2).getInteger(0))),
+ inputRows.get(2)),
+ KV.of(new BeamRecord(keyType, Arrays.<Object>asList(inputRows.get(3).getInteger(0))),
+ inputRows.get(3)));
+ }
+
+ /**
+ * expected results after {@link BeamAggregationTransforms.AggregationCombineFn}.
+ */
+ private List<KV<BeamRecord, BeamRecord>> prepareResultOfAggregationCombineFn()
+ throws ParseException {
+ return Arrays.asList(
+ KV.of(new BeamRecord(keyType, Arrays.<Object>asList(inputRows.get(0).getInteger(0))),
+ new BeamRecord(aggPartType, Arrays.<Object>asList(
+ 4L,
+ 10000L, 2500L, 4000L, 1000L,
+ (short) 10, (short) 2, (short) 4, (short) 1,
+ (byte) 10, (byte) 2, (byte) 4, (byte) 1,
+ 10.0F, 2.5F, 4.0F, 1.0F,
+ 10.0, 2.5, 4.0, 1.0,
+ format.parse("2017-01-01 02:04:03"), format.parse("2017-01-01 01:01:03"),
+ 10, 2, 4, 1
+ )))
+ );
+ }
+
+ /**
+ * Row type of final output row.
+ */
+ private BeamRecordSqlType prepareFinalRowType() {
+ FieldInfoBuilder builder = BeamQueryPlanner.TYPE_FACTORY.builder();
+ List<KV<String, SqlTypeName>> columnMetadata =
+ Arrays.asList(KV.of("f_int", SqlTypeName.INTEGER), KV.of("count", SqlTypeName.BIGINT),
+
+ KV.of("sum1", SqlTypeName.BIGINT), KV.of("avg1", SqlTypeName.BIGINT),
+ KV.of("max1", SqlTypeName.BIGINT), KV.of("min1", SqlTypeName.BIGINT),
+
+ KV.of("sum2", SqlTypeName.SMALLINT), KV.of("avg2", SqlTypeName.SMALLINT),
+ KV.of("max2", SqlTypeName.SMALLINT), KV.of("min2", SqlTypeName.SMALLINT),
+
+ KV.of("sum3", SqlTypeName.TINYINT), KV.of("avg3", SqlTypeName.TINYINT),
+ KV.of("max3", SqlTypeName.TINYINT), KV.of("min3", SqlTypeName.TINYINT),
+
+ KV.of("sum4", SqlTypeName.FLOAT), KV.of("avg4", SqlTypeName.FLOAT),
+ KV.of("max4", SqlTypeName.FLOAT), KV.of("min4", SqlTypeName.FLOAT),
+
+ KV.of("sum5", SqlTypeName.DOUBLE), KV.of("avg5", SqlTypeName.DOUBLE),
+ KV.of("max5", SqlTypeName.DOUBLE), KV.of("min5", SqlTypeName.DOUBLE),
+
+ KV.of("max7", SqlTypeName.TIMESTAMP), KV.of("min7", SqlTypeName.TIMESTAMP),
+
+ KV.of("sum8", SqlTypeName.INTEGER), KV.of("avg8", SqlTypeName.INTEGER),
+ KV.of("max8", SqlTypeName.INTEGER), KV.of("min8", SqlTypeName.INTEGER)
+ );
+ for (KV<String, SqlTypeName> cm : columnMetadata) {
+ builder.add(cm.getKey(), cm.getValue());
+ }
+ return CalciteUtils.toBeamRowType(builder.build());
+ }
+
+ /**
+ * expected results after {@link BeamAggregationTransforms.MergeAggregationRecord}.
+ */
+ private BeamRecord prepareResultOfMergeAggregationRecord() throws ParseException {
+ return new BeamRecord(outputType, Arrays.<Object>asList(
+ 1, 4L,
+ 10000L, 2500L, 4000L, 1000L,
+ (short) 10, (short) 2, (short) 4, (short) 1,
+ (byte) 10, (byte) 2, (byte) 4, (byte) 1,
+ 10.0F, 2.5F, 4.0F, 1.0F,
+ 10.0, 2.5, 4.0, 1.0,
+ format.parse("2017-01-01 02:04:03"), format.parse("2017-01-01 01:01:03"),
+ 10, 2, 4, 1
+ ));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamTransformBaseTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamTransformBaseTest.java
new file mode 100644
index 0000000..3c8f040
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamTransformBaseTest.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema.transform;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.BeforeClass;
+
+/**
+ * shared methods to test PTransforms which execute Beam SQL steps.
+ *
+ */
+public class BeamTransformBaseTest {
+ public static DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+ public static BeamRecordSqlType inputRowType;
+ public static List<BeamRecord> inputRows;
+
+ @BeforeClass
+ public static void prepareInput() throws NumberFormatException, ParseException{
+ List<KV<String, SqlTypeName>> columnMetadata = Arrays.asList(
+ KV.of("f_int", SqlTypeName.INTEGER), KV.of("f_long", SqlTypeName.BIGINT),
+ KV.of("f_short", SqlTypeName.SMALLINT), KV.of("f_byte", SqlTypeName.TINYINT),
+ KV.of("f_float", SqlTypeName.FLOAT), KV.of("f_double", SqlTypeName.DOUBLE),
+ KV.of("f_string", SqlTypeName.VARCHAR), KV.of("f_timestamp", SqlTypeName.TIMESTAMP),
+ KV.of("f_int2", SqlTypeName.INTEGER)
+ );
+ inputRowType = initTypeOfSqlRow(columnMetadata);
+ inputRows = Arrays.asList(
+ initBeamSqlRow(columnMetadata,
+ Arrays.<Object>asList(1, 1000L, Short.valueOf("1"), Byte.valueOf("1"), 1.0F, 1.0,
+ "string_row1", format.parse("2017-01-01 01:01:03"), 1)),
+ initBeamSqlRow(columnMetadata,
+ Arrays.<Object>asList(1, 2000L, Short.valueOf("2"), Byte.valueOf("2"), 2.0F, 2.0,
+ "string_row2", format.parse("2017-01-01 01:02:03"), 2)),
+ initBeamSqlRow(columnMetadata,
+ Arrays.<Object>asList(1, 3000L, Short.valueOf("3"), Byte.valueOf("3"), 3.0F, 3.0,
+ "string_row3", format.parse("2017-01-01 01:03:03"), 3)),
+ initBeamSqlRow(columnMetadata, Arrays.<Object>asList(1, 4000L, Short.valueOf("4"),
+ Byte.valueOf("4"), 4.0F, 4.0, "string_row4", format.parse("2017-01-01 02:04:03"), 4)));
+ }
+
+ /**
+ * create a {@code BeamSqlRowType} for given column metadata.
+ */
+ public static BeamRecordSqlType initTypeOfSqlRow(List<KV<String, SqlTypeName>> columnMetadata){
+ FieldInfoBuilder builder = BeamQueryPlanner.TYPE_FACTORY.builder();
+ for (KV<String, SqlTypeName> cm : columnMetadata) {
+ builder.add(cm.getKey(), cm.getValue());
+ }
+ return CalciteUtils.toBeamRowType(builder.build());
+ }
+
+ /**
+ * Create an empty row with given column metadata.
+ */
+ public static BeamRecord initBeamSqlRow(List<KV<String, SqlTypeName>> columnMetadata) {
+ return initBeamSqlRow(columnMetadata, Arrays.asList());
+ }
+
+ /**
+ * Create a row with given column metadata, and values for each column.
+ *
+ */
+ public static BeamRecord initBeamSqlRow(List<KV<String, SqlTypeName>> columnMetadata,
+ List<Object> rowValues){
+ BeamRecordSqlType rowType = initTypeOfSqlRow(columnMetadata);
+
+ return new BeamRecord(rowType, rowValues);
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtilsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtilsTest.java
new file mode 100644
index 0000000..1a14256
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtilsTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.utils;
+
+import static org.apache.beam.sdk.extensions.sql.impl.utils.SqlTypeUtils.findExpressionOfType;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Optional;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for {@link SqlTypeUtils}.
+ */
+public class SqlTypeUtilsTest {
+ private static final BigDecimal DECIMAL_THREE = new BigDecimal(3);
+ private static final BigDecimal DECIMAL_FOUR = new BigDecimal(4);
+
+ private static final List<BeamSqlExpression> EXPRESSIONS = Arrays.<BeamSqlExpression> asList(
+ BeamSqlPrimitive.of(SqlTypeName.INTERVAL_DAY, DECIMAL_THREE),
+ BeamSqlPrimitive.of(SqlTypeName.INTERVAL_MONTH, DECIMAL_FOUR),
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4),
+ BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+
+ @Test public void testFindExpressionOfType_success() {
+ Optional<BeamSqlExpression> typeName = findExpressionOfType(EXPRESSIONS, SqlTypeName.INTEGER);
+
+ assertTrue(typeName.isPresent());
+ assertEquals(SqlTypeName.INTEGER, typeName.get().getOutputType());
+ }
+
+ @Test public void testFindExpressionOfType_failure() {
+ Optional<BeamSqlExpression> typeName = findExpressionOfType(EXPRESSIONS, SqlTypeName.VARCHAR);
+
+ assertFalse(typeName.isPresent());
+ }
+
+ @Test public void testFindExpressionOfTypes_success() {
+ Optional<BeamSqlExpression> typeName = findExpressionOfType(EXPRESSIONS, SqlTypeName.INT_TYPES);
+
+ assertTrue(typeName.isPresent());
+ assertEquals(SqlTypeName.INTEGER, typeName.get().getOutputType());
+ }
+
+ @Test public void testFindExpressionOfTypes_failure() {
+ Optional<BeamSqlExpression> typeName =
+ findExpressionOfType(EXPRESSIONS, SqlTypeName.CHAR_TYPES);
+
+ assertFalse(typeName.isPresent());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlArithmeticOperatorsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlArithmeticOperatorsIntegrationTest.java
new file mode 100644
index 0000000..5e626a2
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlArithmeticOperatorsIntegrationTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import org.junit.Test;
+
+/**
+ * Integration test for arithmetic operators.
+ */
+public class BeamSqlArithmeticOperatorsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+
+ private static final BigDecimal ZERO = BigDecimal.valueOf(0.0);
+ private static final BigDecimal ONE0 = BigDecimal.valueOf(1);
+ private static final BigDecimal ONE = BigDecimal.valueOf(1.0);
+ private static final BigDecimal ONE2 = BigDecimal.valueOf(1.0).multiply(BigDecimal.valueOf(1.0));
+ private static final BigDecimal ONE10 = BigDecimal.ONE.divide(
+ BigDecimal.ONE, 10, RoundingMode.HALF_EVEN);
+ private static final BigDecimal TWO = BigDecimal.valueOf(2.0);
+
+ @Test
+ public void testPlus() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("1 + 1", 2)
+ .addExpr("1.0 + 1", TWO)
+ .addExpr("1 + 1.0", TWO)
+ .addExpr("1.0 + 1.0", TWO)
+ .addExpr("c_tinyint + c_tinyint", (byte) 2)
+ .addExpr("c_smallint + c_smallint", (short) 2)
+ .addExpr("c_bigint + c_bigint", 2L)
+ .addExpr("c_decimal + c_decimal", TWO)
+ .addExpr("c_tinyint + c_decimal", TWO)
+ .addExpr("c_float + c_decimal", 2.0)
+ .addExpr("c_double + c_decimal", 2.0)
+ .addExpr("c_float + c_float", 2.0f)
+ .addExpr("c_double + c_float", 2.0)
+ .addExpr("c_double + c_double", 2.0)
+ .addExpr("c_float + c_bigint", 2.0f)
+ .addExpr("c_double + c_bigint", 2.0)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testPlus_overflow() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_max + c_tinyint_max", (byte) -2)
+ .addExpr("c_smallint_max + c_smallint_max", (short) -2)
+ .addExpr("c_integer_max + c_integer_max", -2)
+ // yeah, I know 384L is strange, but since it is already overflowed
+ // what the actualy result is not so important, it is wrong any way.
+ .addExpr("c_bigint_max + c_bigint_max", 384L)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testMinus() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("1 - 1", 0)
+ .addExpr("1.0 - 1", ZERO)
+ .addExpr("1 - 0.0", ONE)
+ .addExpr("1.0 - 1.0", ZERO)
+ .addExpr("c_tinyint - c_tinyint", (byte) 0)
+ .addExpr("c_smallint - c_smallint", (short) 0)
+ .addExpr("c_bigint - c_bigint", 0L)
+ .addExpr("c_decimal - c_decimal", ZERO)
+ .addExpr("c_tinyint - c_decimal", ZERO)
+ .addExpr("c_float - c_decimal", 0.0)
+ .addExpr("c_double - c_decimal", 0.0)
+ .addExpr("c_float - c_float", 0.0f)
+ .addExpr("c_double - c_float", 0.0)
+ .addExpr("c_double - c_double", 0.0)
+ .addExpr("c_float - c_bigint", 0.0f)
+ .addExpr("c_double - c_bigint", 0.0)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testMultiply() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("1 * 1", 1)
+ .addExpr("1.0 * 1", ONE2)
+ .addExpr("1 * 1.0", ONE2)
+ .addExpr("1.0 * 1.0", ONE2)
+ .addExpr("c_tinyint * c_tinyint", (byte) 1)
+ .addExpr("c_smallint * c_smallint", (short) 1)
+ .addExpr("c_bigint * c_bigint", 1L)
+ .addExpr("c_decimal * c_decimal", ONE2)
+ .addExpr("c_tinyint * c_decimal", ONE2)
+ .addExpr("c_float * c_decimal", 1.0)
+ .addExpr("c_double * c_decimal", 1.0)
+ .addExpr("c_float * c_float", 1.0f)
+ .addExpr("c_double * c_float", 1.0)
+ .addExpr("c_double * c_double", 1.0)
+ .addExpr("c_float * c_bigint", 1.0f)
+ .addExpr("c_double * c_bigint", 1.0)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testDivide() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("1 / 1", 1)
+ .addExpr("1.0 / 1", ONE10)
+ .addExpr("1 / 1.0", ONE10)
+ .addExpr("1.0 / 1.0", ONE10)
+ .addExpr("c_tinyint / c_tinyint", (byte) 1)
+ .addExpr("c_smallint / c_smallint", (short) 1)
+ .addExpr("c_bigint / c_bigint", 1L)
+ .addExpr("c_decimal / c_decimal", ONE10)
+ .addExpr("c_tinyint / c_decimal", ONE10)
+ .addExpr("c_float / c_decimal", 1.0)
+ .addExpr("c_double / c_decimal", 1.0)
+ .addExpr("c_float / c_float", 1.0f)
+ .addExpr("c_double / c_float", 1.0)
+ .addExpr("c_double / c_double", 1.0)
+ .addExpr("c_float / c_bigint", 1.0f)
+ .addExpr("c_double / c_bigint", 1.0)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testMod() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("mod(1, 1)", 0)
+ .addExpr("mod(1.0, 1)", 0)
+ .addExpr("mod(1, 1.0)", ZERO)
+ .addExpr("mod(1.0, 1.0)", ZERO)
+ .addExpr("mod(c_tinyint, c_tinyint)", (byte) 0)
+ .addExpr("mod(c_smallint, c_smallint)", (short) 0)
+ .addExpr("mod(c_bigint, c_bigint)", 0L)
+ .addExpr("mod(c_decimal, c_decimal)", ZERO)
+ .addExpr("mod(c_tinyint, c_decimal)", ZERO)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
new file mode 100644
index 0000000..3395269
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import com.google.common.base.Joiner;
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSql;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.util.Pair;
+import org.junit.Rule;
+
+/**
+ * Base class for all built-in functions integration tests.
+ */
+public class BeamSqlBuiltinFunctionsIntegrationTestBase {
+ private static final Map<Class, Integer> JAVA_CLASS_TO_SQL_TYPE = new HashMap<>();
+ static {
+ JAVA_CLASS_TO_SQL_TYPE.put(Byte.class, Types.TINYINT);
+ JAVA_CLASS_TO_SQL_TYPE.put(Short.class, Types.SMALLINT);
+ JAVA_CLASS_TO_SQL_TYPE.put(Integer.class, Types.INTEGER);
+ JAVA_CLASS_TO_SQL_TYPE.put(Long.class, Types.BIGINT);
+ JAVA_CLASS_TO_SQL_TYPE.put(Float.class, Types.FLOAT);
+ JAVA_CLASS_TO_SQL_TYPE.put(Double.class, Types.DOUBLE);
+ JAVA_CLASS_TO_SQL_TYPE.put(BigDecimal.class, Types.DECIMAL);
+ JAVA_CLASS_TO_SQL_TYPE.put(String.class, Types.VARCHAR);
+ JAVA_CLASS_TO_SQL_TYPE.put(Date.class, Types.DATE);
+ JAVA_CLASS_TO_SQL_TYPE.put(Boolean.class, Types.BOOLEAN);
+ }
+
+ @Rule
+ public final TestPipeline pipeline = TestPipeline.create();
+
+ protected PCollection<BeamRecord> getTestPCollection() {
+ BeamRecordSqlType type = BeamRecordSqlType.create(
+ Arrays.asList("ts", "c_tinyint", "c_smallint",
+ "c_integer", "c_bigint", "c_float", "c_double", "c_decimal",
+ "c_tinyint_max", "c_smallint_max", "c_integer_max", "c_bigint_max"),
+ Arrays.asList(Types.DATE, Types.TINYINT, Types.SMALLINT,
+ Types.INTEGER, Types.BIGINT, Types.FLOAT, Types.DOUBLE, Types.DECIMAL,
+ Types.TINYINT, Types.SMALLINT, Types.INTEGER, Types.BIGINT)
+ );
+ try {
+ return MockedBoundedTable
+ .of(type)
+ .addRows(
+ parseDate("1986-02-15 11:35:26"),
+ (byte) 1,
+ (short) 1,
+ 1,
+ 1L,
+ 1.0f,
+ 1.0,
+ BigDecimal.ONE,
+ (byte) 127,
+ (short) 32767,
+ 2147483647,
+ 9223372036854775807L
+ )
+ .buildIOReader(pipeline)
+ .setCoder(type.getRecordCoder());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected static Date parseDate(String str) {
+ try {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+ return sdf.parse(str);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Helper class to make write integration test for built-in functions easier.
+ *
+ * <p>example usage:
+ * <pre>{@code
+ * ExpressionChecker checker = new ExpressionChecker()
+ * .addExpr("1 + 1", 2)
+ * .addExpr("1.0 + 1", 2.0)
+ * .addExpr("1 + 1.0", 2.0)
+ * .addExpr("1.0 + 1.0", 2.0)
+ * .addExpr("c_tinyint + c_tinyint", (byte) 2);
+ * checker.buildRunAndCheck(inputCollections);
+ * }</pre>
+ */
+ public class ExpressionChecker {
+ private transient List<Pair<String, Object>> exps = new ArrayList<>();
+
+ public ExpressionChecker addExpr(String expression, Object expectedValue) {
+ exps.add(Pair.of(expression, expectedValue));
+ return this;
+ }
+
+ private String getSql() {
+ List<String> expStrs = new ArrayList<>();
+ for (Pair<String, Object> pair : exps) {
+ expStrs.add(pair.getKey());
+ }
+ return "SELECT " + Joiner.on(",\n ").join(expStrs) + " FROM PCOLLECTION";
+ }
+
+ /**
+ * Build the corresponding SQL, compile to Beam Pipeline, run it, and check the result.
+ */
+ public void buildRunAndCheck() {
+ PCollection<BeamRecord> inputCollection = getTestPCollection();
+ System.out.println("SQL:>\n" + getSql());
+ try {
+ List<String> names = new ArrayList<>();
+ List<Integer> types = new ArrayList<>();
+ List<Object> values = new ArrayList<>();
+
+ for (Pair<String, Object> pair : exps) {
+ names.add(pair.getKey());
+ types.add(JAVA_CLASS_TO_SQL_TYPE.get(pair.getValue().getClass()));
+ values.add(pair.getValue());
+ }
+
+ PCollection<BeamRecord> rows = inputCollection.apply(BeamSql.query(getSql()));
+ PAssert.that(rows).containsInAnyOrder(
+ TestUtils.RowsBuilder
+ .of(BeamRecordSqlType.create(names, types))
+ .addRows(values)
+ .getRows()
+ );
+ inputCollection.getPipeline().run();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlComparisonOperatorsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlComparisonOperatorsIntegrationTest.java
new file mode 100644
index 0000000..a836f79
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlComparisonOperatorsIntegrationTest.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.util.Arrays;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Test;
+
+/**
+ * Integration test for comparison operators.
+ */
+public class BeamSqlComparisonOperatorsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+
+ @Test
+ public void testEquals() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_1 = c_tinyint_1", true)
+ .addExpr("c_tinyint_1 = c_tinyint_2", false)
+ .addExpr("c_smallint_1 = c_smallint_1", true)
+ .addExpr("c_smallint_1 = c_smallint_2", false)
+ .addExpr("c_integer_1 = c_integer_1", true)
+ .addExpr("c_integer_1 = c_integer_2", false)
+ .addExpr("c_bigint_1 = c_bigint_1", true)
+ .addExpr("c_bigint_1 = c_bigint_2", false)
+ .addExpr("c_float_1 = c_float_1", true)
+ .addExpr("c_float_1 = c_float_2", false)
+ .addExpr("c_double_1 = c_double_1", true)
+ .addExpr("c_double_1 = c_double_2", false)
+ .addExpr("c_decimal_1 = c_decimal_1", true)
+ .addExpr("c_decimal_1 = c_decimal_2", false)
+ .addExpr("c_varchar_1 = c_varchar_1", true)
+ .addExpr("c_varchar_1 = c_varchar_2", false)
+ .addExpr("c_boolean_true = c_boolean_true", true)
+ .addExpr("c_boolean_true = c_boolean_false", false)
+
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testNotEquals() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_1 <> c_tinyint_1", false)
+ .addExpr("c_tinyint_1 <> c_tinyint_2", true)
+ .addExpr("c_smallint_1 <> c_smallint_1", false)
+ .addExpr("c_smallint_1 <> c_smallint_2", true)
+ .addExpr("c_integer_1 <> c_integer_1", false)
+ .addExpr("c_integer_1 <> c_integer_2", true)
+ .addExpr("c_bigint_1 <> c_bigint_1", false)
+ .addExpr("c_bigint_1 <> c_bigint_2", true)
+ .addExpr("c_float_1 <> c_float_1", false)
+ .addExpr("c_float_1 <> c_float_2", true)
+ .addExpr("c_double_1 <> c_double_1", false)
+ .addExpr("c_double_1 <> c_double_2", true)
+ .addExpr("c_decimal_1 <> c_decimal_1", false)
+ .addExpr("c_decimal_1 <> c_decimal_2", true)
+ .addExpr("c_varchar_1 <> c_varchar_1", false)
+ .addExpr("c_varchar_1 <> c_varchar_2", true)
+ .addExpr("c_boolean_true <> c_boolean_true", false)
+ .addExpr("c_boolean_true <> c_boolean_false", true)
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testGreaterThan() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_2 > c_tinyint_1", true)
+ .addExpr("c_tinyint_1 > c_tinyint_1", false)
+ .addExpr("c_tinyint_1 > c_tinyint_2", false)
+
+ .addExpr("c_smallint_2 > c_smallint_1", true)
+ .addExpr("c_smallint_1 > c_smallint_1", false)
+ .addExpr("c_smallint_1 > c_smallint_2", false)
+
+ .addExpr("c_integer_2 > c_integer_1", true)
+ .addExpr("c_integer_1 > c_integer_1", false)
+ .addExpr("c_integer_1 > c_integer_2", false)
+
+ .addExpr("c_bigint_2 > c_bigint_1", true)
+ .addExpr("c_bigint_1 > c_bigint_1", false)
+ .addExpr("c_bigint_1 > c_bigint_2", false)
+
+ .addExpr("c_float_2 > c_float_1", true)
+ .addExpr("c_float_1 > c_float_1", false)
+ .addExpr("c_float_1 > c_float_2", false)
+
+ .addExpr("c_double_2 > c_double_1", true)
+ .addExpr("c_double_1 > c_double_1", false)
+ .addExpr("c_double_1 > c_double_2", false)
+
+ .addExpr("c_decimal_2 > c_decimal_1", true)
+ .addExpr("c_decimal_1 > c_decimal_1", false)
+ .addExpr("c_decimal_1 > c_decimal_2", false)
+
+ .addExpr("c_varchar_2 > c_varchar_1", true)
+ .addExpr("c_varchar_1 > c_varchar_1", false)
+ .addExpr("c_varchar_1 > c_varchar_2", false)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGreaterThanException() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_boolean_false > c_boolean_true", false);
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testGreaterThanOrEquals() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_2 >= c_tinyint_1", true)
+ .addExpr("c_tinyint_1 >= c_tinyint_1", true)
+ .addExpr("c_tinyint_1 >= c_tinyint_2", false)
+
+ .addExpr("c_smallint_2 >= c_smallint_1", true)
+ .addExpr("c_smallint_1 >= c_smallint_1", true)
+ .addExpr("c_smallint_1 >= c_smallint_2", false)
+
+ .addExpr("c_integer_2 >= c_integer_1", true)
+ .addExpr("c_integer_1 >= c_integer_1", true)
+ .addExpr("c_integer_1 >= c_integer_2", false)
+
+ .addExpr("c_bigint_2 >= c_bigint_1", true)
+ .addExpr("c_bigint_1 >= c_bigint_1", true)
+ .addExpr("c_bigint_1 >= c_bigint_2", false)
+
+ .addExpr("c_float_2 >= c_float_1", true)
+ .addExpr("c_float_1 >= c_float_1", true)
+ .addExpr("c_float_1 >= c_float_2", false)
+
+ .addExpr("c_double_2 >= c_double_1", true)
+ .addExpr("c_double_1 >= c_double_1", true)
+ .addExpr("c_double_1 >= c_double_2", false)
+
+ .addExpr("c_decimal_2 >= c_decimal_1", true)
+ .addExpr("c_decimal_1 >= c_decimal_1", true)
+ .addExpr("c_decimal_1 >= c_decimal_2", false)
+
+ .addExpr("c_varchar_2 >= c_varchar_1", true)
+ .addExpr("c_varchar_1 >= c_varchar_1", true)
+ .addExpr("c_varchar_1 >= c_varchar_2", false)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGreaterThanOrEqualsException() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_boolean_false >= c_boolean_true", false);
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testLessThan() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_2 < c_tinyint_1", false)
+ .addExpr("c_tinyint_1 < c_tinyint_1", false)
+ .addExpr("c_tinyint_1 < c_tinyint_2", true)
+
+ .addExpr("c_smallint_2 < c_smallint_1", false)
+ .addExpr("c_smallint_1 < c_smallint_1", false)
+ .addExpr("c_smallint_1 < c_smallint_2", true)
+
+ .addExpr("c_integer_2 < c_integer_1", false)
+ .addExpr("c_integer_1 < c_integer_1", false)
+ .addExpr("c_integer_1 < c_integer_2", true)
+
+ .addExpr("c_bigint_2 < c_bigint_1", false)
+ .addExpr("c_bigint_1 < c_bigint_1", false)
+ .addExpr("c_bigint_1 < c_bigint_2", true)
+
+ .addExpr("c_float_2 < c_float_1", false)
+ .addExpr("c_float_1 < c_float_1", false)
+ .addExpr("c_float_1 < c_float_2", true)
+
+ .addExpr("c_double_2 < c_double_1", false)
+ .addExpr("c_double_1 < c_double_1", false)
+ .addExpr("c_double_1 < c_double_2", true)
+
+ .addExpr("c_decimal_2 < c_decimal_1", false)
+ .addExpr("c_decimal_1 < c_decimal_1", false)
+ .addExpr("c_decimal_1 < c_decimal_2", true)
+
+ .addExpr("c_varchar_2 < c_varchar_1", false)
+ .addExpr("c_varchar_1 < c_varchar_1", false)
+ .addExpr("c_varchar_1 < c_varchar_2", true)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testLessThanException() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_boolean_false < c_boolean_true", false);
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testLessThanOrEquals() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_tinyint_2 <= c_tinyint_1", false)
+ .addExpr("c_tinyint_1 <= c_tinyint_1", true)
+ .addExpr("c_tinyint_1 <= c_tinyint_2", true)
+
+ .addExpr("c_smallint_2 <= c_smallint_1", false)
+ .addExpr("c_smallint_1 <= c_smallint_1", true)
+ .addExpr("c_smallint_1 <= c_smallint_2", true)
+
+ .addExpr("c_integer_2 <= c_integer_1", false)
+ .addExpr("c_integer_1 <= c_integer_1", true)
+ .addExpr("c_integer_1 <= c_integer_2", true)
+
+ .addExpr("c_bigint_2 <= c_bigint_1", false)
+ .addExpr("c_bigint_1 <= c_bigint_1", true)
+ .addExpr("c_bigint_1 <= c_bigint_2", true)
+
+ .addExpr("c_float_2 <= c_float_1", false)
+ .addExpr("c_float_1 <= c_float_1", true)
+ .addExpr("c_float_1 <= c_float_2", true)
+
+ .addExpr("c_double_2 <= c_double_1", false)
+ .addExpr("c_double_1 <= c_double_1", true)
+ .addExpr("c_double_1 <= c_double_2", true)
+
+ .addExpr("c_decimal_2 <= c_decimal_1", false)
+ .addExpr("c_decimal_1 <= c_decimal_1", true)
+ .addExpr("c_decimal_1 <= c_decimal_2", true)
+
+ .addExpr("c_varchar_2 <= c_varchar_1", false)
+ .addExpr("c_varchar_1 <= c_varchar_1", true)
+ .addExpr("c_varchar_1 <= c_varchar_2", true)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testLessThanOrEqualsException() {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_boolean_false <= c_boolean_true", false);
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testIsNullAndIsNotNull() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("1 IS NOT NULL", true)
+ .addExpr("NULL IS NOT NULL", false)
+
+ .addExpr("1 IS NULL", false)
+ .addExpr("NULL IS NULL", true)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Override protected PCollection<BeamRecord> getTestPCollection() {
+ BeamRecordSqlType type = BeamRecordSqlType.create(
+ Arrays.asList(
+ "c_tinyint_0", "c_tinyint_1", "c_tinyint_2",
+ "c_smallint_0", "c_smallint_1", "c_smallint_2",
+ "c_integer_0", "c_integer_1", "c_integer_2",
+ "c_bigint_0", "c_bigint_1", "c_bigint_2",
+ "c_float_0", "c_float_1", "c_float_2",
+ "c_double_0", "c_double_1", "c_double_2",
+ "c_decimal_0", "c_decimal_1", "c_decimal_2",
+ "c_varchar_0", "c_varchar_1", "c_varchar_2",
+ "c_boolean_false", "c_boolean_true"
+ ),
+ Arrays.asList(
+ Types.TINYINT, Types.TINYINT, Types.TINYINT,
+ Types.SMALLINT, Types.SMALLINT, Types.SMALLINT,
+ Types.INTEGER, Types.INTEGER, Types.INTEGER,
+ Types.BIGINT, Types.BIGINT, Types.BIGINT,
+ Types.FLOAT, Types.FLOAT, Types.FLOAT,
+ Types.DOUBLE, Types.DOUBLE, Types.DOUBLE,
+ Types.DECIMAL, Types.DECIMAL, Types.DECIMAL,
+ Types.VARCHAR, Types.VARCHAR, Types.VARCHAR,
+ Types.BOOLEAN, Types.BOOLEAN
+ )
+ );
+ try {
+ return MockedBoundedTable
+ .of(type)
+ .addRows(
+ (byte) 0, (byte) 1, (byte) 2,
+ (short) 0, (short) 1, (short) 2,
+ 0, 1, 2,
+ 0L, 1L, 2L,
+ 0.0f, 1.0f, 2.0f,
+ 0.0, 1.0, 2.0,
+ BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.ONE.add(BigDecimal.ONE),
+ "a", "b", "c",
+ false, true
+ )
+ .buildIOReader(pipeline)
+ .setCoder(type.getRecordCoder());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
new file mode 100644
index 0000000..f4416ce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import org.junit.Test;
+
+/**
+ * Integration test for conditional functions.
+ */
+public class BeamSqlConditionalFunctionsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+ @Test
+ public void testConditionalFunctions() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr(
+ "CASE 1 WHEN 1 THEN 'hello' ELSE 'world' END",
+ "hello"
+ )
+ .addExpr(
+ "CASE 2 "
+ + "WHEN 1 THEN 'hello' "
+ + "WHEN 3 THEN 'bond' "
+ + "ELSE 'world' END",
+ "world"
+ )
+ .addExpr(
+ "CASE "
+ + "WHEN 1 = 1 THEN 'hello' "
+ + "ELSE 'world' END",
+ "hello"
+ )
+ .addExpr(
+ "CASE "
+ + "WHEN 1 > 1 THEN 'hello' "
+ + "ELSE 'world' END",
+ "world"
+ )
+ .addExpr("NULLIF(5, 4) ", 5)
+ .addExpr("COALESCE(1, 5) ", 1)
+ .addExpr("COALESCE(NULL, 5) ", 5)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
new file mode 100644
index 0000000..ec5b295
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Date;
+import java.util.Iterator;
+
+import org.apache.beam.sdk.extensions.sql.BeamSql;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Test;
+
+/**
+ * Integration test for date functions.
+ */
+public class BeamSqlDateFunctionsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+ @Test public void testBasicDateTimeFunctions() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("EXTRACT(YEAR FROM ts)", 1986L)
+ .addExpr("YEAR(ts)", 1986L)
+ .addExpr("QUARTER(ts)", 1L)
+ .addExpr("MONTH(ts)", 2L)
+ .addExpr("WEEK(ts)", 7L)
+ .addExpr("DAYOFMONTH(ts)", 15L)
+ .addExpr("DAYOFYEAR(ts)", 46L)
+ .addExpr("DAYOFWEEK(ts)", 7L)
+ .addExpr("HOUR(ts)", 11L)
+ .addExpr("MINUTE(ts)", 35L)
+ .addExpr("SECOND(ts)", 26L)
+ .addExpr("FLOOR(ts TO YEAR)", parseDate("1986-01-01 00:00:00"))
+ .addExpr("CEIL(ts TO YEAR)", parseDate("1987-01-01 00:00:00"))
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test public void testDatetimePlusFunction() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TIMESTAMPADD(SECOND, 3, TIMESTAMP '1984-04-19 01:02:03')",
+ parseDate("1984-04-19 01:02:06"))
+ .addExpr("TIMESTAMPADD(MINUTE, 3, TIMESTAMP '1984-04-19 01:02:03')",
+ parseDate("1984-04-19 01:05:03"))
+ .addExpr("TIMESTAMPADD(HOUR, 3, TIMESTAMP '1984-04-19 01:02:03')",
+ parseDate("1984-04-19 04:02:03"))
+ .addExpr("TIMESTAMPADD(DAY, 3, TIMESTAMP '1984-04-19 01:02:03')",
+ parseDate("1984-04-22 01:02:03"))
+ .addExpr("TIMESTAMPADD(MONTH, 2, TIMESTAMP '1984-01-19 01:02:03')",
+ parseDate("1984-03-19 01:02:03"))
+ .addExpr("TIMESTAMPADD(YEAR, 2, TIMESTAMP '1985-01-19 01:02:03')",
+ parseDate("1987-01-19 01:02:03"))
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test public void testDatetimeInfixPlus() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '3' SECOND",
+ parseDate("1984-01-19 01:02:06"))
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '2' MINUTE",
+ parseDate("1984-01-19 01:04:03"))
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '2' HOUR",
+ parseDate("1984-01-19 03:02:03"))
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '2' DAY",
+ parseDate("1984-01-21 01:02:03"))
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '2' MONTH",
+ parseDate("1984-03-19 01:02:03"))
+ .addExpr("TIMESTAMP '1984-01-19 01:02:03' + INTERVAL '2' YEAR",
+ parseDate("1986-01-19 01:02:03"))
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test public void testTimestampDiff() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TIMESTAMPDIFF(SECOND, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:01:58')", 0)
+ .addExpr("TIMESTAMPDIFF(SECOND, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:01:59')", 1)
+ .addExpr("TIMESTAMPDIFF(SECOND, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:02:00')", 2)
+
+ .addExpr("TIMESTAMPDIFF(MINUTE, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:02:57')", 0)
+ .addExpr("TIMESTAMPDIFF(MINUTE, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:02:58')", 1)
+ .addExpr("TIMESTAMPDIFF(MINUTE, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 01:03:58')", 2)
+
+ .addExpr("TIMESTAMPDIFF(HOUR, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 02:01:57')", 0)
+ .addExpr("TIMESTAMPDIFF(HOUR, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 02:01:58')", 1)
+ .addExpr("TIMESTAMPDIFF(HOUR, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-19 03:01:58')", 2)
+
+ .addExpr("TIMESTAMPDIFF(DAY, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-20 01:01:57')", 0)
+ .addExpr("TIMESTAMPDIFF(DAY, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-20 01:01:58')", 1)
+ .addExpr("TIMESTAMPDIFF(DAY, TIMESTAMP '1984-04-19 01:01:58', "
+ + "TIMESTAMP '1984-04-21 01:01:58')", 2)
+
+ .addExpr("TIMESTAMPDIFF(MONTH, TIMESTAMP '1984-01-19 01:01:58', "
+ + "TIMESTAMP '1984-02-19 01:01:57')", 0)
+ .addExpr("TIMESTAMPDIFF(MONTH, TIMESTAMP '1984-01-19 01:01:58', "
+ + "TIMESTAMP '1984-02-19 01:01:58')", 1)
+ .addExpr("TIMESTAMPDIFF(MONTH, TIMESTAMP '1984-01-19 01:01:58', "
+ + "TIMESTAMP '1984-03-19 01:01:58')", 2)
+
+ .addExpr("TIMESTAMPDIFF(YEAR, TIMESTAMP '1981-01-19 01:01:58', "
+ + "TIMESTAMP '1982-01-19 01:01:57')", 0)
+ .addExpr("TIMESTAMPDIFF(YEAR, TIMESTAMP '1981-01-19 01:01:58', "
+ + "TIMESTAMP '1982-01-19 01:01:58')", 1)
+ .addExpr("TIMESTAMPDIFF(YEAR, TIMESTAMP '1981-01-19 01:01:58', "
+ + "TIMESTAMP '1983-01-19 01:01:58')", 2)
+
+ .addExpr("TIMESTAMPDIFF(YEAR, TIMESTAMP '1981-01-19 01:01:58', "
+ + "TIMESTAMP '1980-01-19 01:01:58')", -1)
+ .addExpr("TIMESTAMPDIFF(YEAR, TIMESTAMP '1981-01-19 01:01:58', "
+ + "TIMESTAMP '1979-01-19 01:01:58')", -2)
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test public void testTimestampMinusInterval() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TIMESTAMP '1984-04-19 01:01:58' - INTERVAL '2' SECOND",
+ parseDate("1984-04-19 01:01:56"))
+ .addExpr("TIMESTAMP '1984-04-19 01:01:58' - INTERVAL '1' MINUTE",
+ parseDate("1984-04-19 01:00:58"))
+ .addExpr("TIMESTAMP '1984-04-19 01:01:58' - INTERVAL '4' HOUR",
+ parseDate("1984-04-18 21:01:58"))
+ .addExpr("TIMESTAMP '1984-04-19 01:01:58' - INTERVAL '5' DAY",
+ parseDate("1984-04-14 01:01:58"))
+ .addExpr("TIMESTAMP '1984-01-19 01:01:58' - INTERVAL '2' MONTH",
+ parseDate("1983-11-19 01:01:58"))
+ .addExpr("TIMESTAMP '1984-01-19 01:01:58' - INTERVAL '1' YEAR",
+ parseDate("1983-01-19 01:01:58"))
+ ;
+ checker.buildRunAndCheck();
+ }
+
+ @Test public void testDateTimeFunctions_currentTime() throws Exception {
+ String sql = "SELECT "
+ + "LOCALTIME as l,"
+ + "LOCALTIMESTAMP as l1,"
+ + "CURRENT_DATE as c1,"
+ + "CURRENT_TIME as c2,"
+ + "CURRENT_TIMESTAMP as c3"
+ + " FROM PCOLLECTION"
+ ;
+ PCollection<BeamRecord> rows = getTestPCollection().apply(
+ BeamSql.query(sql));
+ PAssert.that(rows).satisfies(new Checker());
+ pipeline.run();
+ }
+
+ private static class Checker implements SerializableFunction<Iterable<BeamRecord>, Void> {
+ @Override public Void apply(Iterable<BeamRecord> input) {
+ Iterator<BeamRecord> iter = input.iterator();
+ assertTrue(iter.hasNext());
+ BeamRecord row = iter.next();
+ // LOCALTIME
+ Date date = new Date();
+ assertTrue(date.getTime() - row.getGregorianCalendar(0).getTime().getTime() < 1000);
+ assertTrue(date.getTime() - row.getDate(1).getTime() < 1000);
+ assertTrue(date.getTime() - row.getDate(2).getTime() < 1000);
+ assertTrue(date.getTime() - row.getGregorianCalendar(3).getTime().getTime() < 1000);
+ assertTrue(date.getTime() - row.getDate(4).getTime() < 1000);
+ assertFalse(iter.hasNext());
+ return null;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
new file mode 100644
index 0000000..b408d78
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import org.junit.Test;
+
+/**
+ * Integration test for logical functions.
+ */
+public class BeamSqlLogicalFunctionsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+ @Test
+ public void testStringFunctions() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("c_integer = 1 AND c_bigint = 1", true)
+ .addExpr("c_integer = 1 OR c_bigint = 2", true)
+ .addExpr("NOT c_bigint = 2", true)
+ .addExpr("(NOT c_bigint = 2) AND (c_integer = 1 OR c_bigint = 3)", true)
+ .addExpr("c_integer = 2 AND c_bigint = 1", false)
+ .addExpr("c_integer = 2 OR c_bigint = 2", false)
+ .addExpr("NOT c_bigint = 1", false)
+ .addExpr("(NOT c_bigint = 2) AND (c_integer = 2 OR c_bigint = 3)", false)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
new file mode 100644
index 0000000..995caaf
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import java.math.BigDecimal;
+import java.util.Random;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.junit.Test;
+
+/**
+ * Integration test for built-in MATH functions.
+ */
+public class BeamSqlMathFunctionsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+ private static final int INTEGER_VALUE = 1;
+ private static final long LONG_VALUE = 1L;
+ private static final short SHORT_VALUE = 1;
+ private static final byte BYTE_VALUE = 1;
+ private static final double DOUBLE_VALUE = 1.0;
+ private static final float FLOAT_VALUE = 1.0f;
+ private static final BigDecimal DECIMAL_VALUE = new BigDecimal(1);
+
+ @Test
+ public void testAbs() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ABS(c_integer)", Math.abs(INTEGER_VALUE))
+ .addExpr("ABS(c_bigint)", Math.abs(LONG_VALUE))
+ .addExpr("ABS(c_smallint)", (short) Math.abs(SHORT_VALUE))
+ .addExpr("ABS(c_tinyint)", (byte) Math.abs(BYTE_VALUE))
+ .addExpr("ABS(c_double)", Math.abs(DOUBLE_VALUE))
+ .addExpr("ABS(c_float)", Math.abs(FLOAT_VALUE))
+ .addExpr("ABS(c_decimal)", new BigDecimal(Math.abs(DECIMAL_VALUE.doubleValue())))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testSqrt() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("SQRT(c_integer)", Math.sqrt(INTEGER_VALUE))
+ .addExpr("SQRT(c_bigint)", Math.sqrt(LONG_VALUE))
+ .addExpr("SQRT(c_smallint)", Math.sqrt(SHORT_VALUE))
+ .addExpr("SQRT(c_tinyint)", Math.sqrt(BYTE_VALUE))
+ .addExpr("SQRT(c_double)", Math.sqrt(DOUBLE_VALUE))
+ .addExpr("SQRT(c_float)", Math.sqrt(FLOAT_VALUE))
+ .addExpr("SQRT(c_decimal)", Math.sqrt(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testRound() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ROUND(c_integer, 0)", SqlFunctions.sround(INTEGER_VALUE, 0))
+ .addExpr("ROUND(c_bigint, 0)", SqlFunctions.sround(LONG_VALUE, 0))
+ .addExpr("ROUND(c_smallint, 0)", (short) SqlFunctions.sround(SHORT_VALUE, 0))
+ .addExpr("ROUND(c_tinyint, 0)", (byte) SqlFunctions.sround(BYTE_VALUE, 0))
+ .addExpr("ROUND(c_double, 0)", SqlFunctions.sround(DOUBLE_VALUE, 0))
+ .addExpr("ROUND(c_float, 0)", (float) SqlFunctions.sround(FLOAT_VALUE, 0))
+ .addExpr("ROUND(c_decimal, 0)",
+ new BigDecimal(SqlFunctions.sround(DECIMAL_VALUE.doubleValue(), 0)))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testLn() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("LN(c_integer)", Math.log(INTEGER_VALUE))
+ .addExpr("LN(c_bigint)", Math.log(LONG_VALUE))
+ .addExpr("LN(c_smallint)", Math.log(SHORT_VALUE))
+ .addExpr("LN(c_tinyint)", Math.log(BYTE_VALUE))
+ .addExpr("LN(c_double)", Math.log(DOUBLE_VALUE))
+ .addExpr("LN(c_float)", Math.log(FLOAT_VALUE))
+ .addExpr("LN(c_decimal)", Math.log(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testLog10() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("LOG10(c_integer)", Math.log10(INTEGER_VALUE))
+ .addExpr("LOG10(c_bigint)", Math.log10(LONG_VALUE))
+ .addExpr("LOG10(c_smallint)", Math.log10(SHORT_VALUE))
+ .addExpr("LOG10(c_tinyint)", Math.log10(BYTE_VALUE))
+ .addExpr("LOG10(c_double)", Math.log10(DOUBLE_VALUE))
+ .addExpr("LOG10(c_float)", Math.log10(FLOAT_VALUE))
+ .addExpr("LOG10(c_decimal)", Math.log10(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testExp() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("EXP(c_integer)", Math.exp(INTEGER_VALUE))
+ .addExpr("EXP(c_bigint)", Math.exp(LONG_VALUE))
+ .addExpr("EXP(c_smallint)", Math.exp(SHORT_VALUE))
+ .addExpr("EXP(c_tinyint)", Math.exp(BYTE_VALUE))
+ .addExpr("EXP(c_double)", Math.exp(DOUBLE_VALUE))
+ .addExpr("EXP(c_float)", Math.exp(FLOAT_VALUE))
+ .addExpr("EXP(c_decimal)", Math.exp(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testAcos() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ACOS(c_integer)", Math.acos(INTEGER_VALUE))
+ .addExpr("ACOS(c_bigint)", Math.acos(LONG_VALUE))
+ .addExpr("ACOS(c_smallint)", Math.acos(SHORT_VALUE))
+ .addExpr("ACOS(c_tinyint)", Math.acos(BYTE_VALUE))
+ .addExpr("ACOS(c_double)", Math.acos(DOUBLE_VALUE))
+ .addExpr("ACOS(c_float)", Math.acos(FLOAT_VALUE))
+ .addExpr("ACOS(c_decimal)", Math.acos(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testAsin() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ASIN(c_integer)", Math.asin(INTEGER_VALUE))
+ .addExpr("ASIN(c_bigint)", Math.asin(LONG_VALUE))
+ .addExpr("ASIN(c_smallint)", Math.asin(SHORT_VALUE))
+ .addExpr("ASIN(c_tinyint)", Math.asin(BYTE_VALUE))
+ .addExpr("ASIN(c_double)", Math.asin(DOUBLE_VALUE))
+ .addExpr("ASIN(c_float)", Math.asin(FLOAT_VALUE))
+ .addExpr("ASIN(c_decimal)", Math.asin(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testAtan() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ATAN(c_integer)", Math.atan(INTEGER_VALUE))
+ .addExpr("ATAN(c_bigint)", Math.atan(LONG_VALUE))
+ .addExpr("ATAN(c_smallint)", Math.atan(SHORT_VALUE))
+ .addExpr("ATAN(c_tinyint)", Math.atan(BYTE_VALUE))
+ .addExpr("ATAN(c_double)", Math.atan(DOUBLE_VALUE))
+ .addExpr("ATAN(c_float)", Math.atan(FLOAT_VALUE))
+ .addExpr("ATAN(c_decimal)", Math.atan(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testCot() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("COT(c_integer)", 1.0d / Math.tan(INTEGER_VALUE))
+ .addExpr("COT(c_bigint)", 1.0d / Math.tan(LONG_VALUE))
+ .addExpr("COT(c_smallint)", 1.0d / Math.tan(SHORT_VALUE))
+ .addExpr("COT(c_tinyint)", 1.0d / Math.tan(BYTE_VALUE))
+ .addExpr("COT(c_double)", 1.0d / Math.tan(DOUBLE_VALUE))
+ .addExpr("COT(c_float)", 1.0d / Math.tan(FLOAT_VALUE))
+ .addExpr("COT(c_decimal)", 1.0d / Math.tan(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testDegrees() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("DEGREES(c_integer)", Math.toDegrees(INTEGER_VALUE))
+ .addExpr("DEGREES(c_bigint)", Math.toDegrees(LONG_VALUE))
+ .addExpr("DEGREES(c_smallint)", Math.toDegrees(SHORT_VALUE))
+ .addExpr("DEGREES(c_tinyint)", Math.toDegrees(BYTE_VALUE))
+ .addExpr("DEGREES(c_double)", Math.toDegrees(DOUBLE_VALUE))
+ .addExpr("DEGREES(c_float)", Math.toDegrees(FLOAT_VALUE))
+ .addExpr("DEGREES(c_decimal)", Math.toDegrees(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testRadians() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("RADIANS(c_integer)", Math.toRadians(INTEGER_VALUE))
+ .addExpr("RADIANS(c_bigint)", Math.toRadians(LONG_VALUE))
+ .addExpr("RADIANS(c_smallint)", Math.toRadians(SHORT_VALUE))
+ .addExpr("RADIANS(c_tinyint)", Math.toRadians(BYTE_VALUE))
+ .addExpr("RADIANS(c_double)", Math.toRadians(DOUBLE_VALUE))
+ .addExpr("RADIANS(c_float)", Math.toRadians(FLOAT_VALUE))
+ .addExpr("RADIANS(c_decimal)", Math.toRadians(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testCos() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("COS(c_integer)", Math.cos(INTEGER_VALUE))
+ .addExpr("COS(c_bigint)", Math.cos(LONG_VALUE))
+ .addExpr("COS(c_smallint)", Math.cos(SHORT_VALUE))
+ .addExpr("COS(c_tinyint)", Math.cos(BYTE_VALUE))
+ .addExpr("COS(c_double)", Math.cos(DOUBLE_VALUE))
+ .addExpr("COS(c_float)", Math.cos(FLOAT_VALUE))
+ .addExpr("COS(c_decimal)", Math.cos(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testSin() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("SIN(c_integer)", Math.sin(INTEGER_VALUE))
+ .addExpr("SIN(c_bigint)", Math.sin(LONG_VALUE))
+ .addExpr("SIN(c_smallint)", Math.sin(SHORT_VALUE))
+ .addExpr("SIN(c_tinyint)", Math.sin(BYTE_VALUE))
+ .addExpr("SIN(c_double)", Math.sin(DOUBLE_VALUE))
+ .addExpr("SIN(c_float)", Math.sin(FLOAT_VALUE))
+ .addExpr("SIN(c_decimal)", Math.sin(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testTan() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TAN(c_integer)", Math.tan(INTEGER_VALUE))
+ .addExpr("TAN(c_bigint)", Math.tan(LONG_VALUE))
+ .addExpr("TAN(c_smallint)", Math.tan(SHORT_VALUE))
+ .addExpr("TAN(c_tinyint)", Math.tan(BYTE_VALUE))
+ .addExpr("TAN(c_double)", Math.tan(DOUBLE_VALUE))
+ .addExpr("TAN(c_float)", Math.tan(FLOAT_VALUE))
+ .addExpr("TAN(c_decimal)", Math.tan(DECIMAL_VALUE.doubleValue()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testSign() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("SIGN(c_integer)", Integer.signum(INTEGER_VALUE))
+ .addExpr("SIGN(c_bigint)", (long) (Long.signum(LONG_VALUE)))
+ .addExpr("SIGN(c_smallint)", (short) (Integer.signum(SHORT_VALUE)))
+ .addExpr("SIGN(c_tinyint)", (byte) Integer.signum(BYTE_VALUE))
+ .addExpr("SIGN(c_double)", Math.signum(DOUBLE_VALUE))
+ .addExpr("SIGN(c_float)", Math.signum(FLOAT_VALUE))
+ .addExpr("SIGN(c_decimal)", BigDecimal.valueOf(DECIMAL_VALUE.signum()))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testPower() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("POWER(c_integer, 2)", Math.pow(INTEGER_VALUE, 2))
+ .addExpr("POWER(c_bigint, 2)", Math.pow(LONG_VALUE, 2))
+ .addExpr("POWER(c_smallint, 2)", Math.pow(SHORT_VALUE, 2))
+ .addExpr("POWER(c_tinyint, 2)", Math.pow(BYTE_VALUE, 2))
+ .addExpr("POWER(c_double, 2)", Math.pow(DOUBLE_VALUE, 2))
+ .addExpr("POWER(c_float, 2)", Math.pow(FLOAT_VALUE, 2))
+ .addExpr("POWER(c_decimal, 2)", Math.pow(DECIMAL_VALUE.doubleValue(), 2))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testPi() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("PI", Math.PI)
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testAtan2() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("ATAN2(c_integer, 2)", Math.atan2(INTEGER_VALUE, 2))
+ .addExpr("ATAN2(c_bigint, 2)", Math.atan2(LONG_VALUE, 2))
+ .addExpr("ATAN2(c_smallint, 2)", Math.atan2(SHORT_VALUE, 2))
+ .addExpr("ATAN2(c_tinyint, 2)", Math.atan2(BYTE_VALUE, 2))
+ .addExpr("ATAN2(c_double, 2)", Math.atan2(DOUBLE_VALUE, 2))
+ .addExpr("ATAN2(c_float, 2)", Math.atan2(FLOAT_VALUE, 2))
+ .addExpr("ATAN2(c_decimal, 2)", Math.atan2(DECIMAL_VALUE.doubleValue(), 2))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testTruncate() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("TRUNCATE(c_integer, 2)", SqlFunctions.struncate(INTEGER_VALUE, 2))
+ .addExpr("TRUNCATE(c_bigint, 2)", SqlFunctions.struncate(LONG_VALUE, 2))
+ .addExpr("TRUNCATE(c_smallint, 2)", (short) SqlFunctions.struncate(SHORT_VALUE, 2))
+ .addExpr("TRUNCATE(c_tinyint, 2)", (byte) SqlFunctions.struncate(BYTE_VALUE, 2))
+ .addExpr("TRUNCATE(c_double, 2)", SqlFunctions.struncate(DOUBLE_VALUE, 2))
+ .addExpr("TRUNCATE(c_float, 2)", (float) SqlFunctions.struncate(FLOAT_VALUE, 2))
+ .addExpr("TRUNCATE(c_decimal, 2)", SqlFunctions.struncate(DECIMAL_VALUE, 2))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testRand() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("RAND(c_integer)", new Random(INTEGER_VALUE).nextDouble())
+ ;
+
+ checker.buildRunAndCheck();
+ }
+
+ @Test
+ public void testRandInteger() throws Exception{
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("RAND_INTEGER(c_integer, c_integer)",
+ new Random(INTEGER_VALUE).nextInt(INTEGER_VALUE))
+ ;
+
+ checker.buildRunAndCheck();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlStringFunctionsIntegrationTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlStringFunctionsIntegrationTest.java
new file mode 100644
index 0000000..7a51a95
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlStringFunctionsIntegrationTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import org.junit.Test;
+
+/**
+ * Integration test for string functions.
+ */
+public class BeamSqlStringFunctionsIntegrationTest
+ extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+ @Test
+ public void testStringFunctions() throws Exception {
+ ExpressionChecker checker = new ExpressionChecker()
+ .addExpr("'hello' || ' world'", "hello world")
+ .addExpr("CHAR_LENGTH('hello')", 5)
+ .addExpr("CHARACTER_LENGTH('hello')", 5)
+ .addExpr("UPPER('hello')", "HELLO")
+ .addExpr("LOWER('HELLO')", "hello")
+
+ .addExpr("POSITION('world' IN 'helloworld')", 5)
+ .addExpr("POSITION('world' IN 'helloworldworld' FROM 7)", 10)
+ .addExpr("TRIM(' hello ')", "hello")
+ .addExpr("TRIM(LEADING ' ' FROM ' hello ')", "hello ")
+ .addExpr("TRIM(TRAILING ' ' FROM ' hello ')", " hello")
+
+ .addExpr("TRIM(BOTH ' ' FROM ' hello ')", "hello")
+ .addExpr("OVERLAY('w3333333rce' PLACING 'resou' FROM 3)", "w3resou3rce")
+ .addExpr("SUBSTRING('hello' FROM 2)", "ello")
+ .addExpr("SUBSTRING('hello' FROM 2 FOR 2)", "el")
+ .addExpr("INITCAP('hello world')", "Hello World")
+ ;
+
+ checker.buildRunAndCheck();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTableTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTableTest.java
new file mode 100644
index 0000000..8ce4364
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTableTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for BeamKafkaCSVTable.
+ */
+public class BeamKafkaCSVTableTest {
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+ public static BeamRecord row1;
+ public static BeamRecord row2;
+
+ @BeforeClass
+ public static void setUp() {
+ row1 = new BeamRecord(genRowType(), 1L, 1, 1.0);
+
+ row2 = new BeamRecord(genRowType(), 2L, 2, 2.0);
+ }
+
+ @Test public void testCsvRecorderDecoder() throws Exception {
+ PCollection<BeamRecord> result = pipeline
+ .apply(
+ Create.of("1,\"1\",1.0", "2,2,2.0")
+ )
+ .apply(ParDo.of(new String2KvBytes()))
+ .apply(
+ new BeamKafkaCSVTable.CsvRecorderDecoder(genRowType(), CSVFormat.DEFAULT)
+ );
+
+ PAssert.that(result).containsInAnyOrder(row1, row2);
+
+ pipeline.run();
+ }
+
+ @Test public void testCsvRecorderEncoder() throws Exception {
+ PCollection<BeamRecord> result = pipeline
+ .apply(
+ Create.of(row1, row2)
+ )
+ .apply(
+ new BeamKafkaCSVTable.CsvRecorderEncoder(genRowType(), CSVFormat.DEFAULT)
+ ).apply(
+ new BeamKafkaCSVTable.CsvRecorderDecoder(genRowType(), CSVFormat.DEFAULT)
+ );
+
+ PAssert.that(result).containsInAnyOrder(row1, row2);
+
+ pipeline.run();
+ }
+
+ private static BeamRecordSqlType genRowType() {
+ return CalciteUtils.toBeamRowType(new RelProtoDataType() {
+
+ @Override public RelDataType apply(RelDataTypeFactory a0) {
+ return a0.builder().add("order_id", SqlTypeName.BIGINT)
+ .add("site_id", SqlTypeName.INTEGER)
+ .add("price", SqlTypeName.DOUBLE).build();
+ }
+ }.apply(BeamQueryPlanner.TYPE_FACTORY));
+ }
+
+ private static class String2KvBytes extends DoFn<String, KV<byte[], byte[]>>
+ implements Serializable {
+ @ProcessElement
+ public void processElement(ProcessContext ctx) {
+ ctx.output(KV.of(new byte[] {}, ctx.element().getBytes()));
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
new file mode 100644
index 0000000..a7c2719
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.kafka;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableList;
+import java.net.URI;
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.junit.Test;
+
+/**
+ * UnitTest for {@link KafkaTableProvider}.
+ */
+public class KafkaTableProviderTest {
+ private KafkaTableProvider provider = new KafkaTableProvider();
+ @Test public void testBuildBeamSqlTable() throws Exception {
+ Table table = mockTable("hello");
+ BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
+
+ assertNotNull(sqlTable);
+ assertTrue(sqlTable instanceof BeamKafkaCSVTable);
+
+ BeamKafkaCSVTable csvTable = (BeamKafkaCSVTable) sqlTable;
+ assertEquals("localhost:9092", csvTable.getBootstrapServers());
+ assertEquals(ImmutableList.of("topic1", "topic2"), csvTable.getTopics());
+ }
+
+ @Test
+ public void testGetTableType() throws Exception {
+ assertEquals("kafka", provider.getTableType());
+ }
+
+ private static Table mockTable(String name) {
+ JSONObject properties = new JSONObject();
+ properties.put("bootstrap.servers", "localhost:9092");
+ JSONArray topics = new JSONArray();
+ topics.add("topic1");
+ topics.add("topic2");
+ properties.put("topics", topics);
+
+ return Table.builder()
+ .name(name)
+ .comment(name + " table")
+ .location(URI.create("kafka://localhost:2181/brokers?topic=test"))
+ .columns(ImmutableList.of(
+ Column.builder().name("id").type(Types.INTEGER).primaryKey(true).build(),
+ Column.builder().name("name").type(Types.VARCHAR).primaryKey(false).build()
+ ))
+ .type("kafka")
+ .properties(properties)
+ .build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableTest.java
new file mode 100644
index 0000000..0f1085f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableTest {
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+ @Rule public TestPipeline pipeline2 = TestPipeline.create();
+
+ /**
+ * testData.
+ *
+ * <p>
+ * The types of the csv fields are:
+ * integer,bigint,float,double,string
+ * </p>
+ */
+ private static Object[] data1 = new Object[] { 1, 1L, 1.1F, 1.1, "james" };
+ private static Object[] data2 = new Object[] { 2, 2L, 2.2F, 2.2, "bond" };
+
+ private static List<Object[]> testData = Arrays.asList(data1, data2);
+ private static List<BeamRecord> testDataRows = new ArrayList<BeamRecord>() {{
+ for (Object[] data : testData) {
+ add(buildRow(data));
+ }
+ }};
+
+ private static Path tempFolder;
+ private static File readerSourceFile;
+ private static File writerTargetFile;
+
+ @Test public void testBuildIOReader() {
+ PCollection<BeamRecord> rows = new BeamTextCSVTable(buildBeamSqlRowType(),
+ readerSourceFile.getAbsolutePath()).buildIOReader(pipeline);
+ PAssert.that(rows).containsInAnyOrder(testDataRows);
+ pipeline.run();
+ }
+
+ @Test public void testBuildIOWriter() {
+ new BeamTextCSVTable(buildBeamSqlRowType(),
+ readerSourceFile.getAbsolutePath()).buildIOReader(pipeline)
+ .apply(new BeamTextCSVTable(buildBeamSqlRowType(), writerTargetFile.getAbsolutePath())
+ .buildIOWriter());
+ pipeline.run();
+
+ PCollection<BeamRecord> rows = new BeamTextCSVTable(buildBeamSqlRowType(),
+ writerTargetFile.getAbsolutePath()).buildIOReader(pipeline2);
+
+ // confirm the two reads match
+ PAssert.that(rows).containsInAnyOrder(testDataRows);
+ pipeline2.run();
+ }
+
+ @BeforeClass public static void setUp() throws IOException {
+ tempFolder = Files.createTempDirectory("BeamTextTableTest");
+ readerSourceFile = writeToFile(testData, "readerSourceFile.txt");
+ writerTargetFile = writeToFile(testData, "writerTargetFile.txt");
+ }
+
+ @AfterClass public static void teardownClass() throws IOException {
+ Files.walkFileTree(tempFolder, new SimpleFileVisitor<Path>() {
+
+ @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+ throws IOException {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ private static File writeToFile(List<Object[]> rows, String filename) throws IOException {
+ File file = tempFolder.resolve(filename).toFile();
+ OutputStream output = new FileOutputStream(file);
+ writeToStreamAndClose(rows, output);
+ return file;
+ }
+
+ /**
+ * Helper that writes the given lines (adding a newline in between) to a stream, then closes the
+ * stream.
+ */
+ private static void writeToStreamAndClose(List<Object[]> rows, OutputStream outputStream) {
+ try (PrintStream writer = new PrintStream(outputStream)) {
+ CSVPrinter printer = CSVFormat.DEFAULT.print(writer);
+ for (Object[] row : rows) {
+ for (Object field : row) {
+ printer.print(field);
+ }
+ printer.println();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private RelProtoDataType buildRowType() {
+ return new RelProtoDataType() {
+
+ @Override public RelDataType apply(RelDataTypeFactory a0) {
+ return a0.builder().add("id", SqlTypeName.INTEGER).add("order_id", SqlTypeName.BIGINT)
+ .add("price", SqlTypeName.FLOAT).add("amount", SqlTypeName.DOUBLE)
+ .add("user_name", SqlTypeName.VARCHAR).build();
+ }
+ };
+ }
+
+ private static RelDataType buildRelDataType() {
+ return BeamQueryPlanner.TYPE_FACTORY.builder().add("id", SqlTypeName.INTEGER)
+ .add("order_id", SqlTypeName.BIGINT).add("price", SqlTypeName.FLOAT)
+ .add("amount", SqlTypeName.DOUBLE).add("user_name", SqlTypeName.VARCHAR).build();
+ }
+
+ private static BeamRecordSqlType buildBeamSqlRowType() {
+ return CalciteUtils.toBeamRowType(buildRelDataType());
+ }
+
+ private static BeamRecord buildRow(Object[] data) {
+ return new BeamRecord(buildBeamSqlRowType(), Arrays.asList(data));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java
new file mode 100644
index 0000000..86edd47
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.provider.text;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableList;
+import java.net.URI;
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.commons.csv.CSVFormat;
+import org.junit.Test;
+
+/**
+ * UnitTest for {@link TextTableProvider}.
+ */
+public class TextTableProviderTest {
+ private TextTableProvider provider = new TextTableProvider();
+
+ @Test
+ public void testGetTableType() throws Exception {
+ assertEquals("text", provider.getTableType());
+ }
+
+ @Test
+ public void testBuildBeamSqlTable() throws Exception {
+ Table table = mockTable("hello", null);
+ BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
+
+ assertNotNull(sqlTable);
+ assertTrue(sqlTable instanceof BeamTextCSVTable);
+
+ BeamTextCSVTable csvTable = (BeamTextCSVTable) sqlTable;
+ assertEquals(CSVFormat.DEFAULT, csvTable.getCsvFormat());
+ assertEquals("/home/admin/hello", csvTable.getFilePattern());
+ }
+
+ @Test
+ public void testBuildBeamSqlTable_customizedFormat() throws Exception {
+ Table table = mockTable("hello", "Excel");
+ BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
+
+ assertNotNull(sqlTable);
+ assertTrue(sqlTable instanceof BeamTextCSVTable);
+
+ BeamTextCSVTable csvTable = (BeamTextCSVTable) sqlTable;
+ assertEquals(CSVFormat.EXCEL, csvTable.getCsvFormat());
+ }
+
+ private static Table mockTable(String name, String format) {
+ JSONObject properties = new JSONObject();
+ if (format != null) {
+ properties.put("format", format);
+ }
+ return Table.builder()
+ .name(name)
+ .comment(name + " table")
+ .location(URI.create("text://home/admin/" + name))
+ .columns(ImmutableList.of(
+ Column.builder().name("id").type(Types.INTEGER).primaryKey(true).build(),
+ Column.builder().name("name").type(Types.VARCHAR).primaryKey(false).build()
+ ))
+ .type("text")
+ .properties(properties)
+ .build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java
new file mode 100644
index 0000000..2be5e8a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.meta.store;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableList;
+import java.net.URI;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Column;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.beam.sdk.extensions.sql.meta.provider.text.TextTableProvider;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * UnitTest for {@link InMemoryMetaStore}.
+ */
+public class InMemoryMetaStoreTest {
+ private InMemoryMetaStore store;
+
+ @Before
+ public void setUp() {
+ store = new InMemoryMetaStore();
+ store.registerProvider(new TextTableProvider());
+ }
+
+ @Test
+ public void testCreateTable() throws Exception {
+ Table table = mockTable("person");
+ store.createTable(table);
+ Table actualTable = store.getTable("person");
+ assertEquals(table, actualTable);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCreateTable_invalidTableType() throws Exception {
+ Table table = mockTable("person", "invalid");
+
+ store.createTable(table);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCreateTable_duplicatedName() throws Exception {
+ Table table = mockTable("person");
+ store.createTable(table);
+ store.createTable(table);
+ }
+
+ @Test
+ public void testGetTable_nullName() throws Exception {
+ Table table = store.getTable(null);
+ assertNull(table);
+ }
+
+ @Test public void testListTables() throws Exception {
+ store.createTable(mockTable("hello"));
+ store.createTable(mockTable("world"));
+
+ assertThat(store.listTables(),
+ Matchers.containsInAnyOrder(mockTable("hello"), mockTable("world")));
+ }
+
+ @Test public void testBuildBeamSqlTable() throws Exception {
+ store.createTable(mockTable("hello"));
+ BeamSqlTable actualSqlTable = store.buildBeamSqlTable("hello");
+ assertNotNull(actualSqlTable);
+ assertEquals(
+ BeamRecordSqlType.create(
+ ImmutableList.of("id", "name"),
+ ImmutableList.of(Types.INTEGER, Types.VARCHAR)
+ ),
+ actualSqlTable.getRowType()
+ );
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBuildBeamSqlTable_tableNotExist() throws Exception {
+ store.buildBeamSqlTable("world");
+ }
+
+ @Test
+ public void testRegisterProvider() throws Exception {
+ store.registerProvider(new MockTableProvider("mock", "hello", "world"));
+ assertNotNull(store.getProviders());
+ assertEquals(2, store.getProviders().size());
+ assertEquals("text", store.getProviders().get("text").getTableType());
+ assertEquals("mock", store.getProviders().get("mock").getTableType());
+
+ assertEquals(2, store.listTables().size());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRegisterProvider_duplicatedTableType() throws Exception {
+ store.registerProvider(new MockTableProvider("mock"));
+ store.registerProvider(new MockTableProvider("mock"));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testRegisterProvider_duplicatedTableName() throws Exception {
+ store.registerProvider(new MockTableProvider("mock", "hello", "world"));
+ store.registerProvider(new MockTableProvider("mock1", "hello", "world"));
+ }
+
+ private static Table mockTable(String name, String type) {
+ return Table.builder()
+ .name(name)
+ .comment(name + " table")
+ .location(URI.create("text://home/admin/" + name))
+ .columns(ImmutableList.of(
+ Column.builder().name("id").type(Types.INTEGER).primaryKey(true).build(),
+ Column.builder().name("name").type(Types.VARCHAR).primaryKey(false).build()
+ ))
+ .type(type)
+ .properties(new JSONObject())
+ .build();
+ }
+
+ private static Table mockTable(String name) {
+ return mockTable(name, "text");
+ }
+
+ private static class MockTableProvider implements TableProvider {
+ private String type;
+ private String[] names;
+ public MockTableProvider(String type, String... names) {
+ this.type = type;
+ this.names = names;
+ }
+
+ @Override public void init() {
+
+ }
+
+ @Override public String getTableType() {
+ return type;
+ }
+
+ @Override public void createTable(Table table) {
+
+ }
+
+ @Override public List<Table> listTables() {
+ List<Table> ret = new ArrayList<>(names.length);
+ for (String name : names) {
+ ret.add(mockTable(name, "mock"));
+ }
+
+ return ret;
+ }
+
+ @Override public BeamSqlTable buildBeamSqlTable(Table table) {
+ return null;
+ }
+
+ @Override public void close() {
+
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
new file mode 100644
index 0000000..cf66268
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.mock;
+
+import static org.apache.beam.sdk.extensions.sql.TestUtils.buildBeamSqlRowType;
+import static org.apache.beam.sdk.extensions.sql.TestUtils.buildRows;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+ * Mocked table for bounded data sources.
+ */
+public class MockedBoundedTable extends MockedTable {
+ /** rows written to this table. */
+ private static final ConcurrentLinkedQueue<BeamRecord> CONTENT = new ConcurrentLinkedQueue<>();
+ /** rows flow out from this table. */
+ private final List<BeamRecord> rows = new ArrayList<>();
+
+ public MockedBoundedTable(BeamRecordSqlType beamSqlRowType) {
+ super(beamSqlRowType);
+ }
+
+ /**
+ * Convenient way to build a mocked bounded table.
+ *
+ * <p>e.g.
+ *
+ * <pre>{@code
+ * MockedUnboundedTable
+ * .of(Types.BIGINT, "order_id",
+ * Types.INTEGER, "site_id",
+ * Types.DOUBLE, "price",
+ * Types.TIMESTAMP, "order_time")
+ * }</pre>
+ */
+ public static MockedBoundedTable of(final Object... args){
+ return new MockedBoundedTable(buildBeamSqlRowType(args));
+ }
+
+ /**
+ * Build a mocked bounded table with the specified type.
+ */
+ public static MockedBoundedTable of(final BeamRecordSqlType type) {
+ return new MockedBoundedTable(type);
+ }
+
+
+ /**
+ * Add rows to the builder.
+ *
+ * <p>Sample usage:
+ *
+ * <pre>{@code
+ * addRows(
+ * 1, 3, "james", -- first row
+ * 2, 5, "bond" -- second row
+ * ...
+ * )
+ * }</pre>
+ */
+ public MockedBoundedTable addRows(Object... args) {
+ List<BeamRecord> rows = buildRows(getRowType(), Arrays.asList(args));
+ this.rows.addAll(rows);
+ return this;
+ }
+
+ @Override
+ public BeamIOType getSourceType() {
+ return BeamIOType.BOUNDED;
+ }
+
+ @Override
+ public PCollection<BeamRecord> buildIOReader(Pipeline pipeline) {
+ return PBegin.in(pipeline).apply(
+ "MockedBoundedTable_Reader_" + COUNTER.incrementAndGet(), Create.of(rows));
+ }
+
+ @Override public PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter() {
+ return new OutputStore();
+ }
+
+ /**
+ * Keep output in {@code CONTENT} for validation.
+ *
+ */
+ public static class OutputStore extends PTransform<PCollection<BeamRecord>, PDone> {
+
+ @Override
+ public PDone expand(PCollection<BeamRecord> input) {
+ input.apply(ParDo.of(new DoFn<BeamRecord, Void>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ CONTENT.add(c.element());
+ }
+
+ @Teardown
+ public void close() {
+ CONTENT.clear();
+ }
+
+ }));
+ return PDone.in(input.getPipeline());
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
new file mode 100644
index 0000000..d661866
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.mock;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+ * Base class for mocked table.
+ */
+public abstract class MockedTable extends BaseBeamTable {
+ public static final AtomicInteger COUNTER = new AtomicInteger();
+ public MockedTable(BeamRecordSqlType beamSqlRowType) {
+ super(beamSqlRowType);
+ }
+
+ @Override
+ public PTransform<? super PCollection<BeamRecord>, PDone> buildIOWriter() {
+ throw new UnsupportedOperationException("buildIOWriter unsupported!");
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
new file mode 100644
index 0000000..2e4790b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.mock;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamIOType;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.calcite.util.Pair;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A mocked unbounded table.
+ */
+public class MockedUnboundedTable extends MockedTable {
+ /** rows flow out from this table with the specified watermark instant. */
+ private final List<Pair<Duration, List<BeamRecord>>> timestampedRows = new ArrayList<>();
+ /** specify the index of column in the row which stands for the event time field. */
+ private int timestampField;
+ private MockedUnboundedTable(BeamRecordSqlType beamSqlRowType) {
+ super(beamSqlRowType);
+ }
+
+ /**
+ * Convenient way to build a mocked unbounded table.
+ *
+ * <p>e.g.
+ *
+ * <pre>{@code
+ * MockedUnboundedTable
+ * .of(Types.BIGINT, "order_id",
+ * Types.INTEGER, "site_id",
+ * Types.DOUBLE, "price",
+ * Types.TIMESTAMP, "order_time")
+ * }</pre>
+ */
+ public static MockedUnboundedTable of(final Object... args){
+ return new MockedUnboundedTable(TestUtils.buildBeamSqlRowType(args));
+ }
+
+ public MockedUnboundedTable timestampColumnIndex(int idx) {
+ this.timestampField = idx;
+ return this;
+ }
+
+ /**
+ * Add rows to the builder.
+ *
+ * <p>Sample usage:
+ *
+ * <pre>{@code
+ * addRows(
+ * duration, -- duration which stands for the corresponding watermark instant
+ * 1, 3, "james", -- first row
+ * 2, 5, "bond" -- second row
+ * ...
+ * )
+ * }</pre>
+ */
+ public MockedUnboundedTable addRows(Duration duration, Object... args) {
+ List<BeamRecord> rows = TestUtils.buildRows(getRowType(), Arrays.asList(args));
+ // record the watermark + rows
+ this.timestampedRows.add(Pair.of(duration, rows));
+ return this;
+ }
+
+ @Override public BeamIOType getSourceType() {
+ return BeamIOType.UNBOUNDED;
+ }
+
+ @Override public PCollection<BeamRecord> buildIOReader(Pipeline pipeline) {
+ TestStream.Builder<BeamRecord> values = TestStream.create(beamRecordSqlType.getRecordCoder());
+
+ for (Pair<Duration, List<BeamRecord>> pair : timestampedRows) {
+ values = values.advanceWatermarkTo(new Instant(0).plus(pair.getKey()));
+ for (int i = 0; i < pair.getValue().size(); i++) {
+ values = values.addElements(TimestampedValue.of(pair.getValue().get(i),
+ new Instant(pair.getValue().get(i).getDate(timestampField))));
+ }
+ }
+
+ return pipeline.begin().apply(
+ "MockedUnboundedTable_" + COUNTER.incrementAndGet(),
+ values.advanceWatermarkToInfinity());
+ }
+}
diff --git a/sdks/java/fn-execution/pom.xml b/sdks/java/fn-execution/pom.xml
new file mode 100644
index 0000000..3bdec38
--- /dev/null
+++ b/sdks/java/fn-execution/pom.xml
@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: Fn Execution</name>
+ <description>Contains code shared across the Beam Java SDK Harness Java Runners to execute using
+ the Beam Portability Framework
+ </description>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <exclude>com.google.guava:guava-jdk5</exclude>
+ <exclude>com.google.protobuf:protobuf-lite</exclude>
+ <!-- Fn Execution contains shared utilities for Runners and Harnesses which use
+ the Portability framework. Runner-side interactions must not require a
+ dependency on any particular SDK, so this library must not introduce such an
+ edge. -->
+ <exclude>org.apache.beam:beam-sdks-java-core</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-x86_64</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/ManagedChannelFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/ManagedChannelFactory.java
new file mode 100644
index 0000000..187cfdb
--- /dev/null
+++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/ManagedChannelFactory.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.channel;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.NettyChannelBuilder;
+import io.netty.channel.epoll.EpollDomainSocketChannel;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollSocketChannel;
+import io.netty.channel.unix.DomainSocketAddress;
+import java.net.SocketAddress;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+
+/**
+ * A Factory which creates an underlying {@link ManagedChannel} implementation.
+ */
+public abstract class ManagedChannelFactory {
+ public static ManagedChannelFactory createDefault() {
+ return new Default();
+ }
+
+ public static ManagedChannelFactory createEpoll() {
+ io.netty.channel.epoll.Epoll.ensureAvailability();
+ return new Epoll();
+ }
+
+ public abstract ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor);
+
+ /**
+ * Creates a {@link ManagedChannel} backed by an {@link EpollDomainSocketChannel} if the address
+ * is a {@link DomainSocketAddress}. Otherwise creates a {@link ManagedChannel} backed by an
+ * {@link EpollSocketChannel}.
+ */
+ private static class Epoll extends ManagedChannelFactory {
+ @Override
+ public ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor) {
+ SocketAddress address = SocketAddressFactory.createFrom(apiServiceDescriptor.getUrl());
+ return NettyChannelBuilder.forAddress(address)
+ .channelType(address instanceof DomainSocketAddress
+ ? EpollDomainSocketChannel.class : EpollSocketChannel.class)
+ .eventLoopGroup(new EpollEventLoopGroup())
+ .usePlaintext(true)
+ // Set the message size to max value here. The actual size is governed by the
+ // buffer size in the layers above.
+ .maxInboundMessageSize(Integer.MAX_VALUE)
+ .build();
+ }
+ }
+
+ /**
+ * Creates a {@link ManagedChannel} relying on the {@link ManagedChannelBuilder} to create
+ * instances.
+ */
+ private static class Default extends ManagedChannelFactory {
+ @Override
+ public ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor) {
+ return ManagedChannelBuilder.forTarget(apiServiceDescriptor.getUrl())
+ .usePlaintext(true)
+ // Set the message size to max value here. The actual size is governed by the
+ // buffer size in the layers above.
+ .maxInboundMessageSize(Integer.MAX_VALUE)
+ .build();
+ }
+ }
+}
diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/SocketAddressFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/SocketAddressFactory.java
new file mode 100644
index 0000000..5253291
--- /dev/null
+++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/SocketAddressFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.channel;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.net.HostAndPort;
+import io.netty.channel.unix.DomainSocketAddress;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+
+/** Creates a {@link SocketAddress} based upon a supplied string. */
+public class SocketAddressFactory {
+ private static final String UNIX_DOMAIN_SOCKET_PREFIX = "unix://";
+
+ /**
+ * Parse a {@link SocketAddress} from the given string.
+ */
+ public static SocketAddress createFrom(String value) {
+ if (value.startsWith(UNIX_DOMAIN_SOCKET_PREFIX)) {
+ // Unix Domain Socket address.
+ // Create the underlying file for the Unix Domain Socket.
+ String filePath = value.substring(UNIX_DOMAIN_SOCKET_PREFIX.length());
+ File file = new File(filePath);
+ if (!file.isAbsolute()) {
+ throw new IllegalArgumentException("File path must be absolute: " + filePath);
+ }
+ try {
+ if (file.createNewFile()) {
+ // If this application created the file, delete it when the application exits.
+ file.deleteOnExit();
+ }
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ // Create the SocketAddress referencing the file.
+ return new DomainSocketAddress(file);
+ } else {
+ // Standard TCP/IP address.
+ HostAndPort hostAndPort = HostAndPort.fromString(value);
+ checkArgument(hostAndPort.hasPort(),
+ "Address must be a unix:// path or be in the form host:port. Got: %s", value);
+ return new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort());
+ }
+ }
+}
diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/package-info.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/package-info.java
new file mode 100644
index 0000000..2a33445
--- /dev/null
+++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * gRPC channel management.
+ */
+package org.apache.beam.harness.channel;
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/ManagedChannelFactoryTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/ManagedChannelFactoryTest.java
new file mode 100644
index 0000000..f73ed80
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/ManagedChannelFactoryTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.channel;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
+
+import io.grpc.ManagedChannel;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+
+/** Tests for {@link ManagedChannelFactory}. */
+@RunWith(JUnit4.class)
+public class ManagedChannelFactoryTest {
+ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Test
+ public void testDefaultChannel() {
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:123").build();
+ ManagedChannel channel =
+ ManagedChannelFactory.createDefault().forDescriptor(apiServiceDescriptor);
+ assertEquals("localhost:123", channel.authority());
+ channel.shutdownNow();
+ }
+
+ @Test
+ public void testEpollHostPortChannel() {
+ assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:123").build();
+ ManagedChannel channel =
+ ManagedChannelFactory.createEpoll().forDescriptor(apiServiceDescriptor);
+ assertEquals("localhost:123", channel.authority());
+ channel.shutdownNow();
+ }
+
+ @Test
+ public void testEpollDomainSocketChannel() throws Exception {
+ assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
+ .setUrl("unix://" + tmpFolder.newFile().getAbsolutePath())
+ .build();
+ ManagedChannel channel =
+ ManagedChannelFactory.createEpoll().forDescriptor(apiServiceDescriptor);
+ assertEquals(apiServiceDescriptor.getUrl().substring("unix://".length()), channel.authority());
+ channel.shutdownNow();
+ }
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/SocketAddressFactoryTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/SocketAddressFactoryTest.java
new file mode 100644
index 0000000..95a7d67
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/SocketAddressFactoryTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.channel;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import io.netty.channel.unix.DomainSocketAddress;
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SocketAddressFactory}. */
+@RunWith(JUnit4.class)
+public class SocketAddressFactoryTest {
+ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Test
+ public void testHostPortSocket() {
+ SocketAddress socketAddress = SocketAddressFactory.createFrom("localhost:123");
+ assertThat(socketAddress, Matchers.instanceOf(InetSocketAddress.class));
+ assertEquals("localhost", ((InetSocketAddress) socketAddress).getHostString());
+ assertEquals(123, ((InetSocketAddress) socketAddress).getPort());
+ }
+
+ @Test
+ public void testDomainSocket() throws Exception {
+ File tmpFile = tmpFolder.newFile();
+ SocketAddress socketAddress = SocketAddressFactory.createFrom(
+ "unix://" + tmpFile.getAbsolutePath());
+ assertThat(socketAddress, Matchers.instanceOf(DomainSocketAddress.class));
+ assertEquals(tmpFile.getAbsolutePath(), ((DomainSocketAddress) socketAddress).path());
+ }
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Consumer.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Consumer.java
new file mode 100644
index 0000000..279fc29
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Consumer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+/**
+ * A fork of the Java 8 consumer interface. This exists to enable migration for existing consumers.
+ */
+public interface Consumer<T> {
+ void accept(T item);
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Supplier.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Supplier.java
new file mode 100644
index 0000000..629afc2
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Supplier.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+/**
+ * A fork of the Java 8 Supplier interface, to enable migrations.
+ */
+public interface Supplier<T> {
+ T get();
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutors.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutors.java
new file mode 100644
index 0000000..ca12d5a
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutors.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+import com.google.common.util.concurrent.ForwardingExecutorService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * A {@link TestRule} that validates that all submitted tasks finished and were completed. This
+ * allows for testing that tasks have exercised the appropriate shutdown logic.
+ */
+public class TestExecutors {
+ public static TestExecutorService from(final ExecutorService staticExecutorService) {
+ return from(new Supplier<ExecutorService>() {
+ @Override
+ public ExecutorService get() {
+ return staticExecutorService;
+ }
+ });
+ }
+
+ public static TestExecutorService from(Supplier<ExecutorService> executorServiceSuppler) {
+ return new FromSupplier(executorServiceSuppler);
+ }
+
+ /** A union of the {@link ExecutorService} and {@link TestRule} interfaces. */
+ public interface TestExecutorService extends ExecutorService, TestRule {}
+
+ private static class FromSupplier extends ForwardingExecutorService
+ implements TestExecutorService {
+ private final Supplier<ExecutorService> executorServiceSupplier;
+ private ExecutorService delegate;
+
+ private FromSupplier(Supplier<ExecutorService> executorServiceSupplier) {
+ this.executorServiceSupplier = executorServiceSupplier;
+ }
+
+ @Override
+ public Statement apply(final Statement statement, Description arg1) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ Throwable thrown = null;
+ delegate = executorServiceSupplier.get();
+ try {
+ statement.evaluate();
+ } catch (Throwable t) {
+ thrown = t;
+ }
+ shutdown();
+ if (!awaitTermination(5, TimeUnit.SECONDS)) {
+ shutdownNow();
+ IllegalStateException e =
+ new IllegalStateException("Test executor failed to shutdown cleanly.");
+ if (thrown != null) {
+ thrown.addSuppressed(e);
+ } else {
+ thrown = e;
+ }
+ }
+ if (thrown != null) {
+ throw thrown;
+ }
+ }
+ };
+ }
+
+ @Override
+ protected ExecutorService delegate() {
+ return delegate;
+ }
+ }
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutorsTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutorsTest.java
new file mode 100644
index 0000000..f0c98e0
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutorsTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.beam.harness.test.TestExecutors.TestExecutorService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.junit.runners.model.Statement;
+
+/** Tests for {@link TestExecutors}. */
+@RunWith(JUnit4.class)
+public class TestExecutorsTest {
+ @Test
+ public void testSuccessfulTermination() throws Throwable {
+ ExecutorService service = Executors.newSingleThreadExecutor();
+ final TestExecutorService testService = TestExecutors.from(service);
+ final AtomicBoolean taskRan = new AtomicBoolean();
+ testService
+ .apply(
+ new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ testService.submit(new Runnable() {
+ @Override
+ public void run() {
+ taskRan.set(true);
+ }
+ });
+ }
+ },
+ null)
+ .evaluate();
+ assertTrue(service.isTerminated());
+ assertTrue(taskRan.get());
+ }
+
+ @Test
+ public void testTaskBlocksForeverCausesFailure() throws Throwable {
+ ExecutorService service = Executors.newSingleThreadExecutor();
+ final TestExecutorService testService = TestExecutors.from(service);
+ final AtomicBoolean taskStarted = new AtomicBoolean();
+ final AtomicBoolean taskWasInterrupted = new AtomicBoolean();
+ try {
+ testService
+ .apply(
+ new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ testService.submit(new Runnable() {
+ @Override
+ public void run() {
+ taskToRun();
+ }
+ });
+ }
+
+ private void taskToRun() {
+ taskStarted.set(true);
+ try {
+ while (true) {
+ Thread.sleep(10000);
+ }
+ } catch (InterruptedException e) {
+ taskWasInterrupted.set(true);
+ return;
+ }
+ }
+ },
+ null)
+ .evaluate();
+ fail();
+ } catch (IllegalStateException e) {
+ assertEquals(IllegalStateException.class, e.getClass());
+ assertEquals("Test executor failed to shutdown cleanly.", e.getMessage());
+ }
+ assertTrue(service.isShutdown());
+ }
+
+ @Test
+ public void testStatementFailurePropagatedCleanly() throws Throwable {
+ ExecutorService service = Executors.newSingleThreadExecutor();
+ final TestExecutorService testService = TestExecutors.from(service);
+ final RuntimeException exceptionToThrow = new RuntimeException();
+ try {
+ testService
+ .apply(
+ new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ throw exceptionToThrow;
+ }
+ },
+ null)
+ .evaluate();
+ fail();
+ } catch (RuntimeException thrownException) {
+ assertSame(exceptionToThrow, thrownException);
+ }
+ assertTrue(service.isShutdown());
+ }
+
+ @Test
+ public void testStatementFailurePropagatedWhenExecutorServiceFailingToTerminate()
+ throws Throwable {
+ ExecutorService service = Executors.newSingleThreadExecutor();
+ final TestExecutorService testService = TestExecutors.from(service);
+ final AtomicBoolean taskStarted = new AtomicBoolean();
+ final AtomicBoolean taskWasInterrupted = new AtomicBoolean();
+ final RuntimeException exceptionToThrow = new RuntimeException();
+ try {
+ testService
+ .apply(
+ new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ testService.submit(new Runnable() {
+ @Override
+ public void run() {
+ taskToRun();
+ }
+ });
+ throw exceptionToThrow;
+ }
+
+ private void taskToRun() {
+ taskStarted.set(true);
+ try {
+ while (true) {
+ Thread.sleep(10000);
+ }
+ } catch (InterruptedException e) {
+ taskWasInterrupted.set(true);
+ return;
+ }
+ }
+ },
+ null)
+ .evaluate();
+ fail();
+ } catch (RuntimeException thrownException) {
+ assertSame(exceptionToThrow, thrownException);
+ assertEquals(1, exceptionToThrow.getSuppressed().length);
+ assertEquals(IllegalStateException.class, exceptionToThrow.getSuppressed()[0].getClass());
+ assertEquals(
+ "Test executor failed to shutdown cleanly.",
+ exceptionToThrow.getSuppressed()[0].getMessage());
+ }
+ assertTrue(service.isShutdown());
+ }
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreams.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreams.java
new file mode 100644
index 0000000..3df743a
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreams.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+import io.grpc.stub.CallStreamObserver;
+import io.grpc.stub.StreamObserver;
+
+/** Utility methods which enable testing of {@link StreamObserver}s. */
+public class TestStreams {
+ /**
+ * Creates a test {@link CallStreamObserver} {@link Builder} that forwards
+ * {@link StreamObserver#onNext} calls to the supplied {@link Consumer}.
+ */
+ public static <T> Builder<T> withOnNext(Consumer<T> onNext) {
+ return new Builder<>(new ForwardingCallStreamObserver<>(
+ onNext,
+ TestStreams.<Throwable>noopConsumer(),
+ TestStreams.noopRunnable(),
+ TestStreams.alwaysTrueSupplier()));
+ }
+
+ /** A builder for a test {@link CallStreamObserver} that performs various callbacks. */
+ public static class Builder<T> {
+ private final ForwardingCallStreamObserver<T> observer;
+ private Builder(ForwardingCallStreamObserver<T> observer) {
+ this.observer = observer;
+ }
+
+ /**
+ * Returns a new {@link Builder} like this one with the specified
+ * {@link CallStreamObserver#isReady} callback.
+ */
+ public Builder<T> withIsReady(Supplier<Boolean> isReady) {
+ return new Builder<>(new ForwardingCallStreamObserver<>(
+ observer.onNext,
+ observer.onError,
+ observer.onCompleted,
+ isReady));
+ }
+
+ /**
+ * Returns a new {@link Builder} like this one with the specified
+ * {@link StreamObserver#onCompleted} callback.
+ */
+ public Builder<T> withOnCompleted(Runnable onCompleted) {
+ return new Builder<>(new ForwardingCallStreamObserver<>(
+ observer.onNext,
+ observer.onError,
+ onCompleted,
+ observer.isReady));
+ }
+
+ /**
+ * Returns a new {@link Builder} like this one with the specified
+ * {@link StreamObserver#onError} callback.
+ */
+ public Builder<T> withOnError(final Runnable onError) {
+ return new Builder<>(new ForwardingCallStreamObserver<>(
+ observer.onNext,
+ new Consumer<Throwable>() {
+ @Override
+ public void accept(Throwable t) {
+ onError.run();
+ }
+ },
+ observer.onCompleted,
+ observer.isReady));
+ }
+
+ /**
+ * Returns a new {@link Builder} like this one with the specified
+ * {@link StreamObserver#onError} consumer.
+ */
+ public Builder<T> withOnError(Consumer<Throwable> onError) {
+ return new Builder<>(new ForwardingCallStreamObserver<>(
+ observer.onNext, onError, observer.onCompleted, observer.isReady));
+ }
+
+ public CallStreamObserver<T> build() {
+ return observer;
+ }
+ }
+
+ private static void noop() {
+ }
+
+ private static Runnable noopRunnable() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ }
+ };
+ }
+
+ private static void noop(Throwable t) {
+ }
+
+ private static <T> Consumer<T> noopConsumer() {
+ return new Consumer<T>() {
+ @Override
+ public void accept(T item) {
+ }
+ };
+ }
+
+ private static boolean returnTrue() {
+ return true;
+ }
+
+ private static Supplier<Boolean> alwaysTrueSupplier() {
+ return new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return true;
+ }
+ };
+ }
+
+ /** A {@link CallStreamObserver} which executes the supplied callbacks. */
+ private static class ForwardingCallStreamObserver<T> extends CallStreamObserver<T> {
+ private final Consumer<T> onNext;
+ private final Supplier<Boolean> isReady;
+ private final Consumer<Throwable> onError;
+ private final Runnable onCompleted;
+
+ public ForwardingCallStreamObserver(
+ Consumer<T> onNext,
+ Consumer<Throwable> onError,
+ Runnable onCompleted,
+ Supplier<Boolean> isReady) {
+ this.onNext = onNext;
+ this.onError = onError;
+ this.onCompleted = onCompleted;
+ this.isReady = isReady;
+ }
+
+ @Override
+ public void onNext(T value) {
+ onNext.accept(value);
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ onError.accept(t);
+ }
+
+ @Override
+ public void onCompleted() {
+ onCompleted.run();
+ }
+
+ @Override
+ public boolean isReady() {
+ return isReady.get();
+ }
+
+ @Override
+ public void setOnReadyHandler(Runnable onReadyHandler) {}
+
+ @Override
+ public void disableAutoInboundFlowControl() {}
+
+ @Override
+ public void request(int count) {}
+
+ @Override
+ public void setMessageCompression(boolean enable) {}
+ }
+}
diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreamsTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreamsTest.java
new file mode 100644
index 0000000..c578397
--- /dev/null
+++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreamsTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.harness.test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link TestStreams}. */
+@RunWith(JUnit4.class)
+public class TestStreamsTest {
+ @Test
+ public void testOnNextIsCalled() {
+ final AtomicBoolean onNextWasCalled = new AtomicBoolean();
+ TestStreams.withOnNext(new Consumer<Boolean>() {
+ @Override
+ public void accept(Boolean item) {
+ onNextWasCalled.set(item);
+ }
+ }).build().onNext(true);
+ assertTrue(onNextWasCalled.get());
+ }
+
+ @Test
+ public void testIsReadyIsCalled() {
+ final AtomicBoolean isReadyWasCalled = new AtomicBoolean();
+ assertFalse(TestStreams.withOnNext(null)
+ .withIsReady(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return isReadyWasCalled.getAndSet(true);
+ }
+ })
+ .build()
+ .isReady());
+ assertTrue(isReadyWasCalled.get());
+ }
+
+ @Test
+ public void testOnCompletedIsCalled() {
+ final AtomicBoolean onCompletedWasCalled = new AtomicBoolean();
+ TestStreams.withOnNext(null)
+ .withOnCompleted(new Runnable() {
+ @Override
+ public void run() {
+ onCompletedWasCalled.set(true);
+ }
+ })
+ .build()
+ .onCompleted();
+ assertTrue(onCompletedWasCalled.get());
+ }
+
+ @Test
+ public void testOnErrorRunnableIsCalled() {
+ RuntimeException throwable = new RuntimeException();
+ final AtomicBoolean onErrorWasCalled = new AtomicBoolean();
+ TestStreams.withOnNext(null)
+ .withOnError(new Runnable() {
+ @Override
+ public void run() {
+ onErrorWasCalled.set(true);
+ }
+ })
+ .build()
+ .onError(throwable);
+ assertTrue(onErrorWasCalled.get());
+ }
+
+ @Test
+ public void testOnErrorConsumerIsCalled() {
+ RuntimeException throwable = new RuntimeException();
+ final Collection<Throwable> onErrorWasCalled = new ArrayList<>();
+ TestStreams.withOnNext(null)
+ .withOnError(new Consumer<Throwable>() {
+ @Override
+ public void accept(Throwable item) {
+ onErrorWasCalled.add(item);
+ }
+ })
+ .build()
+ .onError(throwable);
+ assertThat(onErrorWasCalled, Matchers.<Throwable>contains(throwable));
+ }
+}
diff --git a/sdks/java/harness/pom.xml b/sdks/java/harness/pom.xml
index 3918fd9..3707730 100644
--- a/sdks/java/harness/pom.xml
+++ b/sdks/java/harness/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,6 +32,93 @@
<description>This contains the SDK Fn Harness for Beam Java</description>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+ <ignoredUnusedDeclaredDependency>
+ com.google.protobuf:protobuf-java
+ </ignoredUnusedDeclaredDependency>
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <!-- java harness dependencies that are not staged -->
+ <include>org.apache.beam:beam-model-pipeline</include>
+ <include>org.apache.beam:beam-model-fn-execution</include>
+ <include>org.apache.beam:beam-runners-core-construction-java</include>
+ <include>org.apache.beam:beam-runners-core-java</include>
+ <include>org.apache.beam:beam-runners-google-cloud-dataflow-java</include>
+ <include>io.netty:netty-transport-native-epoll</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.fn.harness.private.com.google.common
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.fn.harness.private.com.google.thirdparty
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty.channel</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.fn.harness.private.io.netty.channel
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.beam.runners</pattern>
+ <!--suppress MavenModelInspection -->
+ <shadedPattern>
+ org.apache.beam.fn.harness.private.org.apache.beam.runners
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<!-- Override Beam parent to allow Java8 -->
@@ -46,13 +133,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <excludes>
- <!-- Flaky in Precommit. See BEAM-1487 https://issues.apache.org/jira/browse/BEAM-1487 -->
- <exclude>org.apache.beam.fn.harness.logging.BeamFnLoggingClientTest</exclude>
- <exclude>org.apache.beam.fn.harness.stream.BufferingStreamObserverTest</exclude>
- </excludes>
- </configuration>
</plugin>
</plugins>
</build>
@@ -60,6 +140,21 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
@@ -76,6 +171,14 @@
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
</dependency>
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-fn-execution</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-core-java</artifactId>
@@ -83,12 +186,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-runners-core-construction-java</artifactId>
</dependency>
<dependency>
@@ -131,6 +229,7 @@
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
+ <scope>runtime</scope>
</dependency>
<dependency>
@@ -142,6 +241,12 @@
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
<classifier>linux-x86_64</classifier>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
</dependency>
<dependency>
@@ -149,6 +254,12 @@
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
<groupId>org.hamcrest</groupId>
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
new file mode 100644
index 0000000..ff3dfb2
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static com.google.common.collect.Iterables.getOnlyElement;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.CoderTranslation;
+import org.apache.beam.runners.core.construction.RehydratedComponents;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Registers as a consumer for data over the Beam Fn API. Multiplexes any received data
+ * to all consumers in the specified output map.
+ *
+ * <p>Can be re-used serially across {@link BeamFnApi.ProcessBundleRequest}s.
+ * For each request, call {@link #registerInputLocation()} to start and call
+ * {@link #blockTillReadFinishes()} to finish.
+ */
+public class BeamFnDataReadRunner<OutputT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataReadRunner.class);
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final String URN = "urn:org.apache.beam:source:runner:0.1";
+
+ /** A registrar which provides a factory to handle reading from the Fn Api Data Plane. */
+ @AutoService(PTransformRunnerFactory.Registrar.class)
+ public static class Registrar implements
+ PTransformRunnerFactory.Registrar {
+
+ @Override
+ public Map<String, PTransformRunnerFactory> getPTransformRunnerFactories() {
+ return ImmutableMap.of(URN, new Factory());
+ }
+ }
+
+ /** A factory for {@link BeamFnDataReadRunner}s. */
+ static class Factory<OutputT>
+ implements PTransformRunnerFactory<BeamFnDataReadRunner<OutputT>> {
+
+ @Override
+ public BeamFnDataReadRunner<OutputT> createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+
+ BeamFnApi.Target target = BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference(pTransformId)
+ .setName(getOnlyElement(pTransform.getOutputsMap().keySet()))
+ .build();
+ RunnerApi.Coder coderSpec =
+ coders.get(
+ pCollections.get(getOnlyElement(pTransform.getOutputsMap().values())).getCoderId());
+ Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers =
+ (Collection) pCollectionIdsToConsumers.get(
+ getOnlyElement(pTransform.getOutputsMap().values()));
+
+ BeamFnDataReadRunner<OutputT> runner = new BeamFnDataReadRunner<>(
+ pTransform.getSpec(),
+ processBundleInstructionId,
+ target,
+ coderSpec,
+ coders,
+ beamFnDataClient,
+ consumers);
+ addStartFunction.accept(runner::registerInputLocation);
+ addFinishFunction.accept(runner::blockTillReadFinishes);
+ return runner;
+ }
+ }
+
+ private final Endpoints.ApiServiceDescriptor apiServiceDescriptor;
+ private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
+ private final Supplier<String> processBundleInstructionIdSupplier;
+ private final BeamFnDataClient beamFnDataClientFactory;
+ private final Coder<WindowedValue<OutputT>> coder;
+ private final BeamFnApi.Target inputTarget;
+
+ private CompletableFuture<Void> readFuture;
+
+ BeamFnDataReadRunner(
+ RunnerApi.FunctionSpec functionSpec,
+ Supplier<String> processBundleInstructionIdSupplier,
+ BeamFnApi.Target inputTarget,
+ RunnerApi.Coder coderSpec,
+ Map<String, RunnerApi.Coder> coders,
+ BeamFnDataClient beamFnDataClientFactory,
+ Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers)
+ throws IOException {
+ this.apiServiceDescriptor =
+ BeamFnApi.RemoteGrpcPort.parseFrom(functionSpec.getPayload()).getApiServiceDescriptor();
+ this.inputTarget = inputTarget;
+ this.processBundleInstructionIdSupplier = processBundleInstructionIdSupplier;
+ this.beamFnDataClientFactory = beamFnDataClientFactory;
+ this.consumers = consumers;
+
+ @SuppressWarnings("unchecked")
+ Coder<WindowedValue<OutputT>> coder =
+ (Coder<WindowedValue<OutputT>>)
+ CoderTranslation.fromProto(
+ coderSpec,
+ RehydratedComponents.forComponents(
+ RunnerApi.Components.newBuilder().putAllCoders(coders).build()));
+ this.coder = coder;
+ }
+
+ public void registerInputLocation() {
+ this.readFuture = beamFnDataClientFactory.forInboundConsumer(
+ apiServiceDescriptor,
+ KV.of(processBundleInstructionIdSupplier.get(), inputTarget),
+ coder,
+ this::multiplexToConsumers);
+ }
+
+ public void blockTillReadFinishes() throws Exception {
+ LOG.debug("Waiting for process bundle instruction {} and target {} to close.",
+ processBundleInstructionIdSupplier.get(), inputTarget);
+ readFuture.get();
+ }
+
+ private void multiplexToConsumers(WindowedValue<OutputT> value) throws Exception {
+ for (ThrowingConsumer<WindowedValue<OutputT>> consumer : consumers) {
+ consumer.accept(value);
+ }
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
new file mode 100644
index 0000000..bf1994e
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static com.google.common.collect.Iterables.getOnlyElement;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.CoderTranslation;
+import org.apache.beam.runners.core.construction.RehydratedComponents;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+
+/**
+ * Registers as a consumer with the Beam Fn Data Api. Consumes elements and encodes them for
+ * transmission.
+ *
+ * <p>Can be re-used serially across {@link BeamFnApi.ProcessBundleRequest}s.
+ * For each request, call {@link #registerForOutput()} to start and call {@link #close()} to finish.
+ */
+public class BeamFnDataWriteRunner<InputT> {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final String URN = "urn:org.apache.beam:sink:runner:0.1";
+
+ /** A registrar which provides a factory to handle writing to the Fn Api Data Plane. */
+ @AutoService(PTransformRunnerFactory.Registrar.class)
+ public static class Registrar implements
+ PTransformRunnerFactory.Registrar {
+
+ @Override
+ public Map<String, PTransformRunnerFactory> getPTransformRunnerFactories() {
+ return ImmutableMap.of(URN, new Factory());
+ }
+ }
+
+ /** A factory for {@link BeamFnDataWriteRunner}s. */
+ static class Factory<InputT>
+ implements PTransformRunnerFactory<BeamFnDataWriteRunner<InputT>> {
+
+ @Override
+ public BeamFnDataWriteRunner<InputT> createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ BeamFnApi.Target target = BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference(pTransformId)
+ .setName(getOnlyElement(pTransform.getInputsMap().keySet()))
+ .build();
+ RunnerApi.Coder coderSpec = coders.get(
+ pCollections.get(getOnlyElement(pTransform.getInputsMap().values())).getCoderId());
+ BeamFnDataWriteRunner<InputT> runner =
+ new BeamFnDataWriteRunner<>(
+ pTransform.getSpec(),
+ processBundleInstructionId,
+ target,
+ coderSpec,
+ coders,
+ beamFnDataClient);
+ addStartFunction.accept(runner::registerForOutput);
+ pCollectionIdsToConsumers.put(
+ getOnlyElement(pTransform.getInputsMap().values()),
+ (ThrowingConsumer)
+ (ThrowingConsumer<WindowedValue<InputT>>) runner::consume);
+ addFinishFunction.accept(runner::close);
+ return runner;
+ }
+ }
+
+ private final Endpoints.ApiServiceDescriptor apiServiceDescriptor;
+ private final BeamFnApi.Target outputTarget;
+ private final Coder<WindowedValue<InputT>> coder;
+ private final BeamFnDataClient beamFnDataClientFactory;
+ private final Supplier<String> processBundleInstructionIdSupplier;
+
+ private CloseableThrowingConsumer<WindowedValue<InputT>> consumer;
+
+ BeamFnDataWriteRunner(
+ RunnerApi.FunctionSpec functionSpec,
+ Supplier<String> processBundleInstructionIdSupplier,
+ BeamFnApi.Target outputTarget,
+ RunnerApi.Coder coderSpec,
+ Map<String, RunnerApi.Coder> coders,
+ BeamFnDataClient beamFnDataClientFactory)
+ throws IOException {
+ this.apiServiceDescriptor =
+ BeamFnApi.RemoteGrpcPort.parseFrom(functionSpec.getPayload()).getApiServiceDescriptor();
+ this.beamFnDataClientFactory = beamFnDataClientFactory;
+ this.processBundleInstructionIdSupplier = processBundleInstructionIdSupplier;
+ this.outputTarget = outputTarget;
+
+ @SuppressWarnings("unchecked")
+ Coder<WindowedValue<InputT>> coder =
+ (Coder<WindowedValue<InputT>>)
+ CoderTranslation.fromProto(
+ coderSpec,
+ RehydratedComponents.forComponents(
+ RunnerApi.Components.newBuilder().putAllCoders(coders).build()));
+ this.coder = coder;
+ }
+
+ public void registerForOutput() {
+ consumer = beamFnDataClientFactory.forOutboundConsumer(
+ apiServiceDescriptor,
+ KV.of(processBundleInstructionIdSupplier.get(), outputTarget),
+ coder);
+ }
+
+ public void close() throws Exception {
+ consumer.close();
+ }
+
+ public void consume(WindowedValue<InputT> value) throws Exception {
+ consumer.accept(value);
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
new file mode 100644
index 0000000..d523365
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Source.Reader;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+
+/**
+ * A runner which creates {@link Reader}s for each {@link BoundedSource} sent as an input and
+ * executes the {@link Reader}s read loop.
+ */
+public class BoundedSourceRunner<InputT extends BoundedSource<OutputT>, OutputT> {
+
+ private static final String URN = "urn:org.apache.beam:source:java:0.1";
+
+ /** A registrar which provides a factory to handle Java {@link BoundedSource}s. */
+ @AutoService(PTransformRunnerFactory.Registrar.class)
+ public static class Registrar implements
+ PTransformRunnerFactory.Registrar {
+
+ @Override
+ public Map<String, PTransformRunnerFactory> getPTransformRunnerFactories() {
+ return ImmutableMap.of(URN, new Factory());
+ }
+ }
+
+ /** A factory for {@link BoundedSourceRunner}. */
+ static class Factory<InputT extends BoundedSource<OutputT>, OutputT>
+ implements PTransformRunnerFactory<BoundedSourceRunner<InputT, OutputT>> {
+ @Override
+ public BoundedSourceRunner<InputT, OutputT> createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) {
+
+ ImmutableList.Builder<ThrowingConsumer<WindowedValue<?>>> consumers = ImmutableList.builder();
+ for (String pCollectionId : pTransform.getOutputsMap().values()) {
+ consumers.addAll(pCollectionIdsToConsumers.get(pCollectionId));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ BoundedSourceRunner<InputT, OutputT> runner = new BoundedSourceRunner(
+ pipelineOptions,
+ pTransform.getSpec(),
+ consumers.build());
+
+ // TODO: Remove and replace with source being sent across gRPC port
+ addStartFunction.accept(runner::start);
+
+ ThrowingConsumer runReadLoop =
+ (ThrowingConsumer<WindowedValue<InputT>>) runner::runReadLoop;
+ for (String pCollectionId : pTransform.getInputsMap().values()) {
+ pCollectionIdsToConsumers.put(
+ pCollectionId,
+ runReadLoop);
+ }
+
+ return runner;
+ }
+ }
+
+ private final PipelineOptions pipelineOptions;
+ private final RunnerApi.FunctionSpec definition;
+ private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
+
+ BoundedSourceRunner(
+ PipelineOptions pipelineOptions,
+ RunnerApi.FunctionSpec definition,
+ Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers) {
+ this.pipelineOptions = pipelineOptions;
+ this.definition = definition;
+ this.consumers = consumers;
+ }
+
+ /**
+ * @deprecated The runner harness is meant to send the source over the Beam Fn Data API which
+ * would be consumed by the {@link #runReadLoop}. Drop this method once the runner harness sends
+ * the source instead of unpacking it from the data block of the function specification.
+ */
+ @Deprecated
+ public void start() throws Exception {
+ try {
+ // The representation here is defined as the java serialized representation of the
+ // bounded source object in a ByteString wrapper.
+ byte[] bytes = definition.getPayload().toByteArray();
+ @SuppressWarnings("unchecked")
+ InputT boundedSource =
+ (InputT) SerializableUtils.deserializeFromByteArray(bytes, definition.toString());
+ runReadLoop(WindowedValue.valueInGlobalWindow(boundedSource));
+ } catch (InvalidProtocolBufferException e) {
+ throw new IOException(String.format("Failed to decode %s", definition.getUrn()), e);
+ }
+ }
+
+ /**
+ * Creates a {@link Reader} for each {@link BoundedSource} and executes the {@link Reader}s
+ * read loop. See {@link Reader} for further details of the read loop.
+ *
+ * <p>Propagates any exceptions caused during reading or processing via a consumer to the
+ * caller.
+ */
+ public void runReadLoop(WindowedValue<InputT> value) throws Exception {
+ try (Reader<OutputT> reader = value.getValue().createReader(pipelineOptions)) {
+ if (!reader.start()) {
+ // Reader has no data, immediately return
+ return;
+ }
+ do {
+ // TODO: Should this use the input window as the window for all the outputs?
+ WindowedValue<OutputT> nextValue = WindowedValue.timestampedValueInGlobalWindow(
+ reader.getCurrent(), reader.getCurrentTimestamp());
+ for (ThrowingConsumer<WindowedValue<OutputT>> consumer : consumers) {
+ consumer.accept(nextValue);
+ }
+ } while (reader.advance());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return definition.toString();
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
new file mode 100644
index 0000000..cad8985
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
@@ -0,0 +1,918 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.service.AutoService;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.BagUserState;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.DoFnRunner;
+import org.apache.beam.runners.core.construction.ParDoTranslation;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ReadableStates;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateBinder;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
+import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
+import org.apache.beam.sdk.util.CombineFnUtil;
+import org.apache.beam.sdk.util.DoFnInfo;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Instant;
+
+/**
+ * A {@link DoFnRunner} specific to integrating with the Fn Api. This is to remove the layers
+ * of abstraction caused by StateInternals/TimerInternals since they model state and timer
+ * concepts differently.
+ */
+public class FnApiDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
+ /**
+ * A registrar which provides a factory to handle Java {@link DoFn}s.
+ */
+ @AutoService(PTransformRunnerFactory.Registrar.class)
+ public static class Registrar implements
+ PTransformRunnerFactory.Registrar {
+
+ @Override
+ public Map<String, PTransformRunnerFactory> getPTransformRunnerFactories() {
+ return ImmutableMap.of(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN, new Factory());
+ }
+ }
+
+ /** A factory for {@link FnApiDoFnRunner}. */
+ static class Factory<InputT, OutputT>
+ implements PTransformRunnerFactory<DoFnRunner<InputT, OutputT>> {
+
+ @Override
+ public DoFnRunner<InputT, OutputT> createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) {
+
+ // For every output PCollection, create a map from output name to Consumer
+ ImmutableMap.Builder<String, Collection<ThrowingConsumer<WindowedValue<?>>>>
+ outputMapBuilder = ImmutableMap.builder();
+ for (Map.Entry<String, String> entry : pTransform.getOutputsMap().entrySet()) {
+ outputMapBuilder.put(
+ entry.getKey(),
+ pCollectionIdsToConsumers.get(entry.getValue()));
+ }
+ ImmutableMap<String, Collection<ThrowingConsumer<WindowedValue<?>>>> outputMap =
+ outputMapBuilder.build();
+
+ // Get the DoFnInfo from the serialized blob.
+ ByteString serializedFn = pTransform.getSpec().getPayload();
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ DoFnInfo<InputT, OutputT> doFnInfo = (DoFnInfo) SerializableUtils.deserializeFromByteArray(
+ serializedFn.toByteArray(), "DoFnInfo");
+
+ // Verify that the DoFnInfo tag to output map matches the output map on the PTransform.
+ checkArgument(
+ Objects.equals(
+ new HashSet<>(Collections2.transform(outputMap.keySet(), Long::parseLong)),
+ doFnInfo.getOutputMap().keySet()),
+ "Unexpected mismatch between transform output map %s and DoFnInfo output map %s.",
+ outputMap.keySet(),
+ doFnInfo.getOutputMap());
+
+ ImmutableMultimap.Builder<TupleTag<?>,
+ ThrowingConsumer<WindowedValue<?>>> tagToOutputMapBuilder =
+ ImmutableMultimap.builder();
+ for (Map.Entry<Long, TupleTag<?>> entry : doFnInfo.getOutputMap().entrySet()) {
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ Collection<ThrowingConsumer<WindowedValue<?>>> consumers =
+ outputMap.get(Long.toString(entry.getKey()));
+ tagToOutputMapBuilder.putAll(entry.getValue(), consumers);
+ }
+
+ ImmutableMultimap<TupleTag<?>, ThrowingConsumer<WindowedValue<?>>> tagToOutputMap =
+ tagToOutputMapBuilder.build();
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ DoFnRunner<InputT, OutputT> runner = new FnApiDoFnRunner<>(
+ pipelineOptions,
+ beamFnStateClient,
+ pTransformId,
+ processBundleInstructionId,
+ doFnInfo.getDoFn(),
+ WindowedValue.getFullCoder(
+ doFnInfo.getInputCoder(),
+ doFnInfo.getWindowingStrategy().getWindowFn().windowCoder()),
+ (Collection<ThrowingConsumer<WindowedValue<OutputT>>>) (Collection)
+ tagToOutputMap.get(doFnInfo.getOutputMap().get(doFnInfo.getMainOutput())),
+ tagToOutputMap,
+ doFnInfo.getWindowingStrategy());
+
+ // Register the appropriate handlers.
+ addStartFunction.accept(runner::startBundle);
+ for (String pcollectionId : pTransform.getInputsMap().values()) {
+ pCollectionIdsToConsumers.put(
+ pcollectionId,
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<InputT>>) runner::processElement);
+ }
+ addFinishFunction.accept(runner::finishBundle);
+ return runner;
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////////////////
+
+ private final PipelineOptions pipelineOptions;
+ private final BeamFnStateClient beamFnStateClient;
+ private final String ptransformId;
+ private final Supplier<String> processBundleInstructionId;
+ private final DoFn<InputT, OutputT> doFn;
+ private final WindowedValueCoder<InputT> inputCoder;
+ private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> mainOutputConsumers;
+ private final Multimap<TupleTag<?>, ThrowingConsumer<WindowedValue<?>>> outputMap;
+ private final WindowingStrategy windowingStrategy;
+ private final DoFnSignature doFnSignature;
+ private final DoFnInvoker<InputT, OutputT> doFnInvoker;
+ private final StateBinder stateBinder;
+ private final StartBundleContext startBundleContext;
+ private final ProcessBundleContext processBundleContext;
+ private final FinishBundleContext finishBundleContext;
+ private final Collection<ThrowingRunnable> stateFinalizers;
+
+ /**
+ * The lifetime of this member is only valid during {@link #processElement}
+ * and is null otherwise.
+ */
+ private WindowedValue<InputT> currentElement;
+
+ /**
+ * The lifetime of this member is only valid during {@link #processElement}
+ * and is null otherwise.
+ */
+ private BoundedWindow currentWindow;
+
+ /**
+ * This member should only be accessed indirectly by calling
+ * {@link #createOrUseCachedBagUserStateKey} and is only valid during {@link #processElement}
+ * and is null otherwise.
+ */
+ private StateKey.BagUserState cachedPartialBagUserStateKey;
+
+
+ FnApiDoFnRunner(
+ PipelineOptions pipelineOptions,
+ BeamFnStateClient beamFnStateClient,
+ String ptransformId,
+ Supplier<String> processBundleInstructionId,
+ DoFn<InputT, OutputT> doFn,
+ WindowedValueCoder<InputT> inputCoder,
+ Collection<ThrowingConsumer<WindowedValue<OutputT>>> mainOutputConsumers,
+ Multimap<TupleTag<?>, ThrowingConsumer<WindowedValue<?>>> outputMap,
+ WindowingStrategy windowingStrategy) {
+ this.pipelineOptions = pipelineOptions;
+ this.beamFnStateClient = beamFnStateClient;
+ this.ptransformId = ptransformId;
+ this.processBundleInstructionId = processBundleInstructionId;
+ this.doFn = doFn;
+ this.inputCoder = inputCoder;
+ this.mainOutputConsumers = mainOutputConsumers;
+ this.outputMap = outputMap;
+ this.windowingStrategy = windowingStrategy;
+ this.doFnSignature = DoFnSignatures.signatureForDoFn(doFn);
+ this.doFnInvoker = DoFnInvokers.invokerFor(doFn);
+ this.stateBinder = new BeamFnStateBinder();
+ this.startBundleContext = new StartBundleContext();
+ this.processBundleContext = new ProcessBundleContext();
+ this.finishBundleContext = new FinishBundleContext();
+ this.stateFinalizers = new ArrayList<>();
+ }
+
+ @Override
+ public void startBundle() {
+ doFnInvoker.invokeStartBundle(startBundleContext);
+ }
+
+ @Override
+ public void processElement(WindowedValue<InputT> elem) {
+ currentElement = elem;
+ try {
+ Iterator<BoundedWindow> windowIterator =
+ (Iterator<BoundedWindow>) elem.getWindows().iterator();
+ while (windowIterator.hasNext()) {
+ currentWindow = windowIterator.next();
+ doFnInvoker.invokeProcessElement(processBundleContext);
+ }
+ } finally {
+ currentElement = null;
+ currentWindow = null;
+ cachedPartialBagUserStateKey = null;
+ }
+ }
+
+ @Override
+ public void onTimer(
+ String timerId,
+ BoundedWindow window,
+ Instant timestamp,
+ TimeDomain timeDomain) {
+ throw new UnsupportedOperationException("TODO: Add support for timers");
+ }
+
+ @Override
+ public void finishBundle() {
+ doFnInvoker.invokeFinishBundle(finishBundleContext);
+
+ // Persist all dirty state cells
+ try {
+ for (ThrowingRunnable runnable : stateFinalizers) {
+ runnable.run();
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Outputs the given element to the specified set of consumers wrapping any exceptions.
+ */
+ private <T> void outputTo(
+ Collection<ThrowingConsumer<WindowedValue<T>>> consumers,
+ WindowedValue<T> output) {
+ Iterator<ThrowingConsumer<WindowedValue<T>>> consumerIterator;
+ try {
+ for (ThrowingConsumer<WindowedValue<T>> consumer : consumers) {
+ consumer.accept(output);
+ }
+ } catch (Throwable t) {
+ throw UserCodeException.wrap(t);
+ }
+ }
+
+ /**
+ * Provides arguments for a {@link DoFnInvoker} for {@link DoFn.StartBundle @StartBundle}.
+ */
+ private class StartBundleContext
+ extends DoFn<InputT, OutputT>.StartBundleContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+
+ private StartBundleContext() {
+ doFn.super();
+ }
+
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public PipelineOptions pipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public BoundedWindow window() {
+ throw new UnsupportedOperationException(
+ "Cannot access window outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(
+ DoFn<InputT, OutputT> doFn) {
+ return this;
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access FinishBundleContext outside of @FinishBundle method.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access ProcessContext outside of @ProcessElement method.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access OnTimerContext outside of @OnTimer methods.");
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ throw new UnsupportedOperationException(
+ "Cannot access RestrictionTracker outside of @ProcessElement method.");
+ }
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException(
+ "Cannot access state outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException(
+ "Cannot access timers outside of @ProcessElement and @OnTimer methods.");
+ }
+ }
+
+ /**
+ * Provides arguments for a {@link DoFnInvoker} for {@link DoFn.ProcessElement @ProcessElement}.
+ */
+ private class ProcessBundleContext
+ extends DoFn<InputT, OutputT>.ProcessContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+
+ private ProcessBundleContext() {
+ doFn.super();
+ }
+
+ @Override
+ public BoundedWindow window() {
+ return currentWindow;
+ }
+
+ @Override
+ public DoFn.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access StartBundleContext outside of @StartBundle method.");
+ }
+
+ @Override
+ public DoFn.FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access FinishBundleContext outside of @FinishBundle method.");
+ }
+
+ @Override
+ public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ return this;
+ }
+
+ @Override
+ public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException("TODO: Add support for timers");
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ throw new UnsupportedOperationException("TODO: Add support for SplittableDoFn");
+ }
+
+ @Override
+ public State state(String stateId) {
+ StateDeclaration stateDeclaration = doFnSignature.stateDeclarations().get(stateId);
+ checkNotNull(stateDeclaration, "No state declaration found for %s", stateId);
+ StateSpec<?> spec;
+ try {
+ spec = (StateSpec<?>) stateDeclaration.field().get(doFn);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ return spec.bind(stateId, stateBinder);
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("TODO: Add support for timers");
+ }
+
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public PipelineOptions pipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public void output(OutputT output) {
+ outputTo(mainOutputConsumers,
+ WindowedValue.of(
+ output,
+ currentElement.getTimestamp(),
+ currentWindow,
+ currentElement.getPane()));
+ }
+
+ @Override
+ public void outputWithTimestamp(OutputT output, Instant timestamp) {
+ outputTo(mainOutputConsumers,
+ WindowedValue.of(
+ output,
+ timestamp,
+ currentWindow,
+ currentElement.getPane()));
+ }
+
+ @Override
+ public <T> void output(TupleTag<T> tag, T output) {
+ Collection<ThrowingConsumer<WindowedValue<T>>> consumers = (Collection) outputMap.get(tag);
+ if (consumers == null) {
+ throw new IllegalArgumentException(String.format("Unknown output tag %s", tag));
+ }
+ outputTo(consumers,
+ WindowedValue.of(
+ output,
+ currentElement.getTimestamp(),
+ currentWindow,
+ currentElement.getPane()));
+ }
+
+ @Override
+ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
+ Collection<ThrowingConsumer<WindowedValue<T>>> consumers = (Collection) outputMap.get(tag);
+ if (consumers == null) {
+ throw new IllegalArgumentException(String.format("Unknown output tag %s", tag));
+ }
+ outputTo(consumers,
+ WindowedValue.of(
+ output,
+ timestamp,
+ currentWindow,
+ currentElement.getPane()));
+ }
+
+ @Override
+ public InputT element() {
+ return currentElement.getValue();
+ }
+
+ @Override
+ public <T> T sideInput(PCollectionView<T> view) {
+ throw new UnsupportedOperationException("TODO: Support side inputs");
+ }
+
+ @Override
+ public Instant timestamp() {
+ return currentElement.getTimestamp();
+ }
+
+ @Override
+ public PaneInfo pane() {
+ return currentElement.getPane();
+ }
+
+ @Override
+ public void updateWatermark(Instant watermark) {
+ throw new UnsupportedOperationException("TODO: Add support for SplittableDoFn");
+ }
+ }
+
+ /**
+ * Provides arguments for a {@link DoFnInvoker} for {@link DoFn.FinishBundle @FinishBundle}.
+ */
+ private class FinishBundleContext
+ extends DoFn<InputT, OutputT>.FinishBundleContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+
+ private FinishBundleContext() {
+ doFn.super();
+ }
+
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public PipelineOptions pipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public BoundedWindow window() {
+ throw new UnsupportedOperationException(
+ "Cannot access window outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(
+ DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access StartBundleContext outside of @StartBundle method.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
+ return this;
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access ProcessContext outside of @ProcessElement method.");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access OnTimerContext outside of @OnTimer methods.");
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ throw new UnsupportedOperationException(
+ "Cannot access RestrictionTracker outside of @ProcessElement method.");
+ }
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException(
+ "Cannot access state outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException(
+ "Cannot access timers outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public void output(OutputT output, Instant timestamp, BoundedWindow window) {
+ outputTo(mainOutputConsumers,
+ WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ }
+
+ @Override
+ public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+ Collection<ThrowingConsumer<WindowedValue<T>>> consumers = (Collection) outputMap.get(tag);
+ if (consumers == null) {
+ throw new IllegalArgumentException(String.format("Unknown output tag %s", tag));
+ }
+ outputTo(consumers,
+ WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ }
+ }
+
+ /**
+ * A {@link StateBinder} that uses the Beam Fn State API to read and write user state.
+ *
+ * <p>TODO: Add support for {@link #bindMap} and {@link #bindSet}. Note that
+ * {@link #bindWatermark} should never be implemented.
+ */
+ private class BeamFnStateBinder implements StateBinder {
+ private final Map<StateKey.BagUserState, Object> stateObjectCache = new HashMap<>();
+
+ @Override
+ public <T> ValueState<T> bindValue(String id, StateSpec<ValueState<T>> spec, Coder<T> coder) {
+ return (ValueState<T>) stateObjectCache.computeIfAbsent(
+ createOrUseCachedBagUserStateKey(id),
+ new Function<StateKey.BagUserState, Object>() {
+ @Override
+ public Object apply(StateKey.BagUserState s) {
+ return new ValueState<T>() {
+ private final BagUserState<T> impl = createBagUserState(id, coder);
+
+ @Override
+ public void clear() {
+ impl.clear();
+ }
+
+ @Override
+ public void write(T input) {
+ impl.clear();
+ impl.append(input);
+ }
+
+ @Override
+ public T read() {
+ Iterator<T> value = impl.get().iterator();
+ if (value.hasNext()) {
+ return value.next();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public ValueState<T> readLater() {
+ // TODO: Support prefetching.
+ return this;
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ public <T> BagState<T> bindBag(String id, StateSpec<BagState<T>> spec, Coder<T> elemCoder) {
+ return (BagState<T>) stateObjectCache.computeIfAbsent(
+ createOrUseCachedBagUserStateKey(id),
+ new Function<StateKey.BagUserState, Object>() {
+ @Override
+ public Object apply(StateKey.BagUserState s) {
+ return new BagState<T>() {
+ private final BagUserState<T> impl = createBagUserState(id, elemCoder);
+
+ @Override
+ public void add(T value) {
+ impl.append(value);
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return ReadableStates.immediate(!impl.get().iterator().hasNext());
+ }
+
+ @Override
+ public Iterable<T> read() {
+ return impl.get();
+ }
+
+ @Override
+ public BagState<T> readLater() {
+ // TODO: Support prefetching.
+ return this;
+ }
+
+ @Override
+ public void clear() {
+ impl.clear();
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ public <T> SetState<T> bindSet(String id, StateSpec<SetState<T>> spec, Coder<T> elemCoder) {
+ throw new UnsupportedOperationException("TODO: Add support for a map state to the Fn API.");
+ }
+
+ @Override
+ public <KeyT, ValueT> MapState<KeyT, ValueT> bindMap(String id,
+ StateSpec<MapState<KeyT, ValueT>> spec, Coder<KeyT> mapKeyCoder,
+ Coder<ValueT> mapValueCoder) {
+ throw new UnsupportedOperationException("TODO: Add support for a map state to the Fn API.");
+ }
+
+ @Override
+ public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT> bindCombining(
+ String id,
+ StateSpec<CombiningState<InputT, AccumT, OutputT>> spec, Coder<AccumT> accumCoder,
+ CombineFn<InputT, AccumT, OutputT> combineFn) {
+ return (CombiningState<InputT, AccumT, OutputT>) stateObjectCache.computeIfAbsent(
+ createOrUseCachedBagUserStateKey(id),
+ new Function<StateKey.BagUserState, Object>() {
+ @Override
+ public Object apply(StateKey.BagUserState s) {
+ // TODO: Support squashing accumulators depending on whether we know of all
+ // remote accumulators and local accumulators or just local accumulators.
+ return new CombiningState<InputT, AccumT, OutputT>() {
+ private final BagUserState<AccumT> impl = createBagUserState(id, accumCoder);
+
+ @Override
+ public AccumT getAccum() {
+ Iterator<AccumT> iterator = impl.get().iterator();
+ if (iterator.hasNext()) {
+ return iterator.next();
+ }
+ return combineFn.createAccumulator();
+ }
+
+ @Override
+ public void addAccum(AccumT accum) {
+ Iterator<AccumT> iterator = impl.get().iterator();
+
+ // Only merge if there was a prior value
+ if (iterator.hasNext()) {
+ accum = combineFn.mergeAccumulators(ImmutableList.of(iterator.next(), accum));
+ // Since there was a prior value, we need to clear.
+ impl.clear();
+ }
+
+ impl.append(accum);
+ }
+
+ @Override
+ public AccumT mergeAccumulators(Iterable<AccumT> accumulators) {
+ return combineFn.mergeAccumulators(accumulators);
+ }
+
+ @Override
+ public CombiningState<InputT, AccumT, OutputT> readLater() {
+ return this;
+ }
+
+ @Override
+ public OutputT read() {
+ Iterator<AccumT> iterator = impl.get().iterator();
+ if (iterator.hasNext()) {
+ return combineFn.extractOutput(iterator.next());
+ }
+ return combineFn.defaultValue();
+ }
+
+ @Override
+ public void add(InputT value) {
+ AccumT newAccumulator = combineFn.addInput(getAccum(), value);
+ impl.clear();
+ impl.append(newAccumulator);
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return ReadableStates.immediate(!impl.get().iterator().hasNext());
+ }
+
+ @Override
+ public void clear() {
+ impl.clear();
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT>
+ bindCombiningWithContext(
+ String id,
+ StateSpec<CombiningState<InputT, AccumT, OutputT>> spec,
+ Coder<AccumT> accumCoder,
+ CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
+ return (CombiningState<InputT, AccumT, OutputT>) stateObjectCache.computeIfAbsent(
+ createOrUseCachedBagUserStateKey(id),
+ new Function<StateKey.BagUserState, Object>() {
+ @Override
+ public Object apply(StateKey.BagUserState s) {
+ return bindCombining(id, spec, accumCoder, CombineFnUtil.bindContext(combineFn,
+ new StateContext<BoundedWindow>() {
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ @Override
+ public <T> T sideInput(PCollectionView<T> view) {
+ return processBundleContext.sideInput(view);
+ }
+
+ @Override
+ public BoundedWindow window() {
+ return currentWindow;
+ }
+ }));
+ }
+ });
+ }
+
+ /**
+ * @deprecated The Fn API has no plans to implement WatermarkHoldState as of this writing
+ * and is waiting on resolution of BEAM-2535.
+ */
+ @Override
+ @Deprecated
+ public WatermarkHoldState bindWatermark(String id, StateSpec<WatermarkHoldState> spec,
+ TimestampCombiner timestampCombiner) {
+ throw new UnsupportedOperationException("WatermarkHoldState is unsupported by the Fn API.");
+ }
+
+ private <T> BagUserState<T> createBagUserState(String id, Coder<T> coder) {
+ BagUserState rval = new BagUserState<T>(
+ beamFnStateClient,
+ id,
+ coder,
+ new Supplier<StateRequest.Builder>() {
+ /** Memoizes the partial state key for the lifetime of the {@link BagUserState}. */
+ private final Supplier<StateKey.BagUserState> memoizingSupplier =
+ Suppliers.memoize(() -> createOrUseCachedBagUserStateKey(id))::get;
+
+ @Override
+ public Builder get() {
+ return StateRequest.newBuilder()
+ .setInstructionReference(processBundleInstructionId.get())
+ .setStateKey(StateKey.newBuilder()
+ .setBagUserState(memoizingSupplier.get()));
+ }
+ });
+ stateFinalizers.add(rval::asyncClose);
+ return rval;
+ }
+ }
+
+ /**
+ * Memoizes a partially built {@link StateKey} saving on the encoding cost of the key and
+ * window across multiple state cells for the lifetime of {@link #processElement}.
+ *
+ * <p>This should only be called during {@link #processElement}.
+ */
+ private <K> StateKey.BagUserState createOrUseCachedBagUserStateKey(String id) {
+ if (cachedPartialBagUserStateKey == null) {
+ checkState(currentElement.getValue() instanceof KV,
+ "Accessing state in unkeyed context. Current element is not a KV: %s.",
+ currentElement);
+ checkState(inputCoder.getCoderArguments().get(0) instanceof KvCoder,
+ "Accessing state in unkeyed context. No keyed coder found.");
+
+ ByteString.Output encodedKeyOut = ByteString.newOutput();
+
+ Coder<K> keyCoder = ((KvCoder<K, ?>) inputCoder.getValueCoder()).getKeyCoder();
+ try {
+ keyCoder.encode(((KV<K, ?>) currentElement.getValue()).getKey(), encodedKeyOut);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+
+ ByteString.Output encodedWindowOut = ByteString.newOutput();
+ try {
+ windowingStrategy.getWindowFn().windowCoder().encode(currentWindow, encodedWindowOut);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+
+ cachedPartialBagUserStateKey = StateKey.BagUserState.newBuilder()
+ .setPtransformId(ptransformId)
+ .setKey(encodedKeyOut.toByteString())
+ .setWindow(encodedWindowOut.toByteString()).buildPartial();
+ }
+ return cachedPartialBagUserStateKey.toBuilder().setUserStateId(id).build();
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
index 05ab44f..e1790fa 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
@@ -20,18 +20,23 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.protobuf.TextFormat;
-import java.io.PrintStream;
import java.util.EnumMap;
-import org.apache.beam.fn.harness.channel.ManagedChannelFactory;
+import java.util.List;
import org.apache.beam.fn.harness.control.BeamFnControlClient;
import org.apache.beam.fn.harness.control.ProcessBundleHandler;
import org.apache.beam.fn.harness.control.RegisterHandler;
import org.apache.beam.fn.harness.data.BeamFnDataGrpcClient;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
import org.apache.beam.fn.harness.logging.BeamFnLoggingClient;
+import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
import org.apache.beam.fn.harness.stream.StreamObserverFactory;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.harness.channel.ManagedChannelFactory;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse.Builder;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
+import org.apache.beam.sdk.options.ExperimentalOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.slf4j.Logger;
@@ -43,10 +48,10 @@
* <p>This entry point expects the following environment variables:
* <ul>
* <li>LOGGING_API_SERVICE_DESCRIPTOR: A
- * {@link org.apache.beam.fn.v1.BeamFnApi.ApiServiceDescriptor} encoded as text
+ * {@link org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor} encoded as text
* representing the endpoint that is to be connected to for the Beam Fn Logging service.</li>
* <li>CONTROL_API_SERVICE_DESCRIPTOR: A
- * {@link org.apache.beam.fn.v1.BeamFnApi.ApiServiceDescriptor} encoded as text
+ * {@link Endpoints.ApiServiceDescriptor} encoded as text
* representing the endpoint that is to be connected to for the Beam Fn Control service.</li>
* <li>PIPELINE_OPTIONS: A serialized form of {@link PipelineOptions}. See {@link PipelineOptions}
* for further details.</li>
@@ -58,10 +63,10 @@
private static final String PIPELINE_OPTIONS = "PIPELINE_OPTIONS";
private static final Logger LOG = LoggerFactory.getLogger(FnHarness.class);
- private static BeamFnApi.ApiServiceDescriptor getApiServiceDescriptor(String env)
+ private static Endpoints.ApiServiceDescriptor getApiServiceDescriptor(String env)
throws TextFormat.ParseException {
- BeamFnApi.ApiServiceDescriptor.Builder apiServiceDescriptorBuilder =
- BeamFnApi.ApiServiceDescriptor.newBuilder();
+ Endpoints.ApiServiceDescriptor.Builder apiServiceDescriptorBuilder =
+ Endpoints.ApiServiceDescriptor.newBuilder();
TextFormat.merge(System.getenv(env), apiServiceDescriptorBuilder);
return apiServiceDescriptorBuilder.build();
}
@@ -77,40 +82,51 @@
PipelineOptions options = objectMapper.readValue(
System.getenv(PIPELINE_OPTIONS), PipelineOptions.class);
- BeamFnApi.ApiServiceDescriptor loggingApiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor loggingApiServiceDescriptor =
getApiServiceDescriptor(LOGGING_API_SERVICE_DESCRIPTOR);
- BeamFnApi.ApiServiceDescriptor controlApiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor controlApiServiceDescriptor =
getApiServiceDescriptor(CONTROL_API_SERVICE_DESCRIPTOR);
main(options, loggingApiServiceDescriptor, controlApiServiceDescriptor);
}
public static void main(PipelineOptions options,
- BeamFnApi.ApiServiceDescriptor loggingApiServiceDescriptor,
- BeamFnApi.ApiServiceDescriptor controlApiServiceDescriptor) throws Exception {
- ManagedChannelFactory channelFactory = ManagedChannelFactory.from(options);
+ Endpoints.ApiServiceDescriptor loggingApiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor controlApiServiceDescriptor) throws Exception {
+ ManagedChannelFactory channelFactory;
+ List<String> experiments = options.as(ExperimentalOptions.class).getExperiments();
+ if (experiments != null && experiments.contains("beam_fn_api_epoll")) {
+ channelFactory = ManagedChannelFactory.createEpoll();
+ } else {
+ channelFactory = ManagedChannelFactory.createDefault();
+ }
StreamObserverFactory streamObserverFactory = StreamObserverFactory.fromOptions(options);
- PrintStream originalErrStream = System.err;
-
try (BeamFnLoggingClient logging = new BeamFnLoggingClient(
options,
loggingApiServiceDescriptor,
- channelFactory::forDescriptor,
- streamObserverFactory::from)) {
+ channelFactory::forDescriptor)) {
LOG.info("Fn Harness started");
EnumMap<BeamFnApi.InstructionRequest.RequestCase,
- ThrowingFunction<BeamFnApi.InstructionRequest,
- BeamFnApi.InstructionResponse.Builder>> handlers =
- new EnumMap<>(BeamFnApi.InstructionRequest.RequestCase.class);
+ ThrowingFunction<InstructionRequest, Builder>>
+ handlers = new EnumMap<>(BeamFnApi.InstructionRequest.RequestCase.class);
RegisterHandler fnApiRegistry = new RegisterHandler();
BeamFnDataGrpcClient beamFnDataMultiplexer = new BeamFnDataGrpcClient(
options, channelFactory::forDescriptor, streamObserverFactory::from);
- ProcessBundleHandler processBundleHandler =
- new ProcessBundleHandler(options, fnApiRegistry::getById, beamFnDataMultiplexer);
+ BeamFnStateGrpcClientCache beamFnStateGrpcClientCache = new BeamFnStateGrpcClientCache(
+ options,
+ IdGenerator::generate,
+ channelFactory::forDescriptor,
+ streamObserverFactory::from);
+
+ ProcessBundleHandler processBundleHandler = new ProcessBundleHandler(
+ options,
+ fnApiRegistry::getById,
+ beamFnDataMultiplexer,
+ beamFnStateGrpcClientCache);
handlers.put(BeamFnApi.InstructionRequest.RequestCase.REGISTER,
fnApiRegistry::register);
handlers.put(BeamFnApi.InstructionRequest.RequestCase.PROCESS_BUNDLE,
@@ -123,9 +139,9 @@
LOG.info("Entering instruction processing loop");
control.processInstructionRequests(options.as(GcsOptions.class).getExecutorService());
} catch (Throwable t) {
- t.printStackTrace(originalErrStream);
+ t.printStackTrace();
} finally {
- originalErrStream.println("Shutting SDK harness down.");
+ System.out.println("Shutting SDK harness down.");
}
}
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/IdGenerator.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/IdGenerator.java
new file mode 100644
index 0000000..1112f43
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/IdGenerator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * An id generator.
+ *
+ * <p>This encapsulation exists to prevent usage of the wrong method on a shared {@link AtomicLong}.
+ */
+public final class IdGenerator {
+ private static final AtomicLong idGenerator = new AtomicLong(-1);
+
+ public static String generate() {
+ return Long.toString(idGenerator.getAndDecrement());
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
new file mode 100644
index 0000000..126055a
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness;
+
+import com.google.common.collect.Multimap;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.WindowedValue;
+
+/**
+ * A factory able to instantiate an appropriate handler for a given PTransform.
+ */
+public interface PTransformRunnerFactory<T> {
+
+ /**
+ * Creates and returns a handler for a given PTransform. Note that the handler must support
+ * processing multiple bundles. The handler will be discarded if an error is thrown during
+ * element processing, or during execution of start/finish.
+ *
+ * @param pipelineOptions Pipeline options
+ * @param beamFnDataClient A client for handling inbound and outbound data streams.
+ * @param beamFnStateClient A client for handling state requests.
+ * @param pTransformId The id of the PTransform.
+ * @param pTransform The PTransform definition.
+ * @param processBundleInstructionId A supplier containing the active process bundle instruction
+ * id.
+ * @param pCollections A mapping from PCollection id to PCollection definition.
+ * @param coders A mapping from coder id to coder definition.
+ * @param pCollectionIdsToConsumers A mapping from PCollection id to a collection of consumers.
+ * Note that if this handler is a consumer, it should register itself within this multimap under
+ * the appropriate PCollection ids. Also note that all output consumers needed by this PTransform
+ * (based on the values of the {@link RunnerApi.PTransform#getOutputsMap()} will have already
+ * registered within this multimap.
+ * @param addStartFunction A consumer to register a start bundle handler with.
+ * @param addFinishFunction A consumer to register a finish bundle handler with.
+ */
+ T createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException;
+
+ /**
+ * A registrar which can return a mapping from {@link RunnerApi.FunctionSpec#getUrn()} to
+ * a factory capable of instantiating an appropriate handler.
+ */
+ interface Registrar {
+ /**
+ * Returns a mapping from {@link RunnerApi.FunctionSpec#getUrn()} to a factory capable of
+ * instantiating an appropriate handler.
+ */
+ Map<String, PTransformRunnerFactory> getPTransformRunnerFactories();
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
deleted file mode 100644
index d26f4a5..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.channel;
-
-import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;
-import io.grpc.netty.NettyChannelBuilder;
-import io.netty.channel.epoll.EpollDomainSocketChannel;
-import io.netty.channel.epoll.EpollEventLoopGroup;
-import io.netty.channel.epoll.EpollSocketChannel;
-import io.netty.channel.unix.DomainSocketAddress;
-import java.net.SocketAddress;
-import java.util.List;
-import org.apache.beam.fn.v1.BeamFnApi.ApiServiceDescriptor;
-import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/**
- * Uses {@link PipelineOptions} to configure which underlying {@link ManagedChannel} implementation
- * to use.
- */
-public abstract class ManagedChannelFactory {
- public static ManagedChannelFactory from(PipelineOptions options) {
- List<String> experiments = options.as(DataflowPipelineDebugOptions.class).getExperiments();
- if (experiments != null && experiments.contains("beam_fn_api_epoll")) {
- io.netty.channel.epoll.Epoll.ensureAvailability();
- return new Epoll();
- }
- return new Default();
- }
-
- public abstract ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor);
-
- /**
- * Creates a {@link ManagedChannel} backed by an {@link EpollDomainSocketChannel} if the address
- * is a {@link DomainSocketAddress}. Otherwise creates a {@link ManagedChannel} backed by an
- * {@link EpollSocketChannel}.
- */
- private static class Epoll extends ManagedChannelFactory {
- @Override
- public ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor) {
- SocketAddress address = SocketAddressFactory.createFrom(apiServiceDescriptor.getUrl());
- return NettyChannelBuilder.forAddress(address)
- .channelType(address instanceof DomainSocketAddress
- ? EpollDomainSocketChannel.class : EpollSocketChannel.class)
- .eventLoopGroup(new EpollEventLoopGroup())
- .usePlaintext(true)
- .build();
- }
- }
-
- /**
- * Creates a {@link ManagedChannel} relying on the {@link ManagedChannelBuilder} to create
- * instances.
- */
- private static class Default extends ManagedChannelFactory {
- @Override
- public ManagedChannel forDescriptor(ApiServiceDescriptor apiServiceDescriptor) {
- return ManagedChannelBuilder.forTarget(apiServiceDescriptor.getUrl())
- .usePlaintext(true)
- .build();
- }
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/SocketAddressFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/SocketAddressFactory.java
deleted file mode 100644
index a27d542..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/SocketAddressFactory.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.channel;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.net.HostAndPort;
-import io.netty.channel.unix.DomainSocketAddress;
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-
-/** Creates a {@link SocketAddress} based upon a supplied string. */
-public class SocketAddressFactory {
- private static final String UNIX_DOMAIN_SOCKET_PREFIX = "unix://";
-
- /**
- * Parse a {@link SocketAddress} from the given string.
- */
- public static SocketAddress createFrom(String value) {
- if (value.startsWith(UNIX_DOMAIN_SOCKET_PREFIX)) {
- // Unix Domain Socket address.
- // Create the underlying file for the Unix Domain Socket.
- String filePath = value.substring(UNIX_DOMAIN_SOCKET_PREFIX.length());
- File file = new File(filePath);
- if (!file.isAbsolute()) {
- throw new IllegalArgumentException("File path must be absolute: " + filePath);
- }
- try {
- if (file.createNewFile()) {
- // If this application created the file, delete it when the application exits.
- file.deleteOnExit();
- }
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- // Create the SocketAddress referencing the file.
- return new DomainSocketAddress(file);
- } else {
- // Standard TCP/IP address.
- HostAndPort hostAndPort = HostAndPort.fromString(value);
- checkArgument(hostAndPort.hasPort(),
- "Address must be a unix:// path or be in the form host:port. Got: %s", value);
- return new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort());
- }
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/package-info.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/package-info.java
deleted file mode 100644
index 6323166..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * gRPC channel management.
- */
-package org.apache.beam.fn.harness.channel;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
index e40bb2f..3c98e77 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
@@ -32,8 +32,9 @@
import java.util.function.BiFunction;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,14 +43,14 @@
* an unbounded number of requests.
*
* <p>Also can delegate to a set of handlers based upon the
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest.RequestCase request type}.
+ * {@link BeamFnApi.InstructionRequest.RequestCase request type}.
*
* <p>When the inbound instruction stream finishes successfully, the {@code onFinish} is
* completed successfully signaling to the caller that this client will not produce any more
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest}s. If the inbound instruction stream
+ * {@link BeamFnApi.InstructionRequest}s. If the inbound instruction stream
* errors, the {@code onFinish} is completed exceptionally propagating the failure reason
* to the caller and signaling that this client will not produce any more
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest}s.
+ * {@link BeamFnApi.InstructionRequest}s.
*/
public class BeamFnControlClient {
private static final String FAKE_INSTRUCTION_ID = "FAKE_INSTRUCTION_ID";
@@ -65,8 +66,8 @@
private final CompletableFuture<Void> onFinish;
public BeamFnControlClient(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
- Function<BeamFnApi.ApiServiceDescriptor, ManagedChannel> channelFactory,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
+ Function<Endpoints.ApiServiceDescriptor, ManagedChannel> channelFactory,
BiFunction<Function<StreamObserver<BeamFnApi.InstructionRequest>,
StreamObserver<BeamFnApi.InstructionResponse>>,
StreamObserver<BeamFnApi.InstructionRequest>,
@@ -89,7 +90,7 @@
private class InboundObserver implements StreamObserver<BeamFnApi.InstructionRequest> {
@Override
public void onNext(BeamFnApi.InstructionRequest value) {
- LOG.info("InstructionRequest received {}", value);
+ LOG.debug("Received InstructionRequest {}", value);
Uninterruptibles.putUninterruptibly(bufferedInstructions, value);
}
@@ -155,6 +156,7 @@
}
public void sendInstructionResponse(BeamFnApi.InstructionResponse value) {
+ LOG.debug("Sending InstructionResponse {}", value);
outboundObserver.onNext(value);
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
index fd9f0df..598583c 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
@@ -18,188 +18,179 @@
package org.apache.beam.fn.harness.control;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.Iterables.getOnlyElement;
-
-import com.google.common.collect.Collections2;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.common.collect.Sets;
import com.google.protobuf.Message;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.function.BiConsumer;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Phaser;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
+import org.apache.beam.fn.harness.PTransformRunnerFactory;
+import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fake.FakeStepContext;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.BeamFnDataReadRunner;
-import org.apache.beam.runners.core.BeamFnDataWriteRunner;
-import org.apache.beam.runners.core.BoundedSourceRunner;
-import org.apache.beam.runners.core.DoFnRunner;
-import org.apache.beam.runners.core.DoFnRunners;
-import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.NullSideInputReader;
-import org.apache.beam.runners.dataflow.util.DoFnInfo;
-import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Processes {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s by materializing
- * the set of required runners for each {@link org.apache.beam.fn.v1.BeamFnApi.FunctionSpec},
+ * Processes {@link BeamFnApi.ProcessBundleRequest}s by materializing
+ * the set of required runners for each {@link RunnerApi.FunctionSpec},
* wiring them together based upon the {@code input} and {@code output} map definitions.
*
* <p>Finally executes the DAG based graph by starting all runners in reverse topological order,
* and finishing all runners in forward topological order.
*/
public class ProcessBundleHandler {
+
// TODO: What should the initial set of URNs be?
private static final String DATA_INPUT_URN = "urn:org.apache.beam:source:runner:0.1";
- private static final String DATA_OUTPUT_URN = "urn:org.apache.beam:sink:runner:0.1";
- private static final String JAVA_DO_FN_URN = "urn:org.apache.beam:dofn:java:0.1";
- private static final String JAVA_SOURCE_URN = "urn:org.apache.beam:source:java:0.1";
+ public static final String JAVA_SOURCE_URN = "urn:org.apache.beam:source:java:0.1";
private static final Logger LOG = LoggerFactory.getLogger(ProcessBundleHandler.class);
+ private static final Map<String, PTransformRunnerFactory> REGISTERED_RUNNER_FACTORIES;
+
+ static {
+ Set<Registrar> pipelineRunnerRegistrars =
+ Sets.newTreeSet(ReflectHelpers.ObjectsClassComparator.INSTANCE);
+ pipelineRunnerRegistrars.addAll(
+ Lists.newArrayList(ServiceLoader.load(Registrar.class,
+ ReflectHelpers.findClassLoader())));
+
+ // Load all registered PTransform runner factories.
+ ImmutableMap.Builder<String, PTransformRunnerFactory> builder =
+ ImmutableMap.builder();
+ for (Registrar registrar : pipelineRunnerRegistrars) {
+ builder.putAll(registrar.getPTransformRunnerFactories());
+ }
+ REGISTERED_RUNNER_FACTORIES = builder.build();
+ }
private final PipelineOptions options;
private final Function<String, Message> fnApiRegistry;
private final BeamFnDataClient beamFnDataClient;
+ private final BeamFnStateGrpcClientCache beamFnStateGrpcClientCache;
+ private final Map<String, PTransformRunnerFactory> urnToPTransformRunnerFactoryMap;
+ private final PTransformRunnerFactory defaultPTransformRunnerFactory;
+
public ProcessBundleHandler(
PipelineOptions options,
Function<String, Message> fnApiRegistry,
- BeamFnDataClient beamFnDataClient) {
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateGrpcClientCache beamFnStateGrpcClientCache) {
+ this(options,
+ fnApiRegistry,
+ beamFnDataClient,
+ beamFnStateGrpcClientCache,
+ REGISTERED_RUNNER_FACTORIES);
+ }
+
+ @VisibleForTesting
+ ProcessBundleHandler(
+ PipelineOptions options,
+ Function<String, Message> fnApiRegistry,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateGrpcClientCache beamFnStateGrpcClientCache,
+ Map<String, PTransformRunnerFactory> urnToPTransformRunnerFactoryMap) {
this.options = options;
this.fnApiRegistry = fnApiRegistry;
this.beamFnDataClient = beamFnDataClient;
+ this.beamFnStateGrpcClientCache = beamFnStateGrpcClientCache;
+ this.urnToPTransformRunnerFactoryMap = urnToPTransformRunnerFactoryMap;
+ this.defaultPTransformRunnerFactory = new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beanFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) {
+ throw new IllegalStateException(String.format(
+ "No factory registered for %s, known factories %s",
+ pTransform.getSpec().getUrn(),
+ urnToPTransformRunnerFactoryMap.keySet()));
+ }
+ };
}
- protected <InputT, OutputT> void createConsumersForPrimitiveTransform(
- BeamFnApi.PrimitiveTransform primitiveTransform,
+ private void createRunnerAndConsumersForPTransformRecursively(
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
Supplier<String> processBundleInstructionId,
- Function<BeamFnApi.Target, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> consumers,
- BiConsumer<BeamFnApi.Target, ThrowingConsumer<WindowedValue<InputT>>> addConsumer,
+ BeamFnApi.ProcessBundleDescriptor processBundleDescriptor,
+ Multimap<String, String> pCollectionIdsToConsumingPTransforms,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
Consumer<ThrowingRunnable> addStartFunction,
Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
- BeamFnApi.FunctionSpec functionSpec = primitiveTransform.getFunctionSpec();
+ // Recursively ensure that all consumers of the output PCollection have been created.
+ // Since we are creating the consumers first, we know that the we are building the DAG
+ // in reverse topological order.
+ for (String pCollectionId : pTransform.getOutputsMap().values()) {
+ // If we have created the consumers for this PCollection we can skip it.
+ if (pCollectionIdsToConsumers.containsKey(pCollectionId)) {
+ continue;
+ }
- // For every output PCollection, create a map from output name to Consumer
- ImmutableMap.Builder<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>>
- outputMapBuilder = ImmutableMap.builder();
- for (Map.Entry<String, BeamFnApi.PCollection> entry :
- primitiveTransform.getOutputsMap().entrySet()) {
- outputMapBuilder.put(
- entry.getKey(),
- consumers.apply(
- BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransform.getId())
- .setName(entry.getKey())
- .build()));
- }
- ImmutableMap<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> outputMap =
- outputMapBuilder.build();
-
- // Based upon the function spec, populate the start/finish/consumer information.
- ThrowingConsumer<WindowedValue<InputT>> consumer;
- switch (functionSpec.getUrn()) {
- default:
- BeamFnApi.Target target;
- BeamFnApi.Coder coderSpec;
- throw new IllegalArgumentException(
- String.format("Unknown FunctionSpec %s", functionSpec));
-
- case DATA_OUTPUT_URN:
- target = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransform.getId())
- .setName(getOnlyElement(primitiveTransform.getOutputsMap().keySet()))
- .build();
- coderSpec = (BeamFnApi.Coder) fnApiRegistry.apply(
- getOnlyElement(primitiveTransform.getOutputsMap().values()).getCoderReference());
- BeamFnDataWriteRunner<InputT> remoteGrpcWriteRunner =
- new BeamFnDataWriteRunner<>(
- functionSpec,
- processBundleInstructionId,
- target,
- coderSpec,
- beamFnDataClient);
- addStartFunction.accept(remoteGrpcWriteRunner::registerForOutput);
- consumer = remoteGrpcWriteRunner::consume;
- addFinishFunction.accept(remoteGrpcWriteRunner::close);
- break;
-
- case DATA_INPUT_URN:
- target = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransform.getId())
- .setName(getOnlyElement(primitiveTransform.getInputsMap().keySet()))
- .build();
- coderSpec = (BeamFnApi.Coder) fnApiRegistry.apply(
- getOnlyElement(primitiveTransform.getOutputsMap().values()).getCoderReference());
- BeamFnDataReadRunner<OutputT> remoteGrpcReadRunner =
- new BeamFnDataReadRunner<>(
- functionSpec,
- processBundleInstructionId,
- target,
- coderSpec,
- beamFnDataClient,
- outputMap);
- addStartFunction.accept(remoteGrpcReadRunner::registerInputLocation);
- consumer = null;
- addFinishFunction.accept(remoteGrpcReadRunner::blockTillReadFinishes);
- break;
-
- case JAVA_DO_FN_URN:
- DoFnRunner<InputT, OutputT> doFnRunner = createDoFnRunner(functionSpec, outputMap);
- addStartFunction.accept(doFnRunner::startBundle);
- addFinishFunction.accept(doFnRunner::finishBundle);
- consumer = doFnRunner::processElement;
- break;
-
- case JAVA_SOURCE_URN:
- @SuppressWarnings({"unchecked", "rawtypes"})
- BoundedSourceRunner<BoundedSource<OutputT>, OutputT> sourceRunner =
- createBoundedSourceRunner(functionSpec, outputMap);
- @SuppressWarnings({"unchecked", "rawtypes"})
- ThrowingConsumer<WindowedValue<?>> sourceConsumer =
- (ThrowingConsumer)
- (ThrowingConsumer<WindowedValue<BoundedSource<OutputT>>>)
- sourceRunner::runReadLoop;
- // TODO: Remove and replace with source being sent across gRPC port
- addStartFunction.accept(sourceRunner::start);
- consumer = (ThrowingConsumer) sourceConsumer;
- break;
- }
-
- if (consumer != null) {
- for (Map.Entry<String, BeamFnApi.Target.List> entry :
- primitiveTransform.getInputsMap().entrySet()) {
- for (BeamFnApi.Target target : entry.getValue().getTargetList()) {
- addConsumer.accept(target, consumer);
- }
+ for (String consumingPTransformId : pCollectionIdsToConsumingPTransforms.get(pCollectionId)) {
+ createRunnerAndConsumersForPTransformRecursively(
+ beamFnStateClient,
+ consumingPTransformId,
+ processBundleDescriptor.getTransformsMap().get(consumingPTransformId),
+ processBundleInstructionId,
+ processBundleDescriptor,
+ pCollectionIdsToConsumingPTransforms,
+ pCollectionIdsToConsumers,
+ addStartFunction,
+ addFinishFunction);
}
}
+
+ urnToPTransformRunnerFactoryMap.getOrDefault(
+ pTransform.getSpec().getUrn(), defaultPTransformRunnerFactory)
+ .createRunnerForPTransform(
+ options,
+ beamFnDataClient,
+ beamFnStateClient,
+ pTransformId,
+ pTransform,
+ processBundleInstructionId,
+ processBundleDescriptor.getPcollectionsMap(),
+ processBundleDescriptor.getCodersMap(),
+ pCollectionIdsToConsumers,
+ addStartFunction,
+ addFinishFunction);
}
public BeamFnApi.InstructionResponse.Builder processBundle(BeamFnApi.InstructionRequest request)
@@ -212,121 +203,125 @@
BeamFnApi.ProcessBundleDescriptor bundleDescriptor =
(BeamFnApi.ProcessBundleDescriptor) fnApiRegistry.apply(bundleId);
- Multimap<BeamFnApi.Target,
- ThrowingConsumer<WindowedValue<Object>>> outputTargetToConsumer =
- HashMultimap.create();
+ Multimap<String, String> pCollectionIdsToConsumingPTransforms = HashMultimap.create();
+ Multimap<String,
+ ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers =
+ HashMultimap.create();
List<ThrowingRunnable> startFunctions = new ArrayList<>();
List<ThrowingRunnable> finishFunctions = new ArrayList<>();
- // We process the primitive transform list in reverse order
- // because we assume that the runner provides it in topologically order.
- // This means that all the start/finish functions will be in reverse topological order.
- for (BeamFnApi.PrimitiveTransform primitiveTransform :
- Lists.reverse(bundleDescriptor.getPrimitiveTransformList())) {
- createConsumersForPrimitiveTransform(
- primitiveTransform,
- request::getInstructionId,
- outputTargetToConsumer::get,
- outputTargetToConsumer::put,
- startFunctions::add,
- finishFunctions::add);
+
+ // Build a multimap of PCollection ids to PTransform ids which consume said PCollections
+ for (Map.Entry<String, RunnerApi.PTransform> entry
+ : bundleDescriptor.getTransformsMap().entrySet()) {
+ for (String pCollectionId : entry.getValue().getInputsMap().values()) {
+ pCollectionIdsToConsumingPTransforms.put(pCollectionId, entry.getKey());
+ }
}
- // Already in reverse order so we don't need to do anything.
- for (ThrowingRunnable startFunction : startFunctions) {
- LOG.debug("Starting function {}", startFunction);
- startFunction.run();
- }
+ // Instantiate a State API call handler depending on whether a State Api service descriptor
+ // was specified.
+ try (HandleStateCallsForBundle beamFnStateClient =
+ bundleDescriptor.hasStateApiServiceDescriptor()
+ ? new BlockTillStateCallsFinish(beamFnStateGrpcClientCache.forApiServiceDescriptor(
+ bundleDescriptor.getStateApiServiceDescriptor()))
+ : new FailAllStateCallsForBundle(request.getProcessBundle())) {
+ // Create a BeamFnStateClient
+ for (Map.Entry<String, RunnerApi.PTransform> entry
+ : bundleDescriptor.getTransformsMap().entrySet()) {
+ // Skip anything which isn't a root
+ // TODO: Remove source as a root and have it be triggered by the Runner.
+ if (!DATA_INPUT_URN.equals(entry.getValue().getSpec().getUrn())
+ && !JAVA_SOURCE_URN.equals(entry.getValue().getSpec().getUrn())) {
+ continue;
+ }
- // Need to reverse this since we want to call finish in topological order.
- for (ThrowingRunnable finishFunction : Lists.reverse(finishFunctions)) {
- LOG.debug("Finishing function {}", finishFunction);
- finishFunction.run();
+ createRunnerAndConsumersForPTransformRecursively(
+ beamFnStateClient,
+ entry.getKey(),
+ entry.getValue(),
+ request::getInstructionId,
+ bundleDescriptor,
+ pCollectionIdsToConsumingPTransforms,
+ pCollectionIdsToConsumers,
+ startFunctions::add,
+ finishFunctions::add);
+ }
+
+ // Already in reverse topological order so we don't need to do anything.
+ for (ThrowingRunnable startFunction : startFunctions) {
+ LOG.debug("Starting function {}", startFunction);
+ startFunction.run();
+ }
+
+ // Need to reverse this since we want to call finish in topological order.
+ for (ThrowingRunnable finishFunction : Lists.reverse(finishFunctions)) {
+ LOG.debug("Finishing function {}", finishFunction);
+ finishFunction.run();
+ }
}
return response;
}
/**
- * Converts a {@link org.apache.beam.fn.v1.BeamFnApi.FunctionSpec} into a {@link DoFnRunner}.
+ * A {@link BeamFnStateClient} which counts the number of outstanding {@link StateRequest}s and
+ * blocks till they are all finished.
*/
- private <InputT, OutputT> DoFnRunner<InputT, OutputT> createDoFnRunner(
- BeamFnApi.FunctionSpec functionSpec,
- Map<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> outputMap) {
- ByteString serializedFn;
- try {
- serializedFn = functionSpec.getData().unpack(BytesValue.class).getValue();
- } catch (InvalidProtocolBufferException e) {
- throw new IllegalArgumentException(
- String.format("Unable to unwrap DoFn %s", functionSpec), e);
+ private class BlockTillStateCallsFinish extends HandleStateCallsForBundle {
+ private final BeamFnStateClient beamFnStateClient;
+ private final Phaser phaser;
+ private int currentPhase;
+
+ private BlockTillStateCallsFinish(BeamFnStateClient beamFnStateClient) {
+ this.beamFnStateClient = beamFnStateClient;
+ this.phaser = new Phaser(1 /* initial party is the process bundle handler */);
+ this.currentPhase = phaser.getPhase();
}
- DoFnInfo<?, ?> doFnInfo =
- (DoFnInfo<?, ?>)
- SerializableUtils.deserializeFromByteArray(serializedFn.toByteArray(), "DoFnInfo");
- checkArgument(
- Objects.equals(
- new HashSet<>(Collections2.transform(outputMap.keySet(), Long::parseLong)),
- doFnInfo.getOutputMap().keySet()),
- "Unexpected mismatch between transform output map %s and DoFnInfo output map %s.",
- outputMap.keySet(),
- doFnInfo.getOutputMap());
-
- ImmutableMultimap.Builder<TupleTag<?>,
- ThrowingConsumer<WindowedValue<OutputT>>> tagToOutput =
- ImmutableMultimap.builder();
- for (Map.Entry<Long, TupleTag<?>> entry : doFnInfo.getOutputMap().entrySet()) {
- tagToOutput.putAll(entry.getValue(), outputMap.get(Long.toString(entry.getKey())));
+ @Override
+ public void close() throws Exception {
+ int unarrivedParties = phaser.getUnarrivedParties();
+ if (unarrivedParties > 0) {
+ LOG.debug("Waiting for {} parties to arrive before closing, current phase {}.",
+ unarrivedParties, currentPhase);
+ }
+ currentPhase = phaser.arriveAndAwaitAdvance();
}
- @SuppressWarnings({"unchecked", "rawtypes"})
- final Map<TupleTag<?>, Collection<ThrowingConsumer<WindowedValue<?>>>> tagBasedOutputMap =
- (Map) tagToOutput.build().asMap();
- OutputManager outputManager =
- new OutputManager() {
- Map<TupleTag<?>, Collection<ThrowingConsumer<WindowedValue<?>>>> tupleTagToOutput =
- tagBasedOutputMap;
-
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- try {
- Collection<ThrowingConsumer<WindowedValue<?>>> consumers =
- tupleTagToOutput.get(tag);
- if (consumers == null) {
- /* This is a normal case, e.g., if a DoFn has output but that output is not
- * consumed. Drop the output. */
- return;
- }
- for (ThrowingConsumer<WindowedValue<?>> consumer : consumers) {
- consumer.accept(output);
- }
- } catch (Throwable t) {
- throw new RuntimeException(t);
- }
- }
- };
-
- @SuppressWarnings({"unchecked", "rawtypes", "deprecation"})
- DoFnRunner<InputT, OutputT> runner =
- DoFnRunners.simpleRunner(
- PipelineOptionsFactory.create(), /* TODO */
- (DoFn) doFnInfo.getDoFn(),
- NullSideInputReader.empty(), /* TODO */
- outputManager,
- (TupleTag) doFnInfo.getOutputMap().get(doFnInfo.getMainOutput()),
- new ArrayList<>(doFnInfo.getOutputMap().values()),
- new FakeStepContext(),
- (WindowingStrategy) doFnInfo.getWindowingStrategy());
- return runner;
+ @Override
+ public void handle(StateRequest.Builder requestBuilder,
+ CompletableFuture<StateResponse> response) {
+ // Register each request with the phaser and arrive and deregister each time a request
+ // completes.
+ phaser.register();
+ response.whenComplete((stateResponse, throwable) -> phaser.arriveAndDeregister());
+ beamFnStateClient.handle(requestBuilder, response);
+ }
}
- private <InputT extends BoundedSource<OutputT>, OutputT>
- BoundedSourceRunner<InputT, OutputT> createBoundedSourceRunner(
- BeamFnApi.FunctionSpec functionSpec,
- Map<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> outputMap) {
+ /**
+ * A {@link BeamFnStateClient} which fails all requests because the {@link ProcessBundleRequest}
+ * does not contain a State API {@link ApiServiceDescriptor}.
+ */
+ private class FailAllStateCallsForBundle extends HandleStateCallsForBundle {
+ private final ProcessBundleRequest request;
- @SuppressWarnings({"rawtypes", "unchecked"})
- BoundedSourceRunner<InputT, OutputT> runner =
- new BoundedSourceRunner(options, functionSpec, outputMap);
- return runner;
+ private FailAllStateCallsForBundle(ProcessBundleRequest request) {
+ this.request = request;
+ }
+
+ @Override
+ public void close() throws Exception {
+ // no-op
+ }
+
+ @Override
+ public void handle(Builder requestBuilder, CompletableFuture<StateResponse> response) {
+ throw new IllegalStateException(String.format("State API calls are unsupported because the "
+ + "ProcessBundleRequest %s does not support state.", request));
+ }
+ }
+
+ private abstract class HandleStateCallsForBundle implements AutoCloseable, BeamFnStateClient {
}
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
index fb06231..503536a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
@@ -19,19 +19,21 @@
package org.apache.beam.fn.harness.control;
import com.google.protobuf.Message;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A handler and datastore for types that be can be registered via the Fn API.
*
- * <p>Allows for {@link org.apache.beam.fn.v1.BeamFnApi.RegisterRequest}s to occur in parallel with
+ * <p>Allows for {@link BeamFnApi.RegisterRequest}s to occur in parallel with
* subsequent requests that may lookup registered values by blocking lookups until registration
* occurs.
*/
@@ -45,6 +47,7 @@
public <T extends Message> T getById(String id) {
try {
+ LOG.debug("Attempting to find {}", id);
@SuppressWarnings("unchecked")
CompletableFuture<T> returnValue = (CompletableFuture<T>) computeIfAbsent(id);
/*
@@ -75,11 +78,12 @@
processBundleDescriptor.getId(),
processBundleDescriptor.getClass());
computeIfAbsent(processBundleDescriptor.getId()).complete(processBundleDescriptor);
- for (BeamFnApi.Coder coder : processBundleDescriptor.getCodersList()) {
+ for (Map.Entry<String, RunnerApi.Coder> entry
+ : processBundleDescriptor.getCodersMap().entrySet()) {
LOG.debug("Registering {} with type {}",
- coder.getFunctionSpec().getId(),
- coder.getClass());
- computeIfAbsent(coder.getFunctionSpec().getId()).complete(coder);
+ entry.getKey(),
+ entry.getValue().getClass());
+ computeIfAbsent(entry.getKey()).complete(entry.getValue());
}
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
index 7223e87..97396e7 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
@@ -24,9 +24,9 @@
import java.util.List;
import java.util.function.Consumer;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.ExperimentalOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
@@ -37,7 +37,7 @@
* A buffering outbound {@link Consumer} for the Beam Fn Data API.
*
* <p>Encodes individually consumed elements with the provided {@link Coder} producing
- * a single {@link org.apache.beam.fn.v1.BeamFnApi.Elements} message when the buffer threshold
+ * a single {@link BeamFnApi.Elements} message when the buffer threshold
* is surpassed.
*
* <p>The default buffer threshold can be overridden by specifying the experiment
@@ -81,7 +81,7 @@
* returns the default buffer limit.
*/
private static int getBufferLimit(PipelineOptions options) {
- List<String> experiments = options.as(DataflowPipelineDebugOptions.class).getExperiments();
+ List<String> experiments = options.as(ExperimentalOptions.class).getExperiments();
for (String experiment : experiments == null ? Collections.<String>emptyList() : experiments) {
if (experiment.startsWith(BEAM_FN_API_DATA_BUFFER_LIMIT)) {
return Integer.parseInt(experiment.substring(BEAM_FN_API_DATA_BUFFER_LIMIT.length()));
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
index 7be96b6..c3b7fd2 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
@@ -21,7 +21,8 @@
import java.util.concurrent.CompletableFuture;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
@@ -43,7 +44,7 @@
* <p>The consumer is not required to be thread safe.
*/
<T> CompletableFuture<Void> forInboundConsumer(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
KV<String, BeamFnApi.Target> inputLocation,
Coder<WindowedValue<T>> coder,
ThrowingConsumer<WindowedValue<T>> consumer);
@@ -58,7 +59,7 @@
* <p>The returned closeable consumer is not thread safe.
*/
<T> CloseableThrowingConsumer<WindowedValue<T>> forOutboundConsumer(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
KV<String, BeamFnApi.Target> outputLocation,
Coder<WindowedValue<T>> coder);
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
index 4137cd7..9333410 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
@@ -27,8 +27,9 @@
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnDataGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
@@ -44,8 +45,8 @@
public class BeamFnDataGrpcClient implements BeamFnDataClient {
private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataGrpcClient.class);
- private final ConcurrentMap<BeamFnApi.ApiServiceDescriptor, BeamFnDataGrpcMultiplexer> cache;
- private final Function<BeamFnApi.ApiServiceDescriptor, ManagedChannel> channelFactory;
+ private final ConcurrentMap<Endpoints.ApiServiceDescriptor, BeamFnDataGrpcMultiplexer> cache;
+ private final Function<Endpoints.ApiServiceDescriptor, ManagedChannel> channelFactory;
private final BiFunction<Function<StreamObserver<BeamFnApi.Elements>,
StreamObserver<BeamFnApi.Elements>>,
StreamObserver<BeamFnApi.Elements>,
@@ -54,7 +55,7 @@
public BeamFnDataGrpcClient(
PipelineOptions options,
- Function<BeamFnApi.ApiServiceDescriptor, ManagedChannel> channelFactory,
+ Function<Endpoints.ApiServiceDescriptor, ManagedChannel> channelFactory,
BiFunction<Function<StreamObserver<BeamFnApi.Elements>, StreamObserver<BeamFnApi.Elements>>,
StreamObserver<BeamFnApi.Elements>,
StreamObserver<BeamFnApi.Elements>> streamObserverFactory) {
@@ -74,11 +75,11 @@
*/
@Override
public <T> CompletableFuture<Void> forInboundConsumer(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
KV<String, BeamFnApi.Target> inputLocation,
Coder<WindowedValue<T>> coder,
ThrowingConsumer<WindowedValue<T>> consumer) {
- LOG.debug("Registering consumer instruction {} for target {}",
+ LOG.debug("Registering consumer for instruction {} and target {}",
inputLocation.getKey(),
inputLocation.getValue());
@@ -101,19 +102,22 @@
*/
@Override
public <T> CloseableThrowingConsumer<WindowedValue<T>> forOutboundConsumer(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
KV<String, BeamFnApi.Target> outputLocation,
Coder<WindowedValue<T>> coder) {
BeamFnDataGrpcMultiplexer client = getClientFor(apiServiceDescriptor);
+ LOG.debug("Creating output consumer for instruction {} and target {}",
+ outputLocation.getKey(),
+ outputLocation.getValue());
return new BeamFnDataBufferingOutboundObserver<>(
options, outputLocation, coder, client.getOutboundObserver());
}
private BeamFnDataGrpcMultiplexer getClientFor(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor) {
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor) {
return cache.computeIfAbsent(apiServiceDescriptor,
- (BeamFnApi.ApiServiceDescriptor descriptor) -> new BeamFnDataGrpcMultiplexer(
+ (Endpoints.ApiServiceDescriptor descriptor) -> new BeamFnDataGrpcMultiplexer(
descriptor,
(StreamObserver<BeamFnApi.Elements> inboundObserver) -> streamObserverFactory.apply(
BeamFnDataGrpc.newStub(channelFactory.apply(apiServiceDescriptor))::data,
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
index 53dfe11..cfe726a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
@@ -26,27 +26,29 @@
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.function.Function;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A gRPC multiplexer for a specific {@link org.apache.beam.fn.v1.BeamFnApi.ApiServiceDescriptor}.
+ * A gRPC multiplexer for a specific {@link
+ * Endpoints.ApiServiceDescriptor}.
*
- * <p>Multiplexes data for inbound consumers based upon their individual
- * {@link org.apache.beam.fn.v1.BeamFnApi.Target}s.
+ * <p>Multiplexes data for inbound consumers based upon their individual {@link
+ * org.apache.beam.model.fnexecution.v1.BeamFnApi.Target}s.
*
- * <p>Multiplexing inbound and outbound streams is as thread safe as the consumers of those
- * streams. For inbound streams, this is as thread safe as the inbound observers. For outbound
- * streams, this is as thread safe as the underlying stream observer.
+ * <p>Multiplexing inbound and outbound streams is as thread safe as the consumers of those streams.
+ * For inbound streams, this is as thread safe as the inbound observers. For outbound streams, this
+ * is as thread safe as the underlying stream observer.
*
- * <p>TODO: Add support for multiplexing over multiple outbound observers by stickying
- * the output location with a specific outbound observer.
+ * <p>TODO: Add support for multiplexing over multiple outbound observers by stickying the output
+ * location with a specific outbound observer.
*/
public class BeamFnDataGrpcMultiplexer {
private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataGrpcMultiplexer.class);
- private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
+ private final Endpoints.ApiServiceDescriptor apiServiceDescriptor;
private final StreamObserver<BeamFnApi.Elements> inboundObserver;
private final StreamObserver<BeamFnApi.Elements> outboundObserver;
@VisibleForTesting
@@ -55,7 +57,7 @@
consumers;
public BeamFnDataGrpcMultiplexer(
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
Function<StreamObserver<BeamFnApi.Elements>,
StreamObserver<BeamFnApi.Elements>> outboundObserverFactory) {
this.apiServiceDescriptor = apiServiceDescriptor;
@@ -84,7 +86,7 @@
KV<String, BeamFnApi.Target> key) {
return consumers.computeIfAbsent(
key,
- (KV<String, BeamFnApi.Target> providedKey) -> new CompletableFuture<>());
+ (KV<String, BeamFnApi.Target> unused) -> new CompletableFuture<>());
}
/**
@@ -102,7 +104,12 @@
try {
KV<String, BeamFnApi.Target> key =
KV.of(data.getInstructionReference(), data.getTarget());
- futureForKey(key).get().accept(data);
+ CompletableFuture<Consumer<BeamFnApi.Elements.Data>> consumer = futureForKey(key);
+ if (!consumer.isDone()) {
+ LOG.debug("Received data for key {} without consumer ready. "
+ + "Waiting for consumer to be registered.", key);
+ }
+ consumer.get().accept(data);
if (data.getData().isEmpty()) {
consumers.remove(key);
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
index ac603bd..64a12e0 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
@@ -21,14 +21,14 @@
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Decodes individually consumed {@link org.apache.beam.fn.v1.BeamFnApi.Elements.Data} with the
+ * Decodes individually consumed {@link BeamFnApi.Elements.Data} with the
* provided {@link Coder} passing the individual decoded elements to the provided consumer.
*/
public class BeamFnDataInboundObserver<T> implements Consumer<BeamFnApi.Elements.Data> {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
deleted file mode 100644
index 9b79d11..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.fake;
-
-import java.io.IOException;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
-import org.apache.beam.runners.core.StateInternals;
-import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/**
- * A fake {@link StepContext} factory that performs no-ops.
- */
-public class FakeStepContext implements StepContext {
- @Override
- public String getStepName() {
- return "TODO";
- }
-
- @Override
- public String getTransformName() {
- return "TODO";
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {
- }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(
- TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder,
- W window,
- Coder<W> windowCoder) throws IOException {
- }
-
- @Override
- public StateInternals stateInternals() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TimerInternals timerInternals() {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/package-info.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/package-info.java
deleted file mode 100644
index cd6eb02..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Fake implementations of bindings used with runners-core.
- */
-package org.apache.beam.fn.harness.fake;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fn/ThrowingBiConsumer.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fn/ThrowingBiConsumer.java
new file mode 100644
index 0000000..fca8f3c
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fn/ThrowingBiConsumer.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness.fn;
+
+import java.util.function.BiConsumer;
+
+/**
+ * A {@link BiConsumer} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8
+ * functional interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingBiConsumer<T1, T2> {
+ void accept(T1 t1, T2 t2) throws Exception;
+}
+
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
index c8d11ed..e7e0c71 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
@@ -24,7 +24,10 @@
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.Timestamp;
import io.grpc.ManagedChannel;
-import io.grpc.stub.StreamObserver;
+import io.grpc.Status;
+import io.grpc.stub.CallStreamObserver;
+import io.grpc.stub.ClientCallStreamObserver;
+import io.grpc.stub.ClientResponseObserver;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -36,9 +39,8 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.Phaser;
import java.util.concurrent.TimeUnit;
-import java.util.function.BiFunction;
-import java.util.function.Consumer;
import java.util.function.Function;
import java.util.logging.Formatter;
import java.util.logging.Handler;
@@ -47,44 +49,39 @@
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
-import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.SdkHarnessOptions;
/**
* Configures {@link java.util.logging} to send all {@link LogRecord}s via the Beam Fn Logging API.
*/
public class BeamFnLoggingClient implements AutoCloseable {
private static final String ROOT_LOGGER_NAME = "";
- private static final ImmutableMap<Level, BeamFnApi.LogEntry.Severity> LOG_LEVEL_MAP =
- ImmutableMap.<Level, BeamFnApi.LogEntry.Severity>builder()
- .put(Level.SEVERE, BeamFnApi.LogEntry.Severity.ERROR)
- .put(Level.WARNING, BeamFnApi.LogEntry.Severity.WARN)
- .put(Level.INFO, BeamFnApi.LogEntry.Severity.INFO)
- .put(Level.FINE, BeamFnApi.LogEntry.Severity.DEBUG)
- .put(Level.FINEST, BeamFnApi.LogEntry.Severity.TRACE)
+ private static final ImmutableMap<Level, BeamFnApi.LogEntry.Severity.Enum> LOG_LEVEL_MAP =
+ ImmutableMap.<Level, BeamFnApi.LogEntry.Severity.Enum>builder()
+ .put(Level.SEVERE, BeamFnApi.LogEntry.Severity.Enum.ERROR)
+ .put(Level.WARNING, BeamFnApi.LogEntry.Severity.Enum.WARN)
+ .put(Level.INFO, BeamFnApi.LogEntry.Severity.Enum.INFO)
+ .put(Level.FINE, BeamFnApi.LogEntry.Severity.Enum.DEBUG)
+ .put(Level.FINEST, BeamFnApi.LogEntry.Severity.Enum.TRACE)
.build();
- private static final ImmutableMap<DataflowWorkerLoggingOptions.Level, Level> LEVEL_CONFIGURATION =
- ImmutableMap.<DataflowWorkerLoggingOptions.Level, Level>builder()
- .put(DataflowWorkerLoggingOptions.Level.OFF, Level.OFF)
- .put(DataflowWorkerLoggingOptions.Level.ERROR, Level.SEVERE)
- .put(DataflowWorkerLoggingOptions.Level.WARN, Level.WARNING)
- .put(DataflowWorkerLoggingOptions.Level.INFO, Level.INFO)
- .put(DataflowWorkerLoggingOptions.Level.DEBUG, Level.FINE)
- .put(DataflowWorkerLoggingOptions.Level.TRACE, Level.FINEST)
+ private static final ImmutableMap<SdkHarnessOptions.LogLevel, Level> LEVEL_CONFIGURATION =
+ ImmutableMap.<SdkHarnessOptions.LogLevel, Level>builder()
+ .put(SdkHarnessOptions.LogLevel.OFF, Level.OFF)
+ .put(SdkHarnessOptions.LogLevel.ERROR, Level.SEVERE)
+ .put(SdkHarnessOptions.LogLevel.WARN, Level.WARNING)
+ .put(SdkHarnessOptions.LogLevel.INFO, Level.INFO)
+ .put(SdkHarnessOptions.LogLevel.DEBUG, Level.FINE)
+ .put(SdkHarnessOptions.LogLevel.TRACE, Level.FINEST)
.build();
private static final Formatter FORMATTER = new SimpleFormatter();
- private static final String FAKE_INSTRUCTION_ID = "FAKE_INSTRUCTION_ID";
-
- /* Used to signal to a thread processing a queue to finish its work gracefully. */
- private static final BeamFnApi.LogEntry POISON_PILL =
- BeamFnApi.LogEntry.newBuilder().setInstructionReference(FAKE_INSTRUCTION_ID).build();
-
/**
* The number of log messages that will be buffered. Assuming log messages are at most 1 KiB,
* this represents a buffer of about 10 MiBs.
@@ -95,24 +92,22 @@
* garbage collected. java.util.logging only has weak references to the loggers
* so if they are garbage collected, our hierarchical configuration will be lost. */
private final Collection<Logger> configuredLoggers;
- private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
+ private final Endpoints.ApiServiceDescriptor apiServiceDescriptor;
private final ManagedChannel channel;
- private final StreamObserver<BeamFnApi.LogEntry.List> outboundObserver;
+ private final CallStreamObserver<BeamFnApi.LogEntry.List> outboundObserver;
private final LogControlObserver inboundObserver;
private final LogRecordHandler logRecordHandler;
private final CompletableFuture<Object> inboundObserverCompletion;
+ private final Phaser phaser;
public BeamFnLoggingClient(
PipelineOptions options,
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor,
- Function<BeamFnApi.ApiServiceDescriptor, ManagedChannel> channelFactory,
- BiFunction<Function<StreamObserver<BeamFnApi.LogControl>,
- StreamObserver<BeamFnApi.LogEntry.List>>,
- StreamObserver<BeamFnApi.LogControl>,
- StreamObserver<BeamFnApi.LogEntry.List>> streamObserverFactory) {
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor,
+ Function<Endpoints.ApiServiceDescriptor, ManagedChannel> channelFactory) {
this.apiServiceDescriptor = apiServiceDescriptor;
this.inboundObserverCompletion = new CompletableFuture<>();
this.configuredLoggers = new ArrayList<>();
+ this.phaser = new Phaser(1);
this.channel = channelFactory.apply(apiServiceDescriptor);
// Reset the global log manager, get the root logger and remove the default log handlers.
@@ -124,14 +119,14 @@
}
// Use the passed in logging options to configure the various logger levels.
- DataflowWorkerLoggingOptions loggingOptions = options.as(DataflowWorkerLoggingOptions.class);
- if (loggingOptions.getDefaultWorkerLogLevel() != null) {
- rootLogger.setLevel(LEVEL_CONFIGURATION.get(loggingOptions.getDefaultWorkerLogLevel()));
+ SdkHarnessOptions loggingOptions = options.as(SdkHarnessOptions.class);
+ if (loggingOptions.getDefaultSdkHarnessLogLevel() != null) {
+ rootLogger.setLevel(LEVEL_CONFIGURATION.get(loggingOptions.getDefaultSdkHarnessLogLevel()));
}
- if (loggingOptions.getWorkerLogLevelOverrides() != null) {
- for (Map.Entry<String, DataflowWorkerLoggingOptions.Level> loggerOverride :
- loggingOptions.getWorkerLogLevelOverrides().entrySet()) {
+ if (loggingOptions.getSdkHarnessLogLevelOverrides() != null) {
+ for (Map.Entry<String, SdkHarnessOptions.LogLevel> loggerOverride :
+ loggingOptions.getSdkHarnessLogLevelOverrides().entrySet()) {
Logger logger = Logger.getLogger(loggerOverride.getKey());
logger.setLevel(LEVEL_CONFIGURATION.get(loggerOverride.getValue()));
configuredLoggers.add(logger);
@@ -142,29 +137,32 @@
inboundObserver = new LogControlObserver();
logRecordHandler = new LogRecordHandler(options.as(GcsOptions.class).getExecutorService());
logRecordHandler.setLevel(Level.ALL);
- outboundObserver = streamObserverFactory.apply(stub::logging, inboundObserver);
+ outboundObserver =
+ (CallStreamObserver<BeamFnApi.LogEntry.List>) stub.logging(inboundObserver);
rootLogger.addHandler(logRecordHandler);
}
@Override
public void close() throws Exception {
- // Hang up with the server
- logRecordHandler.close();
+ try {
+ // Reset the logging configuration to what it is at startup
+ for (Logger logger : configuredLoggers) {
+ logger.setLevel(null);
+ }
+ configuredLoggers.clear();
+ LogManager.getLogManager().readConfiguration();
- // Wait for the server to hang up
- inboundObserverCompletion.get();
+ // Hang up with the server
+ logRecordHandler.close();
- // Reset the logging configuration to what it is at startup
- for (Logger logger : configuredLoggers) {
- logger.setLevel(null);
- }
- configuredLoggers.clear();
- LogManager.getLogManager().readConfiguration();
-
- // Shut the channel down
- channel.shutdown();
- if (!channel.awaitTermination(10, TimeUnit.SECONDS)) {
- channel.shutdownNow();
+ // Wait for the server to hang up
+ inboundObserverCompletion.get();
+ } finally {
+ // Shut the channel down
+ channel.shutdown();
+ if (!channel.awaitTermination(10, TimeUnit.SECONDS)) {
+ channel.shutdownNow();
+ }
}
}
@@ -179,16 +177,19 @@
private final BlockingDeque<BeamFnApi.LogEntry> bufferedLogEntries =
new LinkedBlockingDeque<>(MAX_BUFFERED_LOG_ENTRY_COUNT);
private final Future<?> bufferedLogWriter;
- private final ThreadLocal<Consumer<BeamFnApi.LogEntry>> logEntryHandler;
+ /**
+ * Safe object publishing is not required since we only care if the thread that set
+ * this field is equal to the thread also attempting to add a log entry.
+ */
+ private Thread logEntryHandlerThread;
private LogRecordHandler(ExecutorService executorService) {
bufferedLogWriter = executorService.submit(this);
- logEntryHandler = new ThreadLocal<>();
}
@Override
public void publish(LogRecord record) {
- BeamFnApi.LogEntry.Severity severity = LOG_LEVEL_MAP.get(record.getLevel());
+ BeamFnApi.LogEntry.Severity.Enum severity = LOG_LEVEL_MAP.get(record.getLevel());
if (severity == null) {
return;
}
@@ -204,19 +205,18 @@
builder.setTrace(getStackTraceAsString(record.getThrown()));
}
// The thread that sends log records should never perform a blocking publish and
- // only insert log records best effort. We detect which thread is logging
- // by using the thread local, defaulting to the blocking publish.
- MoreObjects.firstNonNull(
- logEntryHandler.get(), this::blockingPublish).accept(builder.build());
- }
-
- /** Blocks caller till enough space exists to publish this log entry. */
- private void blockingPublish(BeamFnApi.LogEntry logEntry) {
- try {
- bufferedLogEntries.put(logEntry);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
+ // only insert log records best effort.
+ if (Thread.currentThread() != logEntryHandlerThread) {
+ // Blocks caller till enough space exists to publish this log entry.
+ try {
+ bufferedLogEntries.put(builder.build());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ } else {
+ // Never blocks caller, will drop log message if buffer is full.
+ bufferedLogEntries.offer(builder.build());
}
}
@@ -225,27 +225,53 @@
// Logging which occurs in this thread will attempt to publish log entries into the
// above handler which should never block if the queue is full otherwise
// this thread will get stuck.
- logEntryHandler.set(bufferedLogEntries::offer);
+ logEntryHandlerThread = Thread.currentThread();
+
List<BeamFnApi.LogEntry> additionalLogEntries =
new ArrayList<>(MAX_BUFFERED_LOG_ENTRY_COUNT);
+ Throwable thrown = null;
try {
- BeamFnApi.LogEntry logEntry;
- while ((logEntry = bufferedLogEntries.take()) != POISON_PILL) {
+ // As long as we haven't yet terminated, then attempt
+ while (!phaser.isTerminated()) {
+ // Try to wait for a message to show up.
+ BeamFnApi.LogEntry logEntry = bufferedLogEntries.poll(1, TimeUnit.SECONDS);
+ // If we don't have a message then we need to try this loop again.
+ if (logEntry == null) {
+ continue;
+ }
+
+ // Attempt to honor flow control. Phaser termination causes await advance to return
+ // immediately.
+ int phase = phaser.getPhase();
+ if (!outboundObserver.isReady()) {
+ phaser.awaitAdvance(phase);
+ }
+
+ // Batch together as many log messages as possible that are held within the buffer
BeamFnApi.LogEntry.List.Builder builder =
BeamFnApi.LogEntry.List.newBuilder().addLogEntries(logEntry);
bufferedLogEntries.drainTo(additionalLogEntries);
- for (int i = 0; i < additionalLogEntries.size(); ++i) {
- if (additionalLogEntries.get(i) == POISON_PILL) {
- additionalLogEntries = additionalLogEntries.subList(0, i);
- break;
- }
- }
builder.addAllLogEntries(additionalLogEntries);
outboundObserver.onNext(builder.build());
+ additionalLogEntries.clear();
}
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IllegalStateException(e);
+
+ // Perform one more final check to see if there are any log entries to guarantee that
+ // if a log entry was added on the thread performing termination that we will send it.
+ bufferedLogEntries.drainTo(additionalLogEntries);
+ if (!additionalLogEntries.isEmpty()) {
+ outboundObserver.onNext(
+ BeamFnApi.LogEntry.List.newBuilder().addAllLogEntries(additionalLogEntries).build());
+ }
+ } catch (Throwable t) {
+ thrown = t;
+ }
+ if (thrown != null) {
+ outboundObserver.onError(
+ Status.INTERNAL.withDescription(getStackTraceAsString(thrown)).asException());
+ throw new IllegalStateException(thrown);
+ } else {
+ outboundObserver.onCompleted();
}
}
@@ -254,31 +280,17 @@
}
@Override
- public void close() {
- synchronized (outboundObserver) {
- // If we are done, then a previous caller has already shutdown the queue processing thread
- // hence we don't need to do it again.
- if (!bufferedLogWriter.isDone()) {
- // We check to see if we were able to successfully insert the poison pill at the end of
- // the queue forcing the remainder of the elements to be processed or if the processing
- // thread is done.
- try {
- // The order of these checks is important because short circuiting will cause us to
- // insert into the queue first and only if it fails do we check that the thread is done.
- while (!bufferedLogEntries.offer(POISON_PILL, 60, TimeUnit.SECONDS)
- || !bufferedLogWriter.isDone()) {
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
- }
- waitTillFinish();
- }
- outboundObserver.onCompleted();
+ public synchronized void close() {
+ // If we are done, then a previous caller has already shutdown the queue processing thread
+ // hence we don't need to do it again.
+ if (phaser.isTerminated()) {
+ return;
}
- }
- private void waitTillFinish() {
+ // Terminate the phaser that we block on when attempting to honor flow control on the
+ // outbound observer.
+ phaser.forceTermination();
+
try {
bufferedLogWriter.get();
} catch (CancellationException e) {
@@ -292,7 +304,14 @@
}
}
- private class LogControlObserver implements StreamObserver<BeamFnApi.LogControl> {
+ private class LogControlObserver
+ implements ClientResponseObserver<BeamFnApi.LogEntry, BeamFnApi.LogControl> {
+
+ @Override
+ public void beforeStart(ClientCallStreamObserver requestStream) {
+ requestStream.setOnReadyHandler(phaser::arrive);
+ }
+
@Override
public void onNext(BeamFnApi.LogControl value) {
}
@@ -306,5 +325,6 @@
public void onCompleted() {
inboundObserverCompletion.complete(null);
}
+
}
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
new file mode 100644
index 0000000..7064db4
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.stream.DataStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateClearRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.sdk.coders.Coder;
+
+/**
+ * An implementation of a bag user state that utilizes the Beam Fn State API to fetch, clear
+ * and persist values.
+ *
+ * <p>Calling {@link #asyncClose()} schedules any required persistence changes. This object should
+ * no longer be used after it is closed.
+ *
+ * <p>TODO: Move to an async persist model where persistence is signalled based upon cache
+ * memory pressure and its need to flush.
+ *
+ * <p>TODO: Support block level caching and prefetch.
+ */
+public class BagUserState<T> {
+ private final BeamFnStateClient beamFnStateClient;
+ private final String stateId;
+ private final Coder<T> coder;
+ private final Supplier<Builder> partialRequestSupplier;
+ private Iterable<T> oldValues;
+ private ArrayList<T> newValues;
+ private List<T> unmodifiableNewValues;
+ private boolean isClosed;
+
+ public BagUserState(
+ BeamFnStateClient beamFnStateClient,
+ String stateId,
+ Coder<T> coder,
+ Supplier<Builder> partialRequestSupplier) {
+ this.beamFnStateClient = beamFnStateClient;
+ this.stateId = stateId;
+ this.coder = coder;
+ this.partialRequestSupplier = partialRequestSupplier;
+ this.oldValues = new LazyCachingIteratorToIterable<>(
+ new DataStreams.DataStreamDecoder(coder,
+ DataStreams.inbound(
+ StateFetchingIterators.usingPartialRequestWithStateKey(
+ beamFnStateClient,
+ partialRequestSupplier))));
+ this.newValues = new ArrayList<>();
+ this.unmodifiableNewValues = Collections.unmodifiableList(newValues);
+ }
+
+ public Iterable<T> get() {
+ checkState(!isClosed,
+ "Bag user state is no longer usable because it is closed for %s", stateId);
+ // If we were cleared we should disregard old values.
+ if (oldValues == null) {
+ return unmodifiableNewValues;
+ }
+ return Iterables.concat(oldValues, unmodifiableNewValues);
+ }
+
+ public void append(T t) {
+ checkState(!isClosed,
+ "Bag user state is no longer usable because it is closed for %s", stateId);
+ newValues.add(t);
+ }
+
+ public void clear() {
+ checkState(!isClosed,
+ "Bag user state is no longer usable because it is closed for %s", stateId);
+ oldValues = null;
+ newValues.clear();
+ }
+
+ public void asyncClose() throws Exception {
+ checkState(!isClosed,
+ "Bag user state is no longer usable because it is closed for %s", stateId);
+ if (oldValues == null) {
+ beamFnStateClient.handle(
+ partialRequestSupplier.get()
+ .setClear(StateClearRequest.getDefaultInstance()),
+ new CompletableFuture<>());
+ }
+ if (!newValues.isEmpty()) {
+ ByteString.Output out = ByteString.newOutput();
+ for (T newValue : newValues) {
+ // TODO: Replace with chunking output stream
+ coder.encode(newValue, out);
+ }
+ beamFnStateClient.handle(
+ partialRequestSupplier.get()
+ .setAppend(StateAppendRequest.newBuilder().setData(out.toByteString())),
+ new CompletableFuture<>());
+ }
+ isClosed = true;
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
new file mode 100644
index 0000000..c2dfd63
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+
+/**
+ * The {@link BeamFnStateClient} is able to forward state requests to a handler which returns
+ * a corresponding response or error if completed unsuccessfully.
+ */
+public interface BeamFnStateClient {
+
+ /**
+ * Consumes a state request populating a unique id returning a future to the response.
+ *
+ * @param requestBuilder A partially completed state request. The id will be populated the client.
+ * @param response A future containing a corresponding {@link StateResponse} for the supplied
+ * request.
+ */
+ void handle(BeamFnApi.StateRequest.Builder requestBuilder,
+ CompletableFuture<StateResponse> response);
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
new file mode 100644
index 0000000..2ca0704
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import io.grpc.ManagedChannel;
+import io.grpc.stub.StreamObserver;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.beam.fn.harness.data.BeamFnDataGrpcClient;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A cache of {@link BeamFnStateClient}s which handle Beam Fn State requests using gRPC.
+ *
+ * <p>TODO: Add the ability to close which cancels any pending and stops any future requests.
+ */
+public class BeamFnStateGrpcClientCache {
+ private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataGrpcClient.class);
+
+ private final ConcurrentMap<ApiServiceDescriptor, BeamFnStateClient> cache;
+ private final Function<ApiServiceDescriptor, ManagedChannel> channelFactory;
+ private final BiFunction<Function<StreamObserver<StateResponse>,
+ StreamObserver<StateRequest>>,
+ StreamObserver<StateResponse>,
+ StreamObserver<StateRequest>> streamObserverFactory;
+ private final PipelineOptions options;
+ private final Supplier<String> idGenerator;
+
+ public BeamFnStateGrpcClientCache(
+ PipelineOptions options,
+ Supplier<String> idGenerator,
+ Function<Endpoints.ApiServiceDescriptor, ManagedChannel> channelFactory,
+ BiFunction<Function<StreamObserver<StateResponse>, StreamObserver<StateRequest>>,
+ StreamObserver<StateResponse>,
+ StreamObserver<StateRequest>> streamObserverFactory) {
+ this.options = options;
+ this.idGenerator = idGenerator;
+ this.channelFactory = channelFactory;
+ this.streamObserverFactory = streamObserverFactory;
+ this.cache = new ConcurrentHashMap<>();
+ }
+
+ /**(
+ * Creates or returns an existing {@link BeamFnStateClient} depending on whether the passed in
+ * {@link ApiServiceDescriptor} currently has a {@link BeamFnStateClient} bound to the same
+ * channel.
+ */
+ public BeamFnStateClient forApiServiceDescriptor(ApiServiceDescriptor apiServiceDescriptor)
+ throws IOException {
+ return cache.computeIfAbsent(apiServiceDescriptor, this::createBeamFnStateClient);
+ }
+
+ private BeamFnStateClient createBeamFnStateClient(ApiServiceDescriptor apiServiceDescriptor) {
+ return new GrpcStateClient(apiServiceDescriptor);
+ }
+
+ /**
+ * A {@link BeamFnStateClient} for a given {@link ApiServiceDescriptor}.
+ */
+ private class GrpcStateClient implements BeamFnStateClient {
+ private final ApiServiceDescriptor apiServiceDescriptor;
+ private final ConcurrentMap<String, CompletableFuture<StateResponse>> outstandingRequests;
+ private final StreamObserver<StateRequest> outboundObserver;
+ private final ManagedChannel channel;
+ private volatile RuntimeException closed;
+
+ private GrpcStateClient(ApiServiceDescriptor apiServiceDescriptor) {
+ this.apiServiceDescriptor = apiServiceDescriptor;
+ this.outstandingRequests = new ConcurrentHashMap<>();
+ this.channel = channelFactory.apply(apiServiceDescriptor);
+ this.outboundObserver = streamObserverFactory.apply(
+ BeamFnStateGrpc.newStub(channel)::state, new InboundObserver());
+ }
+
+ @Override
+ public void handle(
+ StateRequest.Builder requestBuilder, CompletableFuture<StateResponse> response) {
+ requestBuilder.setId(idGenerator.get());
+ StateRequest request = requestBuilder.build();
+ outstandingRequests.put(request.getId(), response);
+
+ // If the server closes, gRPC will throw an error if onNext is called.
+ LOG.debug("Sending StateRequest {}", request);
+ outboundObserver.onNext(request);
+ }
+
+ private synchronized void closeAndCleanUp(RuntimeException cause) {
+ if (closed != null) {
+ return;
+ }
+ cache.remove(apiServiceDescriptor);
+ closed = cause;
+
+ // Make a copy of the map to make the view of the outstanding requests consistent.
+ Map<String, CompletableFuture<StateResponse>> outstandingRequestsCopy =
+ new ConcurrentHashMap<>(outstandingRequests);
+
+ if (outstandingRequestsCopy.isEmpty()) {
+ outboundObserver.onCompleted();
+ return;
+ }
+
+ outstandingRequests.clear();
+ LOG.error("BeamFnState failed, clearing outstanding requests {}", outstandingRequestsCopy);
+
+ for (CompletableFuture<StateResponse> entry : outstandingRequestsCopy.values()) {
+ entry.completeExceptionally(cause);
+ }
+ }
+
+ /**
+ * A {@link StreamObserver} which propagates any server side state request responses by
+ * completing the outstanding response future.
+ *
+ * <p>Also propagates server side failures and closes completing any outstanding requests
+ * exceptionally.
+ */
+ private class InboundObserver implements StreamObserver<StateResponse> {
+ @Override
+ public void onNext(StateResponse value) {
+ LOG.debug("Received StateResponse {}", value);
+ CompletableFuture<StateResponse> responseFuture = outstandingRequests.remove(value.getId());
+ if (responseFuture != null) {
+ if (value.getError().isEmpty()) {
+ responseFuture.complete(value);
+ } else {
+ responseFuture.completeExceptionally(new IllegalStateException(value.getError()));
+ }
+ }
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ closeAndCleanUp(t instanceof RuntimeException
+ ? (RuntimeException) t
+ : new RuntimeException(t));
+ }
+
+ @Override
+ public void onCompleted() {
+ closeAndCleanUp(new RuntimeException("Server hanged up."));
+ }
+ }
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java
new file mode 100644
index 0000000..0a43317
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Converts an iterator to an iterable lazily loading values from the underlying iterator
+ * and caching them to support reiteration.
+ */
+class LazyCachingIteratorToIterable<T> implements Iterable<T> {
+ private final List<T> cachedElements;
+ private final Iterator<T> iterator;
+
+ public LazyCachingIteratorToIterable(Iterator<T> iterator) {
+ this.cachedElements = new ArrayList<>();
+ this.iterator = iterator;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return new CachingIterator();
+ }
+
+ /** An {@link Iterator} which adds and fetched values into the cached elements list. */
+ private class CachingIterator implements Iterator<T> {
+ private int position = 0;
+
+ private CachingIterator() {
+ }
+
+ @Override
+ public boolean hasNext() {
+ // The order of the short circuit is important below.
+ return position < cachedElements.size() || iterator.hasNext();
+ }
+
+ @Override
+ public T next() {
+ if (position < cachedElements.size()) {
+ return cachedElements.get(position++);
+ }
+
+ if (!iterator.hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ T rval = iterator.next();
+ cachedElements.add(rval);
+ position += 1;
+ return rval;
+ }
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
new file mode 100644
index 0000000..b64c946
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import com.google.common.base.Throwables;
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Supplier;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+
+/**
+ * Adapters which convert a a logical series of chunks using continuation tokens over the Beam
+ * Fn State API into an {@link Iterator} of {@link ByteString}s.
+ */
+public class StateFetchingIterators {
+
+ // do not instantiate
+ private StateFetchingIterators() {}
+
+ /**
+ * This adapter handles using the continuation token to provide iteration over all the chunks
+ * returned by the Beam Fn State API using the supplied state client and partially filled
+ * out state request containing a state key.
+ *
+ * @param beamFnStateClient A client for handling state requests.
+ * @param partialStateRequestBuilder A {@link StateRequest} with the
+ * {@link StateRequest#getStateKey()} already set.
+ * @return An {@code Iterator<ByteString>} representing all the requested data.
+ */
+ public static Iterator<ByteString> usingPartialRequestWithStateKey(
+ BeamFnStateClient beamFnStateClient,
+ Supplier<StateRequest.Builder> partialStateRequestBuilder) {
+ return new LazyBlockingStateFetchingIterator(beamFnStateClient, partialStateRequestBuilder);
+ }
+
+ /**
+ * An {@link Iterator} which fetches {@link ByteString} chunks using the State API.
+ *
+ * <p>This iterator will only request a chunk on first access. Also it does not eagerly
+ * pre-fetch any future chunks and blocks whenever required to fetch the next block.
+ */
+ static class LazyBlockingStateFetchingIterator implements Iterator<ByteString> {
+ private enum State { READ_REQUIRED, HAS_NEXT, EOF };
+ private final BeamFnStateClient beamFnStateClient;
+ /** Allows for the partially built state request to be memoized across many requests. */
+ private final Supplier<Builder> stateRequestSupplier;
+ private State currentState;
+ private ByteString continuationToken;
+ private ByteString next;
+
+ LazyBlockingStateFetchingIterator(
+ BeamFnStateClient beamFnStateClient,
+ Supplier<StateRequest.Builder> stateRequestSupplier) {
+ this.currentState = State.READ_REQUIRED;
+ this.beamFnStateClient = beamFnStateClient;
+ this.stateRequestSupplier = stateRequestSupplier;
+ this.continuationToken = ByteString.EMPTY;
+ }
+
+ @Override
+ public boolean hasNext() {
+ switch (currentState) {
+ case EOF:
+ return false;
+ case READ_REQUIRED:
+ CompletableFuture<StateResponse> stateResponseFuture = new CompletableFuture<>();
+ beamFnStateClient.handle(
+ stateRequestSupplier.get().setGet(
+ StateGetRequest.newBuilder().setContinuationToken(continuationToken)),
+ stateResponseFuture);
+ StateResponse stateResponse;
+ try {
+ stateResponse = stateResponseFuture.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ } catch (ExecutionException e) {
+ if (e.getCause() == null) {
+ throw new IllegalStateException(e);
+ }
+ Throwables.throwIfUnchecked(e.getCause());
+ throw new IllegalStateException(e.getCause());
+ }
+ continuationToken = stateResponse.getGet().getContinuationToken();
+ next = stateResponse.getGet().getData();
+ currentState = State.HAS_NEXT;
+ return true;
+ case HAS_NEXT:
+ return true;
+ }
+ throw new IllegalStateException(String.format("Unknown state %s", currentState));
+ }
+
+ @Override
+ public ByteString next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ // If the continuation token is empty, that means we have reached EOF.
+ currentState = ByteString.EMPTY.equals(continuationToken) ? State.EOF : State.READ_REQUIRED;
+ return next;
+ }
+ }
+
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/package-info.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/package-info.java
new file mode 100644
index 0000000..feadb7d
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * State client and state caching.
+ */
+package org.apache.beam.fn.harness.state;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/BufferingStreamObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/BufferingStreamObserver.java
index cda3a4b..cd96440 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/BufferingStreamObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/BufferingStreamObserver.java
@@ -105,10 +105,10 @@
// We check to see if we were able to successfully insert the poison pill at the front of
// the queue to cancel the processing thread eagerly or if the processing thread is done.
try {
- // The order of these checks is important because short circuiting will cause us to
- // insert into the queue first and only if it fails do we check that the thread is done.
- while (!queue.offerFirst((T) POISON_PILL, 60, TimeUnit.SECONDS)
- || !queueDrainer.isDone()) {
+ // We shouldn't attempt to insert into the queue if the queue drainer thread is done
+ // since the queue may be full and nothing will be emptying it.
+ while (!queueDrainer.isDone()
+ && !queue.offerFirst((T) POISON_PILL, 60, TimeUnit.SECONDS)) {
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -130,10 +130,10 @@
// the queue forcing the remainder of the elements to be processed or if the processing
// thread is done.
try {
- // The order of these checks is important because short circuiting will cause us to
- // insert into the queue first and only if it fails do we check that the thread is done.
- while (!queue.offer((T) POISON_PILL, 60, TimeUnit.SECONDS)
- || !queueDrainer.isDone()) {
+ // We shouldn't attempt to insert into the queue if the queue drainer thread is done
+ // since the queue may be full and nothing will be emptying it.
+ while (!queueDrainer.isDone()
+ && !queue.offerLast((T) POISON_PILL, 60, TimeUnit.SECONDS)) {
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/DataStreams.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/DataStreams.java
new file mode 100644
index 0000000..3ecd303
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/DataStreams.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.stream;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CountingInputStream;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PushbackInputStream;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.BlockingQueue;
+import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
+import org.apache.beam.sdk.coders.Coder;
+
+/**
+ * {@link #inbound(Iterator)} treats multiple {@link ByteString}s as a single input stream and
+ * {@link #outbound(CloseableThrowingConsumer)} treats a single {@link OutputStream} as multiple
+ * {@link ByteString}s.
+ */
+public class DataStreams {
+ /**
+ * Converts multiple {@link ByteString}s into a single {@link InputStream}.
+ *
+ * <p>The iterator is accessed lazily. The supplied {@link Iterator} should block until
+ * either it knows that no more values will be provided or it has the next {@link ByteString}.
+ */
+ public static InputStream inbound(Iterator<ByteString> bytes) {
+ return new Inbound(bytes);
+ }
+
+ /**
+ * Converts a single {@link OutputStream} into multiple {@link ByteString}s.
+ */
+ public static OutputStream outbound(CloseableThrowingConsumer<ByteString> consumer) {
+ // TODO: Migrate logic from BeamFnDataBufferingOutboundObserver
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * An input stream which concatenates multiple {@link ByteString}s. Lazily accesses the
+ * first {@link Iterator} on first access of this input stream.
+ *
+ * <p>Closing this input stream has no effect.
+ */
+ private static class Inbound<T> extends InputStream {
+ private static final InputStream EMPTY_STREAM = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return -1;
+ }
+ };
+
+ private final Iterator<ByteString> bytes;
+ private InputStream currentStream;
+
+ public Inbound(Iterator<ByteString> bytes) {
+ this.currentStream = EMPTY_STREAM;
+ this.bytes = bytes;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int rval = -1;
+ // Move on to the next stream if we have read nothing
+ while ((rval = currentStream.read()) == -1 && bytes.hasNext()) {
+ currentStream = bytes.next().newInput();
+ }
+ return rval;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int remainingLen = len;
+ while ((remainingLen -= ByteStreams.read(
+ currentStream, b, off + len - remainingLen, remainingLen)) > 0) {
+ if (bytes.hasNext()) {
+ currentStream = bytes.next().newInput();
+ } else {
+ int bytesRead = len - remainingLen;
+ return bytesRead > 0 ? bytesRead : -1;
+ }
+ }
+ return len - remainingLen;
+ }
+ }
+
+ /**
+ * An adapter which converts an {@link InputStream} to an {@link Iterator} of {@code T} values
+ * using the specified {@link Coder}.
+ *
+ * <p>Note that this adapter follows the Beam Fn API specification for forcing values that decode
+ * consuming zero bytes to consuming exactly one byte.
+ *
+ * <p>Note that access to the underlying {@link InputStream} is lazy and will only be invoked on
+ * first access to {@link #next()} or {@link #hasNext()}.
+ */
+ public static class DataStreamDecoder<T> implements Iterator<T> {
+ private enum State { READ_REQUIRED, HAS_NEXT, EOF };
+
+ private final CountingInputStream countingInputStream;
+ private final PushbackInputStream pushbackInputStream;
+ private final Coder<T> coder;
+ private State currentState;
+ private T next;
+ public DataStreamDecoder(Coder<T> coder, InputStream inputStream) {
+ this.currentState = State.READ_REQUIRED;
+ this.coder = coder;
+ this.pushbackInputStream = new PushbackInputStream(inputStream, 1);
+ this.countingInputStream = new CountingInputStream(pushbackInputStream);
+ }
+
+ @Override
+ public boolean hasNext() {
+ switch (currentState) {
+ case EOF:
+ return false;
+ case READ_REQUIRED:
+ try {
+ int nextByte = pushbackInputStream.read();
+ if (nextByte == -1) {
+ currentState = State.EOF;
+ return false;
+ }
+
+ pushbackInputStream.unread(nextByte);
+ long count = countingInputStream.getCount();
+ next = coder.decode(countingInputStream);
+ // Skip one byte if decoding the value consumed 0 bytes.
+ if (countingInputStream.getCount() - count == 0) {
+ checkState(countingInputStream.read() != -1, "Unexpected EOF reached");
+ }
+ currentState = State.HAS_NEXT;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ return true;
+ case HAS_NEXT:
+ return true;
+ }
+ throw new IllegalStateException(String.format("Unknown state %s", currentState));
+ }
+
+ @Override
+ public T next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ currentState = State.READ_REQUIRED;
+ return next;
+ }
+ }
+
+ /**
+ * Allows for one or more writing threads to append values to this iterator while one reading
+ * thread reads values. {@link #hasNext()} and {@link #next()} will block until a value is
+ * available or this has been closed.
+ *
+ * <p>External synchronization must be provided if multiple readers would like to access the
+ * {@link Iterator#hasNext()} and {@link Iterator#next()} methods.
+ *
+ * <p>The order or values which are appended to this iterator is nondeterministic when multiple
+ * threads call {@link #accept(Object)}.
+ */
+ public static class BlockingQueueIterator<T> implements
+ CloseableThrowingConsumer<T>, Iterator<T> {
+ private static final Object POISION_PILL = new Object();
+ private final BlockingQueue<T> queue;
+
+ /** Only accessed by {@link Iterator#hasNext()} and {@link Iterator#next()} methods. */
+ private T currentElement;
+
+ public BlockingQueueIterator(BlockingQueue<T> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public void close() throws Exception {
+ queue.put((T) POISION_PILL);
+ }
+
+ @Override
+ public void accept(T t) throws Exception {
+ queue.put(t);
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (currentElement == null) {
+ try {
+ currentElement = queue.take();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ return currentElement != POISION_PILL;
+ }
+
+ @Override
+ public T next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ T rval = currentElement;
+ currentElement = null;
+ return rval;
+ }
+ }
+}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java
index 063d5af..99e33c2 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/StreamObserverFactory.java
@@ -23,8 +23,8 @@
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.function.Function;
-import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
+import org.apache.beam.sdk.options.ExperimentalOptions;
import org.apache.beam.sdk.options.PipelineOptions;
/**
@@ -33,7 +33,7 @@
*/
public abstract class StreamObserverFactory {
public static StreamObserverFactory fromOptions(PipelineOptions options) {
- List<String> experiments = options.as(DataflowPipelineDebugOptions.class).getExperiments();
+ List<String> experiments = options.as(ExperimentalOptions.class).getExperiments();
if (experiments != null && experiments.contains("beam_fn_api_buffered_stream")) {
int bufferSize = Buffered.DEFAULT_BUFFER_SIZE;
for (String experiment : experiments) {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
deleted file mode 100644
index e6928d1..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.BytesValue;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.function.Supplier;
-import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.util.CloudObject;
-import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Registers as a consumer for data over the Beam Fn API. Multiplexes any received data
- * to all consumers in the specified output map.
- *
- * <p>Can be re-used serially across {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s.
- * For each request, call {@link #registerInputLocation()} to start and call
- * {@link #blockTillReadFinishes()} to finish.
- */
-public class BeamFnDataReadRunner<OutputT> {
- private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataReadRunner.class);
-
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
- private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
- private final Supplier<String> processBundleInstructionIdSupplier;
- private final BeamFnDataClient beamFnDataClientFactory;
- private final Coder<WindowedValue<OutputT>> coder;
- private final BeamFnApi.Target inputTarget;
-
- private CompletableFuture<Void> readFuture;
-
- public BeamFnDataReadRunner(
- BeamFnApi.FunctionSpec functionSpec,
- Supplier<String> processBundleInstructionIdSupplier,
- BeamFnApi.Target inputTarget,
- BeamFnApi.Coder coderSpec,
- BeamFnDataClient beamFnDataClientFactory,
- Map<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> outputMap)
- throws IOException {
- this.apiServiceDescriptor = functionSpec.getData().unpack(BeamFnApi.RemoteGrpcPort.class)
- .getApiServiceDescriptor();
- this.inputTarget = inputTarget;
- this.processBundleInstructionIdSupplier = processBundleInstructionIdSupplier;
- this.beamFnDataClientFactory = beamFnDataClientFactory;
- this.consumers = ImmutableList.copyOf(FluentIterable.concat(outputMap.values()));
-
- @SuppressWarnings("unchecked")
- Coder<WindowedValue<OutputT>> coder =
- (Coder<WindowedValue<OutputT>>)
- CloudObjects.coderFromCloudObject(
- CloudObject.fromSpec(
- OBJECT_MAPPER.readValue(
- coderSpec
- .getFunctionSpec()
- .getData()
- .unpack(BytesValue.class)
- .getValue()
- .newInput(),
- Map.class)));
- this.coder = coder;
- }
-
- public void registerInputLocation() {
- this.readFuture = beamFnDataClientFactory.forInboundConsumer(
- apiServiceDescriptor,
- KV.of(processBundleInstructionIdSupplier.get(), inputTarget),
- coder,
- this::multiplexToConsumers);
- }
-
- public void blockTillReadFinishes() throws Exception {
- LOG.debug("Waiting for process bundle instruction {} and target {} to close.",
- processBundleInstructionIdSupplier.get(), inputTarget);
- readFuture.get();
- }
-
- private void multiplexToConsumers(WindowedValue<OutputT> value) throws Exception {
- for (ThrowingConsumer<WindowedValue<OutputT>> consumer : consumers) {
- consumer.accept(value);
- }
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
deleted file mode 100644
index a78da5d..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.protobuf.BytesValue;
-import java.io.IOException;
-import java.util.Map;
-import java.util.function.Supplier;
-import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.util.CloudObject;
-import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-
-/**
- * Registers as a consumer with the Beam Fn Data API. Propagates and elements consumed to
- * the the registered consumer.
- *
- * <p>Can be re-used serially across {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s.
- * For each request, call {@link #registerForOutput()} to start and call {@link #close()} to finish.
- */
-public class BeamFnDataWriteRunner<InputT> {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
- private final BeamFnApi.Target outputTarget;
- private final Coder<WindowedValue<InputT>> coder;
- private final BeamFnDataClient beamFnDataClientFactory;
- private final Supplier<String> processBundleInstructionIdSupplier;
-
- private CloseableThrowingConsumer<WindowedValue<InputT>> consumer;
-
- public BeamFnDataWriteRunner(
- BeamFnApi.FunctionSpec functionSpec,
- Supplier<String> processBundleInstructionIdSupplier,
- BeamFnApi.Target outputTarget,
- BeamFnApi.Coder coderSpec,
- BeamFnDataClient beamFnDataClientFactory)
- throws IOException {
- this.apiServiceDescriptor = functionSpec.getData().unpack(BeamFnApi.RemoteGrpcPort.class)
- .getApiServiceDescriptor();
- this.beamFnDataClientFactory = beamFnDataClientFactory;
- this.processBundleInstructionIdSupplier = processBundleInstructionIdSupplier;
- this.outputTarget = outputTarget;
-
- @SuppressWarnings("unchecked")
- Coder<WindowedValue<InputT>> coder =
- (Coder<WindowedValue<InputT>>)
- CloudObjects.coderFromCloudObject(
- CloudObject.fromSpec(
- OBJECT_MAPPER.readValue(
- coderSpec
- .getFunctionSpec()
- .getData()
- .unpack(BytesValue.class)
- .getValue()
- .newInput(),
- Map.class)));
- this.coder = coder;
- }
-
- public void registerForOutput() {
- consumer = beamFnDataClientFactory.forOutboundConsumer(
- apiServiceDescriptor,
- KV.of(processBundleInstructionIdSupplier.get(), outputTarget),
- coder);
- }
-
- public void close() throws Exception {
- consumer.close();
- }
-
- public void consume(WindowedValue<InputT> value) throws Exception {
- consumer.accept(value);
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BoundedSourceRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BoundedSourceRunner.java
deleted file mode 100644
index 9d9c433..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BoundedSourceRunner.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.BytesValue;
-import com.google.protobuf.InvalidProtocolBufferException;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.Source.Reader;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowedValue;
-
-/**
- * A runner which creates {@link Reader}s for each {@link BoundedSource} and executes
- * the {@link Reader}s read loop.
- */
-public class BoundedSourceRunner<InputT extends BoundedSource<OutputT>, OutputT> {
- private final PipelineOptions pipelineOptions;
- private final BeamFnApi.FunctionSpec definition;
- private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
-
- public BoundedSourceRunner(
- PipelineOptions pipelineOptions,
- BeamFnApi.FunctionSpec definition,
- Map<String, Collection<ThrowingConsumer<WindowedValue<OutputT>>>> outputMap) {
- this.pipelineOptions = pipelineOptions;
- this.definition = definition;
- this.consumers = ImmutableList.copyOf(FluentIterable.concat(outputMap.values()));
- }
-
- /**
- * The runner harness is meant to send the source over the Beam Fn Data API which would be
- * consumed by the {@link #runReadLoop}. Drop this method once the runner harness sends the
- * source instead of unpacking it from the data block of the function specification.
- */
- @Deprecated
- public void start() throws Exception {
- try {
- // The representation here is defined as the java serialized representation of the
- // bounded source object packed into a protobuf Any using a protobuf BytesValue wrapper.
- byte[] bytes = definition.getData().unpack(BytesValue.class).getValue().toByteArray();
- @SuppressWarnings("unchecked")
- InputT boundedSource =
- (InputT) SerializableUtils.deserializeFromByteArray(bytes, definition.toString());
- runReadLoop(WindowedValue.valueInGlobalWindow(boundedSource));
- } catch (InvalidProtocolBufferException e) {
- throw new IOException(
- String.format("Failed to decode %s, expected %s",
- definition.getData().getTypeUrl(), BytesValue.getDescriptor().getFullName()),
- e);
- }
- }
-
- /**
- * Creates a {@link Reader} for each {@link BoundedSource} and executes the {@link Reader}s
- * read loop. See {@link Reader} for further details of the read loop.
- *
- * <p>Propagates any exceptions caused during reading or processing via a consumer to the
- * caller.
- */
- public void runReadLoop(WindowedValue<InputT> value) throws Exception {
- try (Reader<OutputT> reader = value.getValue().createReader(pipelineOptions)) {
- if (!reader.start()) {
- // Reader has no data, immediately return
- return;
- }
- do {
- // TODO: Should this use the input window as the window for all the outputs?
- WindowedValue<OutputT> nextValue = WindowedValue.timestampedValueInGlobalWindow(
- reader.getCurrent(), reader.getCurrentTimestamp());
- for (ThrowingConsumer<WindowedValue<OutputT>> consumer : consumers) {
- consumer.accept(nextValue);
- }
- } while (reader.advance());
- }
- }
-
- @Override
- public String toString() {
- return definition.toString();
- }
-}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/package-info.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/package-info.java
deleted file mode 100644
index d250a6a..0000000
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Provides utilities for Beam runner authors.
- */
-package org.apache.beam.runners.core;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
new file mode 100644
index 0000000..f00346d
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.Uninterruptibles;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.harness.test.TestExecutors;
+import org.apache.beam.harness.test.TestExecutors.TestExecutorService;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
+import org.apache.beam.runners.core.construction.CoderTranslation;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.hamcrest.collection.IsMapContaining;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Tests for {@link BeamFnDataReadRunner}. */
+@RunWith(JUnit4.class)
+public class BeamFnDataReadRunnerTest {
+
+ private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
+ .setApiServiceDescriptor(Endpoints.ApiServiceDescriptor.getDefaultInstance()).build();
+ private static final RunnerApi.FunctionSpec FUNCTION_SPEC = RunnerApi.FunctionSpec.newBuilder()
+ .setPayload(PORT_SPEC.toByteString()).build();
+ private static final Coder<WindowedValue<String>> CODER =
+ WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
+ private static final String CODER_SPEC_ID = "string-coder-id";
+ private static final RunnerApi.Coder CODER_SPEC;
+ private static final RunnerApi.Components COMPONENTS;
+ private static final String URN = "urn:org.apache.beam:source:runner:0.1";
+
+ static {
+ try {
+ MessageWithComponents coderAndComponents = CoderTranslation.toProto(CODER);
+ CODER_SPEC = coderAndComponents.getCoder();
+ COMPONENTS =
+ coderAndComponents
+ .getComponents()
+ .toBuilder()
+ .putCoders(CODER_SPEC_ID, CODER_SPEC)
+ .build();
+ } catch (IOException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+ private static final BeamFnApi.Target INPUT_TARGET = BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference("1")
+ .setName("out")
+ .build();
+
+ @Rule public TestExecutorService executor = TestExecutors.from(Executors::newCachedThreadPool);
+ @Mock private BeamFnDataClient mockBeamFnDataClient;
+ @Captor private ArgumentCaptor<ThrowingConsumer<WindowedValue<String>>> consumerCaptor;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @Test
+ public void testCreatingAndProcessingBeamFnDataReadRunner() throws Exception {
+ String bundleId = "57";
+ String outputId = "101";
+
+ List<WindowedValue<String>> outputValues = new ArrayList<>();
+
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> consumers = HashMultimap.create();
+ consumers.put("outputPC",
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<String>>) outputValues::add);
+ List<ThrowingRunnable> startFunctions = new ArrayList<>();
+ List<ThrowingRunnable> finishFunctions = new ArrayList<>();
+
+ RunnerApi.FunctionSpec functionSpec = RunnerApi.FunctionSpec.newBuilder()
+ .setUrn("urn:org.apache.beam:source:runner:0.1")
+ .setPayload(PORT_SPEC.toByteString())
+ .build();
+
+ RunnerApi.PTransform pTransform = RunnerApi.PTransform.newBuilder()
+ .setSpec(functionSpec)
+ .putOutputs(outputId, "outputPC")
+ .build();
+
+ new BeamFnDataReadRunner.Factory<String>().createRunnerForPTransform(
+ PipelineOptionsFactory.create(),
+ mockBeamFnDataClient,
+ null /* beamFnStateClient */,
+ "pTransformId",
+ pTransform,
+ Suppliers.ofInstance(bundleId)::get,
+ ImmutableMap.of("outputPC",
+ RunnerApi.PCollection.newBuilder().setCoderId(CODER_SPEC_ID).build()),
+ COMPONENTS.getCodersMap(),
+ consumers,
+ startFunctions::add,
+ finishFunctions::add);
+
+ verifyZeroInteractions(mockBeamFnDataClient);
+
+ CompletableFuture<Void> completionFuture = new CompletableFuture<>();
+ when(mockBeamFnDataClient.forInboundConsumer(any(), any(), any(), any()))
+ .thenReturn(completionFuture);
+ Iterables.getOnlyElement(startFunctions).run();
+ verify(mockBeamFnDataClient).forInboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId, BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference("pTransformId")
+ .setName(outputId)
+ .build())),
+ eq(CODER),
+ consumerCaptor.capture());
+
+ consumerCaptor.getValue().accept(valueInGlobalWindow("TestValue"));
+ assertThat(outputValues, contains(valueInGlobalWindow("TestValue")));
+ outputValues.clear();
+
+ assertThat(consumers.keySet(), containsInAnyOrder("outputPC"));
+
+ completionFuture.complete(null);
+ Iterables.getOnlyElement(finishFunctions).run();
+
+ verifyNoMoreInteractions(mockBeamFnDataClient);
+ }
+
+ @Test
+ public void testReuseForMultipleBundles() throws Exception {
+ CompletableFuture<Void> bundle1Future = new CompletableFuture<>();
+ CompletableFuture<Void> bundle2Future = new CompletableFuture<>();
+ when(mockBeamFnDataClient.forInboundConsumer(
+ any(),
+ any(),
+ any(),
+ any())).thenReturn(bundle1Future).thenReturn(bundle2Future);
+ List<WindowedValue<String>> valuesA = new ArrayList<>();
+ List<WindowedValue<String>> valuesB = new ArrayList<>();
+
+ AtomicReference<String> bundleId = new AtomicReference<>("0");
+ BeamFnDataReadRunner<String> readRunner = new BeamFnDataReadRunner<>(
+ FUNCTION_SPEC,
+ bundleId::get,
+ INPUT_TARGET,
+ CODER_SPEC,
+ COMPONENTS.getCodersMap(),
+ mockBeamFnDataClient,
+ ImmutableList.of(valuesA::add, valuesB::add));
+
+ // Process for bundle id 0
+ readRunner.registerInputLocation();
+
+ verify(mockBeamFnDataClient).forInboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId.get(), INPUT_TARGET)),
+ eq(CODER),
+ consumerCaptor.capture());
+
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ // Sleep for some small amount of time simulating the parent blocking
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ try {
+ consumerCaptor.getValue().accept(valueInGlobalWindow("ABC"));
+ consumerCaptor.getValue().accept(valueInGlobalWindow("DEF"));
+ } catch (Exception e) {
+ bundle1Future.completeExceptionally(e);
+ } finally {
+ bundle1Future.complete(null);
+ }
+ }
+ });
+
+ readRunner.blockTillReadFinishes();
+ assertThat(valuesA, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
+ assertThat(valuesB, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
+
+ // Process for bundle id 1
+ bundleId.set("1");
+ valuesA.clear();
+ valuesB.clear();
+ readRunner.registerInputLocation();
+
+ verify(mockBeamFnDataClient).forInboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId.get(), INPUT_TARGET)),
+ eq(CODER),
+ consumerCaptor.capture());
+
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ // Sleep for some small amount of time simulating the parent blocking
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ try {
+ consumerCaptor.getValue().accept(valueInGlobalWindow("GHI"));
+ consumerCaptor.getValue().accept(valueInGlobalWindow("JKL"));
+ } catch (Exception e) {
+ bundle2Future.completeExceptionally(e);
+ } finally {
+ bundle2Future.complete(null);
+ }
+ }
+ });
+
+ readRunner.blockTillReadFinishes();
+ assertThat(valuesA, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
+ assertThat(valuesB, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
+
+ verifyNoMoreInteractions(mockBeamFnDataClient);
+ }
+
+ @Test
+ public void testRegistration() {
+ for (Registrar registrar :
+ ServiceLoader.load(Registrar.class)) {
+ if (registrar instanceof BeamFnDataReadRunner.Registrar) {
+ assertThat(registrar.getPTransformRunnerFactories(), IsMapContaining.hasKey(URN));
+ return;
+ }
+ }
+ fail("Expected registrar not found.");
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
new file mode 100644
index 0000000..486f114
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
+import org.apache.beam.fn.harness.data.BeamFnDataClient;
+import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
+import org.apache.beam.runners.core.construction.CoderTranslation;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.hamcrest.collection.IsMapContaining;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Matchers;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Tests for {@link BeamFnDataWriteRunner}. */
+@RunWith(JUnit4.class)
+public class BeamFnDataWriteRunnerTest {
+
+ private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
+ .setApiServiceDescriptor(Endpoints.ApiServiceDescriptor.getDefaultInstance()).build();
+ private static final RunnerApi.FunctionSpec FUNCTION_SPEC = RunnerApi.FunctionSpec.newBuilder()
+ .setPayload(PORT_SPEC.toByteString()).build();
+ private static final String CODER_ID = "string-coder-id";
+ private static final Coder<WindowedValue<String>> CODER =
+ WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
+ private static final RunnerApi.Coder CODER_SPEC;
+ private static final RunnerApi.Components COMPONENTS;
+ private static final String URN = "urn:org.apache.beam:sink:runner:0.1";
+
+ static {
+ try {
+ MessageWithComponents coderAndComponents = CoderTranslation.toProto(CODER);
+ CODER_SPEC = coderAndComponents.getCoder();
+ COMPONENTS =
+ coderAndComponents.getComponents().toBuilder().putCoders(CODER_ID, CODER_SPEC).build();
+ } catch (IOException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+ private static final BeamFnApi.Target OUTPUT_TARGET = BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference("1")
+ .setName("out")
+ .build();
+
+ @Mock private BeamFnDataClient mockBeamFnDataClient;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+
+ @Test
+ public void testCreatingAndProcessingBeamFnDataWriteRunner() throws Exception {
+ String bundleId = "57L";
+ String inputId = "100L";
+
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> consumers = HashMultimap.create();
+ List<ThrowingRunnable> startFunctions = new ArrayList<>();
+ List<ThrowingRunnable> finishFunctions = new ArrayList<>();
+
+ RunnerApi.FunctionSpec functionSpec = RunnerApi.FunctionSpec.newBuilder()
+ .setUrn("urn:org.apache.beam:sink:runner:0.1")
+ .setPayload(PORT_SPEC.toByteString())
+ .build();
+
+ RunnerApi.PTransform pTransform = RunnerApi.PTransform.newBuilder()
+ .setSpec(functionSpec)
+ .putInputs(inputId, "inputPC")
+ .build();
+
+ new BeamFnDataWriteRunner.Factory<String>().createRunnerForPTransform(
+ PipelineOptionsFactory.create(),
+ mockBeamFnDataClient,
+ null /* beamFnStateClient */,
+ "ptransformId",
+ pTransform,
+ Suppliers.ofInstance(bundleId)::get,
+ ImmutableMap.of("inputPC",
+ RunnerApi.PCollection.newBuilder().setCoderId(CODER_ID).build()),
+ COMPONENTS.getCodersMap(),
+ consumers,
+ startFunctions::add,
+ finishFunctions::add);
+
+ verifyZeroInteractions(mockBeamFnDataClient);
+
+ List<WindowedValue<String>> outputValues = new ArrayList<>();
+ AtomicBoolean wasCloseCalled = new AtomicBoolean();
+ CloseableThrowingConsumer<WindowedValue<String>> outputConsumer =
+ new CloseableThrowingConsumer<WindowedValue<String>>(){
+ @Override
+ public void close() throws Exception {
+ wasCloseCalled.set(true);
+ }
+
+ @Override
+ public void accept(WindowedValue<String> t) throws Exception {
+ outputValues.add(t);
+ }
+ };
+
+ when(mockBeamFnDataClient.forOutboundConsumer(
+ any(),
+ any(),
+ Matchers.<Coder<WindowedValue<String>>>any())).thenReturn(outputConsumer);
+ Iterables.getOnlyElement(startFunctions).run();
+ verify(mockBeamFnDataClient).forOutboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId, BeamFnApi.Target.newBuilder()
+ .setPrimitiveTransformReference("ptransformId")
+ .setName(inputId)
+ .build())),
+ eq(CODER));
+
+ assertThat(consumers.keySet(), containsInAnyOrder("inputPC"));
+ Iterables.getOnlyElement(consumers.get("inputPC")).accept(valueInGlobalWindow("TestValue"));
+ assertThat(outputValues, contains(valueInGlobalWindow("TestValue")));
+ outputValues.clear();
+
+ assertFalse(wasCloseCalled.get());
+ Iterables.getOnlyElement(finishFunctions).run();
+ assertTrue(wasCloseCalled.get());
+
+ verifyNoMoreInteractions(mockBeamFnDataClient);
+ }
+
+ @Test
+ public void testReuseForMultipleBundles() throws Exception {
+ RecordingConsumer<WindowedValue<String>> valuesA = new RecordingConsumer<>();
+ RecordingConsumer<WindowedValue<String>> valuesB = new RecordingConsumer<>();
+ when(mockBeamFnDataClient.forOutboundConsumer(
+ any(),
+ any(),
+ Matchers.<Coder<WindowedValue<String>>>any())).thenReturn(valuesA).thenReturn(valuesB);
+ AtomicReference<String> bundleId = new AtomicReference<>("0");
+ BeamFnDataWriteRunner<String> writeRunner = new BeamFnDataWriteRunner<>(
+ FUNCTION_SPEC,
+ bundleId::get,
+ OUTPUT_TARGET,
+ CODER_SPEC,
+ COMPONENTS.getCodersMap(),
+ mockBeamFnDataClient);
+
+ // Process for bundle id 0
+ writeRunner.registerForOutput();
+
+ verify(mockBeamFnDataClient).forOutboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId.get(), OUTPUT_TARGET)),
+ eq(CODER));
+
+ writeRunner.consume(valueInGlobalWindow("ABC"));
+ writeRunner.consume(valueInGlobalWindow("DEF"));
+ writeRunner.close();
+
+ assertTrue(valuesA.closed);
+ assertThat(valuesA, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
+
+ // Process for bundle id 1
+ bundleId.set("1");
+ valuesA.clear();
+ valuesB.clear();
+ writeRunner.registerForOutput();
+
+ verify(mockBeamFnDataClient).forOutboundConsumer(
+ eq(PORT_SPEC.getApiServiceDescriptor()),
+ eq(KV.of(bundleId.get(), OUTPUT_TARGET)),
+ eq(CODER));
+
+ writeRunner.consume(valueInGlobalWindow("GHI"));
+ writeRunner.consume(valueInGlobalWindow("JKL"));
+ writeRunner.close();
+
+ assertTrue(valuesB.closed);
+ assertThat(valuesB, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
+ verifyNoMoreInteractions(mockBeamFnDataClient);
+ }
+
+ private static class RecordingConsumer<T> extends ArrayList<T>
+ implements CloseableThrowingConsumer<T> {
+ private boolean closed;
+ @Override
+ public void close() throws Exception {
+ closed = true;
+ }
+
+ @Override
+ public void accept(T t) throws Exception {
+ if (closed) {
+ throw new IllegalStateException("Consumer is closed but attempting to consume " + t);
+ }
+ add(t);
+ }
+ }
+
+ @Test
+ public void testRegistration() {
+ for (Registrar registrar :
+ ServiceLoader.load(Registrar.class)) {
+ if (registrar instanceof BeamFnDataWriteRunner.Registrar) {
+ assertThat(registrar.getPTransformRunnerFactories(), IsMapContaining.hasKey(URN));
+ return;
+ }
+ }
+ fail("Expected registrar not found.");
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
new file mode 100644
index 0000000..50009c0
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.protobuf.ByteString;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.ServiceLoader;
+import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.hamcrest.Matchers;
+import org.hamcrest.collection.IsMapContaining;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link BoundedSourceRunner}. */
+@RunWith(JUnit4.class)
+public class BoundedSourceRunnerTest {
+
+ public static final String URN = "urn:org.apache.beam:source:java:0.1";
+
+ @Test
+ public void testRunReadLoopWithMultipleSources() throws Exception {
+ List<WindowedValue<Long>> out1Values = new ArrayList<>();
+ List<WindowedValue<Long>> out2Values = new ArrayList<>();
+ Collection<ThrowingConsumer<WindowedValue<Long>>> consumers =
+ ImmutableList.of(out1Values::add, out2Values::add);
+
+ BoundedSourceRunner<BoundedSource<Long>, Long> runner = new BoundedSourceRunner<>(
+ PipelineOptionsFactory.create(),
+ RunnerApi.FunctionSpec.getDefaultInstance(),
+ consumers);
+
+ runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(2)));
+ runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(1)));
+
+ assertThat(out1Values,
+ contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(0L)));
+ assertThat(out2Values,
+ contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(0L)));
+ }
+
+ @Test
+ public void testRunReadLoopWithEmptySource() throws Exception {
+ List<WindowedValue<Long>> outValues = new ArrayList<>();
+ Collection<ThrowingConsumer<WindowedValue<Long>>> consumers =
+ ImmutableList.of(outValues::add);
+
+ BoundedSourceRunner<BoundedSource<Long>, Long> runner = new BoundedSourceRunner<>(
+ PipelineOptionsFactory.create(),
+ RunnerApi.FunctionSpec.getDefaultInstance(),
+ consumers);
+
+ runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(0)));
+
+ assertThat(outValues, empty());
+ }
+
+ @Test
+ public void testStart() throws Exception {
+ List<WindowedValue<Long>> outValues = new ArrayList<>();
+ Collection<ThrowingConsumer<WindowedValue<Long>>> consumers =
+ ImmutableList.of(outValues::add);
+
+ ByteString encodedSource =
+ ByteString.copyFrom(SerializableUtils.serializeToByteArray(CountingSource.upTo(3)));
+
+ BoundedSourceRunner<BoundedSource<Long>, Long> runner = new BoundedSourceRunner<>(
+ PipelineOptionsFactory.create(),
+ RunnerApi.FunctionSpec.newBuilder().setPayload(encodedSource).build(),
+ consumers);
+
+ runner.start();
+
+ assertThat(outValues,
+ contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(2L)));
+ }
+
+ @Test
+ public void testCreatingAndProcessingSourceFromFactory() throws Exception {
+ List<WindowedValue<String>> outputValues = new ArrayList<>();
+
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> consumers = HashMultimap.create();
+ consumers.put("outputPC",
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<String>>) outputValues::add);
+ List<ThrowingRunnable> startFunctions = new ArrayList<>();
+ List<ThrowingRunnable> finishFunctions = new ArrayList<>();
+
+ RunnerApi.FunctionSpec functionSpec =
+ RunnerApi.FunctionSpec.newBuilder()
+ .setUrn("urn:org.apache.beam:source:java:0.1")
+ .setPayload(
+ ByteString.copyFrom(SerializableUtils.serializeToByteArray(CountingSource.upTo(3))))
+ .build();
+
+ RunnerApi.PTransform pTransform = RunnerApi.PTransform.newBuilder()
+ .setSpec(functionSpec)
+ .putInputs("input", "inputPC")
+ .putOutputs("output", "outputPC")
+ .build();
+
+ new BoundedSourceRunner.Factory<>().createRunnerForPTransform(
+ PipelineOptionsFactory.create(),
+ null /* beamFnDataClient */,
+ null /* beamFnStateClient */,
+ "pTransformId",
+ pTransform,
+ Suppliers.ofInstance("57L")::get,
+ ImmutableMap.of(),
+ ImmutableMap.of(),
+ consumers,
+ startFunctions::add,
+ finishFunctions::add);
+
+ // This is testing a deprecated way of running sources and should be removed
+ // once all source definitions are instead propagated along the input edge.
+ Iterables.getOnlyElement(startFunctions).run();
+ assertThat(outputValues, contains(
+ valueInGlobalWindow(0L),
+ valueInGlobalWindow(1L),
+ valueInGlobalWindow(2L)));
+ outputValues.clear();
+
+ // Check that when passing a source along as an input, the source is processed.
+ assertThat(consumers.keySet(), containsInAnyOrder("inputPC", "outputPC"));
+ Iterables.getOnlyElement(consumers.get("inputPC")).accept(
+ valueInGlobalWindow(CountingSource.upTo(2)));
+ assertThat(outputValues, contains(
+ valueInGlobalWindow(0L),
+ valueInGlobalWindow(1L)));
+
+ assertThat(finishFunctions, Matchers.empty());
+ }
+
+ @Test
+ public void testRegistration() {
+ for (Registrar registrar :
+ ServiceLoader.load(Registrar.class)) {
+ if (registrar instanceof BoundedSourceRunner.Registrar) {
+ assertThat(registrar.getPTransformRunnerFactories(), IsMapContaining.hasKey(URN));
+ return;
+ }
+ }
+ fail("Expected registrar not found.");
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
new file mode 100644
index 0000000..e4422a3
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.fn.harness;
+
+import static org.apache.beam.sdk.util.WindowedValue.timestampedValueInGlobalWindow;
+import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
+import org.apache.beam.fn.harness.fn.ThrowingConsumer;
+import org.apache.beam.fn.harness.fn.ThrowingRunnable;
+import org.apache.beam.fn.harness.state.FakeBeamFnStateClient;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.ParDoTranslation;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
+import org.apache.beam.sdk.transforms.CombineWithContext.Context;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.DoFnInfo;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.collection.IsMapContaining;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link FnApiDoFnRunner}. */
+@RunWith(JUnit4.class)
+public class FnApiDoFnRunnerTest {
+
+ public static final String TEST_PTRANSFORM_ID = "pTransformId";
+
+ private static class TestDoFn extends DoFn<String, String> {
+ private static final TupleTag<String> mainOutput = new TupleTag<>("mainOutput");
+ private static final TupleTag<String> additionalOutput = new TupleTag<>("output");
+
+ private BoundedWindow window;
+
+ @ProcessElement
+ public void processElement(ProcessContext context, BoundedWindow window) {
+ context.output("MainOutput" + context.element());
+ context.output(additionalOutput, "AdditionalOutput" + context.element());
+ this.window = window;
+ }
+
+ @FinishBundle
+ public void finishBundle(FinishBundleContext context) {
+ if (window != null) {
+ context.output("FinishBundle", window.maxTimestamp(), window);
+ window = null;
+ }
+ }
+ }
+
+ /**
+ * Create a DoFn that has 3 inputs (inputATarget1, inputATarget2, inputBTarget) and 2 outputs
+ * (mainOutput, output). Validate that inputs are fed to the {@link DoFn} and that outputs
+ * are directed to the correct consumers.
+ */
+ @Test
+ public void testCreatingAndProcessingDoFn() throws Exception {
+ String pTransformId = "pTransformId";
+ String mainOutputId = "101";
+ String additionalOutputId = "102";
+
+ DoFnInfo<?, ?> doFnInfo = DoFnInfo.forFn(
+ new TestDoFn(),
+ WindowingStrategy.globalDefault(),
+ ImmutableList.of(),
+ StringUtf8Coder.of(),
+ Long.parseLong(mainOutputId),
+ ImmutableMap.of(
+ Long.parseLong(mainOutputId), TestDoFn.mainOutput,
+ Long.parseLong(additionalOutputId), TestDoFn.additionalOutput));
+ RunnerApi.FunctionSpec functionSpec =
+ RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnInfo)))
+ .build();
+ RunnerApi.PTransform pTransform = RunnerApi.PTransform.newBuilder()
+ .setSpec(functionSpec)
+ .putInputs("inputA", "inputATarget")
+ .putInputs("inputB", "inputBTarget")
+ .putOutputs(mainOutputId, "mainOutputTarget")
+ .putOutputs(additionalOutputId, "additionalOutputTarget")
+ .build();
+
+ List<WindowedValue<String>> mainOutputValues = new ArrayList<>();
+ List<WindowedValue<String>> additionalOutputValues = new ArrayList<>();
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> consumers = HashMultimap.create();
+ consumers.put("mainOutputTarget",
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<String>>) mainOutputValues::add);
+ consumers.put("additionalOutputTarget",
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<String>>) additionalOutputValues::add);
+ List<ThrowingRunnable> startFunctions = new ArrayList<>();
+ List<ThrowingRunnable> finishFunctions = new ArrayList<>();
+
+ new FnApiDoFnRunner.Factory<>().createRunnerForPTransform(
+ PipelineOptionsFactory.create(),
+ null /* beamFnDataClient */,
+ null /* beamFnStateClient */,
+ pTransformId,
+ pTransform,
+ Suppliers.ofInstance("57L")::get,
+ ImmutableMap.of(),
+ ImmutableMap.of(),
+ consumers,
+ startFunctions::add,
+ finishFunctions::add);
+
+ Iterables.getOnlyElement(startFunctions).run();
+ mainOutputValues.clear();
+
+ assertThat(consumers.keySet(), containsInAnyOrder(
+ "inputATarget", "inputBTarget", "mainOutputTarget", "additionalOutputTarget"));
+
+ Iterables.getOnlyElement(consumers.get("inputATarget")).accept(valueInGlobalWindow("A1"));
+ Iterables.getOnlyElement(consumers.get("inputATarget")).accept(valueInGlobalWindow("A2"));
+ Iterables.getOnlyElement(consumers.get("inputATarget")).accept(valueInGlobalWindow("B"));
+ assertThat(mainOutputValues, contains(
+ valueInGlobalWindow("MainOutputA1"),
+ valueInGlobalWindow("MainOutputA2"),
+ valueInGlobalWindow("MainOutputB")));
+ assertThat(additionalOutputValues, contains(
+ valueInGlobalWindow("AdditionalOutputA1"),
+ valueInGlobalWindow("AdditionalOutputA2"),
+ valueInGlobalWindow("AdditionalOutputB")));
+ mainOutputValues.clear();
+ additionalOutputValues.clear();
+
+ Iterables.getOnlyElement(finishFunctions).run();
+ assertThat(
+ mainOutputValues,
+ contains(
+ timestampedValueInGlobalWindow("FinishBundle", GlobalWindow.INSTANCE.maxTimestamp())));
+ mainOutputValues.clear();
+ }
+
+ private static class ConcatCombineFn extends CombineFn<String, String, String> {
+ @Override
+ public String createAccumulator() {
+ return "";
+ }
+
+ @Override
+ public String addInput(String accumulator, String input) {
+ return accumulator.concat(input);
+ }
+
+ @Override
+ public String mergeAccumulators(Iterable<String> accumulators) {
+ StringBuilder builder = new StringBuilder();
+ for (String value : accumulators) {
+ builder.append(value);
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String extractOutput(String accumulator) {
+ return accumulator;
+ }
+ }
+
+ private static class ConcatCombineFnWithContext
+ extends CombineFnWithContext<String, String, String> {
+ @Override
+ public String createAccumulator(Context c) {
+ return "";
+ }
+
+ @Override
+ public String addInput(String accumulator, String input, Context c) {
+ return accumulator.concat(input);
+ }
+
+ @Override
+ public String mergeAccumulators(Iterable<String> accumulators, Context c) {
+ StringBuilder builder = new StringBuilder();
+ for (String value : accumulators) {
+ builder.append(value);
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String extractOutput(String accumulator, Context c) {
+ return accumulator;
+ }
+ }
+
+ private static class TestStatefulDoFn extends DoFn<KV<String, String>, String> {
+ private static final TupleTag<String> mainOutput = new TupleTag<>("mainOutput");
+ private static final TupleTag<String> additionalOutput = new TupleTag<>("output");
+
+ @StateId("value")
+ private final StateSpec<ValueState<String>> valueStateSpec =
+ StateSpecs.value(StringUtf8Coder.of());
+ @StateId("bag")
+ private final StateSpec<BagState<String>> bagStateSpec =
+ StateSpecs.bag(StringUtf8Coder.of());
+ @StateId("combine")
+ private final StateSpec<CombiningState<String, String, String>> combiningStateSpec =
+ StateSpecs.combining(StringUtf8Coder.of(), new ConcatCombineFn());
+ @StateId("combineWithContext")
+ private final StateSpec<CombiningState<String, String, String>> combiningWithContextStateSpec =
+ StateSpecs.combining(StringUtf8Coder.of(), new ConcatCombineFnWithContext());
+
+ @ProcessElement
+ public void processElement(ProcessContext context,
+ @StateId("value") ValueState<String> valueState,
+ @StateId("bag") BagState<String> bagState,
+ @StateId("combine") CombiningState<String, String, String> combiningState,
+ @StateId("combineWithContext")
+ CombiningState<String, String, String> combiningWithContextState) {
+ context.output("value:" + valueState.read());
+ valueState.write(context.element().getValue());
+
+ context.output("bag:" + Iterables.toString(bagState.read()));
+ bagState.add(context.element().getValue());
+
+ context.output("combine:" + combiningState.read());
+ combiningState.add(context.element().getValue());
+
+ context.output("combineWithContext:" + combiningWithContextState.read());
+ combiningWithContextState.add(context.element().getValue());
+ }
+ }
+
+ @Test
+ public void testUsingUserState() throws Exception {
+ String mainOutputId = "101";
+
+ DoFnInfo<?, ?> doFnInfo = DoFnInfo.forFn(
+ new TestStatefulDoFn(),
+ WindowingStrategy.globalDefault(),
+ ImmutableList.of(),
+ KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()),
+ Long.parseLong(mainOutputId),
+ ImmutableMap.of(Long.parseLong(mainOutputId), new TupleTag<String>("mainOutput")));
+ RunnerApi.FunctionSpec functionSpec =
+ RunnerApi.FunctionSpec.newBuilder()
+ .setUrn(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN)
+ .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnInfo)))
+ .build();
+ RunnerApi.PTransform pTransform = RunnerApi.PTransform.newBuilder()
+ .setSpec(functionSpec)
+ .putInputs("input", "inputTarget")
+ .putOutputs(mainOutputId, "mainOutputTarget")
+ .build();
+
+ FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
+ key("value", "X"), encode("X0"),
+ key("bag", "X"), encode("X0"),
+ key("combine", "X"), encode("X0"),
+ key("combineWithContext", "X"), encode("X0")
+ ));
+
+ List<WindowedValue<String>> mainOutputValues = new ArrayList<>();
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> consumers = HashMultimap.create();
+ consumers.put("mainOutputTarget",
+ (ThrowingConsumer) (ThrowingConsumer<WindowedValue<String>>) mainOutputValues::add);
+ List<ThrowingRunnable> startFunctions = new ArrayList<>();
+ List<ThrowingRunnable> finishFunctions = new ArrayList<>();
+
+ new FnApiDoFnRunner.Factory<>().createRunnerForPTransform(
+ PipelineOptionsFactory.create(),
+ null /* beamFnDataClient */,
+ fakeClient,
+ TEST_PTRANSFORM_ID,
+ pTransform,
+ Suppliers.ofInstance("57L")::get,
+ ImmutableMap.of(),
+ ImmutableMap.of(),
+ consumers,
+ startFunctions::add,
+ finishFunctions::add);
+
+ Iterables.getOnlyElement(startFunctions).run();
+ mainOutputValues.clear();
+
+ assertThat(consumers.keySet(), containsInAnyOrder("inputTarget", "mainOutputTarget"));
+
+ // Ensure that bag user state that is initially empty or populated works.
+ // Ensure that the key order does not matter when we traverse over KV pairs.
+ ThrowingConsumer<WindowedValue<?>> mainInput =
+ Iterables.getOnlyElement(consumers.get("inputTarget"));
+ mainInput.accept(valueInGlobalWindow(KV.of("X", "X1")));
+ mainInput.accept(valueInGlobalWindow(KV.of("Y", "Y1")));
+ mainInput.accept(valueInGlobalWindow(KV.of("X", "X2")));
+ mainInput.accept(valueInGlobalWindow(KV.of("Y", "Y2")));
+ assertThat(mainOutputValues, contains(
+ valueInGlobalWindow("value:X0"),
+ valueInGlobalWindow("bag:[X0]"),
+ valueInGlobalWindow("combine:X0"),
+ valueInGlobalWindow("combineWithContext:X0"),
+ valueInGlobalWindow("value:null"),
+ valueInGlobalWindow("bag:[]"),
+ valueInGlobalWindow("combine:"),
+ valueInGlobalWindow("combineWithContext:"),
+ valueInGlobalWindow("value:X1"),
+ valueInGlobalWindow("bag:[X0, X1]"),
+ valueInGlobalWindow("combine:X0X1"),
+ valueInGlobalWindow("combineWithContext:X0X1"),
+ valueInGlobalWindow("value:Y1"),
+ valueInGlobalWindow("bag:[Y1]"),
+ valueInGlobalWindow("combine:Y1"),
+ valueInGlobalWindow("combineWithContext:Y1")));
+ mainOutputValues.clear();
+
+ Iterables.getOnlyElement(finishFunctions).run();
+ assertThat(mainOutputValues, empty());
+
+ assertEquals(
+ ImmutableMap.<StateKey, ByteString>builder()
+ .put(key("value", "X"), encode("X2"))
+ .put(key("bag", "X"), encode("X0", "X1", "X2"))
+ .put(key("combine", "X"), encode("X0X1X2"))
+ .put(key("combineWithContext", "X"), encode("X0X1X2"))
+ .put(key("value", "Y"), encode("Y2"))
+ .put(key("bag", "Y"), encode("Y1", "Y2"))
+ .put(key("combine", "Y"), encode("Y1Y2"))
+ .put(key("combineWithContext", "Y"), encode("Y1Y2"))
+ .build(),
+ fakeClient.getData());
+ mainOutputValues.clear();
+ }
+
+ /** Produces a {@link StateKey} for the test PTransform id in the Global Window. */
+ private StateKey key(String userStateId, String key) throws IOException {
+ return StateKey.newBuilder().setBagUserState(
+ StateKey.BagUserState.newBuilder()
+ .setPtransformId(TEST_PTRANSFORM_ID)
+ .setUserStateId(userStateId)
+ .setKey(encode(key))
+ .setWindow(ByteString.copyFrom(
+ CoderUtils.encodeToByteArray(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE))))
+ .build();
+ }
+
+ private ByteString encode(String ... values) throws IOException {
+ ByteString.Output out = ByteString.newOutput();
+ for (String value : values) {
+ StringUtf8Coder.of().encode(value, out);
+ }
+ return out.toByteString();
+ }
+
+ @Test
+ public void testRegistration() {
+ for (Registrar registrar :
+ ServiceLoader.load(Registrar.class)) {
+ if (registrar instanceof FnApiDoFnRunner.Registrar) {
+ assertThat(registrar.getPTransformRunnerFactories(),
+ IsMapContaining.hasKey(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN));
+ return;
+ }
+ }
+ fail("Expected registrar not found.");
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
index d92ba72..c926414 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
@@ -28,14 +28,15 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
-import java.util.function.Consumer;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.InstructionRequest;
-import org.apache.beam.fn.v1.BeamFnApi.InstructionResponse;
-import org.apache.beam.fn.v1.BeamFnApi.LogControl;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
+import org.apache.beam.harness.test.Consumer;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.LogControl;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -90,7 +91,7 @@
responseObserver.onCompleted();
}
});
- return TestStreams.withOnNext(new Consumer<BeamFnApi.InstructionResponse>() {
+ return TestStreams.withOnNext(new Consumer<InstructionResponse>() {
@Override
public void accept(InstructionResponse t) {
instructionResponses.add(t);
@@ -106,14 +107,12 @@
Server controlServer = ServerBuilder.forPort(0).addService(controlService).build();
controlServer.start();
try {
- BeamFnApi.ApiServiceDescriptor loggingDescriptor = BeamFnApi.ApiServiceDescriptor
+ Endpoints.ApiServiceDescriptor loggingDescriptor = Endpoints.ApiServiceDescriptor
.newBuilder()
- .setId("1L")
.setUrl("localhost:" + loggingServer.getPort())
.build();
- BeamFnApi.ApiServiceDescriptor controlDescriptor = BeamFnApi.ApiServiceDescriptor
+ Endpoints.ApiServiceDescriptor controlDescriptor = Endpoints.ApiServiceDescriptor
.newBuilder()
- .setId("2L")
.setUrl("localhost:" + controlServer.getPort())
.build();
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/IdGeneratorTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/IdGeneratorTest.java
new file mode 100644
index 0000000..10ce393
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/IdGeneratorTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link IdGenerator}. */
+@RunWith(JUnit4.class)
+public class IdGeneratorTest {
+ @Test
+ public void testGenerationNeverMatches() {
+ final int numToGenerate = 10000;
+ Set<String> generatedValues = new HashSet<>();
+ for (int i = 0; i < numToGenerate; ++i) {
+ generatedValues.add(IdGenerator.generate());
+ }
+ assertEquals(numToGenerate, generatedValues.size());
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
deleted file mode 100644
index 9f634c9..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.channel;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assume.assumeTrue;
-
-import io.grpc.ManagedChannel;
-import org.apache.beam.fn.v1.BeamFnApi.ApiServiceDescriptor;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link ManagedChannelFactory}. */
-@RunWith(JUnit4.class)
-public class ManagedChannelFactoryTest {
- @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Test
- public void testDefaultChannel() {
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("localhost:123")
- .build();
- ManagedChannel channel = ManagedChannelFactory.from(PipelineOptionsFactory.create())
- .forDescriptor(apiServiceDescriptor);
- assertEquals("localhost:123", channel.authority());
- channel.shutdownNow();
- }
-
- @Test
- public void testEpollHostPortChannel() {
- assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("localhost:123")
- .build();
- ManagedChannel channel = ManagedChannelFactory.from(
- PipelineOptionsFactory.fromArgs(new String[]{ "--experiments=beam_fn_api_epoll" }).create())
- .forDescriptor(apiServiceDescriptor);
- assertEquals("localhost:123", channel.authority());
- channel.shutdownNow();
- }
-
- @Test
- public void testEpollDomainSocketChannel() throws Exception {
- assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("unix://" + tmpFolder.newFile().getAbsolutePath())
- .build();
- ManagedChannel channel = ManagedChannelFactory.from(
- PipelineOptionsFactory.fromArgs(new String[]{ "--experiments=beam_fn_api_epoll" }).create())
- .forDescriptor(apiServiceDescriptor);
- assertEquals(apiServiceDescriptor.getUrl().substring("unix://".length()), channel.authority());
- channel.shutdownNow();
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/SocketAddressFactoryTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/SocketAddressFactoryTest.java
deleted file mode 100644
index 610a8ea..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/SocketAddressFactoryTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.channel;
-
-import static org.hamcrest.Matchers.instanceOf;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-import io.netty.channel.unix.DomainSocketAddress;
-import java.io.File;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link SocketAddressFactory}. */
-@RunWith(JUnit4.class)
-public class SocketAddressFactoryTest {
- @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Test
- public void testHostPortSocket() {
- SocketAddress socketAddress = SocketAddressFactory.createFrom("localhost:123");
- assertThat(socketAddress, instanceOf(InetSocketAddress.class));
- assertEquals("localhost", ((InetSocketAddress) socketAddress).getHostString());
- assertEquals(123, ((InetSocketAddress) socketAddress).getPort());
- }
-
- @Test
- public void testDomainSocket() throws Exception {
- File tmpFile = tmpFolder.newFile();
- SocketAddress socketAddress = SocketAddressFactory.createFrom(
- "unix://" + tmpFile.getAbsolutePath());
- assertThat(socketAddress, instanceOf(DomainSocketAddress.class));
- assertEquals(tmpFile.getAbsolutePath(), ((DomainSocketAddress) socketAddress).path());
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
index edb7903..56ae7ed 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
@@ -39,9 +39,10 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -91,8 +92,8 @@
TestStreams.withOnNext(values::add)
.withOnCompleted(() -> clientClosedStream.set(true)).build();
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor =
- BeamFnApi.ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -136,7 +137,7 @@
BeamFnControlClient client = new BeamFnControlClient(
apiServiceDescriptor,
- (BeamFnApi.ApiServiceDescriptor descriptor) -> channel,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel,
this::createStreamForTest,
handlers);
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 748ffea..15b5866 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -18,67 +18,40 @@
package org.apache.beam.fn.harness.control;
-import static org.apache.beam.sdk.util.WindowedValue.timestampedValueInGlobalWindow;
-import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Suppliers;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
+import com.google.common.util.concurrent.Uninterruptibles;
import com.google.protobuf.Message;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BiConsumer;
+import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
-import java.util.function.Function;
import java.util.function.Supplier;
+import org.apache.beam.fn.harness.PTransformRunnerFactory;
import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.runners.dataflow.util.DoFnInfo;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.fn.harness.state.BeamFnStateClient;
+import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -87,50 +60,17 @@
import org.junit.runners.JUnit4;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
-import org.mockito.Matchers;
import org.mockito.Mock;
+import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
/** Tests for {@link ProcessBundleHandler}. */
@RunWith(JUnit4.class)
public class ProcessBundleHandlerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private static final Coder<WindowedValue<String>> STRING_CODER =
- WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
- private static final String LONG_CODER_SPEC_ID = "998L";
- private static final String STRING_CODER_SPEC_ID = "999L";
- private static final BeamFnApi.RemoteGrpcPort REMOTE_PORT = BeamFnApi.RemoteGrpcPort.newBuilder()
- .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.newBuilder()
- .setId("58L")
- .setUrl("TestUrl"))
- .build();
- private static final BeamFnApi.Coder LONG_CODER_SPEC;
- private static final BeamFnApi.Coder STRING_CODER_SPEC;
- static {
- try {
- STRING_CODER_SPEC =
- BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
- .setId(STRING_CODER_SPEC_ID)
- .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
- OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(STRING_CODER)))).build())))
- .build();
- LONG_CODER_SPEC =
- BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
- .setId(STRING_CODER_SPEC_ID)
- .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
- OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(WindowedValue.getFullCoder(
- VarLongCoder.of(), GlobalWindow.Coder.INSTANCE))))).build())))
- .build();
- } catch (IOException e) {
- throw new ExceptionInInitializerError(e);
- }
- }
-
private static final String DATA_INPUT_URN = "urn:org.apache.beam:source:runner:0.1";
private static final String DATA_OUTPUT_URN = "urn:org.apache.beam:sink:runner:0.1";
- private static final String JAVA_DO_FN_URN = "urn:org.apache.beam:dofn:java:0.1";
- private static final String JAVA_SOURCE_URN = "urn:org.apache.beam:source:java:0.1";
@Rule public ExpectedException thrown = ExpectedException.none();
@@ -146,537 +86,326 @@
public void testOrderOfStartAndFinishCalls() throws Exception {
BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
BeamFnApi.ProcessBundleDescriptor.newBuilder()
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("2L"))
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("3L"))
- .build();
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .putOutputs("2L-output", "2L-output-pc")
+ .build())
+ .putTransforms("3L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_OUTPUT_URN).build())
+ .putInputs("3L-input", "2L-output-pc")
+ .build())
+ .putPcollections("2L-output-pc", RunnerApi.PCollection.getDefaultInstance())
+ .build();
Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
- List<BeamFnApi.PrimitiveTransform> transformsProcessed = new ArrayList<>();
+ List<RunnerApi.PTransform> transformsProcessed = new ArrayList<>();
List<String> orderOfOperations = new ArrayList<>();
+ PTransformRunnerFactory<Object> startFinishRecorder = new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+
+ assertThat(processBundleInstructionId.get(), equalTo("999L"));
+
+ transformsProcessed.add(pTransform);
+ addStartFunction.accept(
+ () -> orderOfOperations.add("Start" + pTransformId));
+ addFinishFunction.accept(
+ () -> orderOfOperations.add("Finish" + pTransformId));
+ return null;
+ }
+ };
+
ProcessBundleHandler handler = new ProcessBundleHandler(
PipelineOptionsFactory.create(),
fnApiRegistry::get,
- beamFnDataClient) {
- @Override
- protected <InputT, OutputT> void createConsumersForPrimitiveTransform(
- BeamFnApi.PrimitiveTransform primitiveTransform,
- Supplier<String> processBundleInstructionId,
- Function<BeamFnApi.Target,
- Collection<ThrowingConsumer<WindowedValue<OutputT>>>> consumers,
- BiConsumer<BeamFnApi.Target, ThrowingConsumer<WindowedValue<InputT>>> addConsumer,
- Consumer<ThrowingRunnable> addStartFunction,
- Consumer<ThrowingRunnable> addFinishFunction)
- throws IOException {
+ beamFnDataClient,
+ null /* beamFnStateClient */,
+ ImmutableMap.of(
+ DATA_INPUT_URN, startFinishRecorder,
+ DATA_OUTPUT_URN, startFinishRecorder));
- assertThat(processBundleInstructionId.get(), equalTo("999L"));
-
- transformsProcessed.add(primitiveTransform);
- addStartFunction.accept(
- () -> orderOfOperations.add("Start" + primitiveTransform.getId()));
- addFinishFunction.accept(
- () -> orderOfOperations.add("Finish" + primitiveTransform.getId()));
- }
- };
handler.processBundle(BeamFnApi.InstructionRequest.newBuilder()
.setInstructionId("999L")
.setProcessBundle(
BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorReference("1L"))
.build());
- // Processing of primitive transforms is performed in reverse order.
+ // Processing of transforms is performed in reverse order.
assertThat(transformsProcessed, contains(
- processBundleDescriptor.getPrimitiveTransform(1),
- processBundleDescriptor.getPrimitiveTransform(0)));
+ processBundleDescriptor.getTransformsMap().get("3L"),
+ processBundleDescriptor.getTransformsMap().get("2L")));
// Start should occur in reverse order while finish calls should occur in forward order
assertThat(orderOfOperations, contains("Start3L", "Start2L", "Finish2L", "Finish3L"));
}
@Test
- public void testCreatingPrimitiveTransformExceptionsArePropagated() throws Exception {
+ public void testCreatingPTransformExceptionsArePropagated() throws Exception {
BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
BeamFnApi.ProcessBundleDescriptor.newBuilder()
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("2L"))
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("3L"))
- .build();
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .build())
+ .build();
Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
ProcessBundleHandler handler = new ProcessBundleHandler(
PipelineOptionsFactory.create(),
fnApiRegistry::get,
- beamFnDataClient) {
- @Override
- protected <InputT, OutputT> void createConsumersForPrimitiveTransform(
- BeamFnApi.PrimitiveTransform primitiveTransform,
- Supplier<String> processBundleInstructionId,
- Function<BeamFnApi.Target,
- Collection<ThrowingConsumer<WindowedValue<OutputT>>>> consumers,
- BiConsumer<BeamFnApi.Target, ThrowingConsumer<WindowedValue<InputT>>> addConsumer,
- Consumer<ThrowingRunnable> addStartFunction,
- Consumer<ThrowingRunnable> addFinishFunction)
- throws IOException {
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("TestException");
- throw new IllegalStateException("TestException");
- }
- };
+ beamFnDataClient,
+ null /* beamFnStateGrpcClientCache */,
+ ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("TestException");
+ throw new IllegalStateException("TestException");
+ }
+ }));
handler.processBundle(
BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(
BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorReference("1L"))
- .build());
+ .build());
}
@Test
- public void testPrimitiveTransformStartExceptionsArePropagated() throws Exception {
+ public void testPTransformStartExceptionsArePropagated() throws Exception {
BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
BeamFnApi.ProcessBundleDescriptor.newBuilder()
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("2L"))
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("3L"))
- .build();
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .build())
+ .build();
Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
ProcessBundleHandler handler = new ProcessBundleHandler(
PipelineOptionsFactory.create(),
fnApiRegistry::get,
- beamFnDataClient) {
- @Override
- protected <InputT, OutputT> void createConsumersForPrimitiveTransform(
- BeamFnApi.PrimitiveTransform primitiveTransform,
- Supplier<String> processBundleInstructionId,
- Function<BeamFnApi.Target,
- Collection<ThrowingConsumer<WindowedValue<OutputT>>>> consumers,
- BiConsumer<BeamFnApi.Target, ThrowingConsumer<WindowedValue<InputT>>> addConsumer,
- Consumer<ThrowingRunnable> addStartFunction,
- Consumer<ThrowingRunnable> addFinishFunction)
- throws IOException {
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("TestException");
- addStartFunction.accept(this::throwException);
- }
-
- private void throwException() {
- throw new IllegalStateException("TestException");
- }
- };
+ beamFnDataClient,
+ null /* beamFnStateGrpcClientCache */,
+ ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("TestException");
+ addStartFunction.accept(ProcessBundleHandlerTest::throwException);
+ return null;
+ }
+ }));
handler.processBundle(
BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(
BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorReference("1L"))
- .build());
+ .build());
}
@Test
- public void testPrimitiveTransformFinishExceptionsArePropagated() throws Exception {
+ public void testPTransformFinishExceptionsArePropagated() throws Exception {
BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
BeamFnApi.ProcessBundleDescriptor.newBuilder()
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("2L"))
- .addPrimitiveTransform(BeamFnApi.PrimitiveTransform.newBuilder().setId("3L"))
- .build();
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .build())
+ .build();
Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
ProcessBundleHandler handler = new ProcessBundleHandler(
PipelineOptionsFactory.create(),
fnApiRegistry::get,
- beamFnDataClient) {
- @Override
- protected <InputT, OutputT> void createConsumersForPrimitiveTransform(
- BeamFnApi.PrimitiveTransform primitiveTransform,
- Supplier<String> processBundleInstructionId,
- Function<BeamFnApi.Target,
- Collection<ThrowingConsumer<WindowedValue<OutputT>>>> consumers,
- BiConsumer<BeamFnApi.Target, ThrowingConsumer<WindowedValue<InputT>>> addConsumer,
- Consumer<ThrowingRunnable> addStartFunction,
- Consumer<ThrowingRunnable> addFinishFunction)
- throws IOException {
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("TestException");
- addFinishFunction.accept(this::throwException);
- }
-
- private void throwException() {
- throw new IllegalStateException("TestException");
- }
- };
+ beamFnDataClient,
+ null /* beamFnStateGrpcClientCache */,
+ ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("TestException");
+ addFinishFunction.accept(ProcessBundleHandlerTest::throwException);
+ return null;
+ }
+ }));
handler.processBundle(
BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(
BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorReference("1L"))
- .build());
- }
-
- private static class TestDoFn extends DoFn<String, String> {
- private static final TupleTag<String> mainOutput = new TupleTag<>("mainOutput");
- private static final TupleTag<String> additionalOutput = new TupleTag<>("output");
-
- private BoundedWindow window;
-
- @ProcessElement
- public void processElement(ProcessContext context, BoundedWindow window) {
- context.output("MainOutput" + context.element());
- context.output(additionalOutput, "AdditionalOutput" + context.element());
- this.window = window;
- }
-
- @FinishBundle
- public void finishBundle(FinishBundleContext context) {
- if (window != null) {
- context.output("FinishBundle", window.maxTimestamp(), window);
- window = null;
- }
- }
- }
-
- /**
- * Create a DoFn that has 3 inputs (inputATarget1, inputATarget2, inputBTarget) and 2 outputs
- * (mainOutput, output). Validate that inputs are fed to the {@link DoFn} and that outputs
- * are directed to the correct consumers.
- */
- @Test
- public void testCreatingAndProcessingDoFn() throws Exception {
- Map<String, Message> fnApiRegistry = ImmutableMap.of(STRING_CODER_SPEC_ID, STRING_CODER_SPEC);
- String primitiveTransformId = "100L";
- long mainOutputId = 101L;
- long additionalOutputId = 102L;
-
- DoFnInfo<?, ?> doFnInfo = DoFnInfo.forFn(
- new TestDoFn(),
- WindowingStrategy.globalDefault(),
- ImmutableList.of(),
- STRING_CODER,
- mainOutputId,
- ImmutableMap.of(
- mainOutputId, TestDoFn.mainOutput,
- additionalOutputId, TestDoFn.additionalOutput));
- BeamFnApi.FunctionSpec functionSpec = BeamFnApi.FunctionSpec.newBuilder()
- .setId("1L")
- .setUrn(JAVA_DO_FN_URN)
- .setData(Any.pack(BytesValue.newBuilder()
- .setValue(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnInfo)))
- .build()))
- .build();
- BeamFnApi.Target inputATarget1 = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1000L")
- .setName("inputATarget1")
- .build();
- BeamFnApi.Target inputATarget2 = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1001L")
- .setName("inputATarget1")
- .build();
- BeamFnApi.Target inputBTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1002L")
- .setName("inputBTarget")
- .build();
- BeamFnApi.PrimitiveTransform primitiveTransform = BeamFnApi.PrimitiveTransform.newBuilder()
- .setId(primitiveTransformId)
- .setFunctionSpec(functionSpec)
- .putInputs("inputA", BeamFnApi.Target.List.newBuilder()
- .addTarget(inputATarget1)
- .addTarget(inputATarget2)
- .build())
- .putInputs("inputB", BeamFnApi.Target.List.newBuilder()
- .addTarget(inputBTarget)
- .build())
- .putOutputs(Long.toString(mainOutputId), BeamFnApi.PCollection.newBuilder()
- .setCoderReference(STRING_CODER_SPEC_ID)
- .build())
- .putOutputs(Long.toString(additionalOutputId), BeamFnApi.PCollection.newBuilder()
- .setCoderReference(STRING_CODER_SPEC_ID)
- .build())
- .build();
-
- List<WindowedValue<String>> mainOutputValues = new ArrayList<>();
- List<WindowedValue<String>> additionalOutputValues = new ArrayList<>();
- BeamFnApi.Target mainOutputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName(Long.toString(mainOutputId))
- .build();
- BeamFnApi.Target additionalOutputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName(Long.toString(additionalOutputId))
- .build();
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> existingConsumers =
- ImmutableMultimap.of(
- mainOutputTarget, mainOutputValues::add,
- additionalOutputTarget, additionalOutputValues::add);
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> newConsumers =
- HashMultimap.create();
- List<ThrowingRunnable> startFunctions = new ArrayList<>();
- List<ThrowingRunnable> finishFunctions = new ArrayList<>();
-
- ProcessBundleHandler handler = new ProcessBundleHandler(
- PipelineOptionsFactory.create(),
- fnApiRegistry::get,
- beamFnDataClient);
- handler.createConsumersForPrimitiveTransform(
- primitiveTransform,
- Suppliers.ofInstance("57L")::get,
- existingConsumers::get,
- newConsumers::put,
- startFunctions::add,
- finishFunctions::add);
-
- Iterables.getOnlyElement(startFunctions).run();
- mainOutputValues.clear();
-
- assertEquals(newConsumers.keySet(),
- ImmutableSet.of(inputATarget1, inputATarget2, inputBTarget));
-
- Iterables.getOnlyElement(newConsumers.get(inputATarget1)).accept(valueInGlobalWindow("A1"));
- Iterables.getOnlyElement(newConsumers.get(inputATarget1)).accept(valueInGlobalWindow("A2"));
- Iterables.getOnlyElement(newConsumers.get(inputATarget1)).accept(valueInGlobalWindow("B"));
- assertThat(mainOutputValues, contains(
- valueInGlobalWindow("MainOutputA1"),
- valueInGlobalWindow("MainOutputA2"),
- valueInGlobalWindow("MainOutputB")));
- assertThat(additionalOutputValues, contains(
- valueInGlobalWindow("AdditionalOutputA1"),
- valueInGlobalWindow("AdditionalOutputA2"),
- valueInGlobalWindow("AdditionalOutputB")));
- mainOutputValues.clear();
- additionalOutputValues.clear();
-
- Iterables.getOnlyElement(finishFunctions).run();
- assertThat(
- mainOutputValues,
- contains(
- timestampedValueInGlobalWindow("FinishBundle", GlobalWindow.INSTANCE.maxTimestamp())));
- mainOutputValues.clear();
+ .build());
}
@Test
- public void testCreatingAndProcessingSource() throws Exception {
- Map<String, Message> fnApiRegistry = ImmutableMap.of(LONG_CODER_SPEC_ID, LONG_CODER_SPEC);
- String primitiveTransformId = "100L";
- long outputId = 101L;
+ public void testPendingStateCallsBlockTillCompletion() throws Exception {
+ BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
+ BeamFnApi.ProcessBundleDescriptor.newBuilder()
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .build())
+ .setStateApiServiceDescriptor(ApiServiceDescriptor.getDefaultInstance())
+ .build();
+ Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
- BeamFnApi.Target inputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1000L")
- .setName("inputTarget")
- .build();
+ CompletableFuture<StateResponse> successfulResponse = new CompletableFuture<>();
+ CompletableFuture<StateResponse> unsuccessfulResponse = new CompletableFuture<>();
- List<WindowedValue<String>> outputValues = new ArrayList<>();
- BeamFnApi.Target outputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName(Long.toString(outputId))
- .build();
+ BeamFnStateGrpcClientCache mockBeamFnStateGrpcClient =
+ Mockito.mock(BeamFnStateGrpcClientCache.class);
+ BeamFnStateClient mockBeamFnStateClient = Mockito.mock(BeamFnStateClient.class);
+ when(mockBeamFnStateGrpcClient.forApiServiceDescriptor(any()))
+ .thenReturn(mockBeamFnStateClient);
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> existingConsumers =
- ImmutableMultimap.of(outputTarget, outputValues::add);
- Multimap<BeamFnApi.Target,
- ThrowingConsumer<WindowedValue<BoundedSource<Long>>>> newConsumers =
- HashMultimap.create();
- List<ThrowingRunnable> startFunctions = new ArrayList<>();
- List<ThrowingRunnable> finishFunctions = new ArrayList<>();
-
- BeamFnApi.FunctionSpec functionSpec = BeamFnApi.FunctionSpec.newBuilder()
- .setId("1L")
- .setUrn(JAVA_SOURCE_URN)
- .setData(Any.pack(BytesValue.newBuilder()
- .setValue(ByteString.copyFrom(
- SerializableUtils.serializeToByteArray(CountingSource.upTo(3))))
- .build()))
- .build();
-
- BeamFnApi.PrimitiveTransform primitiveTransform = BeamFnApi.PrimitiveTransform.newBuilder()
- .setId(primitiveTransformId)
- .setFunctionSpec(functionSpec)
- .putInputs("input",
- BeamFnApi.Target.List.newBuilder().addTarget(inputTarget).build())
- .putOutputs(Long.toString(outputId),
- BeamFnApi.PCollection.newBuilder().setCoderReference(LONG_CODER_SPEC_ID).build())
- .build();
-
- ProcessBundleHandler handler = new ProcessBundleHandler(
- PipelineOptionsFactory.create(),
- fnApiRegistry::get,
- beamFnDataClient);
-
- handler.createConsumersForPrimitiveTransform(
- primitiveTransform,
- Suppliers.ofInstance("57L")::get,
- existingConsumers::get,
- newConsumers::put,
- startFunctions::add,
- finishFunctions::add);
-
- // This is testing a deprecated way of running sources and should be removed
- // once all source definitions are instead propagated along the input edge.
- Iterables.getOnlyElement(startFunctions).run();
- assertThat(outputValues, contains(
- valueInGlobalWindow(0L),
- valueInGlobalWindow(1L),
- valueInGlobalWindow(2L)));
- outputValues.clear();
-
- // Check that when passing a source along as an input, the source is processed.
- assertEquals(newConsumers.keySet(), ImmutableSet.of(inputTarget));
- Iterables.getOnlyElement(newConsumers.get(inputTarget)).accept(
- valueInGlobalWindow(CountingSource.upTo(2)));
- assertThat(outputValues, contains(
- valueInGlobalWindow(0L),
- valueInGlobalWindow(1L)));
-
- assertThat(finishFunctions, empty());
- }
-
- @Test
- public void testCreatingAndProcessingBeamFnDataReadRunner() throws Exception {
- Map<String, Message> fnApiRegistry = ImmutableMap.of(STRING_CODER_SPEC_ID, STRING_CODER_SPEC);
- String bundleId = "57L";
- String primitiveTransformId = "100L";
- long outputId = 101L;
-
- List<WindowedValue<String>> outputValues = new ArrayList<>();
- BeamFnApi.Target outputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName(Long.toString(outputId))
- .build();
-
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> existingConsumers =
- ImmutableMultimap.of(outputTarget, outputValues::add);
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> newConsumers =
- HashMultimap.create();
- List<ThrowingRunnable> startFunctions = new ArrayList<>();
- List<ThrowingRunnable> finishFunctions = new ArrayList<>();
-
- BeamFnApi.FunctionSpec functionSpec = BeamFnApi.FunctionSpec.newBuilder()
- .setId("1L")
- .setUrn(DATA_INPUT_URN)
- .setData(Any.pack(REMOTE_PORT))
- .build();
-
- BeamFnApi.PrimitiveTransform primitiveTransform = BeamFnApi.PrimitiveTransform.newBuilder()
- .setId(primitiveTransformId)
- .setFunctionSpec(functionSpec)
- .putInputs("input", BeamFnApi.Target.List.getDefaultInstance())
- .putOutputs(Long.toString(outputId),
- BeamFnApi.PCollection.newBuilder().setCoderReference(STRING_CODER_SPEC_ID).build())
- .build();
-
- ProcessBundleHandler handler = new ProcessBundleHandler(
- PipelineOptionsFactory.create(),
- fnApiRegistry::get,
- beamFnDataClient);
-
- handler.createConsumersForPrimitiveTransform(
- primitiveTransform,
- Suppliers.ofInstance(bundleId)::get,
- existingConsumers::get,
- newConsumers::put,
- startFunctions::add,
- finishFunctions::add);
-
- verifyZeroInteractions(beamFnDataClient);
-
- CompletableFuture<Void> completionFuture = new CompletableFuture<>();
- when(beamFnDataClient.forInboundConsumer(any(), any(), any(), any()))
- .thenReturn(completionFuture);
- Iterables.getOnlyElement(startFunctions).run();
- verify(beamFnDataClient).forInboundConsumer(
- eq(REMOTE_PORT.getApiServiceDescriptor()),
- eq(KV.of(bundleId, BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName("input")
- .build())),
- eq(STRING_CODER),
- consumerCaptor.capture());
-
- consumerCaptor.getValue().accept(valueInGlobalWindow("TestValue"));
- assertThat(outputValues, contains(valueInGlobalWindow("TestValue")));
- outputValues.clear();
-
- assertThat(newConsumers.keySet(), empty());
-
- completionFuture.complete(null);
- Iterables.getOnlyElement(finishFunctions).run();
-
- verifyNoMoreInteractions(beamFnDataClient);
- }
-
- @Test
- public void testCreatingAndProcessingBeamFnDataWriteRunner() throws Exception {
- Map<String, Message> fnApiRegistry = ImmutableMap.of(STRING_CODER_SPEC_ID, STRING_CODER_SPEC);
- String bundleId = "57L";
- String primitiveTransformId = "100L";
- long outputId = 101L;
-
- BeamFnApi.Target inputTarget = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1000L")
- .setName("inputTarget")
- .build();
-
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> existingConsumers =
- ImmutableMultimap.of();
- Multimap<BeamFnApi.Target, ThrowingConsumer<WindowedValue<String>>> newConsumers =
- HashMultimap.create();
- List<ThrowingRunnable> startFunctions = new ArrayList<>();
- List<ThrowingRunnable> finishFunctions = new ArrayList<>();
-
- BeamFnApi.FunctionSpec functionSpec = BeamFnApi.FunctionSpec.newBuilder()
- .setId("1L")
- .setUrn(DATA_OUTPUT_URN)
- .setData(Any.pack(REMOTE_PORT))
- .build();
-
- BeamFnApi.PrimitiveTransform primitiveTransform = BeamFnApi.PrimitiveTransform.newBuilder()
- .setId(primitiveTransformId)
- .setFunctionSpec(functionSpec)
- .putInputs("input", BeamFnApi.Target.List.newBuilder().addTarget(inputTarget).build())
- .putOutputs(Long.toString(outputId),
- BeamFnApi.PCollection.newBuilder().setCoderReference(STRING_CODER_SPEC_ID).build())
- .build();
-
- ProcessBundleHandler handler = new ProcessBundleHandler(
- PipelineOptionsFactory.create(),
- fnApiRegistry::get,
- beamFnDataClient);
-
- handler.createConsumersForPrimitiveTransform(
- primitiveTransform,
- Suppliers.ofInstance(bundleId)::get,
- existingConsumers::get,
- newConsumers::put,
- startFunctions::add,
- finishFunctions::add);
-
- verifyZeroInteractions(beamFnDataClient);
-
- List<WindowedValue<String>> outputValues = new ArrayList<>();
- AtomicBoolean wasCloseCalled = new AtomicBoolean();
- CloseableThrowingConsumer<WindowedValue<String>> outputConsumer =
- new CloseableThrowingConsumer<WindowedValue<String>>(){
+ doAnswer(new Answer<Void>() {
@Override
- public void close() throws Exception {
- wasCloseCalled.set(true);
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ StateRequest.Builder stateRequestBuilder =
+ (StateRequest.Builder) invocation.getArguments()[0];
+ CompletableFuture<StateResponse> completableFuture =
+ (CompletableFuture<StateResponse>) invocation.getArguments()[1];
+ new Thread() {
+ @Override
+ public void run() {
+ // Simulate sleeping which introduces a race which most of the time requires
+ // the ProcessBundleHandler to block.
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+ switch (stateRequestBuilder.getInstructionReference()) {
+ case "SUCCESS":
+ completableFuture.complete(StateResponse.getDefaultInstance());
+ break;
+ case "FAIL":
+ completableFuture.completeExceptionally(new RuntimeException("TEST ERROR"));
+ }
+ }
+ }.start();
+ return null;
}
+ }).when(mockBeamFnStateClient).handle(any(), any());
- @Override
- public void accept(WindowedValue<String> t) throws Exception {
- outputValues.add(t);
- }
- };
+ ProcessBundleHandler handler = new ProcessBundleHandler(
+ PipelineOptionsFactory.create(),
+ fnApiRegistry::get,
+ beamFnDataClient,
+ mockBeamFnStateGrpcClient,
+ ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ addStartFunction.accept(() -> doStateCalls(beamFnStateClient));
+ return null;
+ }
- when(beamFnDataClient.forOutboundConsumer(
- any(),
- any(),
- Matchers.<Coder<WindowedValue<String>>>any())).thenReturn(outputConsumer);
- Iterables.getOnlyElement(startFunctions).run();
- verify(beamFnDataClient).forOutboundConsumer(
- eq(REMOTE_PORT.getApiServiceDescriptor()),
- eq(KV.of(bundleId, BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference(primitiveTransformId)
- .setName(Long.toString(outputId))
- .build())),
- eq(STRING_CODER));
+ private void doStateCalls(BeamFnStateClient beamFnStateClient) {
+ beamFnStateClient.handle(StateRequest.newBuilder().setInstructionReference("SUCCESS"),
+ successfulResponse);
+ beamFnStateClient.handle(StateRequest.newBuilder().setInstructionReference("FAIL"),
+ unsuccessfulResponse);
+ }
+ }));
+ handler.processBundle(
+ BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(
+ BeamFnApi.ProcessBundleRequest.newBuilder()
+ .setProcessBundleDescriptorReference("1L"))
+ .build());
- assertEquals(newConsumers.keySet(), ImmutableSet.of(inputTarget));
- Iterables.getOnlyElement(newConsumers.get(inputTarget)).accept(
- valueInGlobalWindow("TestValue"));
- assertThat(outputValues, contains(valueInGlobalWindow("TestValue")));
- outputValues.clear();
+ assertTrue(successfulResponse.isDone());
+ assertTrue(unsuccessfulResponse.isDone());
+ }
- assertFalse(wasCloseCalled.get());
- Iterables.getOnlyElement(finishFunctions).run();
- assertTrue(wasCloseCalled.get());
+ @Test
+ public void testStateCallsFailIfNoStateApiServiceDescriptorSpecified() throws Exception {
+ BeamFnApi.ProcessBundleDescriptor processBundleDescriptor =
+ BeamFnApi.ProcessBundleDescriptor.newBuilder()
+ .putTransforms("2L", RunnerApi.PTransform.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build())
+ .build())
+ .build();
+ Map<String, Message> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
- verifyNoMoreInteractions(beamFnDataClient);
+ ProcessBundleHandler handler = new ProcessBundleHandler(
+ PipelineOptionsFactory.create(),
+ fnApiRegistry::get,
+ beamFnDataClient,
+ null /* beamFnStateGrpcClientCache */,
+ ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {
+ @Override
+ public Object createRunnerForPTransform(
+ PipelineOptions pipelineOptions,
+ BeamFnDataClient beamFnDataClient,
+ BeamFnStateClient beamFnStateClient,
+ String pTransformId,
+ RunnerApi.PTransform pTransform,
+ Supplier<String> processBundleInstructionId,
+ Map<String, RunnerApi.PCollection> pCollections,
+ Map<String, RunnerApi.Coder> coders,
+ Multimap<String, ThrowingConsumer<WindowedValue<?>>> pCollectionIdsToConsumers,
+ Consumer<ThrowingRunnable> addStartFunction,
+ Consumer<ThrowingRunnable> addFinishFunction) throws IOException {
+ addStartFunction.accept(() -> doStateCalls(beamFnStateClient));
+ return null;
+ }
+
+ private void doStateCalls(BeamFnStateClient beamFnStateClient) {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("State API calls are unsupported");
+ beamFnStateClient.handle(StateRequest.newBuilder().setInstructionReference("SUCCESS"),
+ new CompletableFuture<>());
+ }
+ }));
+ handler.processBundle(
+ BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(
+ BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorReference("1L"))
+ .build());
+ }
+
+
+ private static void throwException() {
+ throw new IllegalStateException("TestException");
}
}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
index c32fcc4..aa1a504 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
@@ -23,10 +23,11 @@
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.beam.fn.harness.test.TestExecutors;
-import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.harness.test.TestExecutors;
+import org.apache.beam.harness.test.TestExecutors.TestExecutorService;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -41,12 +42,21 @@
BeamFnApi.InstructionRequest.newBuilder()
.setInstructionId("1L")
.setRegister(BeamFnApi.RegisterRequest.newBuilder()
- .addProcessBundleDescriptor(BeamFnApi.ProcessBundleDescriptor.newBuilder().setId("1L")
- .addCoders(BeamFnApi.Coder.newBuilder().setFunctionSpec(
- BeamFnApi.FunctionSpec.newBuilder().setId("10L")).build()))
+ .addProcessBundleDescriptor(BeamFnApi.ProcessBundleDescriptor.newBuilder()
+ .setId("1L")
+ .putCoders("10L", RunnerApi.Coder.newBuilder()
+ .setSpec(RunnerApi.SdkFunctionSpec.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn("urn:10L").build())
+ .build())
+ .build())
+ .build())
.addProcessBundleDescriptor(BeamFnApi.ProcessBundleDescriptor.newBuilder().setId("2L")
- .addCoders(BeamFnApi.Coder.newBuilder().setFunctionSpec(
- BeamFnApi.FunctionSpec.newBuilder().setId("20L")).build()))
+ .putCoders("20L", RunnerApi.Coder.newBuilder()
+ .setSpec(RunnerApi.SdkFunctionSpec.newBuilder()
+ .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn("urn:20L").build())
+ .build())
+ .build())
+ .build())
.build())
.build();
private static final BeamFnApi.InstructionResponse REGISTER_RESPONSE =
@@ -71,9 +81,11 @@
handler.getById("1L"));
assertEquals(REGISTER_REQUEST.getRegister().getProcessBundleDescriptor(1),
handler.getById("2L"));
- assertEquals(REGISTER_REQUEST.getRegister().getProcessBundleDescriptor(0).getCoders(0),
+ assertEquals(
+ REGISTER_REQUEST.getRegister().getProcessBundleDescriptor(0).getCodersOrThrow("10L"),
handler.getById("10L"));
- assertEquals(REGISTER_REQUEST.getRegister().getProcessBundleDescriptor(1).getCoders(0),
+ assertEquals(
+ REGISTER_REQUEST.getRegister().getProcessBundleDescriptor(1).getCodersOrThrow("20L"),
handler.getById("20L"));
assertEquals(REGISTER_RESPONSE, responseFuture.get());
}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
index c2b4542..81b1aa4 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
@@ -29,8 +29,8 @@
import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
index 31eb0db..9e21398 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
@@ -41,13 +41,15 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnDataGrpc;
+import org.apache.beam.harness.test.Consumer;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements;
+import org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -129,8 +131,8 @@
CallStreamObserver<BeamFnApi.Elements> inboundServerObserver =
TestStreams.withOnNext(inboundServerValues::add).build();
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor =
- BeamFnApi.ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -151,7 +153,7 @@
BeamFnDataGrpcClient clientFactory = new BeamFnDataGrpcClient(
PipelineOptionsFactory.create(),
- (BeamFnApi.ApiServiceDescriptor descriptor) -> channel,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel,
this::createStreamForTest);
CompletableFuture<Void> readFutureA = clientFactory.forInboundConsumer(
@@ -197,8 +199,8 @@
CallStreamObserver<BeamFnApi.Elements> inboundServerObserver =
TestStreams.withOnNext(inboundServerValues::add).build();
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor =
- BeamFnApi.ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -220,7 +222,7 @@
BeamFnDataGrpcClient clientFactory = new BeamFnDataGrpcClient(
PipelineOptionsFactory.create(),
- (BeamFnApi.ApiServiceDescriptor descriptor) -> channel,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel,
this::createStreamForTest);
CompletableFuture<Void> readFuture = clientFactory.forInboundConsumer(
@@ -262,7 +264,7 @@
Collection<BeamFnApi.Elements> inboundServerValues = new ConcurrentLinkedQueue<>();
CallStreamObserver<BeamFnApi.Elements> inboundServerObserver =
TestStreams.withOnNext(
- new Consumer<BeamFnApi.Elements>() {
+ new Consumer<Elements>() {
@Override
public void accept(BeamFnApi.Elements t) {
inboundServerValues.add(t);
@@ -271,8 +273,8 @@
}
).build();
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor =
- BeamFnApi.ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -292,7 +294,7 @@
BeamFnDataGrpcClient clientFactory = new BeamFnDataGrpcClient(
PipelineOptionsFactory.fromArgs(
new String[]{ "--experiments=beam_fn_api_data_buffer_limit=20" }).create(),
- (BeamFnApi.ApiServiceDescriptor descriptor) -> channel,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel,
this::createStreamForTest);
try (CloseableThrowingConsumer<WindowedValue<String>> consumer =
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
index a9095ae..6a12ed0 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
@@ -30,15 +30,16 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.junit.Test;
/** Tests for {@link BeamFnDataGrpcMultiplexer}. */
public class BeamFnDataGrpcMultiplexerTest {
- private static final BeamFnApi.ApiServiceDescriptor DESCRIPTOR =
- BeamFnApi.ApiServiceDescriptor.newBuilder().setUrl("test").build();
+ private static final Endpoints.ApiServiceDescriptor DESCRIPTOR =
+ Endpoints.ApiServiceDescriptor.newBuilder().setUrl("test").build();
private static final KV<String, BeamFnApi.Target> OUTPUT_LOCATION =
KV.of(
"777L",
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
index 54aba8b..c939423 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
@@ -32,7 +32,7 @@
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
index bb6a501..1e68b18 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
@@ -28,6 +28,7 @@
import com.google.protobuf.Timestamp;
import io.grpc.ManagedChannel;
import io.grpc.Server;
+import io.grpc.Status;
import io.grpc.inprocess.InProcessChannelBuilder;
import io.grpc.inprocess.InProcessServerBuilder;
import io.grpc.stub.CallStreamObserver;
@@ -37,15 +38,17 @@
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.LogRecord;
-import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -74,7 +77,7 @@
private static final BeamFnApi.LogEntry TEST_ENTRY =
BeamFnApi.LogEntry.newBuilder()
- .setSeverity(BeamFnApi.LogEntry.Severity.DEBUG)
+ .setSeverity(BeamFnApi.LogEntry.Severity.Enum.DEBUG)
.setMessage("Message")
.setThread("12345")
.setTimestamp(Timestamp.newBuilder().setSeconds(1234567).setNanos(890000000).build())
@@ -82,13 +85,14 @@
.build();
private static final BeamFnApi.LogEntry TEST_ENTRY_WITH_EXCEPTION =
BeamFnApi.LogEntry.newBuilder()
- .setSeverity(BeamFnApi.LogEntry.Severity.WARN)
+ .setSeverity(BeamFnApi.LogEntry.Severity.Enum.WARN)
.setMessage("MessageWithException")
.setTrace(getStackTraceAsString(TEST_RECORD_WITH_EXCEPTION.getThrown()))
.setThread("12345")
.setTimestamp(Timestamp.newBuilder().setSeconds(1234567).setNanos(890000000).build())
.setLogLocation("LoggerName")
.build();
+ @Rule public ExpectedException thrown = ExpectedException.none();
@Test
public void testLogging() throws Exception {
@@ -108,8 +112,8 @@
}
}).build();
- BeamFnApi.ApiServiceDescriptor apiServiceDescriptor =
- BeamFnApi.ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -123,18 +127,18 @@
})
.build();
server.start();
+
+ ManagedChannel channel =
+ InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build();
try {
- ManagedChannel channel =
- InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build();
BeamFnLoggingClient client = new BeamFnLoggingClient(
PipelineOptionsFactory.fromArgs(new String[] {
- "--defaultWorkerLogLevel=OFF",
- "--workerLogLevelOverrides={\"ConfiguredLogger\": \"DEBUG\"}"
+ "--defaultSdkHarnessLogLevel=OFF",
+ "--sdkHarnessLogLevelOverrides={\"ConfiguredLogger\": \"DEBUG\"}"
}).create(),
apiServiceDescriptor,
- (BeamFnApi.ApiServiceDescriptor descriptor) -> channel,
- this::createStreamForTest);
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel);
// Ensure that log levels were correctly set.
assertEquals(Level.OFF,
@@ -161,9 +165,105 @@
}
}
- private <ReqT, RespT> StreamObserver<RespT> createStreamForTest(
- Function<StreamObserver<ReqT>, StreamObserver<RespT>> clientFactory,
- StreamObserver<ReqT> handler) {
- return clientFactory.apply(handler);
+ @Test
+ public void testWhenServerFailsThatClientIsAbleToCleanup() throws Exception {
+ AtomicBoolean clientClosedStream = new AtomicBoolean();
+ Collection<BeamFnApi.LogEntry> values = new ConcurrentLinkedQueue<>();
+ AtomicReference<StreamObserver<BeamFnApi.LogControl>> outboundServerObserver =
+ new AtomicReference<>();
+ CallStreamObserver<BeamFnApi.LogEntry.List> inboundServerObserver = TestStreams.withOnNext(
+ (BeamFnApi.LogEntry.List logEntries) -> values.addAll(logEntries.getLogEntriesList()))
+ .build();
+
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
+ .setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
+ .build();
+ Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
+ .addService(new BeamFnLoggingGrpc.BeamFnLoggingImplBase() {
+ @Override
+ public StreamObserver<BeamFnApi.LogEntry.List> logging(
+ StreamObserver<BeamFnApi.LogControl> outboundObserver) {
+ outboundServerObserver.set(outboundObserver);
+ outboundObserver.onError(Status.INTERNAL.withDescription("TEST ERROR").asException());
+ return inboundServerObserver;
+ }
+ })
+ .build();
+ server.start();
+
+ ManagedChannel channel =
+ InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build();
+ try {
+ BeamFnLoggingClient client = new BeamFnLoggingClient(
+ PipelineOptionsFactory.fromArgs(new String[] {
+ "--defaultSdkHarnessLogLevel=OFF",
+ "--sdkHarnessLogLevelOverrides={\"ConfiguredLogger\": \"DEBUG\"}"
+ }).create(),
+ apiServiceDescriptor,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel);
+
+ thrown.expectMessage("TEST ERROR");
+ client.close();
+ } finally {
+ // Verify that after close, log levels are reset.
+ assertEquals(Level.INFO, LogManager.getLogManager().getLogger("").getLevel());
+ assertNull(LogManager.getLogManager().getLogger("ConfiguredLogger").getLevel());
+
+ assertTrue(channel.isShutdown());
+
+ server.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testWhenServerHangsUpEarlyThatClientIsAbleCleanup() throws Exception {
+ AtomicBoolean clientClosedStream = new AtomicBoolean();
+ Collection<BeamFnApi.LogEntry> values = new ConcurrentLinkedQueue<>();
+ AtomicReference<StreamObserver<BeamFnApi.LogControl>> outboundServerObserver =
+ new AtomicReference<>();
+ CallStreamObserver<BeamFnApi.LogEntry.List> inboundServerObserver =
+ TestStreams.withOnNext(
+ (BeamFnApi.LogEntry.List logEntries) -> values.addAll(logEntries.getLogEntriesList()))
+ .build();
+
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
+ .setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
+ .build();
+ Server server = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
+ .addService(new BeamFnLoggingGrpc.BeamFnLoggingImplBase() {
+ @Override
+ public StreamObserver<BeamFnApi.LogEntry.List> logging(
+ StreamObserver<BeamFnApi.LogControl> outboundObserver) {
+ outboundServerObserver.set(outboundObserver);
+ outboundObserver.onCompleted();
+ return inboundServerObserver;
+ }
+ })
+ .build();
+ server.start();
+
+ ManagedChannel channel =
+ InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build();
+ try {
+ BeamFnLoggingClient client = new BeamFnLoggingClient(
+ PipelineOptionsFactory.fromArgs(new String[] {
+ "--defaultSdkHarnessLogLevel=OFF",
+ "--sdkHarnessLogLevelOverrides={\"ConfiguredLogger\": \"DEBUG\"}"
+ }).create(),
+ apiServiceDescriptor,
+ (Endpoints.ApiServiceDescriptor descriptor) -> channel);
+
+ client.close();
+ } finally {
+ // Verify that after close, log levels are reset.
+ assertEquals(Level.INFO, LogManager.getLogManager().getLogger("").getLevel());
+ assertNull(LogManager.getLogManager().getLogger("ConfiguredLogger").getLevel());
+
+ assertTrue(channel.isShutdown());
+
+ server.shutdownNow();
+ }
}
}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
new file mode 100644
index 0000000..6d3e078
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link BagUserState}. */
+@RunWith(JUnit4.class)
+public class BagUserStateTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testGet() throws Exception {
+ FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
+ key("A"), encode("A1", "A2", "A3")));
+ BagUserState<String> userState =
+ new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));
+ assertArrayEquals(new String[]{ "A1", "A2", "A3" },
+ Iterables.toArray(userState.get(), String.class));
+
+ userState.asyncClose();
+ thrown.expect(IllegalStateException.class);
+ userState.get();
+ }
+
+ @Test
+ public void testAppend() throws Exception {
+ FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
+ key("A"), encode("A1")));
+ BagUserState<String> userState =
+ new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));
+ userState.append("A2");
+ userState.append("A3");
+ userState.asyncClose();
+
+ assertEquals(encode("A1", "A2", "A3"), fakeClient.getData().get(key("A")));
+ thrown.expect(IllegalStateException.class);
+ userState.append("A4");
+ }
+
+ @Test
+ public void testClear() throws Exception {
+ FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
+ key("A"), encode("A1", "A2", "A3")));
+ BagUserState<String> userState =
+ new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));
+
+ userState.clear();
+ userState.append("A1");
+ userState.clear();
+ userState.asyncClose();
+
+ assertNull(fakeClient.getData().get(key("A")));
+ thrown.expect(IllegalStateException.class);
+ userState.clear();
+ }
+
+ private StateRequest.Builder requestForId(String id) {
+ return StateRequest.newBuilder().setStateKey(
+ StateKey.newBuilder().setBagUserState(
+ StateKey.BagUserState.newBuilder().setKey(ByteString.copyFromUtf8(id))));
+ }
+
+ private StateKey key(String id) {
+ return StateKey.newBuilder().setBagUserState(
+ StateKey.BagUserState.newBuilder().setKey(ByteString.copyFromUtf8(id))).build();
+ }
+
+ private ByteString encode(String ... values) throws IOException {
+ ByteString.Output out = ByteString.newOutput();
+ for (String value : values) {
+ StringUtf8Coder.of().encode(value, out);
+ }
+ return out.toByteString();
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
new file mode 100644
index 0000000..12c9c43
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import io.grpc.ManagedChannel;
+import io.grpc.Server;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
+import io.grpc.stub.CallStreamObserver;
+import io.grpc.stub.StreamObserver;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Function;
+import org.apache.beam.fn.harness.IdGenerator;
+import org.apache.beam.harness.test.TestStreams;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link BeamFnStateGrpcClientCache}. */
+@RunWith(JUnit4.class)
+public class BeamFnStateGrpcClientCacheTest {
+ private static final String SUCCESS = "SUCCESS";
+ private static final String FAIL = "FAIL";
+ private static final String TEST_ERROR = "TEST ERROR";
+ private static final String SERVER_ERROR = "SERVER ERROR";
+
+ private Endpoints.ApiServiceDescriptor apiServiceDescriptor;
+ private ManagedChannel testChannel;
+ private Server testServer;
+ private BeamFnStateGrpcClientCache clientCache;
+ private BlockingQueue<StreamObserver<StateResponse>> outboundServerObservers;
+ private BlockingQueue<StateRequest> values;
+
+ @Before
+ public void setUp() throws Exception {
+ values = new LinkedBlockingQueue<>();
+ outboundServerObservers = new LinkedBlockingQueue<>();
+ CallStreamObserver<StateRequest> inboundServerObserver =
+ TestStreams.withOnNext(values::add).build();
+
+ apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
+ .setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
+ .build();
+ testServer = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
+ .addService(new BeamFnStateGrpc.BeamFnStateImplBase() {
+ @Override
+ public StreamObserver<StateRequest> state(
+ StreamObserver<StateResponse> outboundObserver) {
+ Uninterruptibles.putUninterruptibly(outboundServerObservers, outboundObserver);
+ return inboundServerObserver;
+ }
+ })
+ .build();
+ testServer.start();
+
+ testChannel = InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build();
+
+ clientCache = new BeamFnStateGrpcClientCache(
+ PipelineOptionsFactory.create(),
+ IdGenerator::generate,
+ (Endpoints.ApiServiceDescriptor descriptor) -> testChannel,
+ this::createStreamForTest);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ testServer.shutdownNow();
+ testChannel.shutdownNow();
+ }
+
+ @Test
+ public void testCachingOfClient() throws Exception {
+ assertSame(clientCache.forApiServiceDescriptor(apiServiceDescriptor),
+ clientCache.forApiServiceDescriptor(apiServiceDescriptor));
+ assertNotSame(clientCache.forApiServiceDescriptor(apiServiceDescriptor),
+ clientCache.forApiServiceDescriptor(
+ Endpoints.ApiServiceDescriptor.getDefaultInstance()));
+ }
+
+ @Test
+ public void testRequestResponses() throws Exception {
+ BeamFnStateClient client = clientCache.forApiServiceDescriptor(apiServiceDescriptor);
+
+ CompletableFuture<StateResponse> successfulResponse = new CompletableFuture<>();
+ CompletableFuture<StateResponse> unsuccessfulResponse = new CompletableFuture<>();
+
+ client.handle(
+ StateRequest.newBuilder().setInstructionReference(SUCCESS), successfulResponse);
+ client.handle(
+ StateRequest.newBuilder().setInstructionReference(FAIL), unsuccessfulResponse);
+
+ // Wait for the client to connect.
+ StreamObserver<StateResponse> outboundServerObserver = outboundServerObservers.take();
+ // Ensure the client doesn't break when sent garbage.
+ outboundServerObserver.onNext(StateResponse.newBuilder().setId("UNKNOWN ID").build());
+
+ // We expect to receive and handle two requests
+ handleServerRequest(outboundServerObserver, values.take());
+ handleServerRequest(outboundServerObserver, values.take());
+
+ // Ensure that the successful and unsuccessful responses were propagated.
+ assertNotNull(successfulResponse.get());
+ try {
+ unsuccessfulResponse.get();
+ fail("Expected unsuccessful response");
+ } catch (ExecutionException e) {
+ assertThat(e.toString(), containsString(TEST_ERROR));
+ }
+ }
+
+ @Test
+ public void testServerErrorCausesPendingAndFutureCallsToFail() throws Exception {
+ BeamFnStateClient client = clientCache.forApiServiceDescriptor(apiServiceDescriptor);
+
+ CompletableFuture<StateResponse> inflight = new CompletableFuture<>();
+ client.handle(StateRequest.newBuilder().setInstructionReference(SUCCESS), inflight);
+
+ // Wait for the client to connect.
+ StreamObserver<StateResponse> outboundServerObserver = outboundServerObservers.take();
+ // Send an error from the server.
+ outboundServerObserver.onError(
+ new StatusRuntimeException(Status.INTERNAL.withDescription(SERVER_ERROR)));
+
+ try {
+ inflight.get();
+ fail("Expected unsuccessful response due to server error");
+ } catch (ExecutionException e) {
+ assertThat(e.toString(), containsString(SERVER_ERROR));
+ }
+
+ // Send a response after the client will have received an error.
+ CompletableFuture<StateResponse> late = new CompletableFuture<>();
+ client.handle(StateRequest.newBuilder().setInstructionReference(SUCCESS), late);
+
+ try {
+ inflight.get();
+ fail("Expected unsuccessful response due to server error");
+ } catch (ExecutionException e) {
+ assertThat(e.toString(), containsString(SERVER_ERROR));
+ }
+ }
+
+ @Test
+ public void testServerCompletionCausesPendingAndFutureCallsToFail() throws Exception {
+ BeamFnStateClient client = clientCache.forApiServiceDescriptor(apiServiceDescriptor);
+
+ CompletableFuture<StateResponse> inflight = new CompletableFuture<>();
+ client.handle(StateRequest.newBuilder().setInstructionReference(SUCCESS), inflight);
+
+ // Wait for the client to connect.
+ StreamObserver<StateResponse> outboundServerObserver = outboundServerObservers.take();
+ // Send that the server is done.
+ outboundServerObserver.onCompleted();
+
+ try {
+ inflight.get();
+ fail("Expected unsuccessful response due to server completion");
+ } catch (ExecutionException e) {
+ assertThat(e.toString(), containsString("Server hanged up"));
+ }
+
+ // Send a response after the client will have received an error.
+ CompletableFuture<StateResponse> late = new CompletableFuture<>();
+ client.handle(StateRequest.newBuilder().setInstructionReference(SUCCESS), late);
+
+ try {
+ inflight.get();
+ fail("Expected unsuccessful response due to server completion");
+ } catch (ExecutionException e) {
+ assertThat(e.toString(), containsString("Server hanged up"));
+ }
+ }
+
+ private void handleServerRequest(
+ StreamObserver<StateResponse> outboundObserver, StateRequest value) {
+ switch (value.getInstructionReference()) {
+ case SUCCESS:
+ outboundObserver.onNext(StateResponse.newBuilder().setId(value.getId()).build());
+ return;
+ case FAIL:
+ outboundObserver.onNext(StateResponse.newBuilder()
+ .setId(value.getId())
+ .setError(TEST_ERROR)
+ .build());
+ return;
+ default:
+ outboundObserver.onNext(StateResponse.newBuilder().setId(value.getId()).build());
+ return;
+ }
+ }
+
+ private <ReqT, RespT> StreamObserver<RespT> createStreamForTest(
+ Function<StreamObserver<ReqT>, StreamObserver<RespT>> clientFactory,
+ StreamObserver<ReqT> handler) {
+ return clientFactory.apply(handler);
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
new file mode 100644
index 0000000..e991db6
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import com.google.protobuf.ByteString;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateClearResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey.TypeCase;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.RequestCase;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+
+/** A fake implementation of a {@link BeamFnStateClient} to aid with testing. */
+public class FakeBeamFnStateClient implements BeamFnStateClient {
+ private final Map<StateKey, ByteString> data;
+ private int currentId;
+
+ public FakeBeamFnStateClient(Map<StateKey, ByteString> initialData) {
+ this.data = new ConcurrentHashMap<>(initialData);
+ }
+
+ public Map<StateKey, ByteString> getData() {
+ return Collections.unmodifiableMap(data);
+ }
+
+ @Override
+ public void handle(StateRequest.Builder requestBuilder,
+ CompletableFuture<StateResponse> responseFuture) {
+ // The id should never be filled out
+ assertEquals("", requestBuilder.getId());
+ requestBuilder.setId(generateId());
+
+ StateRequest request = requestBuilder.build();
+ StateKey key = request.getStateKey();
+ StateResponse.Builder response;
+
+ assertNotEquals(RequestCase.REQUEST_NOT_SET, request.getRequestCase());
+ assertNotEquals(TypeCase.TYPE_NOT_SET, key.getTypeCase());
+ // multimap side input and runner based state keys only support get requests
+ if (key.getTypeCase() == TypeCase.MULTIMAP_SIDE_INPUT
+ || key.getTypeCase() == TypeCase.RUNNER) {
+ assertEquals(RequestCase.GET, request.getRequestCase());
+ }
+
+ switch (request.getRequestCase()) {
+ case GET:
+ // Chunk gets into 5 byte return blocks
+ ByteString byteString = data.getOrDefault(request.getStateKey(), ByteString.EMPTY);
+ int block = 0;
+ if (request.getGet().getContinuationToken().size() > 0) {
+ block = Integer.parseInt(request.getGet().getContinuationToken().toStringUtf8());
+ }
+ ByteString returnBlock = byteString.substring(
+ block * 5, Math.min(byteString.size(), (block + 1) * 5));
+ ByteString continuationToken = ByteString.EMPTY;
+ if (byteString.size() > (block + 1) * 5) {
+ continuationToken = ByteString.copyFromUtf8(Integer.toString(block + 1));
+ }
+ response = StateResponse.newBuilder().setGet(StateGetResponse.newBuilder()
+ .setData(returnBlock)
+ .setContinuationToken(continuationToken));
+ break;
+
+ case CLEAR:
+ data.remove(request.getStateKey());
+ response = StateResponse.newBuilder().setClear(StateClearResponse.getDefaultInstance());
+ break;
+
+ case APPEND:
+ ByteString previousValue = data.getOrDefault(request.getStateKey(), ByteString.EMPTY);
+ data.put(request.getStateKey(), previousValue.concat(request.getAppend().getData()));
+ response = StateResponse.newBuilder().setAppend(StateAppendResponse.getDefaultInstance());
+ break;
+
+ default:
+ throw new IllegalStateException(
+ String.format("Unknown request type %s", request.getRequestCase()));
+ }
+
+ responseFuture.complete(response.setId(requestBuilder.getId()).build());
+ }
+
+ private String generateId() {
+ return Integer.toString(++currentId);
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java
new file mode 100644
index 0000000..53eefb4
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link LazyCachingIteratorToIterable}. */
+@RunWith(JUnit4.class)
+public class LazyCachingIteratorToIterableTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testEmptyIterator() {
+ Iterable<Object> iterable = new LazyCachingIteratorToIterable<>(Iterators.forArray());
+ assertArrayEquals(new Object[0], Iterables.toArray(iterable, Object.class));
+ // iterate multiple times
+ assertArrayEquals(new Object[0], Iterables.toArray(iterable, Object.class));
+
+ thrown.expect(NoSuchElementException.class);
+ iterable.iterator().next();
+ }
+
+ @Test
+ public void testInterleavedIteration() {
+ Iterable<String> iterable =
+ new LazyCachingIteratorToIterable<>(Iterators.forArray("A", "B", "C"));
+
+ Iterator<String> iterator1 = iterable.iterator();
+ assertTrue(iterator1.hasNext());
+ assertEquals("A", iterator1.next());
+ Iterator<String> iterator2 = iterable.iterator();
+ assertTrue(iterator2.hasNext());
+ assertEquals("A", iterator2.next());
+ assertTrue(iterator2.hasNext());
+ assertEquals("B", iterator2.next());
+ assertTrue(iterator1.hasNext());
+ assertEquals("B", iterator1.next());
+ assertTrue(iterator1.hasNext());
+ assertEquals("C", iterator1.next());
+ assertFalse(iterator1.hasNext());
+ assertTrue(iterator2.hasNext());
+ assertEquals("C", iterator2.next());
+ assertFalse(iterator2.hasNext());
+
+ thrown.expect(NoSuchElementException.class);
+ iterator1.next();
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
new file mode 100644
index 0000000..0c2f922
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.state;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import com.google.common.collect.Iterators;
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import java.util.concurrent.CompletableFuture;
+import org.apache.beam.fn.harness.state.StateFetchingIterators.LazyBlockingStateFetchingIterator;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link StateFetchingIterators}. */
+@RunWith(Enclosed.class)
+public class StateFetchingIteratorsTest {
+ /** Tests for {@link StateFetchingIterators.LazyBlockingStateFetchingIterator}. */
+ @RunWith(JUnit4.class)
+ public static class LazyBlockingStateFetchingIteratorTest {
+
+ @Test
+ public void testEmpty() throws Exception {
+ testFetch(ByteString.EMPTY);
+ }
+
+ @Test
+ public void testNonEmpty() throws Exception {
+ testFetch(ByteString.copyFromUtf8("A"));
+ }
+
+ @Test
+ public void testWithLastByteStringBeingEmpty() throws Exception {
+ testFetch(ByteString.copyFromUtf8("A"), ByteString.EMPTY);
+ }
+
+ @Test
+ public void testMulti() throws Exception {
+ testFetch(ByteString.copyFromUtf8("BC"), ByteString.copyFromUtf8("DEF"));
+ }
+
+ @Test
+ public void testMultiWithEmptyByteStrings() throws Exception {
+ testFetch(ByteString.EMPTY, ByteString.copyFromUtf8("BC"), ByteString.EMPTY,
+ ByteString.EMPTY, ByteString.copyFromUtf8("DEF"), ByteString.EMPTY);
+ }
+
+ private void testFetch(ByteString... expected) {
+ BeamFnStateClient fakeStateClient = new BeamFnStateClient() {
+ @Override
+ public void handle(
+ StateRequest.Builder requestBuilder, CompletableFuture<StateResponse> response) {
+ ByteString continuationToken = requestBuilder.getGet().getContinuationToken();
+ StateGetResponse.Builder builder = StateGetResponse.newBuilder();
+
+ int requestedPosition = 0; // Default position is 0
+ if (!ByteString.EMPTY.equals(continuationToken)) {
+ requestedPosition = Integer.parseInt(continuationToken.toStringUtf8());
+ }
+
+ // Compute the new continuation token
+ ByteString newContinuationToken = ByteString.EMPTY;
+ if (requestedPosition != expected.length - 1) {
+ newContinuationToken = ByteString.copyFromUtf8(Integer.toString(requestedPosition + 1));
+ }
+ response.complete(StateResponse.newBuilder()
+ .setId(requestBuilder.getId())
+ .setGet(StateGetResponse.newBuilder()
+ .setData(expected[requestedPosition])
+ .setContinuationToken(newContinuationToken))
+ .build());
+ }
+ };
+ Iterator<ByteString> byteStrings =
+ new LazyBlockingStateFetchingIterator(fakeStateClient, StateRequest::newBuilder);
+ assertArrayEquals(expected, Iterators.toArray(byteStrings, Object.class));
+ }
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/BufferingStreamObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/BufferingStreamObserverTest.java
index 76b7ef0..96648e9 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/BufferingStreamObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/BufferingStreamObserverTest.java
@@ -31,10 +31,10 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
-import org.apache.beam.fn.harness.test.TestExecutors;
-import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.harness.test.TestStreams;
+import org.apache.beam.harness.test.Consumer;
+import org.apache.beam.harness.test.TestExecutors;
+import org.apache.beam.harness.test.TestExecutors.TestExecutorService;
+import org.apache.beam.harness.test.TestStreams;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -61,7 +61,7 @@
// critical section. Any thread that enters purposefully blocks by sleeping
// to increase the contention between threads artificially.
assertFalse(isCriticalSectionShared.getAndSet(true));
- Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
onNextValues.add(t);
assertTrue(isCriticalSectionShared.getAndSet(false));
}
@@ -134,7 +134,7 @@
}
// Have them wait and then flip that we do allow elements and wake up those awaiting
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
elementsAllowed.set(true);
phaser.arrive();
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DataStreamsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DataStreamsTest.java
new file mode 100644
index 0000000..f7a87e1
--- /dev/null
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DataStreamsTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.stream;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.common.collect.Iterators;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CountingOutputStream;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.SynchronousQueue;
+import org.apache.beam.fn.harness.stream.DataStreams.BlockingQueueIterator;
+import org.apache.beam.fn.harness.stream.DataStreams.DataStreamDecoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link DataStreams}. */
+@RunWith(Enclosed.class)
+public class DataStreamsTest {
+
+ /** Tests for {@link DataStreams.Inbound}. */
+ @RunWith(JUnit4.class)
+ public static class InboundTest {
+ private static final ByteString BYTES_A = ByteString.copyFromUtf8("TestData");
+ private static final ByteString BYTES_B = ByteString.copyFromUtf8("SomeOtherTestData");
+
+ @Test
+ public void testEmptyRead() throws Exception {
+ assertEquals(ByteString.EMPTY, read());
+ assertEquals(ByteString.EMPTY, read(ByteString.EMPTY));
+ assertEquals(ByteString.EMPTY, read(ByteString.EMPTY, ByteString.EMPTY));
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ assertEquals(BYTES_A.concat(BYTES_B), read(BYTES_A, BYTES_B));
+ assertEquals(BYTES_A.concat(BYTES_B), read(BYTES_A, ByteString.EMPTY, BYTES_B));
+ assertEquals(BYTES_A.concat(BYTES_B), read(BYTES_A, BYTES_B, ByteString.EMPTY));
+ }
+
+ private static ByteString read(ByteString... bytes) throws IOException {
+ return ByteString.readFrom(DataStreams.inbound(Arrays.asList(bytes).iterator()));
+ }
+ }
+
+ /** Tests for {@link DataStreams.BlockingQueueIterator}. */
+ @RunWith(JUnit4.class)
+ public static class BlockingQueueIteratorTest {
+ @Test(timeout = 10_000)
+ public void testBlockingQueueIteratorWithoutBlocking() throws Exception {
+ BlockingQueueIterator<String> iterator =
+ new BlockingQueueIterator<>(new ArrayBlockingQueue<>(3));
+
+ iterator.accept("A");
+ iterator.accept("B");
+ iterator.close();
+
+ assertEquals(Arrays.asList("A", "B"),
+ Arrays.asList(Iterators.toArray(iterator, String.class)));
+ }
+
+ @Test(timeout = 10_000)
+ public void testBlockingQueueIteratorWithBlocking() throws Exception {
+ // The synchronous queue only allows for one element to transfer at a time and blocks
+ // the sending/receiving parties until both parties are there.
+ final BlockingQueueIterator<String> iterator =
+ new BlockingQueueIterator<>(new SynchronousQueue<>());
+ final CompletableFuture<List<String>> valuesFuture = new CompletableFuture<>();
+ Thread appender = new Thread() {
+ @Override
+ public void run() {
+ valuesFuture.complete(Arrays.asList(Iterators.toArray(iterator, String.class)));
+ }
+ };
+ appender.start();
+ iterator.accept("A");
+ iterator.accept("B");
+ iterator.close();
+ assertEquals(Arrays.asList("A", "B"), valuesFuture.get());
+ appender.join();
+ }
+ }
+
+ /** Tests for {@link DataStreams.DataStreamDecoder}. */
+ @RunWith(JUnit4.class)
+ public static class DataStreamDecoderTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testEmptyInputStream() throws Exception {
+ testDecoderWith(StringUtf8Coder.of());
+ }
+
+ @Test
+ public void testNonEmptyInputStream() throws Exception {
+ testDecoderWith(StringUtf8Coder.of(), "A", "BC", "DEF", "GHIJ");
+ }
+
+ @Test
+ public void testNonEmptyInputStreamWithZeroLengthCoder() throws Exception {
+ CountingOutputStream countingOutputStream =
+ new CountingOutputStream(ByteStreams.nullOutputStream());
+ GlobalWindow.Coder.INSTANCE.encode(GlobalWindow.INSTANCE, countingOutputStream);
+ assumeTrue(countingOutputStream.getCount() == 0);
+
+ testDecoderWith(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, GlobalWindow.INSTANCE);
+ }
+
+ private <T> void testDecoderWith(Coder<T> coder, T... expected) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (T value : expected) {
+ int size = baos.size();
+ coder.encode(value, baos);
+ // Pad an arbitrary byte when values encode to zero bytes
+ if (baos.size() - size == 0) {
+ baos.write(0);
+ }
+ }
+
+ Iterator<T> decoder =
+ new DataStreamDecoder<>(coder, new ByteArrayInputStream(baos.toByteArray()));
+
+ Object[] actual = Iterators.toArray(decoder, Object.class);
+ assertArrayEquals(expected, actual);
+
+ assertFalse(decoder.hasNext());
+ assertFalse(decoder.hasNext());
+
+ thrown.expect(NoSuchElementException.class);
+ decoder.next();
+ }
+ }
+}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DirectStreamObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DirectStreamObserverTest.java
index b5d3ec1..05d8d5a 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DirectStreamObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DirectStreamObserverTest.java
@@ -31,10 +31,10 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
-import org.apache.beam.fn.harness.test.TestExecutors;
-import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.harness.test.TestStreams;
+import org.apache.beam.harness.test.Consumer;
+import org.apache.beam.harness.test.TestExecutors;
+import org.apache.beam.harness.test.TestExecutors.TestExecutorService;
+import org.apache.beam.harness.test.TestStreams;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutors.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutors.java
deleted file mode 100644
index f846466..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutors.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.test;
-
-import com.google.common.util.concurrent.ForwardingExecutorService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-/**
- * A {@link TestRule} that validates that all submitted tasks finished and were completed. This
- * allows for testing that tasks have exercised the appropriate shutdown logic.
- */
-public class TestExecutors {
- public static TestExecutorService from(Supplier<ExecutorService> executorServiceSuppler) {
- return new FromSupplier(executorServiceSuppler);
- }
-
- /** A union of the {@link ExecutorService} and {@link TestRule} interfaces. */
- public interface TestExecutorService extends ExecutorService, TestRule {}
-
- private static class FromSupplier extends ForwardingExecutorService
- implements TestExecutorService {
- private final Supplier<ExecutorService> executorServiceSupplier;
- private ExecutorService delegate;
-
- private FromSupplier(Supplier<ExecutorService> executorServiceSupplier) {
- this.executorServiceSupplier = executorServiceSupplier;
- }
-
- @Override
- public Statement apply(Statement statement, Description arg1) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- Throwable thrown = null;
- delegate = executorServiceSupplier.get();
- try {
- statement.evaluate();
- } catch (Throwable t) {
- thrown = t;
- }
- shutdown();
- if (!awaitTermination(5, TimeUnit.SECONDS)) {
- shutdownNow();
- IllegalStateException e =
- new IllegalStateException("Test executor failed to shutdown cleanly.");
- if (thrown != null) {
- thrown.addSuppressed(e);
- } else {
- thrown = e;
- }
- }
- if (thrown != null) {
- throw thrown;
- }
- }
- };
- }
-
- @Override
- protected ExecutorService delegate() {
- return delegate;
- }
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutorsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutorsTest.java
deleted file mode 100644
index 85c64d0..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutorsTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.junit.runners.model.Statement;
-
-/** Tests for {@link TestExecutors}. */
-@RunWith(JUnit4.class)
-public class TestExecutorsTest {
- @Test
- public void testSuccessfulTermination() throws Throwable {
- ExecutorService service = Executors.newSingleThreadExecutor();
- final TestExecutorService testService = TestExecutors.from(() -> service);
- final AtomicBoolean taskRan = new AtomicBoolean();
- testService
- .apply(
- new Statement() {
- @Override
- public void evaluate() throws Throwable {
- testService.submit(() -> taskRan.set(true));
- }
- },
- null)
- .evaluate();
- assertTrue(service.isTerminated());
- assertTrue(taskRan.get());
- }
-
- @Test
- public void testTaskBlocksForeverCausesFailure() throws Throwable {
- ExecutorService service = Executors.newSingleThreadExecutor();
- final TestExecutorService testService = TestExecutors.from(() -> service);
- final AtomicBoolean taskStarted = new AtomicBoolean();
- final AtomicBoolean taskWasInterrupted = new AtomicBoolean();
- try {
- testService
- .apply(
- new Statement() {
- @Override
- public void evaluate() throws Throwable {
- testService.submit(this::taskToRun);
- }
-
- private void taskToRun() {
- taskStarted.set(true);
- try {
- while (true) {
- Thread.sleep(10000);
- }
- } catch (InterruptedException e) {
- taskWasInterrupted.set(true);
- return;
- }
- }
- },
- null)
- .evaluate();
- fail();
- } catch (IllegalStateException e) {
- assertEquals(IllegalStateException.class, e.getClass());
- assertEquals("Test executor failed to shutdown cleanly.", e.getMessage());
- }
- assertTrue(service.isShutdown());
- }
-
- @Test
- public void testStatementFailurePropagatedCleanly() throws Throwable {
- ExecutorService service = Executors.newSingleThreadExecutor();
- final TestExecutorService testService = TestExecutors.from(() -> service);
- final RuntimeException exceptionToThrow = new RuntimeException();
- try {
- testService
- .apply(
- new Statement() {
- @Override
- public void evaluate() throws Throwable {
- throw exceptionToThrow;
- }
- },
- null)
- .evaluate();
- fail();
- } catch (RuntimeException thrownException) {
- assertSame(exceptionToThrow, thrownException);
- }
- assertTrue(service.isShutdown());
- }
-
- @Test
- public void testStatementFailurePropagatedWhenExecutorServiceFailingToTerminate()
- throws Throwable {
- ExecutorService service = Executors.newSingleThreadExecutor();
- final TestExecutorService testService = TestExecutors.from(() -> service);
- final AtomicBoolean taskStarted = new AtomicBoolean();
- final AtomicBoolean taskWasInterrupted = new AtomicBoolean();
- final RuntimeException exceptionToThrow = new RuntimeException();
- try {
- testService
- .apply(
- new Statement() {
- @Override
- public void evaluate() throws Throwable {
- testService.submit(this::taskToRun);
- throw exceptionToThrow;
- }
-
- private void taskToRun() {
- taskStarted.set(true);
- try {
- while (true) {
- Thread.sleep(10000);
- }
- } catch (InterruptedException e) {
- taskWasInterrupted.set(true);
- return;
- }
- }
- },
- null)
- .evaluate();
- fail();
- } catch (RuntimeException thrownException) {
- assertSame(exceptionToThrow, thrownException);
- assertEquals(1, exceptionToThrow.getSuppressed().length);
- assertEquals(IllegalStateException.class, exceptionToThrow.getSuppressed()[0].getClass());
- assertEquals(
- "Test executor failed to shutdown cleanly.",
- exceptionToThrow.getSuppressed()[0].getMessage());
- }
- assertTrue(service.isShutdown());
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreams.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreams.java
deleted file mode 100644
index f398286..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreams.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.test;
-
-import io.grpc.stub.CallStreamObserver;
-import io.grpc.stub.StreamObserver;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-/** Utility methods which enable testing of {@link StreamObserver}s. */
-public class TestStreams {
- /**
- * Creates a test {@link CallStreamObserver} {@link Builder} that forwards
- * {@link StreamObserver#onNext} calls to the supplied {@link Consumer}.
- */
- public static <T> Builder<T> withOnNext(Consumer<T> onNext) {
- return new Builder<>(new ForwardingCallStreamObserver<>(
- onNext,
- TestStreams::noop,
- TestStreams::noop,
- TestStreams::returnTrue));
- }
-
- /** A builder for a test {@link CallStreamObserver} that performs various callbacks. */
- public static class Builder<T> {
- private final ForwardingCallStreamObserver<T> observer;
- private Builder(ForwardingCallStreamObserver<T> observer) {
- this.observer = observer;
- }
-
- /**
- * Returns a new {@link Builder} like this one with the specified
- * {@link CallStreamObserver#isReady} callback.
- */
- public Builder<T> withIsReady(Supplier<Boolean> isReady) {
- return new Builder<>(new ForwardingCallStreamObserver<>(
- observer.onNext,
- observer.onError,
- observer.onCompleted,
- isReady));
- }
-
- /**
- * Returns a new {@link Builder} like this one with the specified
- * {@link StreamObserver#onCompleted} callback.
- */
- public Builder<T> withOnCompleted(Runnable onCompleted) {
- return new Builder<>(new ForwardingCallStreamObserver<>(
- observer.onNext,
- observer.onError,
- onCompleted,
- observer.isReady));
- }
-
- /**
- * Returns a new {@link Builder} like this one with the specified
- * {@link StreamObserver#onError} callback.
- */
- public Builder<T> withOnError(Runnable onError) {
- return new Builder<>(new ForwardingCallStreamObserver<>(
- observer.onNext,
- new Consumer<Throwable>() {
- @Override
- public void accept(Throwable t) {
- onError.run();
- }
- },
- observer.onCompleted,
- observer.isReady));
- }
-
- /**
- * Returns a new {@link Builder} like this one with the specified
- * {@link StreamObserver#onError} consumer.
- */
- public Builder<T> withOnError(Consumer<Throwable> onError) {
- return new Builder<>(new ForwardingCallStreamObserver<>(
- observer.onNext, onError, observer.onCompleted, observer.isReady));
- }
-
- public CallStreamObserver<T> build() {
- return observer;
- }
- }
-
- private static void noop() {
- }
-
- private static void noop(Throwable t) {
- }
-
- private static boolean returnTrue() {
- return true;
- }
-
- /** A {@link CallStreamObserver} which executes the supplied callbacks. */
- private static class ForwardingCallStreamObserver<T> extends CallStreamObserver<T> {
- private final Consumer<T> onNext;
- private final Supplier<Boolean> isReady;
- private final Consumer<Throwable> onError;
- private final Runnable onCompleted;
-
- public ForwardingCallStreamObserver(
- Consumer<T> onNext,
- Consumer<Throwable> onError,
- Runnable onCompleted,
- Supplier<Boolean> isReady) {
- this.onNext = onNext;
- this.onError = onError;
- this.onCompleted = onCompleted;
- this.isReady = isReady;
- }
-
- @Override
- public void onNext(T value) {
- onNext.accept(value);
- }
-
- @Override
- public void onError(Throwable t) {
- onError.accept(t);
- }
-
- @Override
- public void onCompleted() {
- onCompleted.run();
- }
-
- @Override
- public boolean isReady() {
- return isReady.get();
- }
-
- @Override
- public void setOnReadyHandler(Runnable onReadyHandler) {}
-
- @Override
- public void disableAutoInboundFlowControl() {}
-
- @Override
- public void request(int count) {}
-
- @Override
- public void setMessageCompression(boolean enable) {}
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreamsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreamsTest.java
deleted file mode 100644
index b684c90..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreamsTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.fn.harness.test;
-
-import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link TestStreams}. */
-@RunWith(JUnit4.class)
-public class TestStreamsTest {
- @Test
- public void testOnNextIsCalled() {
- AtomicBoolean onNextWasCalled = new AtomicBoolean();
- TestStreams.withOnNext(onNextWasCalled::set).build().onNext(true);
- assertTrue(onNextWasCalled.get());
- }
-
- @Test
- public void testIsReadyIsCalled() {
- final AtomicBoolean isReadyWasCalled = new AtomicBoolean();
- assertFalse(TestStreams.withOnNext(null)
- .withIsReady(() -> isReadyWasCalled.getAndSet(true))
- .build()
- .isReady());
- assertTrue(isReadyWasCalled.get());
- }
-
- @Test
- public void testOnCompletedIsCalled() {
- AtomicBoolean onCompletedWasCalled = new AtomicBoolean();
- TestStreams.withOnNext(null)
- .withOnCompleted(() -> onCompletedWasCalled.set(true))
- .build()
- .onCompleted();
- assertTrue(onCompletedWasCalled.get());
- }
-
- @Test
- public void testOnErrorRunnableIsCalled() {
- RuntimeException throwable = new RuntimeException();
- AtomicBoolean onErrorWasCalled = new AtomicBoolean();
- TestStreams.withOnNext(null)
- .withOnError(() -> onErrorWasCalled.set(true))
- .build()
- .onError(throwable);
- assertTrue(onErrorWasCalled.get());
- }
-
- @Test
- public void testOnErrorConsumerIsCalled() {
- RuntimeException throwable = new RuntimeException();
- Collection<Throwable> onErrorWasCalled = new ArrayList<>();
- TestStreams.withOnNext(null)
- .withOnError(onErrorWasCalled::add)
- .build()
- .onError(throwable);
- assertThat(onErrorWasCalled, contains(throwable));
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
deleted file mode 100644
index a3d4a1b..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
-import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.harness.test.TestExecutors;
-import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-/** Tests for {@link BeamFnDataReadRunner}. */
-@RunWith(JUnit4.class)
-public class BeamFnDataReadRunnerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
- .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
- private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()
- .setData(Any.pack(PORT_SPEC)).build();
- private static final Coder<WindowedValue<String>> CODER =
- WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
- private static final BeamFnApi.Coder CODER_SPEC;
- static {
- try {
- CODER_SPEC = BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
- .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
- OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(CODER)))).build())))
- .build();
- } catch (IOException e) {
- throw new ExceptionInInitializerError(e);
- }
- }
- private static final BeamFnApi.Target INPUT_TARGET = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1")
- .setName("out")
- .build();
-
- @Rule public TestExecutorService executor = TestExecutors.from(Executors::newCachedThreadPool);
- @Mock private BeamFnDataClient mockBeamFnDataClientFactory;
- @Captor private ArgumentCaptor<ThrowingConsumer<WindowedValue<String>>> consumerCaptor;
-
- @Before
- public void setUp() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test
- public void testReuseForMultipleBundles() throws Exception {
- CompletableFuture<Void> bundle1Future = new CompletableFuture<>();
- CompletableFuture<Void> bundle2Future = new CompletableFuture<>();
- when(mockBeamFnDataClientFactory.forInboundConsumer(
- any(),
- any(),
- any(),
- any())).thenReturn(bundle1Future).thenReturn(bundle2Future);
- List<WindowedValue<String>> valuesA = new ArrayList<>();
- List<WindowedValue<String>> valuesB = new ArrayList<>();
- Map<String, Collection<ThrowingConsumer<WindowedValue<String>>>> outputMap = ImmutableMap.of(
- "outA", ImmutableList.of(valuesA::add),
- "outB", ImmutableList.of(valuesB::add));
- AtomicReference<String> bundleId = new AtomicReference<>("0");
- BeamFnDataReadRunner<String> readRunner = new BeamFnDataReadRunner<>(
- FUNCTION_SPEC,
- bundleId::get,
- INPUT_TARGET,
- CODER_SPEC,
- mockBeamFnDataClientFactory,
- outputMap);
-
- // Process for bundle id 0
- readRunner.registerInputLocation();
-
- verify(mockBeamFnDataClientFactory).forInboundConsumer(
- eq(PORT_SPEC.getApiServiceDescriptor()),
- eq(KV.of(bundleId.get(), INPUT_TARGET)),
- eq(CODER),
- consumerCaptor.capture());
-
- executor.submit(new Runnable() {
- @Override
- public void run() {
- // Sleep for some small amount of time simulating the parent blocking
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
- try {
- consumerCaptor.getValue().accept(valueInGlobalWindow("ABC"));
- consumerCaptor.getValue().accept(valueInGlobalWindow("DEF"));
- } catch (Exception e) {
- bundle1Future.completeExceptionally(e);
- } finally {
- bundle1Future.complete(null);
- }
- }
- });
-
- readRunner.blockTillReadFinishes();
- assertThat(valuesA, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
- assertThat(valuesB, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
-
- // Process for bundle id 1
- bundleId.set("1");
- valuesA.clear();
- valuesB.clear();
- readRunner.registerInputLocation();
-
- verify(mockBeamFnDataClientFactory).forInboundConsumer(
- eq(PORT_SPEC.getApiServiceDescriptor()),
- eq(KV.of(bundleId.get(), INPUT_TARGET)),
- eq(CODER),
- consumerCaptor.capture());
-
- executor.submit(new Runnable() {
- @Override
- public void run() {
- // Sleep for some small amount of time simulating the parent blocking
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
- try {
- consumerCaptor.getValue().accept(valueInGlobalWindow("GHI"));
- consumerCaptor.getValue().accept(valueInGlobalWindow("JKL"));
- } catch (Exception e) {
- bundle2Future.completeExceptionally(e);
- } finally {
- bundle2Future.complete(null);
- }
- }
- });
-
- readRunner.blockTillReadFinishes();
- assertThat(valuesA, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
- assertThat(valuesB, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
-
- verifyNoMoreInteractions(mockBeamFnDataClientFactory);
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
deleted file mode 100644
index 3383966..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
-import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.beam.fn.harness.data.BeamFnDataClient;
-import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.dataflow.util.CloudObjects;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Matchers;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-/** Tests for {@link BeamFnDataWriteRunner}. */
-@RunWith(JUnit4.class)
-public class BeamFnDataWriteRunnerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
- .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
- private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()
- .setData(Any.pack(PORT_SPEC)).build();
- private static final Coder<WindowedValue<String>> CODER =
- WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
- private static final BeamFnApi.Coder CODER_SPEC;
- static {
- try {
- CODER_SPEC = BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
- .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
- OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(CODER)))).build())))
- .build();
- } catch (IOException e) {
- throw new ExceptionInInitializerError(e);
- }
- }
- private static final BeamFnApi.Target OUTPUT_TARGET = BeamFnApi.Target.newBuilder()
- .setPrimitiveTransformReference("1")
- .setName("out")
- .build();
-
- @Mock private BeamFnDataClient mockBeamFnDataClientFactory;
-
- @Before
- public void setUp() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test
- public void testReuseForMultipleBundles() throws Exception {
- RecordingConsumer<WindowedValue<String>> valuesA = new RecordingConsumer<>();
- RecordingConsumer<WindowedValue<String>> valuesB = new RecordingConsumer<>();
- when(mockBeamFnDataClientFactory.forOutboundConsumer(
- any(),
- any(),
- Matchers.<Coder<WindowedValue<String>>>any())).thenReturn(valuesA).thenReturn(valuesB);
- AtomicReference<String> bundleId = new AtomicReference<>("0");
- BeamFnDataWriteRunner<String> writeRunner = new BeamFnDataWriteRunner<>(
- FUNCTION_SPEC,
- bundleId::get,
- OUTPUT_TARGET,
- CODER_SPEC,
- mockBeamFnDataClientFactory);
-
- // Process for bundle id 0
- writeRunner.registerForOutput();
-
- verify(mockBeamFnDataClientFactory).forOutboundConsumer(
- eq(PORT_SPEC.getApiServiceDescriptor()),
- eq(KV.of(bundleId.get(), OUTPUT_TARGET)),
- eq(CODER));
-
- writeRunner.consume(valueInGlobalWindow("ABC"));
- writeRunner.consume(valueInGlobalWindow("DEF"));
- writeRunner.close();
-
- assertTrue(valuesA.closed);
- assertThat(valuesA, contains(valueInGlobalWindow("ABC"), valueInGlobalWindow("DEF")));
-
- // Process for bundle id 1
- bundleId.set("1");
- valuesA.clear();
- valuesB.clear();
- writeRunner.registerForOutput();
-
- verify(mockBeamFnDataClientFactory).forOutboundConsumer(
- eq(PORT_SPEC.getApiServiceDescriptor()),
- eq(KV.of(bundleId.get(), OUTPUT_TARGET)),
- eq(CODER));
-
- writeRunner.consume(valueInGlobalWindow("GHI"));
- writeRunner.consume(valueInGlobalWindow("JKL"));
- writeRunner.close();
-
- assertTrue(valuesB.closed);
- assertThat(valuesB, contains(valueInGlobalWindow("GHI"), valueInGlobalWindow("JKL")));
- verifyNoMoreInteractions(mockBeamFnDataClientFactory);
- }
-
- private static class RecordingConsumer<T> extends ArrayList<T>
- implements CloseableThrowingConsumer<T> {
- private boolean closed;
- @Override
- public void close() throws Exception {
- closed = true;
- }
-
- @Override
- public void accept(T t) throws Exception {
- if (closed) {
- throw new IllegalStateException("Consumer is closed but attempting to consume " + t);
- }
- add(t);
- }
-
- }
-}
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BoundedSourceRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BoundedSourceRunnerTest.java
deleted file mode 100644
index 73860ef..0000000
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BoundedSourceRunnerTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core;
-
-import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.collection.IsEmptyCollection.empty;
-import static org.junit.Assert.assertThat;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.BytesValue;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.CountingSource;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link BoundedSourceRunner}. */
-@RunWith(JUnit4.class)
-public class BoundedSourceRunnerTest {
- @Test
- public void testRunReadLoopWithMultipleSources() throws Exception {
- List<WindowedValue<Long>> out1ValuesA = new ArrayList<>();
- List<WindowedValue<Long>> out1ValuesB = new ArrayList<>();
- List<WindowedValue<Long>> out2Values = new ArrayList<>();
- Map<String, Collection<ThrowingConsumer<WindowedValue<Long>>>> outputMap = ImmutableMap.of(
- "out1", ImmutableList.of(out1ValuesA::add, out1ValuesB::add),
- "out2", ImmutableList.of(out2Values::add));
-
- BoundedSourceRunner<BoundedSource<Long>, Long> runner =
- new BoundedSourceRunner<>(
- PipelineOptionsFactory.create(),
- BeamFnApi.FunctionSpec.getDefaultInstance(),
- outputMap);
-
- runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(2)));
- runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(1)));
-
- assertThat(out1ValuesA,
- contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(0L)));
- assertThat(out1ValuesB,
- contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(0L)));
- assertThat(out2Values,
- contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(0L)));
- }
-
- @Test
- public void testRunReadLoopWithEmptySource() throws Exception {
- List<WindowedValue<Long>> out1Values = new ArrayList<>();
- Map<String, Collection<ThrowingConsumer<WindowedValue<Long>>>> outputMap = ImmutableMap.of(
- "out1", ImmutableList.of(out1Values::add));
-
- BoundedSourceRunner<BoundedSource<Long>, Long> runner =
- new BoundedSourceRunner<>(
- PipelineOptionsFactory.create(),
- BeamFnApi.FunctionSpec.getDefaultInstance(),
- outputMap);
-
- runner.runReadLoop(valueInGlobalWindow(CountingSource.upTo(0)));
-
- assertThat(out1Values, empty());
- }
-
- @Test
- public void testStart() throws Exception {
- List<WindowedValue<Long>> outValues = new ArrayList<>();
- Map<String, Collection<ThrowingConsumer<WindowedValue<Long>>>> outputMap = ImmutableMap.of(
- "out", ImmutableList.of(outValues::add));
-
- ByteString encodedSource =
- ByteString.copyFrom(SerializableUtils.serializeToByteArray(CountingSource.upTo(3)));
-
- BoundedSourceRunner<BoundedSource<Long>, Long> runner =
- new BoundedSourceRunner<>(
- PipelineOptionsFactory.create(),
- BeamFnApi.FunctionSpec.newBuilder().setData(
- Any.pack(BytesValue.newBuilder().setValue(encodedSource).build())).build(),
- outputMap);
-
- runner.start();
-
- assertThat(outValues,
- contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(2L)));
- }
-}
diff --git a/sdks/java/io/amqp/pom.xml b/sdks/java/io/amqp/pom.xml
new file mode 100644
index 0000000..218c5cd
--- /dev/null
+++ b/sdks/java/io/amqp/pom.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-amqp</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: AMQP</name>
+ <description>IO to read and write using AMQP 1.0 protocol (http://www.amqp.org).</description>
+
+ <properties>
+ <activemq.version>5.13.1</activemq.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.qpid</groupId>
+ <artifactId>proton-j</artifactId>
+ <version>0.13.1</version>
+ </dependency>
+
+ <!-- compile dependencies -->
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-broker</artifactId>
+ <version>${activemq.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-amqp</artifactId>
+ <version>${activemq.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq.tooling</groupId>
+ <artifactId>activemq-junit</artifactId>
+ <version>${activemq.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java
new file mode 100644
index 0000000..d2e059b
--- /dev/null
+++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.amqp;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Joiner;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.qpid.proton.message.Message;
+import org.apache.qpid.proton.messenger.Messenger;
+import org.apache.qpid.proton.messenger.Tracker;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * AmqpIO supports AMQP 1.0 protocol using the Apache QPid Proton-J library.
+ *
+ * <p>It's also possible to use AMQP 1.0 protocol via Apache Qpid JMS connection factory and the
+ * Apache Beam JmsIO.
+ *
+ * <h3>Binding AMQP and receive messages</h3>
+ *
+ * <p>The {@link AmqpIO} {@link Read} can bind a AMQP listener endpoint and receive messages. It can
+ * also connect to a AMPQ broker (such as Apache Qpid or Apache ActiveMQ).
+ *
+ * <p>{@link AmqpIO} {@link Read} returns an unbounded {@link PCollection} of {@link Message}
+ * containing the received messages.
+ *
+ * <p>To configure a AMQP source, you have to provide a list of addresses where it will receive
+ * messages. An address has the following form: {@code
+ * [amqp[s]://][user[:password]@]domain[/[name]]} where {@code domain} can be one of {@code
+ * host | host:port | ip | ip:port | name}. NB: the {@code ~} character allows to bind a AMQP
+ * listener instead of connecting to a remote broker. For instance {@code amqp://~0.0.0.0:1234}
+ * will bind a AMQP listener on any network interface on the 1234 port number.
+ *
+ * <p>The following example illustrates how to configure a AMQP source:
+ *
+ * <pre>{@code
+ *
+ * pipeline.apply(AmqpIO.read()
+ * .withAddresses(Collections.singletonList("amqp://host:1234")))
+ *
+ * }</pre>
+ *
+ * <h3>Sending messages to a AMQP endpoint</h3>
+ *
+ * <p>{@link AmqpIO} provides a sink to send {@link PCollection} elements as messages.
+ *
+ * <p>As for the {@link Read}, {@link AmqpIO} {@link Write} requires a list of addresses where to
+ * send messages. The following example illustrates how to configure the {@link AmqpIO}
+ * {@link Write}:
+ *
+ * <pre>{@code
+ *
+ * pipeline
+ * .apply(...) // provide PCollection<Message>
+ * .apply(AmqpIO.write());
+ *
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class AmqpIO {
+
+ public static Read read() {
+ return new AutoValue_AmqpIO_Read.Builder().setMaxNumRecords(Long.MAX_VALUE).build();
+ }
+
+ public static Write write() {
+ return new AutoValue_AmqpIO_Write();
+ }
+
+ private AmqpIO() {
+ }
+
+ /**
+ * A {@link PTransform} to read/receive messages using AMQP 1.0 protocol.
+ */
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<Message>> {
+
+ @Nullable abstract List<String> addresses();
+ abstract long maxNumRecords();
+ @Nullable abstract Duration maxReadTime();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setAddresses(List<String> addresses);
+ abstract Builder setMaxNumRecords(long maxNumRecords);
+ abstract Builder setMaxReadTime(Duration maxReadTime);
+ abstract Read build();
+ }
+
+ /**
+ * Define the AMQP addresses where to receive messages.
+ */
+ public Read withAddresses(List<String> addresses) {
+ checkArgument(addresses != null, "addresses can not be null");
+ checkArgument(!addresses.isEmpty(), "addresses can not be empty");
+ return builder().setAddresses(addresses).build();
+ }
+
+ /**
+ * Define the max number of records received by the {@link Read}.
+ * When the max number of records is lower than {@code Long.MAX_VALUE}, the {@link Read} will
+ * provide a bounded {@link PCollection}.
+ */
+ public Read withMaxNumRecords(long maxNumRecords) {
+ return builder().setMaxNumRecords(maxNumRecords).build();
+ }
+
+ /**
+ * Define the max read time (duration) while the {@link Read} will receive messages.
+ * When this max read time is not null, the {@link Read} will provide a bounded
+ * {@link PCollection}.
+ */
+ public Read withMaxReadTime(Duration maxReadTime) {
+ return builder().setMaxReadTime(maxReadTime).build();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("addresses", Joiner.on(" ").join(addresses())));
+ }
+
+ @Override
+ public PCollection<Message> expand(PBegin input) {
+ checkArgument(addresses() != null, "withAddresses() is required");
+ checkArgument(
+ maxReadTime() == null || maxNumRecords() == Long.MAX_VALUE,
+ "withMaxNumRecords() and withMaxReadTime() are exclusive");
+
+ org.apache.beam.sdk.io.Read.Unbounded<Message> unbounded =
+ org.apache.beam.sdk.io.Read.from(new UnboundedAmqpSource(this));
+
+ PTransform<PBegin, PCollection<Message>> transform = unbounded;
+
+ if (maxNumRecords() != Long.MAX_VALUE) {
+ transform = unbounded.withMaxNumRecords(maxNumRecords());
+ } else if (maxReadTime() != null) {
+ transform = unbounded.withMaxReadTime(maxReadTime());
+ }
+
+ return input.getPipeline().apply(transform);
+ }
+
+ }
+
+ private static class AmqpCheckpointMark implements UnboundedSource.CheckpointMark, Serializable {
+
+ private transient Messenger messenger;
+ private transient List<Tracker> trackers = new ArrayList<>();
+
+ public AmqpCheckpointMark() {
+ }
+
+ @Override
+ public void finalizeCheckpoint() {
+ for (Tracker tracker : trackers) {
+ // flag as not cumulative
+ messenger.accept(tracker, 0);
+ }
+ trackers.clear();
+ }
+
+ // set an empty list to messages when deserialize
+ private void readObject(java.io.ObjectInputStream stream)
+ throws java.io.IOException, ClassNotFoundException {
+ trackers = new ArrayList<>();
+ }
+
+ }
+
+ private static class UnboundedAmqpSource
+ extends UnboundedSource<Message, AmqpCheckpointMark> {
+
+ private final Read spec;
+
+ public UnboundedAmqpSource(Read spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ public List<UnboundedAmqpSource> split(int desiredNumSplits,
+ PipelineOptions pipelineOptions) {
+ // amqp is a queue system, so, it's possible to have multiple concurrent sources, even if
+ // they bind the listener
+ List<UnboundedAmqpSource> sources = new ArrayList<>();
+ for (int i = 0; i < Math.max(1, desiredNumSplits); ++i) {
+ sources.add(new UnboundedAmqpSource(spec));
+ }
+ return sources;
+ }
+
+ @Override
+ public UnboundedReader<Message> createReader(PipelineOptions pipelineOptions,
+ AmqpCheckpointMark checkpointMark) {
+ return new UnboundedAmqpReader(this, checkpointMark);
+ }
+
+ @Override
+ public Coder<Message> getOutputCoder() {
+ return new AmqpMessageCoder();
+ }
+
+ @Override
+ public Coder<AmqpCheckpointMark> getCheckpointMarkCoder() {
+ return SerializableCoder.of(AmqpCheckpointMark.class);
+ }
+ }
+
+ private static class UnboundedAmqpReader extends UnboundedSource.UnboundedReader<Message> {
+
+ private final UnboundedAmqpSource source;
+
+ private Messenger messenger;
+ private Message current;
+ private Instant currentTimestamp;
+ private Instant watermark = new Instant(Long.MIN_VALUE);
+ private AmqpCheckpointMark checkpointMark;
+
+ public UnboundedAmqpReader(UnboundedAmqpSource source, AmqpCheckpointMark checkpointMark) {
+ this.source = source;
+ this.current = null;
+ if (checkpointMark != null) {
+ this.checkpointMark = checkpointMark;
+ } else {
+ this.checkpointMark = new AmqpCheckpointMark();
+ }
+ }
+
+ @Override
+ public Instant getWatermark() {
+ return watermark;
+ }
+
+ @Override
+ public Instant getCurrentTimestamp() {
+ if (current == null) {
+ throw new NoSuchElementException();
+ }
+ return currentTimestamp;
+ }
+
+ @Override
+ public Message getCurrent() {
+ if (current == null) {
+ throw new NoSuchElementException();
+ }
+ return current;
+ }
+
+ @Override
+ public UnboundedSource.CheckpointMark getCheckpointMark() {
+ return checkpointMark;
+ }
+
+ @Override
+ public UnboundedAmqpSource getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ Read spec = source.spec;
+ messenger = Messenger.Factory.create();
+ messenger.start();
+ for (String address : spec.addresses()) {
+ messenger.subscribe(address);
+ }
+ checkpointMark.messenger = messenger;
+ return advance();
+ }
+
+ @Override
+ public boolean advance() {
+ messenger.recv();
+ if (messenger.incoming() <= 0) {
+ current = null;
+ return false;
+ }
+ Message message = messenger.get();
+ Tracker tracker = messenger.incomingTracker();
+ checkpointMark.trackers.add(tracker);
+ currentTimestamp = new Instant(message.getCreationTime());
+ watermark = currentTimestamp;
+ current = message;
+ return true;
+ }
+
+ @Override
+ public void close() {
+ if (messenger != null) {
+ messenger.stop();
+ }
+ }
+
+ }
+
+ /**
+ * A {@link PTransform} to send messages using AMQP 1.0 protocol.
+ */
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<Message>, PDone> {
+
+ @Override
+ public PDone expand(PCollection<Message> input) {
+ input.apply(ParDo.of(new WriteFn(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ private static class WriteFn extends DoFn<Message, Void> {
+
+ private final Write spec;
+
+ private transient Messenger messenger;
+
+ public WriteFn(Write spec) {
+ this.spec = spec;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ messenger = Messenger.Factory.create();
+ messenger.start();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext processContext) throws Exception {
+ Message message = processContext.element();
+ messenger.put(message);
+ messenger.send();
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ if (messenger != null) {
+ messenger.stop();
+ }
+ }
+
+ }
+
+ }
+
+}
diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java
new file mode 100644
index 0000000..5a55260
--- /dev/null
+++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.amqp;
+
+import com.google.common.io.ByteStreams;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.BufferOverflowException;
+
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.util.VarInt;
+import org.apache.qpid.proton.message.Message;
+
+/**
+ * A coder for AMQP message.
+ */
+public class AmqpMessageCoder extends CustomCoder<Message> {
+
+ private static final int[] MESSAGE_SIZES = new int[]{
+ 8 * 1024,
+ 64 * 1024,
+ 1 * 1024 * 1024,
+ 64 * 1024 * 1024
+ };
+
+ static AmqpMessageCoder of() {
+ return new AmqpMessageCoder();
+ }
+
+ @Override
+ public void encode(Message value, OutputStream outStream) throws CoderException, IOException {
+ for (int maxMessageSize : MESSAGE_SIZES) {
+ try {
+ encode(value, outStream, maxMessageSize);
+ return;
+ } catch (Exception e) {
+ continue;
+ }
+ }
+ throw new CoderException("Message is larger than the max size supported by the coder");
+ }
+
+ private void encode(Message value, OutputStream outStream, int messageSize) throws
+ IOException, BufferOverflowException {
+ byte[] data = new byte[messageSize];
+ int bytesWritten = value.encode(data, 0, data.length);
+ VarInt.encode(bytesWritten, outStream);
+ outStream.write(data, 0, bytesWritten);
+ }
+
+ @Override
+ public Message decode(InputStream inStream) throws CoderException, IOException {
+ Message message = Message.Factory.create();
+ int bytesToRead = VarInt.decodeInt(inStream);
+ byte[] encodedMessage = new byte[bytesToRead];
+ ByteStreams.readFully(inStream, encodedMessage);
+ message.decode(encodedMessage, 0, encodedMessage.length);
+ return message;
+ }
+
+}
diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java
new file mode 100644
index 0000000..bc3445c
--- /dev/null
+++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.amqp;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.qpid.proton.message.Message;
+
+/**
+ * A {@link CoderProviderRegistrar} for standard types used with {@link AmqpIO}.
+ */
+@AutoService(CoderProviderRegistrar.class)
+public class AmqpMessageCoderProviderRegistrar implements CoderProviderRegistrar {
+
+ @Override
+ public List<CoderProvider> getCoderProviders() {
+ return ImmutableList.of(
+ CoderProviders.forCoder(TypeDescriptor.of(Message.class),
+ AmqpMessageCoder.of()));
+ }
+
+}
diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/package-info.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/package-info.java
new file mode 100644
index 0000000..091f234
--- /dev/null
+++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transforms for reading and writing using AMQP 1.0 protocol.
+ */
+package org.apache.beam.sdk.io.amqp;
diff --git a/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpIOTest.java b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpIOTest.java
new file mode 100644
index 0000000..947929f
--- /dev/null
+++ b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpIOTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.amqp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.activemq.junit.EmbeddedActiveMQBroker;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.proton.amqp.messaging.AmqpValue;
+import org.apache.qpid.proton.message.Message;
+import org.apache.qpid.proton.messenger.Messenger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests on {@link AmqpIO}.
+ */
+@RunWith(JUnit4.class)
+public class AmqpIOTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AmqpIOTest.class);
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @Rule public EmbeddedAmqpBroker broker = new EmbeddedAmqpBroker();
+
+ @Test
+ public void testRead() throws Exception {
+ PCollection<Message> output = pipeline.apply(AmqpIO.read()
+ .withMaxNumRecords(100)
+ .withAddresses(Collections.singletonList(broker.getQueueUri("testRead"))));
+ PAssert.thatSingleton(output.apply(Count.<Message>globally())).isEqualTo(100L);
+
+ Messenger sender = Messenger.Factory.create();
+ sender.start();
+ for (int i = 0; i < 100; i++) {
+ Message message = Message.Factory.create();
+ message.setAddress(broker.getQueueUri("testRead"));
+ message.setBody(new AmqpValue("Test " + i));
+ sender.put(message);
+ sender.send();
+ }
+ sender.stop();
+
+ pipeline.run();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ List<Message> data = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ Message message = Message.Factory.create();
+ message.setBody(new AmqpValue("Test " + i));
+ message.setAddress(broker.getQueueUri("testWrite"));
+ message.setSubject("test");
+ data.add(message);
+ }
+ pipeline.apply(Create.of(data).withCoder(AmqpMessageCoder.of())).apply(AmqpIO.write());
+ pipeline.run().waitUntilFinish();
+
+ List<String> received = new ArrayList<>();
+ Messenger messenger = Messenger.Factory.create();
+ messenger.start();
+ messenger.subscribe(broker.getQueueUri("testWrite"));
+ while (received.size() < 100) {
+ messenger.recv();
+ while (messenger.incoming() > 0) {
+ Message message = messenger.get();
+ LOG.info("Received: " + message.getBody().toString());
+ received.add(message.getBody().toString());
+ }
+ }
+ messenger.stop();
+
+ assertEquals(100, received.size());
+ for (int i = 0; i < 100; i++) {
+ assertTrue(received.contains("AmqpValue{Test " + i + "}"));
+ }
+ }
+
+ private static class EmbeddedAmqpBroker extends EmbeddedActiveMQBroker {
+ @Override
+ protected void configure() {
+ try {
+ getBrokerService().addConnector("amqp://localhost:0");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getQueueUri(String queueName) {
+ return getBrokerService().getDefaultSocketURIString() + "/" + queueName;
+ }
+ }
+
+}
diff --git a/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java
new file mode 100644
index 0000000..7a8efeb
--- /dev/null
+++ b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.amqp;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Joiner;
+
+import java.util.Collections;
+
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.qpid.proton.amqp.messaging.AmqpValue;
+import org.apache.qpid.proton.message.Message;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test on {@link AmqpMessageCoder}.
+ */
+@RunWith(JUnit4.class)
+public class AmqpMessageCoderTest {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void encodeDecode() throws Exception {
+ Message message = Message.Factory.create();
+ message.setBody(new AmqpValue("body"));
+ message.setAddress("address");
+ message.setSubject("test");
+ AmqpMessageCoder coder = AmqpMessageCoder.of();
+
+ Message clone = CoderUtils.clone(coder, message);
+
+ assertEquals("AmqpValue{body}", clone.getBody().toString());
+ assertEquals("address", clone.getAddress());
+ assertEquals("test", clone.getSubject());
+ }
+
+ @Test
+ public void encodeDecodeTooMuchLargerMessage() throws Exception {
+ thrown.expect(CoderException.class);
+ Message message = Message.Factory.create();
+ message.setAddress("address");
+ message.setSubject("subject");
+ String body = Joiner.on("").join(Collections.nCopies(64 * 1024 * 1024, " "));
+ message.setBody(new AmqpValue(body));
+
+ AmqpMessageCoder coder = AmqpMessageCoder.of();
+
+ byte[] encoded = CoderUtils.encodeToByteArray(coder, message);
+ }
+
+ @Test
+ public void encodeDecodeLargeMessage() throws Exception {
+ Message message = Message.Factory.create();
+ message.setAddress("address");
+ message.setSubject("subject");
+ String body = Joiner.on("").join(Collections.nCopies(32 * 1024 * 1024, " "));
+ message.setBody(new AmqpValue(body));
+
+ AmqpMessageCoder coder = AmqpMessageCoder.of();
+
+ Message clone = CoderUtils.clone(coder, message);
+
+ clone.getBody().toString().equals(message.getBody().toString());
+ }
+
+}
diff --git a/sdks/java/io/cassandra/pom.xml b/sdks/java/io/cassandra/pom.xml
new file mode 100644
index 0000000..6ba2eda
--- /dev/null
+++ b/sdks/java/io/cassandra/pom.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-cassandra</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Cassandra</name>
+ <description>IO to read and write with Apache Cassandra database</description>
+
+ <properties>
+ <cassandra.driver.version>3.2.0</cassandra.driver.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-mapping</artifactId>
+ <version>${cassandra.driver.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>${cassandra.driver.version}</version>
+ </dependency>
+
+ <!-- compile dependencies -->
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java
new file mode 100644
index 0000000..b8309c0
--- /dev/null
+++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java
@@ -0,0 +1,486 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An IO to read from Apache Cassandra.
+ *
+ * <h3>Reading from Apache Cassandra</h3>
+ *
+ * <p>{@code CassandraIO} provides a source to read and returns a bounded collection of
+ * entities as {@code PCollection<Entity>}. An entity is built by Cassandra mapper
+ * ({@code com.datastax.driver.mapping.EntityMapper}) based on a
+ * POJO containing annotations (as described http://docs.datastax
+ * .com/en/developer/java-driver/2.1/manual/object_mapper/creating/").
+ *
+ * <p>The following example illustrates various options for configuring the IO:
+ *
+ * <pre>{@code
+ * pipeline.apply(CassandraIO.<Person>read()
+ * .withHosts(Arrays.asList("host1", "host2"))
+ * .withPort(9042)
+ * .withKeyspace("beam")
+ * .withTable("Person")
+ * .withEntity(Person.class)
+ * .withCoder(SerializableCoder.of(Person.class))
+ * // above options are the minimum set, returns PCollection<Person>
+ *
+ * }</pre>
+ *
+ * <h3>Writing to Apache Cassandra</h3>
+ *
+ * <p>{@code CassandraIO} provides a sink to write a collection of entities to Apache Cassandra.
+ *
+ * <p>The following example illustrates various options for configuring the IO write:
+ *
+ * <pre>{@code
+ * pipeline
+ * .apply(...) // provides a PCollection<Person> where Person is an entity
+ * .apply(CassandraIO.<Person>write()
+ * .withHosts(Arrays.asList("host1", "host2"))
+ * .withPort(9042)
+ * .withKeyspace("beam")
+ * .withEntity(Person.class));
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class CassandraIO {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CassandraIO.class);
+
+ private CassandraIO() {}
+
+ /**
+ * Provide a {@link Read} {@link PTransform} to read data from a Cassandra database.
+ */
+ public static <T> Read<T> read() {
+ return new AutoValue_CassandraIO_Read.Builder<T>().build();
+ }
+
+ /**
+ * Provide a {@link Write} {@link PTransform} to write data to a Cassandra database.
+ */
+ public static <T> Write<T> write() {
+ return new AutoValue_CassandraIO_Write.Builder<T>().build();
+ }
+
+ /**
+ * A {@link PTransform} to read data from Apache Cassandra. See {@link CassandraIO} for more
+ * information on usage and configuration.
+ */
+ @AutoValue
+ public abstract static class Read<T> extends PTransform<PBegin, PCollection<T>> {
+
+ @Nullable abstract List<String> hosts();
+ @Nullable abstract Integer port();
+ @Nullable abstract String keyspace();
+ @Nullable abstract String table();
+ @Nullable abstract Class<T> entity();
+ @Nullable abstract Coder<T> coder();
+ @Nullable abstract String username();
+ @Nullable abstract String password();
+ @Nullable abstract String localDc();
+ @Nullable abstract String consistencyLevel();
+ @Nullable abstract CassandraService<T> cassandraService();
+ abstract Builder<T> builder();
+
+ /**
+ * Specify the hosts of the Apache Cassandra instances.
+ */
+ public Read<T> withHosts(List<String> hosts) {
+ checkArgument(hosts != null, "hosts can not be null");
+ checkArgument(!hosts.isEmpty(), "hosts can not be empty");
+ return builder().setHosts(hosts).build();
+ }
+
+ /**
+ * Specify the port number of the Apache Cassandra instances.
+ */
+ public Read<T> withPort(int port) {
+ checkArgument(port > 0, "port must be > 0, but was: %d", port);
+ return builder().setPort(port).build();
+ }
+
+ /**
+ * Specify the Cassandra keyspace where to read data.
+ */
+ public Read<T> withKeyspace(String keyspace) {
+ checkArgument(keyspace != null, "keyspace can not be null");
+ return builder().setKeyspace(keyspace).build();
+ }
+
+ /**
+ * Specify the Cassandra table where to read data.
+ */
+ public Read<T> withTable(String table) {
+ checkArgument(table != null, "table can not be null");
+ return builder().setTable(table).build();
+ }
+
+ /**
+ * Specify the entity class (annotated POJO). The {@link CassandraIO} will read the data and
+ * convert the data as entity instances. The {@link PCollection} resulting from the read will
+ * contains entity elements.
+ */
+ public Read<T> withEntity(Class<T> entity) {
+ checkArgument(entity != null, "entity can not be null");
+ return builder().setEntity(entity).build();
+ }
+
+ /**
+ * Specify the {@link Coder} used to serialize the entity in the {@link PCollection}.
+ */
+ public Read<T> withCoder(Coder<T> coder) {
+ checkArgument(coder != null, "coder can not be null");
+ return builder().setCoder(coder).build();
+ }
+
+ /**
+ * Specify the username for authentication.
+ */
+ public Read<T> withUsername(String username) {
+ checkArgument(username != null, "username can not be null");
+ return builder().setUsername(username).build();
+ }
+
+ /**
+ * Specify the password for authentication.
+ */
+ public Read<T> withPassword(String password) {
+ checkArgument(password != null, "password can not be null");
+ return builder().setPassword(password).build();
+ }
+
+ /**
+ * Specify the local DC used for the load balancing.
+ */
+ public Read<T> withLocalDc(String localDc) {
+ checkArgument(localDc != null, "localDc can not be null");
+ return builder().setLocalDc(localDc).build();
+ }
+
+ public Read<T> withConsistencyLevel(String consistencyLevel) {
+ checkArgument(consistencyLevel != null, "consistencyLevel can not be null");
+ return builder().setConsistencyLevel(consistencyLevel).build();
+ }
+
+ /**
+ * Specify an instance of {@link CassandraService} used to connect and read from Cassandra
+ * database.
+ */
+ public Read<T> withCassandraService(CassandraService<T> cassandraService) {
+ checkArgument(cassandraService != null, "cassandraService can not be null");
+ return builder().setCassandraService(cassandraService).build();
+ }
+
+ @Override
+ public PCollection<T> expand(PBegin input) {
+ checkArgument(
+ (hosts() != null && port() != null) || cassandraService() != null,
+ "Either withHosts() and withPort(), or withCassandraService() is required");
+ checkArgument(keyspace() != null, "withKeyspace() is required");
+ checkArgument(table() != null, "withTable() is required");
+ checkArgument(entity() != null, "withEntity() is required");
+ checkArgument(coder() != null, "withCoder() is required");
+
+ return input.apply(org.apache.beam.sdk.io.Read.from(
+ new CassandraSource<T>(this, null)));
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setHosts(List<String> hosts);
+ abstract Builder<T> setPort(Integer port);
+ abstract Builder<T> setKeyspace(String keyspace);
+ abstract Builder<T> setTable(String table);
+ abstract Builder<T> setEntity(Class<T> entity);
+ abstract Builder<T> setCoder(Coder<T> coder);
+ abstract Builder<T> setUsername(String username);
+ abstract Builder<T> setPassword(String password);
+ abstract Builder<T> setLocalDc(String localDc);
+ abstract Builder<T> setConsistencyLevel(String consistencyLevel);
+ abstract Builder<T> setCassandraService(CassandraService<T> cassandraService);
+ abstract Read<T> build();
+ }
+
+ /**
+ * Helper function to either get a fake/mock Cassandra service provided by
+ * {@link #withCassandraService(CassandraService)} or creates and returns an implementation
+ * of a concrete Cassandra service dealing with a Cassandra instance.
+ */
+ @VisibleForTesting
+ CassandraService<T> getCassandraService() {
+ if (cassandraService() != null) {
+ return cassandraService();
+ }
+ return new CassandraServiceImpl<>();
+ }
+
+ }
+
+ @VisibleForTesting
+ static class CassandraSource<T> extends BoundedSource<T> {
+
+ protected final Read<T> spec;
+ protected final String splitQuery;
+
+ CassandraSource(Read<T> spec,
+ String splitQuery) {
+ this.spec = spec;
+ this.splitQuery = splitQuery;
+ }
+
+ @Override
+ public Coder<T> getOutputCoder() {
+ return spec.coder();
+ }
+
+ @Override
+ public BoundedReader<T> createReader(PipelineOptions pipelineOptions) {
+ return spec.getCassandraService().createReader(this);
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
+ return spec.getCassandraService().getEstimatedSizeBytes(spec);
+ }
+
+ @Override
+ public List<BoundedSource<T>> split(long desiredBundleSizeBytes,
+ PipelineOptions pipelineOptions) {
+ return spec.getCassandraService()
+ .split(spec, desiredBundleSizeBytes);
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ if (spec.hosts() != null) {
+ builder.add(DisplayData.item("hosts", spec.hosts().toString()));
+ }
+ if (spec.port() != null) {
+ builder.add(DisplayData.item("port", spec.port()));
+ }
+ builder.addIfNotNull(DisplayData.item("keyspace", spec.keyspace()));
+ builder.addIfNotNull(DisplayData.item("table", spec.table()));
+ builder.addIfNotNull(DisplayData.item("username", spec.username()));
+ builder.addIfNotNull(DisplayData.item("localDc", spec.localDc()));
+ builder.addIfNotNull(DisplayData.item("consistencyLevel", spec.consistencyLevel()));
+ }
+ }
+
+ /**
+ * A {@link PTransform} to write into Apache Cassandra. See {@link CassandraIO} for details on
+ * usage and configuration.
+ */
+ @AutoValue
+ public abstract static class Write<T> extends PTransform<PCollection<T>, PDone> {
+
+ @Nullable abstract List<String> hosts();
+ @Nullable abstract Integer port();
+ @Nullable abstract String keyspace();
+ @Nullable abstract Class<T> entity();
+ @Nullable abstract String username();
+ @Nullable abstract String password();
+ @Nullable abstract String localDc();
+ @Nullable abstract String consistencyLevel();
+ @Nullable abstract CassandraService<T> cassandraService();
+ abstract Builder<T> builder();
+
+ /**
+ * Specify the Cassandra instance hosts where to write data.
+ */
+ public Write<T> withHosts(List<String> hosts) {
+ checkArgument(hosts != null, "CassandraIO.write().withHosts(hosts) called with null hosts");
+ checkArgument(!hosts.isEmpty(), "CassandraIO.write().withHosts(hosts) called with empty "
+ + "hosts list");
+ return builder().setHosts(hosts).build();
+ }
+
+ /**
+ * Specify the Cassandra instance port number where to write data.
+ */
+ public Write<T> withPort(int port) {
+ checkArgument(port > 0, "CassandraIO.write().withPort(port) called with invalid port "
+ + "number (%d)", port);
+ return builder().setPort(port).build();
+ }
+
+ /**
+ * Specify the Cassandra keyspace where to write data.
+ */
+ public Write<T> withKeyspace(String keyspace) {
+ checkArgument(keyspace != null, "CassandraIO.write().withKeyspace(keyspace) called with "
+ + "null keyspace");
+ return builder().setKeyspace(keyspace).build();
+ }
+
+ /**
+ * Specify the entity class in the input {@link PCollection}. The {@link CassandraIO} will
+ * map this entity to the Cassandra table thanks to the annotations.
+ */
+ public Write<T> withEntity(Class<T> entity) {
+ checkArgument(entity != null, "CassandraIO.write().withEntity(entity) called with null "
+ + "entity");
+ return builder().setEntity(entity).build();
+ }
+
+ /**
+ * Specify the username used for authentication.
+ */
+ public Write<T> withUsername(String username) {
+ checkArgument(username != null, "CassandraIO.write().withUsername(username) called with "
+ + "null username");
+ return builder().setUsername(username).build();
+ }
+
+ /**
+ * Specify the password used for authentication.
+ */
+ public Write<T> withPassword(String password) {
+ checkArgument(password != null, "CassandraIO.write().withPassword(password) called with "
+ + "null password");
+ return builder().setPassword(password).build();
+ }
+
+ /**
+ * Specify the local DC used by the load balancing policy.
+ */
+ public Write<T> withLocalDc(String localDc) {
+ checkArgument(localDc != null, "CassandraIO.write().withLocalDc(localDc) called with null"
+ + " localDc");
+ return builder().setLocalDc(localDc).build();
+ }
+
+ public Write<T> withConsistencyLevel(String consistencyLevel) {
+ checkArgument(consistencyLevel != null, "CassandraIO.write().withConsistencyLevel"
+ + "(consistencyLevel) called with null consistencyLevel");
+ return builder().setConsistencyLevel(consistencyLevel).build();
+ }
+
+ /**
+ * Specify the {@link CassandraService} used to connect and write into the Cassandra database.
+ */
+ public Write<T> withCassandraService(CassandraService<T> cassandraService) {
+ checkArgument(cassandraService != null, "CassandraIO.write().withCassandraService"
+ + "(service) called with null service");
+ return builder().setCassandraService(cassandraService).build();
+ }
+
+ @Override
+ public void validate(PipelineOptions pipelineOptions) {
+ checkState(hosts() != null || cassandraService() != null,
+ "CassandraIO.write() requires a list of hosts to be set via withHosts(hosts) or a "
+ + "Cassandra service to be set via withCassandraService(service)");
+ checkState(port() != null || cassandraService() != null, "CassandraIO.write() requires a "
+ + "valid port number to be set via withPort(port) or a Cassandra service to be set via "
+ + "withCassandraService(service)");
+ checkState(keyspace() != null, "CassandraIO.write() requires a keyspace to be set via "
+ + "withKeyspace(keyspace)");
+ checkState(entity() != null, "CassandraIO.write() requires an entity to be set via "
+ + "withEntity(entity)");
+ }
+
+ @Override
+ public PDone expand(PCollection<T> input) {
+ input.apply(ParDo.of(new WriteFn<T>(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setHosts(List<String> hosts);
+ abstract Builder<T> setPort(Integer port);
+ abstract Builder<T> setKeyspace(String keyspace);
+ abstract Builder<T> setEntity(Class<T> entity);
+ abstract Builder<T> setUsername(String username);
+ abstract Builder<T> setPassword(String password);
+ abstract Builder<T> setLocalDc(String localDc);
+ abstract Builder<T> setConsistencyLevel(String consistencyLevel);
+ abstract Builder<T> setCassandraService(CassandraService<T> cassandraService);
+ abstract Write<T> build();
+ }
+
+ /**
+ * Helper function to either get a fake/mock Cassandra service provided by
+ * {@link #withCassandraService(CassandraService)} or creates and returns an implementation
+ * of a concrete Cassandra service dealing with a Cassandra instance.
+ */
+ @VisibleForTesting
+ CassandraService<T> getCassandraService() {
+ if (cassandraService() != null) {
+ return cassandraService();
+ }
+ return new CassandraServiceImpl<>();
+ }
+
+ }
+
+ private static class WriteFn<T> extends DoFn<T, Void> {
+
+ private final Write<T> spec;
+ private CassandraService.Writer writer;
+
+ public WriteFn(Write<T> spec) {
+ this.spec = spec;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ writer = spec.getCassandraService().createWriter(spec);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext processContext) {
+ T entity = processContext.element();
+ writer.write(entity);
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ writer.close();
+ writer = null;
+ }
+
+ }
+
+}
diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraService.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraService.java
new file mode 100644
index 0000000..5071762
--- /dev/null
+++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraService.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.beam.sdk.io.BoundedSource;
+
+/**
+ * An interface for real or fake implementations of Cassandra.
+ */
+public interface CassandraService<T> extends Serializable {
+
+ /**
+ * Returns a {@link org.apache.beam.sdk.io.BoundedSource.BoundedReader} that will read from
+ * Cassandra using the spec from
+ * {@link org.apache.beam.sdk.io.cassandra.CassandraIO.CassandraSource}.
+ */
+ BoundedSource.BoundedReader<T> createReader(CassandraIO.CassandraSource<T> source);
+
+ /**
+ * Returns an estimation of the size that could be read.
+ */
+ long getEstimatedSizeBytes(CassandraIO.Read<T> spec);
+
+ /**
+ * Split a table read into several sources.
+ */
+ List<BoundedSource<T>> split(CassandraIO.Read<T> spec,
+ long desiredBundleSizeBytes);
+
+ /**
+ * Create a {@link Writer} that writes entities into the Cassandra instance.
+ */
+ Writer createWriter(CassandraIO.Write<T> spec) throws Exception;
+
+ /**
+ * Writer for an entity.
+ */
+ interface Writer<T> extends AutoCloseable {
+
+ /**
+ * This method should be synchronous. It means you have to be sure that the entity is fully
+ * stored (and committed) into the Cassandra instance when you exit from this method.
+ */
+ void write(T entity);
+
+ }
+
+}
diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImpl.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImpl.java
new file mode 100644
index 0000000..63c8ef4
--- /dev/null
+++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImpl.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.PlainTextAuthProvider;
+import com.datastax.driver.core.QueryOptions;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;
+import com.datastax.driver.core.policies.RoundRobinPolicy;
+import com.datastax.driver.core.policies.TokenAwarePolicy;
+import com.datastax.driver.core.querybuilder.QueryBuilder;
+import com.datastax.driver.core.querybuilder.Select;
+import com.datastax.driver.mapping.Mapper;
+import com.datastax.driver.mapping.MappingManager;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.beam.sdk.io.BoundedSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the {@link CassandraService} that actually use a Cassandra instance.
+ */
+public class CassandraServiceImpl<T> implements CassandraService<T> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CassandraServiceImpl.class);
+
+ private static final long MIN_TOKEN = Long.MIN_VALUE;
+ private static final long MAX_TOKEN = Long.MAX_VALUE;
+ private static final BigInteger TOTAL_TOKEN_COUNT =
+ BigInteger.valueOf(MAX_TOKEN).subtract(BigInteger.valueOf(MIN_TOKEN));
+
+ private class CassandraReaderImpl<T> extends BoundedSource.BoundedReader<T> {
+
+ private final CassandraIO.CassandraSource<T> source;
+
+ private Cluster cluster;
+ private Session session;
+ private ResultSet resultSet;
+ private Iterator<T> iterator;
+ private T current;
+
+ public CassandraReaderImpl(CassandraIO.CassandraSource<T> source) {
+ this.source = source;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ LOG.debug("Starting Cassandra reader");
+ cluster = getCluster(source.spec.hosts(), source.spec.port(), source.spec.username(),
+ source.spec.password(), source.spec.localDc(), source.spec.consistencyLevel());
+ session = cluster.connect();
+ LOG.debug("Query: " + source.splitQuery);
+ resultSet = session.execute(source.splitQuery);
+
+ final MappingManager mappingManager = new MappingManager(session);
+ Mapper mapper = mappingManager.mapper(source.spec.entity());
+ iterator = mapper.map(resultSet).iterator();
+ return advance();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ if (iterator.hasNext()) {
+ current = iterator.next();
+ return true;
+ }
+ current = null;
+ return false;
+ }
+
+ @Override
+ public void close() {
+ LOG.debug("Closing Cassandra reader");
+ if (session != null) {
+ session.close();
+ }
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+
+ @Override
+ public T getCurrent() throws NoSuchElementException {
+ if (current == null) {
+ throw new NoSuchElementException();
+ }
+ return current;
+ }
+
+ @Override
+ public CassandraIO.CassandraSource<T> getCurrentSource() {
+ return source;
+ }
+
+ }
+
+ @Override
+ public CassandraReaderImpl<T> createReader(CassandraIO.CassandraSource<T> source) {
+ return new CassandraReaderImpl<>(source);
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(CassandraIO.Read<T> spec) {
+ try (Cluster cluster = getCluster(spec.hosts(), spec.port(), spec.username(), spec.password(),
+ spec.localDc(), spec.consistencyLevel())) {
+ if (isMurmur3Partitioner(cluster)) {
+ try {
+ List<TokenRange> tokenRanges = getTokenRanges(cluster,
+ spec.keyspace(),
+ spec.table());
+ return getEstimatedSizeBytes(tokenRanges);
+ } catch (Exception e) {
+ LOG.warn("Can't estimate the size", e);
+ return 0L;
+ }
+ } else {
+ LOG.warn("Only Murmur3 partitioner is supported, can't estimate the size");
+ return 0L;
+ }
+ }
+ }
+
+ /**
+ * Actually estimate the size of the data to read on the cluster, based on the given token
+ * ranges to address.
+ */
+ @VisibleForTesting
+ protected static long getEstimatedSizeBytes(List<TokenRange> tokenRanges) {
+ long size = 0L;
+ for (TokenRange tokenRange : tokenRanges) {
+ size = size + tokenRange.meanPartitionSize * tokenRange.partitionCount;
+ }
+ return Math.round(size / getRingFraction(tokenRanges));
+ }
+
+ @Override
+ public List<BoundedSource<T>> split(CassandraIO.Read<T> spec,
+ long desiredBundleSizeBytes) {
+ try (Cluster cluster = getCluster(spec.hosts(), spec.port(), spec.username(), spec.password(),
+ spec.localDc(), spec.consistencyLevel())) {
+ if (isMurmur3Partitioner(cluster)) {
+ LOG.info("Murmur3Partitioner detected, splitting");
+ return split(spec, desiredBundleSizeBytes, getEstimatedSizeBytes(spec));
+ } else {
+ LOG.warn("Only Murmur3Partitioner is supported for splitting, using an unique source for "
+ + "the read");
+ String splitQuery = QueryBuilder.select().from(spec.keyspace(), spec.table()).toString();
+ List<BoundedSource<T>> sources = new ArrayList<>();
+ sources.add(new CassandraIO.CassandraSource<T>(spec, splitQuery));
+ return sources;
+ }
+ }
+ }
+
+ /**
+ * Compute the number of splits based on the estimated size and the desired bundle size, and
+ * create several sources.
+ */
+ @VisibleForTesting
+ protected List<BoundedSource<T>> split(CassandraIO.Read<T> spec,
+ long desiredBundleSizeBytes,
+ long estimatedSizeBytes) {
+ long numSplits = 1;
+ List<BoundedSource<T>> sourceList = new ArrayList<>();
+ if (desiredBundleSizeBytes > 0) {
+ numSplits = estimatedSizeBytes / desiredBundleSizeBytes;
+ }
+ if (numSplits <= 0) {
+ LOG.warn("Number of splits is less than 0 ({}), fallback to 1", numSplits);
+ numSplits = 1;
+ }
+
+ LOG.info("Number of splits is {}", numSplits);
+
+ double startRange = MIN_TOKEN;
+ double endRange = MAX_TOKEN;
+ double startToken, endToken;
+
+ endToken = startRange;
+ double incrementValue = endRange - startRange / numSplits;
+ String splitQuery;
+ if (numSplits == 1) {
+ // we have an unique split
+ splitQuery = QueryBuilder.select().from(spec.keyspace(), spec.table()).toString();
+ sourceList.add(new CassandraIO.CassandraSource<T>(spec, splitQuery));
+ } else {
+ // we have more than one split
+ for (int i = 0; i < numSplits; i++) {
+ startToken = endToken;
+ endToken = (i == numSplits) ? endRange : (startToken + incrementValue);
+ Select.Where builder = QueryBuilder.select().from(spec.keyspace(), spec.table()).where();
+ if (i > 0) {
+ builder = builder.and(QueryBuilder.gte("token($pk)", startToken));
+ }
+ if (i < (numSplits - 1)) {
+ builder = builder.and(QueryBuilder.lt("token($pk)", endToken));
+ }
+ sourceList.add(new CassandraIO.CassandraSource(spec, builder.toString()));
+ }
+ }
+ return sourceList;
+ }
+
+ /**
+ * Get a Cassandra cluster using hosts and port.
+ */
+ private Cluster getCluster(List<String> hosts, int port, String username, String password,
+ String localDc, String consistencyLevel) {
+ Cluster.Builder builder = Cluster.builder()
+ .addContactPoints(hosts.toArray(new String[0]))
+ .withPort(port);
+
+ if (username != null) {
+ builder.withAuthProvider(new PlainTextAuthProvider(username, password));
+ }
+
+ if (localDc != null) {
+ builder.withLoadBalancingPolicy(
+ new TokenAwarePolicy(new DCAwareRoundRobinPolicy.Builder().withLocalDc(localDc).build()));
+ } else {
+ builder.withLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
+ }
+
+ if (consistencyLevel != null) {
+ builder.withQueryOptions(
+ new QueryOptions().setConsistencyLevel(ConsistencyLevel.valueOf(consistencyLevel)));
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Gets the list of token ranges that a table occupies on a give Cassandra node.
+ *
+ * <p>NB: This method is compatible with Cassandra 2.1.5 and greater.
+ */
+ private static List<TokenRange> getTokenRanges(Cluster cluster, String keyspace, String table) {
+ try (Session session = cluster.newSession()) {
+ ResultSet resultSet =
+ session.execute(
+ "SELECT range_start, range_end, partitions_count, mean_partition_size FROM "
+ + "system.size_estimates WHERE keyspace_name = ? AND table_name = ?",
+ keyspace,
+ table);
+
+ ArrayList<TokenRange> tokenRanges = new ArrayList<>();
+ for (Row row : resultSet) {
+ TokenRange tokenRange =
+ new TokenRange(
+ row.getLong("partitions_count"),
+ row.getLong("mean_partition_size"),
+ row.getLong("range_start"),
+ row.getLong("range_end"));
+ tokenRanges.add(tokenRange);
+ }
+ // The table may not contain the estimates yet
+ // or have partitions_count and mean_partition_size fields = 0
+ // if the data was just inserted and the amount of data in the table was small.
+ // This is very common situation during tests,
+ // when we insert a few rows and immediately query them.
+ // However, for tiny data sets the lack of size estimates is not a problem at all,
+ // because we don't want to split tiny data anyways.
+ // Therefore, we're not issuing a warning if the result set was empty
+ // or mean_partition_size and partitions_count = 0.
+ return tokenRanges;
+ }
+ }
+
+ /**
+ * Compute the percentage of token addressed compared with the whole tokens in the cluster.
+ */
+ @VisibleForTesting
+ protected static double getRingFraction(List<TokenRange> tokenRanges) {
+ double ringFraction = 0;
+ for (TokenRange tokenRange : tokenRanges) {
+ ringFraction = ringFraction + (distance(tokenRange.rangeStart, tokenRange.rangeEnd)
+ .doubleValue() / TOTAL_TOKEN_COUNT.doubleValue());
+ }
+ return ringFraction;
+ }
+
+ /**
+ * Measure distance between two tokens.
+ */
+ @VisibleForTesting
+ protected static BigInteger distance(long left, long right) {
+ if (right > left) {
+ return BigInteger.valueOf(right).subtract(BigInteger.valueOf(left));
+ } else {
+ return BigInteger.valueOf(right).subtract(BigInteger.valueOf(left)).add(TOTAL_TOKEN_COUNT);
+ }
+ }
+
+ /**
+ * Check if the current partitioner is the Murmur3 (default in Cassandra version newer than 2).
+ */
+ @VisibleForTesting
+ protected static boolean isMurmur3Partitioner(Cluster cluster) {
+ return cluster.getMetadata().getPartitioner()
+ .equals("org.apache.cassandra.dht.Murmur3Partitioner");
+ }
+
+ /**
+ * Represent a token range in Cassandra instance, wrapping the partition count, size and token
+ * range.
+ */
+ @VisibleForTesting
+ protected static class TokenRange {
+ private final long partitionCount;
+ private final long meanPartitionSize;
+ private final long rangeStart;
+ private final long rangeEnd;
+
+ public TokenRange(
+ long partitionCount, long meanPartitionSize, long rangeStart, long
+ rangeEnd) {
+ this.partitionCount = partitionCount;
+ this.meanPartitionSize = meanPartitionSize;
+ this.rangeStart = rangeStart;
+ this.rangeEnd = rangeEnd;
+ }
+ }
+
+ /**
+ * Writer storing an entity into Apache Cassandra database.
+ */
+ protected class WriterImpl<T> implements Writer<T> {
+
+ private final CassandraIO.Write<T> spec;
+
+ private final Cluster cluster;
+ private final Session session;
+ private final MappingManager mappingManager;
+
+ public WriterImpl(CassandraIO.Write<T> spec) {
+ this.spec = spec;
+ this.cluster = getCluster(spec.hosts(), spec.port(), spec.username(), spec.password(),
+ spec.localDc(), spec.consistencyLevel());
+ this.session = cluster.connect(spec.keyspace());
+ this.mappingManager = new MappingManager(session);
+ }
+
+ /**
+ * Write the entity to the Cassandra instance, using {@link Mapper} obtained with the
+ * {@link MappingManager}. This method use {@link Mapper#save(Object)} method, which is
+ * synchronous. It means the entity is guaranteed to be reliably committed to Cassandra.
+ */
+ @Override
+ public void write(T entity) {
+ Mapper<T> mapper = (Mapper<T>) mappingManager.mapper(entity.getClass());
+ mapper.save(entity);
+ }
+
+ @Override
+ public void close() {
+ if (session != null) {
+ session.close();
+ }
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+
+ }
+
+ @Override
+ public Writer createWriter(CassandraIO.Write<T> spec) {
+ return new WriterImpl(spec);
+ }
+
+}
diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/package-info.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/package-info.java
new file mode 100644
index 0000000..6659b62
--- /dev/null
+++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transforms for reading and writing from Apache Cassandra database.
+ */
+package org.apache.beam.sdk.io.cassandra;
diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOIT.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOIT.java
new file mode 100644
index 0000000..e67d305
--- /dev/null
+++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOIT.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import static org.junit.Assert.assertEquals;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.mapping.annotations.Column;
+import com.datastax.driver.mapping.annotations.PartitionKey;
+import com.datastax.driver.mapping.annotations.Table;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SerializableMatcher;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test of {@link CassandraIO} on a concrete and independent Cassandra instance.
+ *
+ * <p>This test requires a running Cassandra instance, and the test dataset must exists.
+ *
+ * <p>You can run this test directly using Maven with:
+ *
+ * <pre>{@code
+ * mvn -e -Pio-it verify -pl sdks/java/io/cassandra -DintegrationTestPipelineOptions='[
+ * "--cassandraHost=1.2.3.4",
+ * "--cassandraPort=9042"]'
+ * }</pre>
+ */
+@RunWith(JUnit4.class)
+public class CassandraIOIT implements Serializable {
+
+ private static IOTestPipelineOptions options;
+
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ PipelineOptionsFactory.register(IOTestPipelineOptions.class);
+ options = TestPipeline.testingPipelineOptions()
+ .as(IOTestPipelineOptions.class);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ // cleanup the write table
+ CassandraTestDataSet.cleanUpDataTable(options);
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ PCollection<Scientist> output = pipeline.apply(CassandraIO.<Scientist>read()
+ .withHosts(Collections.singletonList(options.getCassandraHost()))
+ .withPort(options.getCassandraPort())
+ .withKeyspace(CassandraTestDataSet.KEYSPACE)
+ .withTable(CassandraTestDataSet.TABLE_READ_NAME)
+ .withEntity(Scientist.class)
+ .withCoder(SerializableCoder.of(Scientist.class)));
+
+ PAssert.thatSingleton(output.apply("Count scientist", Count.<Scientist>globally()))
+ .isEqualTo(1000L);
+
+ PCollection<KV<String, Integer>> mapped =
+ output.apply(
+ MapElements.via(
+ new SimpleFunction<Scientist, KV<String, Integer>>() {
+ public KV<String, Integer> apply(Scientist scientist) {
+ KV<String, Integer> kv = KV.of(scientist.name, scientist.id);
+ return kv;
+ }
+ }
+ )
+ );
+ PAssert.that(mapped.apply("Count occurrences per scientist", Count.<String, Integer>perKey()))
+ .satisfies(
+ new SerializableFunction<Iterable<KV<String, Long>>, Void>() {
+ @Override
+ public Void apply(Iterable<KV<String, Long>> input) {
+ for (KV<String, Long> element : input) {
+ assertEquals(element.getKey(), 1000 / 10, element.getValue().longValue());
+ }
+ return null;
+ }
+ }
+ );
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ IOTestPipelineOptions options =
+ TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
+
+ options.setOnSuccessMatcher(
+ new CassandraMatcher(
+ CassandraTestDataSet.getCluster(options),
+ CassandraTestDataSet.TABLE_WRITE_NAME));
+
+ TestPipeline.convertToArgs(options);
+
+ ArrayList<ScientistForWrite> data = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ ScientistForWrite scientist = new ScientistForWrite();
+ scientist.id = i;
+ scientist.name = "Name " + i;
+ data.add(scientist);
+ }
+
+ pipeline
+ .apply(Create.of(data))
+ .apply(CassandraIO.<ScientistForWrite>write()
+ .withHosts(Collections.singletonList(options.getCassandraHost()))
+ .withPort(options.getCassandraPort())
+ .withKeyspace(CassandraTestDataSet.KEYSPACE)
+ .withEntity(ScientistForWrite.class));
+
+ pipeline.run().waitUntilFinish();
+ }
+
+ /**
+ * Simple matcher.
+ */
+ public class CassandraMatcher extends TypeSafeMatcher<PipelineResult>
+ implements SerializableMatcher<PipelineResult> {
+
+ private String tableName;
+ private Cluster cluster;
+
+ public CassandraMatcher(Cluster cluster, String tableName) {
+ this.cluster = cluster;
+ this.tableName = tableName;
+ }
+
+ @Override
+ protected boolean matchesSafely(PipelineResult pipelineResult) {
+ pipelineResult.waitUntilFinish();
+ Session session = cluster.connect();
+ ResultSet result = session.execute("select id,name from " + CassandraTestDataSet.KEYSPACE
+ + "." + tableName);
+ List<Row> rows = result.all();
+ if (rows.size() != 1000) {
+ return false;
+ }
+ for (Row row : rows) {
+ if (!row.getString("name").matches("Name.*")) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("Expected Cassandra record pattern is (Name.*)");
+ }
+ }
+
+ /**
+ * Simple Cassandra entity representing a scientist. Used for read test.
+ */
+ @Table(name = CassandraTestDataSet.TABLE_READ_NAME, keyspace = CassandraTestDataSet.KEYSPACE)
+ public static class Scientist implements Serializable {
+
+ @PartitionKey
+ @Column(name = "id")
+ private final int id;
+
+ @Column(name = "name")
+ private final String name;
+
+ public Scientist() {
+ this(0, "");
+ }
+
+ public Scientist(int id) {
+ this(0, "");
+ }
+
+ public Scientist(int id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ /**
+ * Simple Cassandra entity representing a scientist, used for write test.
+ */
+ @Table(name = CassandraTestDataSet.TABLE_WRITE_NAME, keyspace = CassandraTestDataSet.KEYSPACE)
+ public class ScientistForWrite implements Serializable {
+
+ @PartitionKey
+ @Column(name = "id")
+ public Integer id;
+
+ @Column(name = "name")
+ public String name;
+
+ public String toString() {
+ return id + ":" + name;
+ }
+
+ }
+
+}
diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
new file mode 100644
index 0000000..cfd78d2
--- /dev/null
+++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import com.datastax.driver.mapping.annotations.Column;
+import com.datastax.driver.mapping.annotations.Table;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Tests of {@link CassandraIO}. */
+public class CassandraIOTest implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CassandraIOTest.class);
+
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ public void testEstimatedSizeBytes() throws Exception {
+ final FakeCassandraService service = new FakeCassandraService();
+ service.load();
+
+ PipelineOptions pipelineOptions = PipelineOptionsFactory.create();
+ CassandraIO.Read spec = CassandraIO.<Scientist>read().withCassandraService(service);
+ CassandraIO.CassandraSource source = new CassandraIO.CassandraSource(
+ spec,
+ null);
+ long estimatedSizeBytes = source.getEstimatedSizeBytes(pipelineOptions);
+ // the size is the sum of the bytes size of the String representation of a scientist in the map
+ assertEquals(113890, estimatedSizeBytes);
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ FakeCassandraService service = new FakeCassandraService();
+ service.load();
+
+ PCollection<Scientist> output = pipeline.apply(CassandraIO
+ .<Scientist>read()
+ .withCassandraService(service)
+ .withKeyspace("beam")
+ .withTable("scientist")
+ .withCoder(SerializableCoder.of(Scientist.class))
+ .withEntity(Scientist.class)
+ );
+
+ PAssert.thatSingleton(output.apply("Count", Count.<Scientist>globally()))
+ .isEqualTo(10000L);
+
+ PCollection<KV<String, Integer>> mapped =
+ output.apply(
+ MapElements.via(
+ new SimpleFunction<Scientist, KV<String, Integer>>() {
+ public KV<String, Integer> apply(Scientist scientist) {
+ return KV.of(scientist.name, scientist.id);
+ }
+ }));
+ PAssert.that(mapped.apply("Count occurrences per scientist", Count.<String, Integer>perKey()))
+ .satisfies(
+ new SerializableFunction<Iterable<KV<String, Long>>, Void>() {
+ @Override
+ public Void apply(Iterable<KV<String, Long>> input) {
+ for (KV<String, Long> element : input) {
+ assertEquals(element.getKey(), 1000, element.getValue().longValue());
+ }
+ return null;
+ }
+ });
+
+ pipeline.run();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ FakeCassandraService service = new FakeCassandraService();
+
+ ArrayList<Scientist> data = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ Scientist scientist = new Scientist();
+ scientist.id = i;
+ scientist.name = "Name " + i;
+ data.add(scientist);
+ }
+
+ pipeline
+ .apply(Create.of(data))
+ .apply(CassandraIO.<Scientist>write().withCassandraService(service)
+ .withKeyspace("beam")
+ .withEntity(Scientist.class));
+ pipeline.run();
+
+ assertEquals(service.getTable().size(), 1000);
+ for (Scientist scientist : service.getTable().values()) {
+ assertTrue(scientist.name.matches("Name (\\d*)"));
+ }
+ }
+
+ /**
+ * A {@link CassandraService} implementation that stores the entity in memory.
+ */
+ private static class FakeCassandraService implements CassandraService<Scientist> {
+
+ private static final Map<Integer, Scientist> table = new ConcurrentHashMap<>();
+
+ public void load() {
+ table.clear();
+ String[] scientists = {
+ "Lovelace",
+ "Franklin",
+ "Meitner",
+ "Hopper",
+ "Curie",
+ "Faraday",
+ "Newton",
+ "Bohr",
+ "Galilei",
+ "Maxwell"
+ };
+ for (int i = 0; i < 10000; i++) {
+ int index = i % scientists.length;
+ Scientist scientist = new Scientist();
+ scientist.id = i;
+ scientist.name = scientists[index];
+ table.put(scientist.id, scientist);
+ }
+ }
+
+ public Map<Integer, Scientist> getTable() {
+ return table;
+ }
+
+ @Override
+ public FakeCassandraReader createReader(CassandraIO.CassandraSource source) {
+ return new FakeCassandraReader(source);
+ }
+
+ static class FakeCassandraReader extends BoundedSource.BoundedReader {
+
+ private final CassandraIO.CassandraSource source;
+
+ private Iterator<Scientist> iterator;
+ private Scientist current;
+
+ public FakeCassandraReader(CassandraIO.CassandraSource source) {
+ this.source = source;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ iterator = table.values().iterator();
+ return advance();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ if (iterator.hasNext()) {
+ current = iterator.next();
+ return true;
+ }
+ current = null;
+ return false;
+ }
+
+ @Override
+ public void close() {
+ iterator = null;
+ current = null;
+ }
+
+ @Override
+ public Scientist getCurrent() throws NoSuchElementException {
+ if (current == null) {
+ throw new NoSuchElementException();
+ }
+ return current;
+ }
+
+ @Override
+ public CassandraIO.CassandraSource getCurrentSource() {
+ return this.source;
+ }
+
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(CassandraIO.Read spec) {
+ long size = 0L;
+ for (Scientist scientist : table.values()) {
+ size = size + scientist.toString().getBytes().length;
+ }
+ return size;
+ }
+
+ @Override
+ public List<BoundedSource<Scientist>> split(CassandraIO.Read spec,
+ long desiredBundleSizeBytes) {
+ List<BoundedSource<Scientist>> sources = new ArrayList<>();
+ sources.add(new CassandraIO.CassandraSource<Scientist>(spec, null));
+ return sources;
+ }
+
+ static class FakeCassandraWriter implements Writer<Scientist> {
+
+ @Override
+ public void write(Scientist scientist) {
+ table.put(scientist.id, scientist);
+ }
+
+ @Override
+ public void close() {
+ // nothing to do
+ }
+
+ }
+
+ @Override
+ public FakeCassandraWriter createWriter(CassandraIO.Write<Scientist> spec) {
+ return new FakeCassandraWriter();
+ }
+
+ }
+
+ /** Simple Cassandra entity used in test. */
+ @Table(name = "scientist", keyspace = "beam")
+ public static class Scientist implements Serializable {
+
+ @Column(name = "person_name")
+ public String name;
+
+ @Column(name = "person_id")
+ public int id;
+
+ public String toString() {
+ return id + ":" + name;
+ }
+ }
+}
diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImplTest.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImplTest.java
new file mode 100644
index 0000000..6a68e90
--- /dev/null
+++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImplTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Metadata;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests on {@link CassandraServiceImplTest}.
+ */
+public class CassandraServiceImplTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CassandraServiceImplTest.class);
+
+ private static final String MURMUR3_PARTITIONER = "org.apache.cassandra.dht.Murmur3Partitioner";
+
+ private Cluster createClusterMock() {
+ Metadata metadata = Mockito.mock(Metadata.class);
+ Mockito.when(metadata.getPartitioner()).thenReturn(MURMUR3_PARTITIONER);
+ Cluster cluster = Mockito.mock(Cluster.class);
+ Mockito.when(cluster.getMetadata()).thenReturn(metadata);
+ return cluster;
+ }
+
+ @Test
+ public void testValidPartitioner() throws Exception {
+ assertTrue(CassandraServiceImpl.isMurmur3Partitioner(createClusterMock()));
+ }
+
+ @Test
+ public void testDistance() throws Exception {
+ BigInteger distance = CassandraServiceImpl.distance(10L, 100L);
+ assertEquals(BigInteger.valueOf(90), distance);
+
+ distance = CassandraServiceImpl.distance(100L, 10L);
+ assertEquals(new BigInteger("18446744073709551525"), distance);
+ }
+
+ @Test
+ public void testRingFraction() throws Exception {
+ // simulate a first range taking "half" of the available tokens
+ List<CassandraServiceImpl.TokenRange> tokenRanges = new ArrayList<>();
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1, Long.MIN_VALUE, 0));
+ assertEquals(0.5, CassandraServiceImpl.getRingFraction(tokenRanges), 0);
+
+ // add a second range to cover all tokens available
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1, 0, Long.MAX_VALUE));
+ assertEquals(1.0, CassandraServiceImpl.getRingFraction(tokenRanges), 0);
+ }
+
+ @Test
+ public void testEstimatedSizeBytes() throws Exception {
+ List<CassandraServiceImpl.TokenRange> tokenRanges = new ArrayList<>();
+ // one partition containing all tokens, the size is actually the size of the partition
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE));
+ assertEquals(1000, CassandraServiceImpl.getEstimatedSizeBytes(tokenRanges));
+
+ // one partition with half of the tokens, we estimate the size to the double of this partition
+ tokenRanges = new ArrayList<>();
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1000, Long.MIN_VALUE, 0));
+ assertEquals(2000, CassandraServiceImpl.getEstimatedSizeBytes(tokenRanges));
+
+ // we have three partitions covering all tokens, the size is the sum of partition size *
+ // partition count
+ tokenRanges = new ArrayList<>();
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1000, Long.MIN_VALUE, -3));
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(1, 1000, -2, 10000));
+ tokenRanges.add(new CassandraServiceImpl.TokenRange(2, 3000, 10001, Long.MAX_VALUE));
+ assertEquals(8000, CassandraServiceImpl.getEstimatedSizeBytes(tokenRanges));
+ }
+
+ @Test
+ public void testThreeSplits() throws Exception {
+ CassandraServiceImpl service = new CassandraServiceImpl();
+ CassandraIO.Read spec = CassandraIO.read().withKeyspace("beam").withTable("test");
+ List<CassandraIO.CassandraSource> sources = service.split(spec, 50, 150);
+ assertEquals(3, sources.size());
+ assertTrue(sources.get(0).splitQuery.matches("SELECT \\* FROM beam.test WHERE token\\"
+ + "(\\$pk\\)<(.*)"));
+ assertTrue(sources.get(1).splitQuery.matches("SELECT \\* FROM beam.test WHERE token\\"
+ + "(\\$pk\\)>=(.*) AND token\\(\\$pk\\)<(.*)"));
+ assertTrue(sources.get(2).splitQuery.matches("SELECT \\* FROM beam.test WHERE token\\"
+ + "(\\$pk\\)>=(.*)"));
+ }
+
+ @Test
+ public void testTwoSplits() throws Exception {
+ CassandraServiceImpl service = new CassandraServiceImpl();
+ CassandraIO.Read spec = CassandraIO.read().withKeyspace("beam").withTable("test");
+ List<CassandraIO.CassandraSource> sources = service.split(spec, 50, 100);
+ assertEquals(2, sources.size());
+ LOG.info("TOKEN: " + ((double) Long.MAX_VALUE / 2));
+ LOG.info(sources.get(0).splitQuery);
+ LOG.info(sources.get(1).splitQuery);
+ assertEquals("SELECT * FROM beam.test WHERE token($pk)<" + ((double) Long.MAX_VALUE / 2) + ";",
+ sources.get(0).splitQuery);
+ assertEquals("SELECT * FROM beam.test WHERE token($pk)>=" + ((double) Long.MAX_VALUE / 2)
+ + ";",
+ sources.get(1).splitQuery);
+ }
+
+ @Test
+ public void testUniqueSplit() throws Exception {
+ CassandraServiceImpl service = new CassandraServiceImpl();
+ CassandraIO.Read spec = CassandraIO.read().withKeyspace("beam").withTable("test");
+ List<CassandraIO.CassandraSource> sources = service.split(spec, 100, 100);
+ assertEquals(1, sources.size());
+ assertEquals("SELECT * FROM beam.test;", sources.get(0).splitQuery);
+ }
+
+}
diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraTestDataSet.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraTestDataSet.java
new file mode 100644
index 0000000..461f5ea
--- /dev/null
+++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraTestDataSet.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.cassandra;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+
+import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Manipulates test data used by the {@link CassandraIO} tests.
+ *
+ * <p>This is independent from the tests so that for read tests it can be run separately after
+ * data store creation rather than every time (which can be more fragile).
+ */
+public class CassandraTestDataSet {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CassandraTestDataSet.class);
+
+ /**
+ * Use this to create the read tables before IT read tests.
+ *
+ * <p>To invoke this class, you can use this command line:
+ * (run from the cassandra root directory)
+ * mvn test-compile exec:java -Dexec.mainClass=org.apache.beam.sdk.io.cassandra
+ * .CassandraTestDataSet \
+ * -Dexec.args="--cassandraHost=localhost --cassandraPort=9042 \
+ * -Dexec.classpathScope=test
+ * @param args Please pass options from IOTestPipelineOptions used for connection to Cassandra as
+ * shown above.
+ */
+ public static void main(String[] args) {
+ PipelineOptionsFactory.register(IOTestPipelineOptions.class);
+ IOTestPipelineOptions options =
+ PipelineOptionsFactory.fromArgs(args).as(IOTestPipelineOptions.class);
+
+ createDataTable(options);
+ }
+
+ public static final String KEYSPACE = "BEAM";
+ public static final String TABLE_READ_NAME = "BEAM_READ_TEST";
+ public static final String TABLE_WRITE_NAME = "BEAM_WRITE_TEST";
+
+ public static void createDataTable(IOTestPipelineOptions options) {
+ createTable(options, TABLE_READ_NAME);
+ insertTestData(options, TABLE_READ_NAME);
+ createTable(options, TABLE_WRITE_NAME);
+ }
+
+ public static Cluster getCluster(IOTestPipelineOptions options) {
+ return Cluster.builder()
+ .addContactPoint(options.getCassandraHost())
+ .withPort(options.getCassandraPort())
+ .build();
+ }
+
+ private static void createTable(IOTestPipelineOptions options, String tableName) {
+ Cluster cluster = null;
+ Session session = null;
+ try {
+ cluster = getCluster(options);
+ session = cluster.connect();
+
+ LOG.info("Create {} keyspace if not exists", KEYSPACE);
+ session.execute("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH REPLICATION = "
+ + "{'class':'SimpleStrategy', 'replication_factor':3};");
+
+ session.execute("USE " + KEYSPACE);
+
+ LOG.info("Create {} table if not exists", tableName);
+ session.execute("CREATE TABLE IF NOT EXISTS " + tableName + "(id int, name text, PRIMARY "
+ + "KEY(id))");
+ } finally {
+ if (session != null) {
+ session.close();
+ }
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+ }
+
+ private static void insertTestData(IOTestPipelineOptions options, String tableName) {
+ Cluster cluster = null;
+ Session session = null;
+ try {
+ cluster = getCluster(options);
+ session = cluster.connect();
+
+ LOG.info("Insert test dataset");
+ String[] scientists = {
+ "Lovelace",
+ "Franklin",
+ "Meitner",
+ "Hopper",
+ "Curie",
+ "Faraday",
+ "Newton",
+ "Bohr",
+ "Galilei",
+ "Maxwell"
+ };
+ for (int i = 0; i < 1000; i++) {
+ int index = i % scientists.length;
+ session.execute("INSERT INTO " + KEYSPACE + "." + tableName + "(id, name) values("
+ + i + ",'" + scientists[index] + "');");
+ }
+ } finally {
+ if (session != null) {
+ session.close();
+ }
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+ }
+
+ public static void cleanUpDataTable(IOTestPipelineOptions options) {
+ Cluster cluster = null;
+ Session session = null;
+ try {
+ cluster = getCluster(options);
+ session = cluster.connect();
+ session.execute("TRUNCATE TABLE " + KEYSPACE + "." + TABLE_WRITE_NAME);
+ } finally {
+ if (session != null) {
+ session.close();
+ }
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+ }
+
+}
diff --git a/sdks/java/io/common/pom.xml b/sdks/java/io/common/pom.xml
index f7525fd..eb79091 100644
--- a/sdks/java/io/common/pom.xml
+++ b/sdks/java/io/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -38,5 +38,15 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
index d3915c9..256c94d 100644
--- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
+++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
@@ -71,9 +71,21 @@
Integer getElasticsearchHttpPort();
void setElasticsearchHttpPort(Integer value);
- @Description("Tcp port for elasticsearch server")
- @Default.Integer(9300)
- Integer getElasticsearchTcpPort();
- void setElasticsearchTcpPort(Integer value);
+ /* Solr */
+ @Description("Address of Zookeeper server for Solr")
+ @Default.String("zookeeper-server")
+ String getSolrZookeeperServer();
+ void setSolrZookeeperServer(String value);
+
+ /* Cassandra */
+ @Description("Host for Cassandra server (host name/ip address)")
+ @Default.String("cassandra-host")
+ String getCassandraHost();
+ void setCassandraHost(String host);
+
+ @Description("Port for Cassandra server")
+ @Default.Integer(7001)
+ Integer getCassandraPort();
+ void setCassandraPort(Integer port);
}
diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java
new file mode 100644
index 0000000..5f0a2fb
--- /dev/null
+++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.common;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/**
+ * Used to pass values around within test pipelines.
+ */
+@AutoValue
+public abstract class TestRow implements Serializable, Comparable<TestRow> {
+ /**
+ * Manually create a test row.
+ */
+ public static TestRow create(Integer id, String name) {
+ return new AutoValue_TestRow(id, name);
+ }
+
+ public abstract Integer id();
+ public abstract String name();
+
+ public int compareTo(TestRow other) {
+ return id().compareTo(other.id());
+ }
+
+ /**
+ * Creates a {@link org.apache.beam.sdk.io.common.TestRow} from the seed value.
+ */
+ public static TestRow fromSeed(Integer seed) {
+ return create(seed, getNameForSeed(seed));
+ }
+
+ /**
+ * Returns the name field value produced from the given seed.
+ */
+ public static String getNameForSeed(Integer seed) {
+ return "Testval" + seed;
+ }
+
+ /**
+ * Returns a range of {@link org.apache.beam.sdk.io.common.TestRow}s for seed values between
+ * rangeStart (inclusive) and rangeEnd (exclusive).
+ */
+ public static Iterable<TestRow> getExpectedValues(int rangeStart, int rangeEnd) {
+ List<TestRow> ret = new ArrayList<TestRow>(rangeEnd - rangeStart + 1);
+ for (int i = rangeStart; i < rangeEnd; i++) {
+ ret.add(fromSeed(i));
+ }
+ return ret;
+ }
+
+ /**
+ * Uses the input Long values as seeds to produce {@link org.apache.beam.sdk.io.common.TestRow}s.
+ */
+ public static class DeterministicallyConstructTestRowFn extends DoFn<Long, TestRow> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(fromSeed(c.element().intValue()));
+ }
+ }
+
+ /**
+ * Outputs just the name stored in the {@link org.apache.beam.sdk.io.common.TestRow}.
+ */
+ public static class SelectNameFn extends DoFn<TestRow, String> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().name());
+ }
+ }
+
+ /**
+ * Precalculated hashes - you can calculate an entry by running HashingFn on
+ * the name() for the rows generated from seeds in [0, n).
+ */
+ private static final Map<Integer, String> EXPECTED_HASHES = ImmutableMap.of(
+ 1000, "7d94d63a41164be058a9680002914358"
+ );
+
+ /**
+ * Returns the hash value that {@link org.apache.beam.sdk.io.common.HashingFn} will return when it
+ * is run on {@link org.apache.beam.sdk.io.common.TestRow}s produced by
+ * getExpectedValues(0, rowCount).
+ */
+ public static String getExpectedHashForRowCount(int rowCount)
+ throws UnsupportedOperationException {
+ String hash = EXPECTED_HASHES.get(rowCount);
+ if (hash == null) {
+ throw new UnsupportedOperationException("No hash for that row count");
+ }
+ return hash;
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/pom.xml b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/pom.xml
new file mode 100644
index 0000000..7793bc6
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-2</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests :: 2.x</name>
+ <description>Tests of ElasticsearchIO on Elasticsearch 2.x</description>
+
+ <properties>
+ <elasticsearch.version>2.4.1</elasticsearch.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/sdks/java/io/elasticsearch/src/test/contrib/create_elk_container.sh b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/contrib/create_elk_container.sh
similarity index 100%
rename from sdks/java/io/elasticsearch/src/test/contrib/create_elk_container.sh
rename to sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/contrib/create_elk_container.sh
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
new file mode 100644
index 0000000..93fdd9b
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.elasticsearch.client.RestClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * A test of {@link ElasticsearchIO} on an independent Elasticsearch v2.x instance.
+ *
+ * <p>This test requires a running instance of Elasticsearch, and the test dataset must exist in the
+ * database.
+ *
+ * <p>You can run this test by doing the following from the beam parent module directory:
+ *
+ * <pre>
+ * mvn -e -Pio-it verify -pl sdks/java/io/elasticsearch -DintegrationTestPipelineOptions='[
+ * "--elasticsearchServer=1.2.3.4",
+ * "--elasticsearchHttpPort=9200"]'
+ * </pre>
+ */
+public class ElasticsearchIOIT {
+ private static RestClient restClient;
+ private static IOTestPipelineOptions options;
+ private static ConnectionConfiguration readConnectionConfiguration;
+ private static ConnectionConfiguration writeConnectionConfiguration;
+ private static ElasticsearchIOTestCommon elasticsearchIOTestCommon;
+
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ PipelineOptionsFactory.register(IOTestPipelineOptions.class);
+ options = TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
+ readConnectionConfiguration = ElasticsearchIOITCommon
+ .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.READ);
+ writeConnectionConfiguration = ElasticsearchIOITCommon
+ .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.WRITE);
+ restClient = readConnectionConfiguration.createClient();
+ elasticsearchIOTestCommon = new ElasticsearchIOTestCommon(readConnectionConfiguration,
+ restClient, true);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ ElasticSearchIOTestUtils.deleteIndex(writeConnectionConfiguration, restClient);
+ restClient.close();
+ }
+
+ @Test
+ public void testSplitsVolume() throws Exception {
+ Read read = ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration);
+ BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null, null,
+ null);
+ //desiredBundleSize is ignored because in ES 2.x there is no way to split shards. So we get
+ // as many bundles as ES shards and bundle size is shard size
+ long desiredBundleSizeBytes = 0;
+ List<? extends BoundedSource<String>> splits = initialSource
+ .split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
+ //this is the number of ES shards
+ // (By default, each index in Elasticsearch is allocated 5 primary shards)
+ long expectedNumSplits = 5;
+ assertEquals(expectedNumSplits, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<String> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ assertEquals(expectedNumSplits, nonEmptySplits);
+ }
+
+ @Test
+ public void testReadVolume() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testRead();
+ }
+
+ @Test
+ public void testWriteVolume() throws Exception {
+ ElasticsearchIOTestCommon elasticsearchIOTestCommonWrite = new ElasticsearchIOTestCommon(
+ writeConnectionConfiguration, restClient, true);
+ elasticsearchIOTestCommonWrite.setPipeline(pipeline);
+ elasticsearchIOTestCommonWrite.testWrite();
+ }
+
+ @Test
+ public void testSizesVolume() throws Exception {
+ elasticsearchIOTestCommon.testSizes();
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
new file mode 100644
index 0000000..06298cd
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_INDEX;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_TYPE;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.NUM_DOCS_UTESTS;
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Tests for {@link ElasticsearchIO} version 2.x. */
+
+@RunWith(JUnit4.class)
+public class ElasticsearchIOTest implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIOTest.class);
+
+ private static final String ES_IP = "127.0.0.1";
+
+ private static Node node;
+ private static RestClient restClient;
+ private static ConnectionConfiguration connectionConfiguration;
+ //cannot use inheritance because ES5 test already extends ESIntegTestCase.
+ private static ElasticsearchIOTestCommon elasticsearchIOTestCommon;
+
+ @ClassRule
+ public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
+
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ ServerSocket serverSocket = new ServerSocket(0);
+ int esHttpPort = serverSocket.getLocalPort();
+ serverSocket.close();
+ LOG.info("Starting embedded Elasticsearch instance ({})", esHttpPort);
+ Settings.Builder settingsBuilder =
+ Settings.settingsBuilder()
+ .put("cluster.name", "beam")
+ .put("http.enabled", "true")
+ .put("node.data", "true")
+ .put("path.data", TEMPORARY_FOLDER.getRoot().getPath())
+ .put("path.home", TEMPORARY_FOLDER.getRoot().getPath())
+ .put("node.name", "beam")
+ .put("network.host", ES_IP)
+ .put("http.port", esHttpPort)
+ .put("index.store.stats_refresh_interval", 0)
+ // had problems with some jdk, embedded ES was too slow for bulk insertion,
+ // and queue of 50 was full. No pb with real ES instance (cf testWrite integration test)
+ .put("threadpool.bulk.queue_size", 100);
+ node = new Node(settingsBuilder.build());
+ LOG.info("Elasticsearch node created");
+ node.start();
+ connectionConfiguration = ConnectionConfiguration
+ .create(new String[] { "http://" + ES_IP + ":" + esHttpPort }, ES_INDEX, ES_TYPE);
+ restClient = connectionConfiguration.createClient();
+ elasticsearchIOTestCommon = new ElasticsearchIOTestCommon(connectionConfiguration, restClient,
+ false);
+ }
+
+ @AfterClass
+ public static void afterClass() throws IOException{
+ restClient.close();
+ node.close();
+ }
+
+ @Before
+ public void before() throws Exception {
+ ElasticSearchIOTestUtils.deleteIndex(connectionConfiguration, restClient);
+ }
+
+ @Test
+ public void testSizes() throws Exception {
+ elasticsearchIOTestCommon.testSizes();
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testRead();
+ }
+
+ @Test
+ public void testReadWithQuery() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testReadWithQuery();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testWrite();
+ }
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testWriteWithErrors() throws Exception {
+ elasticsearchIOTestCommon.setExpectedException(expectedException);
+ elasticsearchIOTestCommon.testWriteWithErrors();
+ }
+
+ @Test
+ public void testWriteWithMaxBatchSize() throws Exception {
+ elasticsearchIOTestCommon.testWriteWithMaxBatchSize();
+ }
+
+ @Test
+ public void testWriteWithMaxBatchSizeBytes() throws Exception {
+ elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
+ }
+
+ @Test
+ public void testSplit() throws Exception {
+ ElasticSearchIOTestUtils
+ .insertTestDocuments(connectionConfiguration, NUM_DOCS_UTESTS, restClient);
+ PipelineOptions options = PipelineOptionsFactory.create();
+ Read read =
+ ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
+ BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null, null,
+ null);
+ //desiredBundleSize is ignored because in ES 2.x there is no way to split shards. So we get
+ // as many bundles as ES shards and bundle size is shard size
+ int desiredBundleSizeBytes = 0;
+ List<? extends BoundedSource<String>> splits =
+ initialSource.split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
+ //this is the number of ES shards
+ // (By default, each index in Elasticsearch is allocated 5 primary shards)
+ int expectedNumSplits = 5;
+ assertEquals(expectedNumSplits, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<String> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ assertEquals("Wrong number of non empty splits", expectedNumSplits, nonEmptySplits);
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/pom.xml b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/pom.xml
new file mode 100644
index 0000000..ba76316
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/pom.xml
@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-5</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests :: 5.x</name>
+ <description>Tests of ElasticsearchIO on Elasticsearch 5.x</description>
+
+ <properties>
+ <elasticsearch.version>5.6.3</elasticsearch.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.elasticsearch.test</groupId>
+ <artifactId>framework</artifactId>
+ <version>${elasticsearch.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!--to enable rest connection on embeded elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>transport-netty4-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.carrotsearch.randomizedtesting</groupId>
+ <artifactId>randomizedtesting-runner</artifactId>
+ <version>2.5.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <!--needed for ESIntegTestCase-->
+ <configuration>
+ <argLine>-Dtests.security.manager=false</argLine>
+ </configuration>
+ </plugin>
+ <!-- Overridden enforcer plugin for JDK1.8 for running tests -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.4.1</version>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <enforceBytecodeVersion>
+ <maxJdkVersion>1.8</maxJdkVersion>
+ <excludes>
+ <!-- Supplied by the user JDK and compiled with matching
+ version. Is not shaded, so safe to ignore. -->
+ <exclude>jdk.tools:jdk.tools</exclude>
+ </excludes>
+ </enforceBytecodeVersion>
+ <requireJavaVersion>
+ <version>[1.8,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>extra-enforcer-rules</artifactId>
+ <version>1.0-beta-6</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/java/io/elasticsearch/src/test/contrib/create_elk_container.sh b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/contrib/create_elk_container.sh
similarity index 100%
copy from sdks/java/io/elasticsearch/src/test/contrib/create_elk_container.sh
copy to sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/contrib/create_elk_container.sh
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
new file mode 100644
index 0000000..7c33740
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.elasticsearch.client.RestClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * A test of {@link ElasticsearchIO} on an independent Elasticsearch v5.x instance.
+ *
+ * <p>This test requires a running instance of Elasticsearch, and the test dataset must exist in the
+ * database.
+ *
+ * <p>You can run this test by doing the following from the beam parent module directory:
+ *
+ * <pre>
+ * mvn -e -Pio-it verify -pl sdks/java/io/elasticsearch -DintegrationTestPipelineOptions='[
+ * "--elasticsearchServer=1.2.3.4",
+ * "--elasticsearchHttpPort=9200"]'
+ * </pre>
+ */
+public class ElasticsearchIOIT {
+ private static RestClient restClient;
+ private static IOTestPipelineOptions options;
+ private static ConnectionConfiguration readConnectionConfiguration;
+ private static ConnectionConfiguration writeConnectionConfiguration;
+ private static ElasticsearchIOTestCommon elasticsearchIOTestCommon;
+
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ PipelineOptionsFactory.register(IOTestPipelineOptions.class);
+ options = TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
+ readConnectionConfiguration = ElasticsearchIOITCommon
+ .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.READ);
+ writeConnectionConfiguration = ElasticsearchIOITCommon
+ .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.WRITE);
+ restClient = readConnectionConfiguration.createClient();
+ elasticsearchIOTestCommon = new ElasticsearchIOTestCommon(readConnectionConfiguration,
+ restClient, true);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ ElasticSearchIOTestUtils.deleteIndex(writeConnectionConfiguration, restClient);
+ restClient.close();
+ }
+
+ @Test
+ public void testSplitsVolume() throws Exception {
+ Read read = ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration);
+ BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null, null,
+ null);
+ int desiredBundleSizeBytes = 10000;
+ List<? extends BoundedSource<String>> splits =
+ initialSource.split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
+ long indexSize = BoundedElasticsearchSource.estimateIndexSize(readConnectionConfiguration);
+ float expectedNumSourcesFloat = (float) indexSize / desiredBundleSizeBytes;
+ int expectedNumSources = (int) Math.ceil(expectedNumSourcesFloat);
+ assertEquals(expectedNumSources, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<String> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ assertEquals("Wrong number of empty splits", expectedNumSources, nonEmptySplits);
+ }
+
+ @Test
+ public void testReadVolume() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testRead();
+ }
+
+ @Test
+ public void testWriteVolume() throws Exception {
+ //cannot share elasticsearchIOTestCommon because tests run in parallel.
+ ElasticsearchIOTestCommon elasticsearchIOTestCommonWrite = new ElasticsearchIOTestCommon(
+ writeConnectionConfiguration, restClient, true);
+ elasticsearchIOTestCommonWrite.setPipeline(pipeline);
+ elasticsearchIOTestCommonWrite.testWrite();
+ }
+
+ @Test
+ public void testSizesVolume() throws Exception {
+ elasticsearchIOTestCommon.testSizes();
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
new file mode 100644
index 0000000..50a8764
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_INDEX;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_TYPE;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.NUM_DOCS_UTESTS;
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/*
+Cannot use @RunWith(JUnit4.class) with ESIntegTestCase
+Cannot have @BeforeClass @AfterClass with ESIntegTestCase
+*/
+
+/** Tests for {@link ElasticsearchIO} version 5. */
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class ElasticsearchIOTest extends ESIntegTestCase implements Serializable {
+
+ private ElasticsearchIOTestCommon elasticsearchIOTestCommon;
+ private ConnectionConfiguration connectionConfiguration;
+
+ private String[] fillAddresses(){
+ ArrayList<String> result = new ArrayList<>();
+ for (InetSocketAddress address : cluster().httpAddresses()){
+ result.add(String.format("http://%s:%s", address.getHostString(), address.getPort()));
+ }
+ return result.toArray(new String[result.size()]);
+ }
+
+
+ @Override
+ protected Settings nodeSettings(int nodeOrdinal) {
+ System.setProperty("es.set.netty.runtime.available.processors", "false");
+ return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+ .put("http.enabled", "true")
+ // had problems with some jdk, embedded ES was too slow for bulk insertion,
+ // and queue of 50 was full. No pb with real ES instance (cf testWrite integration test)
+ .put("thread_pool.bulk.queue_size", 100)
+ .build();
+ }
+
+ @Override
+ public Settings indexSettings() {
+ return Settings.builder().put(super.indexSettings())
+ //useful to have updated sizes for getEstimatedSize
+ .put("index.store.stats_refresh_interval", 0)
+ .build();
+ }
+
+ @Override
+ protected Collection<Class<? extends Plugin>> nodePlugins() {
+ ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>();
+ plugins.add(Netty4Plugin.class);
+ return plugins;
+ }
+
+ @Before
+ public void setup(){
+ if (connectionConfiguration == null){
+ connectionConfiguration = ConnectionConfiguration.create(fillAddresses(), ES_INDEX, ES_TYPE);
+ elasticsearchIOTestCommon = new ElasticsearchIOTestCommon(connectionConfiguration,
+ getRestClient(), false);
+ }
+ }
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
+ @Test
+ public void testSizes() throws Exception {
+ // need to create the index using the helper method (not create it at first insertion)
+ // for the indexSettings() to be run
+ createIndex(ES_INDEX);
+ elasticsearchIOTestCommon.testSizes();
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ // need to create the index using the helper method (not create it at first insertion)
+ // for the indexSettings() to be run
+ createIndex(ES_INDEX);
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testRead();
+ }
+
+ @Test
+ public void testReadWithQuery() throws Exception {
+ // need to create the index using the helper method (not create it at first insertion)
+ // for the indexSettings() to be run
+ createIndex(ES_INDEX);
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testReadWithQuery();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ elasticsearchIOTestCommon.setPipeline(pipeline);
+ elasticsearchIOTestCommon.testWrite();
+ }
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testWriteWithErrors() throws Exception {
+ elasticsearchIOTestCommon.setExpectedException(expectedException);
+ elasticsearchIOTestCommon.testWriteWithErrors();
+ }
+
+ @Test
+ public void testWriteWithMaxBatchSize() throws Exception {
+ elasticsearchIOTestCommon.testWriteWithMaxBatchSize();
+ }
+
+ @Test
+ public void testWriteWithMaxBatchSizeBytes() throws Exception {
+ elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
+ }
+
+ @Test
+ public void testSplit() throws Exception {
+ //need to create the index using the helper method (not create it at first insertion)
+ // for the indexSettings() to be run
+ createIndex(ES_INDEX);
+ ElasticSearchIOTestUtils
+ .insertTestDocuments(connectionConfiguration, NUM_DOCS_UTESTS, getRestClient());
+ PipelineOptions options = PipelineOptionsFactory.create();
+ Read read =
+ ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
+ BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null, null,
+ null);
+ int desiredBundleSizeBytes = 1000;
+ List<? extends BoundedSource<String>> splits =
+ initialSource.split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
+ long indexSize = BoundedElasticsearchSource.estimateIndexSize(connectionConfiguration);
+ float expectedNumSourcesFloat = (float) indexSize / desiredBundleSizeBytes;
+ int expectedNumSources = (int) Math.ceil(expectedNumSourcesFloat);
+ assertEquals(expectedNumSources, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<String> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ assertEquals("Wrong number of non empty splits", expectedNumSources, nonEmptySplits);
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/elasticsearch/bootstrap/JarHell.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/elasticsearch/bootstrap/JarHell.java
new file mode 100644
index 0000000..c359d1d
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/elasticsearch/bootstrap/JarHell.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.elasticsearch.bootstrap;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * We need a real Elasticsearch instance to properly test the IO
+ * (split, slice API, scroll API, ...). Starting at ES 5, to have Elasticsearch embedded,
+ * we are forced to use Elasticsearch test framework. But this framework checks for class duplicates
+ * in classpath and it cannot be deactivated. When the class duplication come from a dependency,
+ * then it cannot be avoided. Elasticsearch community does not provide a way of deactivating
+ * the jar hell test, so skip it by making this hack. In this case duplicate class is
+ * class: org.apache.maven.surefire.report.SafeThrowable
+ * jar1: surefire-api-2.20.jar
+ * jar2: surefire-junit47-2.20.jar
+ */
+class JarHell {
+
+
+ public static void checkJarHell() throws IOException, URISyntaxException {
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml
new file mode 100644
index 0000000..b30764a
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-common</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests :: Common </name>
+ <description>Common test classes for ElasticsearchIO </description>
+
+ <properties>
+ <commons-io.version>1.3.2</commons-io.version>
+ <log4j.version>2.6.2</log4j.version>
+ <httpcomponents.core.version>4.4.5</httpcomponents.core.version>
+ <httpcomponents.httpasyncclient.version>4.1.2</httpcomponents.httpasyncclient.version>
+ <httpcomponents.httpclient.version>4.5.2</httpcomponents.httpclient.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.core.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.core.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
new file mode 100644
index 0000000..06cfc24
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+
+/** Test utilities to use with {@link ElasticsearchIO}. */
+class ElasticSearchIOTestUtils {
+
+ /** Enumeration that specifies whether to insert malformed documents. */
+ public enum InjectionMode {
+ INJECT_SOME_INVALID_DOCS,
+ DO_NOT_INJECT_INVALID_DOCS
+ }
+
+ /** Deletes the given index synchronously. */
+ static void deleteIndex(ConnectionConfiguration connectionConfiguration,
+ RestClient restClient) throws IOException {
+ try {
+ restClient.performRequest("DELETE", String.format("/%s", connectionConfiguration.getIndex()));
+ } catch (IOException e) {
+ // it is fine to ignore this expression as deleteIndex occurs in @before,
+ // so when the first tests is run, the index does not exist yet
+ if (!e.getMessage().contains("index_not_found_exception")){
+ throw e;
+ }
+ }
+ }
+
+ /** Inserts the given number of test documents into Elasticsearch. */
+ static void insertTestDocuments(ConnectionConfiguration connectionConfiguration,
+ long numDocs, RestClient restClient) throws IOException {
+ List<String> data =
+ ElasticSearchIOTestUtils.createDocuments(
+ numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
+ StringBuilder bulkRequest = new StringBuilder();
+ int i = 0;
+ for (String document : data) {
+ bulkRequest.append(String.format(
+ "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\" } }%n%s%n",
+ connectionConfiguration.getIndex(), connectionConfiguration.getType(), i++, document));
+ }
+ String endPoint = String.format("/%s/%s/_bulk", connectionConfiguration.getIndex(),
+ connectionConfiguration.getType());
+ HttpEntity requestBody =
+ new NStringEntity(bulkRequest.toString(), ContentType.APPLICATION_JSON);
+ Response response = restClient.performRequest("POST", endPoint,
+ Collections.singletonMap("refresh", "true"), requestBody);
+ ElasticsearchIO
+ .checkForErrors(response, ElasticsearchIO.getBackendVersion(connectionConfiguration));
+ }
+
+ /**
+ * Forces a refresh of the given index to make recently inserted documents available for search.
+ *
+ * @return The number of docs in the index
+ */
+ static long refreshIndexAndGetCurrentNumDocs(
+ ConnectionConfiguration connectionConfiguration, RestClient restClient) throws IOException {
+ long result = 0;
+ try {
+ String endPoint = String.format("/%s/_refresh", connectionConfiguration.getIndex());
+ restClient.performRequest("POST", endPoint);
+
+ endPoint = String.format("/%s/%s/_search", connectionConfiguration.getIndex(),
+ connectionConfiguration.getType());
+ Response response = restClient.performRequest("GET", endPoint);
+ JsonNode searchResult = ElasticsearchIO.parseResponse(response);
+ result = searchResult.path("hits").path("total").asLong();
+ } catch (IOException e) {
+ // it is fine to ignore bellow exceptions because in testWriteWithBatchSize* sometimes,
+ // we call upgrade before any doc have been written
+ // (when there are fewer docs processed than batchSize).
+ // In that cases index/type has not been created (created upon first doc insertion)
+ if (!e.getMessage().contains("index_not_found_exception")){
+ throw e;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Generates a list of test documents for insertion.
+ *
+ * @param numDocs Number of docs to generate
+ * @param injectionMode {@link InjectionMode} that specifies whether to insert malformed documents
+ * @return the list of json String representing the documents
+ */
+ static List<String> createDocuments(long numDocs, InjectionMode injectionMode) {
+ String[] scientists = {
+ "Einstein",
+ "Darwin",
+ "Copernicus",
+ "Pasteur",
+ "Curie",
+ "Faraday",
+ "Newton",
+ "Bohr",
+ "Galilei",
+ "Maxwell"
+ };
+ ArrayList<String> data = new ArrayList<>();
+ for (int i = 0; i < numDocs; i++) {
+ int index = i % scientists.length;
+ // insert 2 malformed documents
+ if (InjectionMode.INJECT_SOME_INVALID_DOCS.equals(injectionMode) && (i == 6 || i == 7)) {
+ data.add(String.format("{\"scientist\";\"%s\", \"id\":%s}", scientists[index], i));
+ } else {
+ data.add(String.format("{\"scientist\":\"%s\", \"id\":%s}", scientists[index], i));
+ }
+ }
+ return data;
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOITCommon.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOITCommon.java
new file mode 100644
index 0000000..391062d4
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOITCommon.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_INDEX;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_TYPE;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.NUM_DOCS_ITESTS;
+
+import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.elasticsearch.client.RestClient;
+
+/**
+ * Manipulates test data used by the {@link ElasticsearchIO}
+ * integration tests.
+ *
+ * <p>This is independent from the tests so that for read tests it can be run separately after data
+ * store creation rather than every time (which can be more fragile.)
+ */
+public class ElasticsearchIOITCommon {
+
+ private static final String writeIndex = ES_INDEX + System.currentTimeMillis();
+
+ /**
+ * Use this to create the index for reading before IT read tests.
+ *
+ * <p>To invoke this class, you can use this command line from elasticsearch io module directory:
+ *
+ * <pre>
+ * mvn test-compile exec:java \
+ * -Dexec.mainClass=ElasticsearchIOITCommon \
+ * -Dexec.args="--elasticsearchServer=1.2.3.4 \
+ * --elasticsearchHttpPort=9200 \
+ * -Dexec.classpathScope=test
+ * </pre>
+ *
+ * @param args Please pass options from ElasticsearchTestOptions used for connection to
+ * Elasticsearch as shown above.
+ */
+ public static void main(String[] args) throws Exception {
+ PipelineOptionsFactory.register(IOTestPipelineOptions.class);
+ IOTestPipelineOptions options =
+ PipelineOptionsFactory.fromArgs(args).as(IOTestPipelineOptions.class);
+ createAndPopulateReadIndex(options);
+ }
+
+ private static void createAndPopulateReadIndex(IOTestPipelineOptions options) throws Exception {
+ // automatically creates the index and insert docs
+ ConnectionConfiguration connectionConfiguration =
+ getConnectionConfiguration(options, ReadOrWrite.READ);
+ try (RestClient restClient = connectionConfiguration.createClient()) {
+ ElasticSearchIOTestUtils
+ .insertTestDocuments(connectionConfiguration, NUM_DOCS_ITESTS, restClient);
+ }
+ }
+
+ static ConnectionConfiguration getConnectionConfiguration(IOTestPipelineOptions options,
+ ReadOrWrite rOw) {
+ ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.create(
+ new String[] {
+ "http://"
+ + options.getElasticsearchServer()
+ + ":"
+ + options.getElasticsearchHttpPort()
+ },
+ (rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex,
+ ES_TYPE);
+ return connectionConfiguration;
+ }
+
+ /** Enum that tells whether we use the index for reading or for writing. */
+ enum ReadOrWrite {
+ READ,
+ WRITE
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java
new file mode 100644
index 0000000..3fb08bb
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Write;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.parseResponse;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.core.Is.isA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.hamcrest.CustomMatcher;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Common test class for {@link ElasticsearchIO}. */
+class ElasticsearchIOTestCommon implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIOTestCommon.class);
+
+ static final String ES_INDEX = "beam";
+ static final String ES_TYPE = "test";
+ static final long NUM_DOCS_UTESTS = 400L;
+ static final long NUM_DOCS_ITESTS = 50000L;
+ private static final long AVERAGE_DOC_SIZE = 25L;
+
+
+ private static final int NUM_SCIENTISTS = 10;
+ private static final long BATCH_SIZE = 200L;
+ private static final long BATCH_SIZE_BYTES = 2048L;
+
+ private long numDocs;
+ private ConnectionConfiguration connectionConfiguration;
+ private RestClient restClient;
+ private boolean useAsITests;
+
+ private TestPipeline pipeline;
+ private ExpectedException expectedException;
+
+ ElasticsearchIOTestCommon(ConnectionConfiguration connectionConfiguration, RestClient restClient,
+ boolean useAsITests) {
+ this.connectionConfiguration = connectionConfiguration;
+ this.restClient = restClient;
+ this.numDocs = useAsITests ? NUM_DOCS_ITESTS : NUM_DOCS_UTESTS;
+ this.useAsITests = useAsITests;
+ }
+
+ // lazy init of the test rules (cannot be static)
+ void setPipeline(TestPipeline pipeline) {
+ this.pipeline = pipeline;
+ }
+
+ void setExpectedException(ExpectedException expectedException) {
+ this.expectedException = expectedException;
+ }
+
+ void testSizes() throws Exception {
+ if (!useAsITests) {
+ ElasticSearchIOTestUtils.insertTestDocuments(connectionConfiguration, numDocs, restClient);
+ }
+ PipelineOptions options = PipelineOptionsFactory.create();
+ Read read =
+ ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
+ BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null, null,
+ null);
+ // can't use equal assert as Elasticsearch indexes never have same size
+ // (due to internal Elasticsearch implementation)
+ long estimatedSize = initialSource.getEstimatedSizeBytes(options);
+ LOG.info("Estimated size: {}", estimatedSize);
+ assertThat("Wrong estimated size", estimatedSize, greaterThan(AVERAGE_DOC_SIZE * numDocs));
+ }
+
+
+ void testRead() throws Exception {
+ if (!useAsITests) {
+ ElasticSearchIOTestUtils.insertTestDocuments(connectionConfiguration, numDocs, restClient);
+ }
+
+ PCollection<String> output =
+ pipeline.apply(
+ ElasticsearchIO.read()
+ .withConnectionConfiguration(connectionConfiguration)
+ //set to default value, useful just to test parameter passing.
+ .withScrollKeepalive("5m")
+ //set to default value, useful just to test parameter passing.
+ .withBatchSize(100L));
+ PAssert.thatSingleton(output.apply("Count", Count.<String>globally())).isEqualTo(numDocs);
+ pipeline.run();
+ }
+
+ void testReadWithQuery() throws Exception {
+ if (!useAsITests){
+ ElasticSearchIOTestUtils.insertTestDocuments(connectionConfiguration, numDocs, restClient);
+ }
+
+ String query =
+ "{\n"
+ + " \"query\": {\n"
+ + " \"match\" : {\n"
+ + " \"scientist\" : {\n"
+ + " \"query\" : \"Einstein\",\n"
+ + " \"type\" : \"boolean\"\n"
+ + " }\n"
+ + " }\n"
+ + " }\n"
+ + "}";
+
+ PCollection<String> output =
+ pipeline.apply(
+ ElasticsearchIO.read()
+ .withConnectionConfiguration(connectionConfiguration)
+ .withQuery(query));
+ PAssert.thatSingleton(output.apply("Count", Count.<String>globally()))
+ .isEqualTo(numDocs / NUM_SCIENTISTS);
+ pipeline.run();
+ }
+
+ void testWrite() throws Exception {
+ List<String> data =
+ ElasticSearchIOTestUtils.createDocuments(
+ numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
+ pipeline
+ .apply(Create.of(data))
+ .apply(ElasticsearchIO.write().withConnectionConfiguration(connectionConfiguration));
+ pipeline.run();
+
+ long currentNumDocs = ElasticSearchIOTestUtils
+ .refreshIndexAndGetCurrentNumDocs(connectionConfiguration, restClient);
+ assertEquals(numDocs, currentNumDocs);
+
+ String requestBody =
+ "{\n"
+ + " \"query\" : {\"match\": {\n"
+ + " \"scientist\": \"Einstein\"\n"
+ + " }}\n"
+ + "}\n";
+ String endPoint = String.format("/%s/%s/_search", connectionConfiguration.getIndex(),
+ connectionConfiguration.getType());
+ HttpEntity httpEntity = new NStringEntity(requestBody, ContentType.APPLICATION_JSON);
+ Response response =
+ restClient.performRequest(
+ "GET",
+ endPoint,
+ Collections.<String, String>emptyMap(),
+ httpEntity);
+ JsonNode searchResult = parseResponse(response);
+ int count = searchResult.path("hits").path("total").asInt();
+ assertEquals(numDocs / NUM_SCIENTISTS, count);
+ }
+
+ void testWriteWithErrors() throws Exception {
+ Write write =
+ ElasticsearchIO.write()
+ .withConnectionConfiguration(connectionConfiguration)
+ .withMaxBatchSize(BATCH_SIZE);
+ // write bundles size is the runner decision, we cannot force a bundle size,
+ // so we test the Writer as a DoFn outside of a runner.
+ DoFnTester<String, Void> fnTester = DoFnTester.of(new Write.WriteFn(write));
+
+ List<String> input =
+ ElasticSearchIOTestUtils.createDocuments(
+ numDocs, ElasticSearchIOTestUtils.InjectionMode.INJECT_SOME_INVALID_DOCS);
+ expectedException.expect(isA(IOException.class));
+ expectedException.expectMessage(
+ new CustomMatcher<String>("RegExp matcher") {
+ @Override
+ public boolean matches(Object o) {
+ String message = (String) o;
+ // This regexp tests that 2 malformed documents are actually in error
+ // and that the message contains their IDs.
+ // It also ensures that root reason, root error type,
+ // caused by reason and caused by error type are present in message.
+ // To avoid flakiness of the test in case of Elasticsearch error message change,
+ // only "failed to parse" root reason is matched,
+ // the other messages are matched using .+
+ return message.matches(
+ "(?is).*Error writing to Elasticsearch, some elements could not be inserted"
+ + ".*Document id .+: failed to parse \\(.+\\).*Caused by: .+ \\(.+\\).*"
+ + "Document id .+: failed to parse \\(.+\\).*Caused by: .+ \\(.+\\).*");
+ }
+ });
+ // inserts into Elasticsearch
+ fnTester.processBundle(input);
+ }
+
+ void testWriteWithMaxBatchSize() throws Exception {
+ Write write =
+ ElasticsearchIO.write()
+ .withConnectionConfiguration(connectionConfiguration)
+ .withMaxBatchSize(BATCH_SIZE);
+ // write bundles size is the runner decision, we cannot force a bundle size,
+ // so we test the Writer as a DoFn outside of a runner.
+ DoFnTester<String, Void> fnTester = DoFnTester.of(new Write.WriteFn(write));
+ List<String> input =
+ ElasticSearchIOTestUtils.createDocuments(
+ numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
+ long numDocsProcessed = 0;
+ long numDocsInserted = 0;
+ for (String document : input) {
+ fnTester.processElement(document);
+ numDocsProcessed++;
+ // test every 100 docs to avoid overloading ES
+ if ((numDocsProcessed % 100) == 0) {
+ // force the index to upgrade after inserting for the inserted docs
+ // to be searchable immediately
+ long currentNumDocs = ElasticSearchIOTestUtils
+ .refreshIndexAndGetCurrentNumDocs(connectionConfiguration, restClient);
+ if ((numDocsProcessed % BATCH_SIZE) == 0) {
+ /* bundle end */
+ assertEquals(
+ "we are at the end of a bundle, we should have inserted all processed documents",
+ numDocsProcessed,
+ currentNumDocs);
+ numDocsInserted = currentNumDocs;
+ } else {
+ /* not bundle end */
+ assertEquals(
+ "we are not at the end of a bundle, we should have inserted no more documents",
+ numDocsInserted,
+ currentNumDocs);
+ }
+ }
+ }
+ }
+
+ void testWriteWithMaxBatchSizeBytes() throws Exception {
+ Write write =
+ ElasticsearchIO.write()
+ .withConnectionConfiguration(connectionConfiguration)
+ .withMaxBatchSizeBytes(BATCH_SIZE_BYTES);
+ // write bundles size is the runner decision, we cannot force a bundle size,
+ // so we test the Writer as a DoFn outside of a runner.
+ DoFnTester<String, Void> fnTester = DoFnTester.of(new Write.WriteFn(write));
+ List<String> input =
+ ElasticSearchIOTestUtils.createDocuments(
+ numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
+ long numDocsProcessed = 0;
+ long sizeProcessed = 0;
+ long numDocsInserted = 0;
+ long batchInserted = 0;
+ for (String document : input) {
+ fnTester.processElement(document);
+ numDocsProcessed++;
+ sizeProcessed += document.getBytes().length;
+ // test every 40 docs to avoid overloading ES
+ if ((numDocsProcessed % 40) == 0) {
+ // force the index to upgrade after inserting for the inserted docs
+ // to be searchable immediately
+ long currentNumDocs = ElasticSearchIOTestUtils
+ .refreshIndexAndGetCurrentNumDocs(connectionConfiguration, restClient);
+ if (sizeProcessed / BATCH_SIZE_BYTES > batchInserted) {
+ /* bundle end */
+ assertThat(
+ "we have passed a bundle size, we should have inserted some documents",
+ currentNumDocs,
+ greaterThan(numDocsInserted));
+ numDocsInserted = currentNumDocs;
+ batchInserted = (sizeProcessed / BATCH_SIZE_BYTES);
+ } else {
+ /* not bundle end */
+ assertEquals(
+ "we are not at the end of a bundle, we should have inserted no more documents",
+ numDocsInserted,
+ currentNumDocs);
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/io/elasticsearch-tests/pom.xml b/sdks/java/io/elasticsearch-tests/pom.xml
new file mode 100644
index 0000000..59ef454
--- /dev/null
+++ b/sdks/java/io/elasticsearch-tests/pom.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-parent</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests </name>
+ <description>Tests for ElasticsearchIO.</description>
+ <packaging>pom</packaging>
+
+ <properties>
+ <commons-io.version>1.3.2</commons-io.version>
+ <jna.version>4.1.0</jna.version>
+ <log4j.version>2.6.2</log4j.version>
+ <elasticsearch.client.rest.version>5.6.3</elasticsearch.client.rest.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- This optional dependency is used by the test framework -->
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>${jna.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <!--do not use 2.7 for ES 5.0-->
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <!--do not use 2.7 for ES 5.0-->
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.client.rest.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <modules>
+ <module>elasticsearch-tests-common</module>
+ <module>elasticsearch-tests-2</module>
+ <module>elasticsearch-tests-5</module>
+ </modules>
+
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/elasticsearch/pom.xml b/sdks/java/io/elasticsearch/pom.xml
index 03632ce..37f3d11 100644
--- a/sdks/java/io/elasticsearch/pom.xml
+++ b/sdks/java/io/elasticsearch/pom.xml
@@ -17,139 +17,91 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
- <artifactId>beam-sdks-java-io-elasticsearch</artifactId>
- <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch</name>
- <description>IO to read and write on Elasticsearch.</description>
+ <artifactId>beam-sdks-java-io-elasticsearch</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Elasticsearch</name>
+ <description>IO to read and write on Elasticsearch</description>
- <dependencies>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
- </dependency>
+ <properties>
+ <elasticsearch.client.rest.version>5.6.3</elasticsearch.client.rest.version>
+ <httpcomponents.core.version>4.4.5</httpcomponents.core.version>
+ <httpcomponents.httpasyncclient.version>4.1.2</httpcomponents.httpasyncclient.version>
+ <httpcomponents.httpclient.version>4.5.2</httpcomponents.httpclient.version>
+ </properties>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
+ <dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.client.rest.version}</version>
+ </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>rest</artifactId>
- <version>5.0.0</version>
- </dependency>
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore-nio</artifactId>
- <version>4.4.5</version>
- </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>4.4.5</version>
- </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpasyncclient</artifactId>
- <version>4.1.2</version>
- </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.2</version>
- </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.core.version}</version>
+ </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.core.version}</version>
+ </dependency>
- <!-- compile dependencies -->
- <dependency>
- <groupId>com.google.auto.value</groupId>
- <artifactId>auto-value</artifactId>
- <scope>provided</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ </dependency>
- <!-- test -->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>2.4.1</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <scope>test</scope>
- </dependency>
+ </dependencies>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>1.3.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jdk14</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-direct-java</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-common</artifactId>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
-
- </dependencies>
-
-</project>
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
index f6ceef2..023eb13 100644
--- a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
+++ b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
@@ -24,11 +24,14 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.Serializable;
-import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyStore;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -39,6 +42,7 @@
import java.util.Map;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
+import javax.net.ssl.SSLContext;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -56,18 +60,19 @@
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
-import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
import org.apache.http.nio.entity.NStringEntity;
+import org.apache.http.ssl.SSLContexts;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
/**
* Transforms for reading and writing data from/to Elasticsearch.
- * This IO is only compatible with Elasticsearch v2.x
*
* <h3>Reading from Elasticsearch</h3>
*
@@ -113,7 +118,7 @@
* <p>Optionally, you can provide {@code withBatchSize()} and {@code withBatchSizeBytes()}
* to specify the size of the write batch in number of documents or in bytes.
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class ElasticsearchIO {
public static Read read() {
@@ -139,25 +144,67 @@
private static final ObjectMapper mapper = new ObjectMapper();
- private static JsonNode parseResponse(Response response) throws IOException {
+ @VisibleForTesting
+ static JsonNode parseResponse(Response response) throws IOException {
return mapper.readValue(response.getEntity().getContent(), JsonNode.class);
}
+ static void checkForErrors(Response response, int backendVersion) throws IOException {
+ JsonNode searchResult = parseResponse(response);
+ boolean errors = searchResult.path("errors").asBoolean();
+ if (errors) {
+ StringBuilder errorMessages =
+ new StringBuilder(
+ "Error writing to Elasticsearch, some elements could not be inserted:");
+ JsonNode items = searchResult.path("items");
+ //some items present in bulk might have errors, concatenate error messages
+ for (JsonNode item : items) {
+ String errorRootName = "";
+ if (backendVersion == 2) {
+ errorRootName = "create";
+ } else if (backendVersion == 5) {
+ errorRootName = "index";
+ }
+ JsonNode errorRoot = item.path(errorRootName);
+ JsonNode error = errorRoot.get("error");
+ if (error != null) {
+ String type = error.path("type").asText();
+ String reason = error.path("reason").asText();
+ String docId = errorRoot.path("_id").asText();
+ errorMessages.append(String.format("%nDocument id %s: %s (%s)", docId, reason, type));
+ JsonNode causedBy = error.get("caused_by");
+ if (causedBy != null) {
+ String cbReason = causedBy.path("reason").asText();
+ String cbType = causedBy.path("type").asText();
+ errorMessages.append(String.format("%nCaused by: %s (%s)", cbReason, cbType));
+ }
+ }
+ }
+ throw new IOException(errorMessages.toString());
+ }
+ }
+
/** A POJO describing a connection configuration to Elasticsearch. */
@AutoValue
public abstract static class ConnectionConfiguration implements Serializable {
- abstract List<String> getAddresses();
+ public abstract List<String> getAddresses();
@Nullable
- abstract String getUsername();
+ public abstract String getUsername();
@Nullable
- abstract String getPassword();
+ public abstract String getPassword();
- abstract String getIndex();
+ @Nullable
+ public abstract String getKeystorePath();
- abstract String getType();
+ @Nullable
+ public abstract String getKeystorePassword();
+
+ public abstract String getIndex();
+
+ public abstract String getType();
abstract Builder builder();
@@ -169,6 +216,10 @@
abstract Builder setPassword(String password);
+ abstract Builder setKeystorePath(String keystorePath);
+
+ abstract Builder setKeystorePassword(String password);
+
abstract Builder setIndex(String index);
abstract Builder setType(String type);
@@ -183,61 +234,29 @@
* @param index the index toward which the requests will be issued
* @param type the document type toward which the requests will be issued
* @return the connection configuration object
- * @throws IOException when it fails to connect to Elasticsearch
*/
- public static ConnectionConfiguration create(String[] addresses, String index, String type)
- throws IOException {
- checkArgument(
- addresses != null,
- "ConnectionConfiguration.create(addresses, index, type) called with null address");
- checkArgument(
- addresses.length != 0,
- "ConnectionConfiguration.create(addresses, "
- + "index, type) called with empty addresses");
- checkArgument(
- index != null,
- "ConnectionConfiguration.create(addresses, index, type) called with null index");
- checkArgument(
- type != null,
- "ConnectionConfiguration.create(addresses, index, type) called with null type");
+ public static ConnectionConfiguration create(String[] addresses, String index, String type){
+ checkArgument(addresses != null, "addresses can not be null");
+ checkArgument(addresses.length > 0, "addresses can not be empty");
+ checkArgument(index != null, "index can not be null");
+ checkArgument(type != null, "type can not be null");
ConnectionConfiguration connectionConfiguration =
new AutoValue_ElasticsearchIO_ConnectionConfiguration.Builder()
.setAddresses(Arrays.asList(addresses))
.setIndex(index)
.setType(type)
.build();
- checkVersion(connectionConfiguration);
return connectionConfiguration;
}
- private static void checkVersion(ConnectionConfiguration connectionConfiguration)
- throws IOException {
- RestClient restClient = connectionConfiguration.createClient();
- Response response = restClient.performRequest("GET", "", new BasicHeader("", ""));
- JsonNode jsonNode = parseResponse(response);
- String version = jsonNode.path("version").path("number").asText();
- boolean version2x = version.startsWith("2.");
- restClient.close();
- checkArgument(
- version2x,
- "ConnectionConfiguration.create(addresses, index, type): "
- + "the Elasticsearch version to connect to is different of 2.x. "
- + "This version of the ElasticsearchIO is only compatible with Elasticsearch v2.x");
- }
-
/**
* If Elasticsearch authentication is enabled, provide the username.
*
* @param username the username used to authenticate to Elasticsearch
- * @return the {@link ConnectionConfiguration} object with username set
*/
public ConnectionConfiguration withUsername(String username) {
- checkArgument(
- username != null,
- "ConnectionConfiguration.create().withUsername(username) called with null username");
- checkArgument(
- !username.isEmpty(),
- "ConnectionConfiguration.create().withUsername(username) called with empty username");
+ checkArgument(username != null, "username can not be null");
+ checkArgument(!username.isEmpty(), "username can not be empty");
return builder().setUsername(username).build();
}
@@ -245,26 +264,46 @@
* If Elasticsearch authentication is enabled, provide the password.
*
* @param password the password used to authenticate to Elasticsearch
- * @return the {@link ConnectionConfiguration} object with password set
*/
public ConnectionConfiguration withPassword(String password) {
- checkArgument(
- password != null,
- "ConnectionConfiguration.create().withPassword(password) called with null password");
- checkArgument(
- !password.isEmpty(),
- "ConnectionConfiguration.create().withPassword(password) called with empty password");
+ checkArgument(password != null, "password can not be null");
+ checkArgument(!password.isEmpty(), "password can not be empty");
return builder().setPassword(password).build();
}
+ /**
+ * If Elasticsearch uses SSL/TLS with mutual authentication (via shield),
+ * provide the keystore containing the client key.
+ *
+ * @param keystorePath the location of the keystore containing the client key.
+ */
+ public ConnectionConfiguration withKeystorePath(String keystorePath) {
+ checkArgument(keystorePath != null, "keystorePath can not be null");
+ checkArgument(!keystorePath.isEmpty(), "keystorePath can not be empty");
+ return builder().setKeystorePath(keystorePath).build();
+ }
+
+ /**
+ * If Elasticsearch uses SSL/TLS with mutual authentication (via shield),
+ * provide the password to open the client keystore.
+ *
+ * @param keystorePassword the password of the client keystore.
+ */
+ public ConnectionConfiguration withKeystorePassword(String keystorePassword) {
+ checkArgument(keystorePassword != null, "keystorePassword can not be null");
+ return builder().setKeystorePassword(keystorePassword).build();
+ }
+
private void populateDisplayData(DisplayData.Builder builder) {
builder.add(DisplayData.item("address", getAddresses().toString()));
builder.add(DisplayData.item("index", getIndex()));
builder.add(DisplayData.item("type", getType()));
builder.addIfNotNull(DisplayData.item("username", getUsername()));
+ builder.addIfNotNull(DisplayData.item("keystore.path", getKeystorePath()));
}
- private RestClient createClient() throws MalformedURLException {
+ @VisibleForTesting
+ RestClient createClient() throws IOException {
HttpHost[] hosts = new HttpHost[getAddresses().size()];
int i = 0;
for (String address : getAddresses()) {
@@ -285,6 +324,28 @@
}
});
}
+ if (getKeystorePath() != null && !getKeystorePath().isEmpty()) {
+ try {
+ KeyStore keyStore = KeyStore.getInstance("jks");
+ try (InputStream is = new FileInputStream(new File(getKeystorePath()))) {
+ String keystorePassword = getKeystorePassword();
+ keyStore.load(is, (keystorePassword == null) ? null : keystorePassword.toCharArray());
+ }
+ final SSLContext sslContext = SSLContexts.custom()
+ .loadTrustMaterial(keyStore, new TrustSelfSignedStrategy()).build();
+ final SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext);
+ restClientBuilder.setHttpClientConfigCallback(
+ new RestClientBuilder.HttpClientConfigCallback() {
+ @Override
+ public HttpAsyncClientBuilder customizeHttpClient(
+ HttpAsyncClientBuilder httpClientBuilder) {
+ return httpClientBuilder.setSSLContext(sslContext).setSSLStrategy(sessionStrategy);
+ }
+ });
+ } catch (Exception e) {
+ throw new IOException("Can't load the client certificate from the keystore", e);
+ }
+ }
return restClientBuilder.build();
}
}
@@ -320,17 +381,9 @@
abstract Read build();
}
- /**
- * Provide the Elasticsearch connection configuration object.
- *
- * @param connectionConfiguration the Elasticsearch {@link ConnectionConfiguration} object
- * @return the {@link Read} with connection configuration set
- */
+ /** Provide the Elasticsearch connection configuration object. */
public Read withConnectionConfiguration(ConnectionConfiguration connectionConfiguration) {
- checkArgument(
- connectionConfiguration != null,
- "ElasticsearchIO.read()"
- + ".withConnectionConfiguration(configuration) called with null configuration");
+ checkArgument(connectionConfiguration != null, "connectionConfiguration can not be null");
return builder().setConnectionConfiguration(connectionConfiguration).build();
}
@@ -340,12 +393,10 @@
* @param query the query. See <a
* href="https://www.elastic.co/guide/en/elasticsearch/reference/2.4/query-dsl.html">Query
* DSL</a>
- * @return the {@link Read} object with query set
*/
public Read withQuery(String query) {
- checkArgument(
- !Strings.isNullOrEmpty(query),
- "ElasticsearchIO.read().withQuery(query) called" + " with null or empty query");
+ checkArgument(query != null, "query can not be null");
+ checkArgument(!query.isEmpty(), "query can not be empty");
return builder().setQuery(query).build();
}
@@ -353,15 +404,10 @@
* Provide a scroll keepalive. See <a
* href="https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html">scroll
* API</a> Default is "5m". Change this only if you get "No search context found" errors.
- *
- * @param scrollKeepalive keepalive duration ex "5m" from 5 minutes
- * @return the {@link Read} with scroll keepalive set
*/
public Read withScrollKeepalive(String scrollKeepalive) {
- checkArgument(
- scrollKeepalive != null && !scrollKeepalive.equals("0m"),
- "ElasticsearchIO.read().withScrollKeepalive(keepalive) called"
- + " with null or \"0m\" keepalive");
+ checkArgument(scrollKeepalive != null, "scrollKeepalive can not be null");
+ checkArgument(!scrollKeepalive.equals("0m"), "scrollKeepalive can not be 0m");
return builder().setScrollKeepalive(scrollKeepalive).build();
}
@@ -373,18 +419,11 @@
* batchSize
*
* @param batchSize number of documents read in each scroll read
- * @return the {@link Read} with batch size set
*/
public Read withBatchSize(long batchSize) {
checkArgument(
- batchSize > 0,
- "ElasticsearchIO.read().withBatchSize(batchSize) called with a negative "
- + "or equal to 0 value: %s",
- batchSize);
- checkArgument(
- batchSize <= MAX_BATCH_SIZE,
- "ElasticsearchIO.read().withBatchSize(batchSize) "
- + "called with a too large value (over %s): %s",
+ batchSize > 0 && batchSize <= MAX_BATCH_SIZE,
+ "batchSize must be > 0 and <= %d, but was: %d",
MAX_BATCH_SIZE,
batchSize);
return builder().setBatchSize(batchSize).build();
@@ -392,86 +431,114 @@
@Override
public PCollection<String> expand(PBegin input) {
- return input.apply(
- org.apache.beam.sdk.io.Read.from(new BoundedElasticsearchSource(this, null)));
- }
-
- @Override
- public void validate(PipelineOptions options) {
+ ConnectionConfiguration connectionConfiguration = getConnectionConfiguration();
checkState(
- getConnectionConfiguration() != null,
- "ElasticsearchIO.read() requires a connection configuration"
- + " to be set via withConnectionConfiguration(configuration)");
+ connectionConfiguration != null,
+ "withConnectionConfiguration() is required");
+ return input.apply(org.apache.beam.sdk.io.Read
+ .from(new BoundedElasticsearchSource(this, null, null, null)));
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder.addIfNotNull(DisplayData.item("query", getQuery()));
+ builder.addIfNotNull(DisplayData.item("batchSize", getBatchSize()));
+ builder.addIfNotNull(DisplayData.item("scrollKeepalive", getScrollKeepalive()));
getConnectionConfiguration().populateDisplayData(builder);
}
}
/** A {@link BoundedSource} reading from Elasticsearch. */
@VisibleForTesting
- static class BoundedElasticsearchSource extends BoundedSource<String> {
+ public static class BoundedElasticsearchSource extends BoundedSource<String> {
- private final ElasticsearchIO.Read spec;
- // shardPreference is the shard number where the source will read the documents
- @Nullable private final String shardPreference;
+ private int backendVersion;
- BoundedElasticsearchSource(Read spec, @Nullable String shardPreference) {
+ private final Read spec;
+ // shardPreference is the shard id where the source will read the documents
+ @Nullable
+ private final String shardPreference;
+ @Nullable
+ private final Integer numSlices;
+ @Nullable
+ private final Integer sliceId;
+
+ //constructor used in split() when we know the backend version
+ private BoundedElasticsearchSource(Read spec, @Nullable String shardPreference,
+ @Nullable Integer numSlices, @Nullable Integer sliceId, int backendVersion) {
+ this.backendVersion = backendVersion;
this.spec = spec;
this.shardPreference = shardPreference;
+ this.numSlices = numSlices;
+ this.sliceId = sliceId;
}
+ @VisibleForTesting
+ BoundedElasticsearchSource(Read spec, @Nullable String shardPreference,
+ @Nullable Integer numSlices, @Nullable Integer sliceId) {
+ this.spec = spec;
+ this.shardPreference = shardPreference;
+ this.numSlices = numSlices;
+ this.sliceId = sliceId;
+ }
@Override
public List<? extends BoundedSource<String>> split(
long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ ConnectionConfiguration connectionConfiguration = spec.getConnectionConfiguration();
+ this.backendVersion = getBackendVersion(connectionConfiguration);
List<BoundedElasticsearchSource> sources = new ArrayList<>();
+ if (backendVersion == 2){
+ // 1. We split per shard :
+ // unfortunately, Elasticsearch 2. x doesn 't provide a way to do parallel reads on a single
+ // shard.So we do not use desiredBundleSize because we cannot split shards.
+ // With the slice API in ES 5.0 we will be able to use desiredBundleSize.
+ // Basically we will just ask the slice API to return data
+ // in nbBundles = estimatedSize / desiredBundleSize chuncks.
+ // So each beam source will read around desiredBundleSize volume of data.
- // 1. We split per shard :
- // unfortunately, Elasticsearch 2. x doesn 't provide a way to do parallel reads on a single
- // shard.So we do not use desiredBundleSize because we cannot split shards.
- // With the slice API in ES 5.0 we will be able to use desiredBundleSize.
- // Basically we will just ask the slice API to return data
- // in nbBundles = estimatedSize / desiredBundleSize chuncks.
- // So each beam source will read around desiredBundleSize volume of data.
+ JsonNode statsJson = BoundedElasticsearchSource.getStats(connectionConfiguration, true);
+ JsonNode shardsJson =
+ statsJson
+ .path("indices")
+ .path(connectionConfiguration.getIndex())
+ .path("shards");
- // 2. Primary and replica shards have the same shard_id, we filter primary
- // to have one source for each shard_id. Even if we specify preference=shards:2,
- // ES load balances (round robin) the request between primary shard 2 and replica shard 2.
- // But, as each shard (replica or primary) is responsible for only one part of the data,
- // there will be no duplicate.
-
- JsonNode statsJson = getStats(true);
- JsonNode shardsJson =
- statsJson
- .path("indices")
- .path(spec.getConnectionConfiguration().getIndex())
- .path("shards");
-
- Iterator<Map.Entry<String, JsonNode>> shards = shardsJson.fields();
- while (shards.hasNext()) {
- Map.Entry<String, JsonNode> shardJson = shards.next();
- String shardId = shardJson.getKey();
- JsonNode value = (JsonNode) shardJson.getValue();
- boolean isPrimaryShard =
- value
- .path(0)
- .path("routing")
- .path("primary")
- .asBoolean();
- if (isPrimaryShard) {
- sources.add(new BoundedElasticsearchSource(spec, shardId));
+ Iterator<Map.Entry<String, JsonNode>> shards = shardsJson.fields();
+ while (shards.hasNext()) {
+ Map.Entry<String, JsonNode> shardJson = shards.next();
+ String shardId = shardJson.getKey();
+ sources.add(new BoundedElasticsearchSource(spec, shardId, null, null, backendVersion));
+ }
+ checkArgument(!sources.isEmpty(), "No shard found");
+ } else if (backendVersion == 5){
+ long indexSize = BoundedElasticsearchSource.estimateIndexSize(connectionConfiguration);
+ float nbBundlesFloat = (float) indexSize / desiredBundleSizeBytes;
+ int nbBundles = (int) Math.ceil(nbBundlesFloat);
+ //ES slice api imposes that the number of slices is <= 1024 even if it can be overloaded
+ if (nbBundles > 1024) {
+ nbBundles = 1024;
+ }
+ // split the index into nbBundles chunks of desiredBundleSizeBytes by creating
+ // nbBundles sources each reading a slice of the index
+ // (see https://goo.gl/MhtSWz)
+ // the slice API allows to split the ES shards
+ // to have bundles closer to desiredBundleSizeBytes
+ for (int i = 0; i < nbBundles; i++) {
+ sources.add(new BoundedElasticsearchSource(spec, null, nbBundles, i, backendVersion));
}
}
- checkArgument(!sources.isEmpty(), "No primary shard found");
return sources;
}
@Override
public long getEstimatedSizeBytes(PipelineOptions options) throws IOException {
+ return estimateIndexSize(spec.getConnectionConfiguration());
+ }
+
+ @VisibleForTesting
+ static long estimateIndexSize(ConnectionConfiguration connectionConfiguration)
+ throws IOException {
// we use indices stats API to estimate size and list the shards
// (https://www.elastic.co/guide/en/elasticsearch/reference/2.4/indices-stats.html)
// as Elasticsearch 2.x doesn't not support any way to do parallel read inside a shard
@@ -480,11 +547,11 @@
// NB: Elasticsearch 5.x now provides the slice API.
// (https://www.elastic.co/guide/en/elasticsearch/reference/5.0/search-request-scroll.html
// #sliced-scroll)
- JsonNode statsJson = getStats(false);
+ JsonNode statsJson = getStats(connectionConfiguration, false);
JsonNode indexStats =
statsJson
.path("indices")
- .path(spec.getConnectionConfiguration().getIndex())
+ .path(connectionConfiguration.getIndex())
.path("primaries");
JsonNode store = indexStats.path("store");
return store.path("size_in_bytes").asLong();
@@ -494,6 +561,8 @@
public void populateDisplayData(DisplayData.Builder builder) {
spec.populateDisplayData(builder);
builder.addIfNotNull(DisplayData.item("shard", shardPreference));
+ builder.addIfNotNull(DisplayData.item("numSlices", numSlices));
+ builder.addIfNotNull(DisplayData.item("sliceId", sliceId));
}
@Override
@@ -507,19 +576,20 @@
}
@Override
- public Coder<String> getDefaultOutputCoder() {
+ public Coder<String> getOutputCoder() {
return StringUtf8Coder.of();
}
- private JsonNode getStats(boolean shardLevel) throws IOException {
+ private static JsonNode getStats(ConnectionConfiguration connectionConfiguration,
+ boolean shardLevel) throws IOException {
HashMap<String, String> params = new HashMap<>();
if (shardLevel) {
params.put("level", "shards");
}
- String endpoint = String.format("/%s/_stats", spec.getConnectionConfiguration().getIndex());
- try (RestClient restClient = spec.getConnectionConfiguration().createClient()) {
+ String endpoint = String.format("/%s/_stats", connectionConfiguration.getIndex());
+ try (RestClient restClient = connectionConfiguration.createClient()) {
return parseResponse(
- restClient.performRequest("GET", endpoint, params, new BasicHeader("", "")));
+ restClient.performRequest("GET", endpoint, params));
}
}
}
@@ -543,9 +613,18 @@
String query = source.spec.getQuery();
if (query == null) {
- query = "{ \"query\": { \"match_all\": {} } }";
+ query = "{\"query\": { \"match_all\": {} }}";
}
-
+ if (source.backendVersion == 5){
+ //if there is more than one slice
+ if (source.numSlices != null && source.numSlices > 1){
+ // add slice to the user query
+ String sliceQuery = String
+ .format("\"slice\": {\"id\": %s,\"max\": %s}", source.sliceId,
+ source.numSlices);
+ query = query.replaceFirst("\\{", "{" + sliceQuery + ",");
+ }
+ }
Response response;
String endPoint =
String.format(
@@ -554,13 +633,16 @@
source.spec.getConnectionConfiguration().getType());
Map<String, String> params = new HashMap<>();
params.put("scroll", source.spec.getScrollKeepalive());
- params.put("size", String.valueOf(source.spec.getBatchSize()));
- if (source.shardPreference != null) {
- params.put("preference", "_shards:" + source.shardPreference);
+ if (source.backendVersion == 2){
+ params.put("size", String.valueOf(source.spec.getBatchSize()));
+ if (source.shardPreference != null) {
+ params.put("preference", "_shards:" + source.shardPreference);
+ }
}
- HttpEntity queryEntity = new NStringEntity(query, ContentType.APPLICATION_JSON);
+ HttpEntity queryEntity = new NStringEntity(query,
+ ContentType.APPLICATION_JSON);
response =
- restClient.performRequest("GET", endPoint, params, queryEntity, new BasicHeader("", ""));
+ restClient.performRequest("GET", endPoint, params, queryEntity);
JsonNode searchResult = parseResponse(response);
updateScrollId(searchResult);
return readNextBatchAndReturnFirstDocument(searchResult);
@@ -586,8 +668,7 @@
"GET",
"/_search/scroll",
Collections.<String, String>emptyMap(),
- scrollEntity,
- new BasicHeader("", ""));
+ scrollEntity);
JsonNode searchResult = parseResponse(response);
updateScrollId(searchResult);
return readNextBatchAndReturnFirstDocument(searchResult);
@@ -631,8 +712,7 @@
"DELETE",
"/_search/scroll",
Collections.<String, String>emptyMap(),
- entity,
- new BasicHeader("", ""));
+ entity);
} finally {
if (restClient != null) {
restClient.close();
@@ -677,10 +757,7 @@
* @return the {@link Write} with connection configuration set
*/
public Write withConnectionConfiguration(ConnectionConfiguration connectionConfiguration) {
- checkArgument(
- connectionConfiguration != null,
- "ElasticsearchIO.write()"
- + ".withConnectionConfiguration(configuration) called with null configuration");
+ checkArgument(connectionConfiguration != null, "connectionConfiguration can not be null");
return builder().setConnectionConfiguration(connectionConfiguration).build();
}
@@ -696,10 +773,7 @@
* @return the {@link Write} with connection batch size set
*/
public Write withMaxBatchSize(long batchSize) {
- checkArgument(
- batchSize > 0,
- "ElasticsearchIO.write()"
- + ".withMaxBatchSize(batchSize) called with incorrect <= 0 value");
+ checkArgument(batchSize > 0, "batchSize must be > 0, but was %d", batchSize);
return builder().setMaxBatchSize(batchSize).build();
}
@@ -715,43 +789,41 @@
* @return the {@link Write} with connection batch size in bytes set
*/
public Write withMaxBatchSizeBytes(long batchSizeBytes) {
- checkArgument(
- batchSizeBytes > 0,
- "ElasticsearchIO.write()"
- + ".withMaxBatchSizeBytes(batchSizeBytes) called with incorrect <= 0 value");
+ checkArgument(batchSizeBytes > 0, "batchSizeBytes must be > 0, but was %d", batchSizeBytes);
return builder().setMaxBatchSizeBytes(batchSizeBytes).build();
}
@Override
- public void validate(PipelineOptions options) {
- checkState(
- getConnectionConfiguration() != null,
- "ElasticsearchIO.write() requires a connection configuration"
- + " to be set via withConnectionConfiguration(configuration)");
- }
-
- @Override
public PDone expand(PCollection<String> input) {
+ ConnectionConfiguration connectionConfiguration = getConnectionConfiguration();
+ checkState(connectionConfiguration != null, "withConnectionConfiguration() is required");
input.apply(ParDo.of(new WriteFn(this)));
return PDone.in(input.getPipeline());
}
+ /**
+ * {@link DoFn} to for the {@link Write} transform.
+ * */
@VisibleForTesting
static class WriteFn extends DoFn<String, Void> {
- private final Write spec;
+ private int backendVersion;
+ private final Write spec;
private transient RestClient restClient;
private ArrayList<String> batch;
private long currentBatchSizeBytes;
+ @VisibleForTesting
WriteFn(Write spec) {
this.spec = spec;
}
@Setup
- public void createClient() throws Exception {
- restClient = spec.getConnectionConfiguration().createClient();
+ public void setup() throws Exception {
+ ConnectionConfiguration connectionConfiguration = spec.getConnectionConfiguration();
+ backendVersion = getBackendVersion(connectionConfiguration);
+ restClient = connectionConfiguration.createClient();
}
@StartBundle
@@ -764,7 +836,7 @@
public void processElement(ProcessContext context) throws Exception {
String document = context.element();
batch.add(String.format("{ \"index\" : {} }%n%s%n", document));
- currentBatchSizeBytes += document.getBytes().length;
+ currentBatchSizeBytes += document.getBytes(StandardCharsets.UTF_8).length;
if (batch.size() >= spec.getMaxBatchSize()
|| currentBatchSizeBytes >= spec.getMaxBatchSizeBytes()) {
flushBatch();
@@ -799,34 +871,8 @@
"POST",
endPoint,
Collections.<String, String>emptyMap(),
- requestBody,
- new BasicHeader("", ""));
- JsonNode searchResult = parseResponse(response);
- boolean errors = searchResult.path("errors").asBoolean();
- if (errors) {
- StringBuilder errorMessages =
- new StringBuilder(
- "Error writing to Elasticsearch, some elements could not be inserted:");
- JsonNode items = searchResult.path("items");
- //some items present in bulk might have errors, concatenate error messages
- for (JsonNode item : items) {
- JsonNode creationObject = item.path("create");
- JsonNode error = creationObject.get("error");
- if (error != null) {
- String type = error.path("type").asText();
- String reason = error.path("reason").asText();
- String docId = creationObject.path("_id").asText();
- errorMessages.append(String.format("%nDocument id %s: %s (%s)", docId, reason, type));
- JsonNode causedBy = error.get("caused_by");
- if (causedBy != null) {
- String cbReason = causedBy.path("reason").asText();
- String cbType = causedBy.path("type").asText();
- errorMessages.append(String.format("%nCaused by: %s (%s)", cbReason, cbType));
- }
- }
- }
- throw new IOException(errorMessages.toString());
- }
+ requestBody);
+ checkForErrors(response, backendVersion);
}
@Teardown
@@ -837,4 +883,21 @@
}
}
}
+ static int getBackendVersion(ConnectionConfiguration connectionConfiguration) {
+ try (RestClient restClient = connectionConfiguration.createClient()) {
+ Response response = restClient.performRequest("GET", "");
+ JsonNode jsonNode = parseResponse(response);
+ int backendVersion = Integer
+ .parseInt(jsonNode.path("version").path("number").asText().substring(0, 1));
+ checkArgument((backendVersion == 2 || backendVersion == 5),
+ "The Elasticsearch version to connect to is %s.x. "
+ + "This version of the ElasticsearchIO is only compatible with "
+ + "Elasticsearch v5.x and v2.x",
+ backendVersion);
+ return backendVersion;
+
+ } catch (IOException e){
+ throw (new IllegalArgumentException("Cannot get Elasticsearch version"));
+ }
+ }
}
diff --git a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/package-info.java b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/package-info.java
index 396705b..73d2166 100644
--- a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/package-info.java
+++ b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/package-info.java
@@ -15,6 +15,5 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
/** Transforms for reading and writing from Elasticsearch. */
package org.apache.beam.sdk.io.elasticsearch;
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
deleted file mode 100644
index b0d161f..0000000
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.elasticsearch;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.IndicesAdminClient;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.index.IndexNotFoundException;
-
-/** Test utilities to use with {@link ElasticsearchIO}. */
-class ElasticSearchIOTestUtils {
-
- /** Enumeration that specifies whether to insert malformed documents. */
- enum InjectionMode {
- INJECT_SOME_INVALID_DOCS,
- DO_NOT_INJECT_INVALID_DOCS
- }
-
- /** Deletes the given index synchronously. */
- static void deleteIndex(String index, Client client) throws Exception {
- IndicesAdminClient indices = client.admin().indices();
- IndicesExistsResponse indicesExistsResponse =
- indices.exists(new IndicesExistsRequest(index)).get();
- if (indicesExistsResponse.isExists()) {
- indices.prepareClose(index).get();
- indices.delete(Requests.deleteIndexRequest(index)).get();
- }
- }
-
- /** Inserts the given number of test documents into Elasticsearch. */
- static void insertTestDocuments(String index, String type, long numDocs, Client client)
- throws Exception {
- final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk().setRefresh(true);
- List<String> data =
- ElasticSearchIOTestUtils.createDocuments(
- numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
- for (String document : data) {
- bulkRequestBuilder.add(client.prepareIndex(index, type, null).setSource(document));
- }
- final BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
- if (bulkResponse.hasFailures()) {
- throw new IOException(
- String.format(
- "Cannot insert test documents in index %s : %s",
- index, bulkResponse.buildFailureMessage()));
- }
- }
-
- /**
- * Forces an upgrade of the given index to make recently inserted documents available for search.
- *
- * @return The number of docs in the index
- */
- static long upgradeIndexAndGetCurrentNumDocs(String index, String type, Client client) {
- try {
- client.admin().indices().upgrade(new UpgradeRequest(index)).actionGet();
- SearchResponse response =
- client.prepareSearch(index).setTypes(type).execute().actionGet(5000);
- return response.getHits().getTotalHits();
- // it is fine to ignore bellow exceptions because in testWriteWithBatchSize* sometimes,
- // we call upgrade before any doc have been written
- // (when there are fewer docs processed than batchSize).
- // In that cases index/type has not been created (created upon first doc insertion)
- } catch (IndexNotFoundException e) {
- } catch (java.lang.IllegalArgumentException e) {
- if (!e.getMessage().contains("No search type")) {
- throw e;
- }
- }
- return 0;
- }
-
- /**
- * Generates a list of test documents for insertion.
- *
- * @param numDocs Number of docs to generate
- * @param injectionMode {@link InjectionMode} that specifies whether to insert malformed documents
- * @return the list of json String representing the documents
- */
- static List<String> createDocuments(long numDocs, InjectionMode injectionMode) {
- String[] scientists = {
- "Einstein",
- "Darwin",
- "Copernicus",
- "Pasteur",
- "Curie",
- "Faraday",
- "Newton",
- "Bohr",
- "Galilei",
- "Maxwell"
- };
- ArrayList<String> data = new ArrayList<>();
- for (int i = 0; i < numDocs; i++) {
- int index = i % scientists.length;
- // insert 2 malformed documents
- if (InjectionMode.INJECT_SOME_INVALID_DOCS.equals(injectionMode) && (i == 6 || i == 7)) {
- data.add(String.format("{\"scientist\";\"%s\", \"id\":%d}", scientists[index], i));
- } else {
- data.add(String.format("{\"scientist\":\"%s\", \"id\":%d}", scientists[index], i));
- }
- }
- return data;
- }
-}
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
deleted file mode 100644
index d968bc2..0000000
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.elasticsearch;
-
-import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollection;
-import org.elasticsearch.client.transport.TransportClient;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A test of {@link ElasticsearchIO} on an independent Elasticsearch instance.
- *
- * <p>This test requires a running instance of Elasticsearch, and the test dataset must exist in the
- * database.
- *
- * <p>You can run this test by doing the following from the beam parent module directory:
- *
- * <pre>
- * mvn -e -Pio-it verify -pl sdks/java/io/elasticsearch -DintegrationTestPipelineOptions='[
- * "--elasticsearchServer=1.2.3.4",
- * "--elasticsearchHttpPort=9200",
- * "--elasticsearchTcpPort=9300" ]'
- * </pre>
- */
-public class ElasticsearchIOIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchIOIT.class);
- private static TransportClient client;
- private static IOTestPipelineOptions options;
- private static ElasticsearchIO.ConnectionConfiguration readConnectionConfiguration;
- @Rule public TestPipeline pipeline = TestPipeline.create();
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- PipelineOptionsFactory.register(IOTestPipelineOptions.class);
- options = TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
- client = ElasticsearchTestDataSet.getClient(options);
- readConnectionConfiguration =
- ElasticsearchTestDataSet.getConnectionConfiguration(
- options, ElasticsearchTestDataSet.ReadOrWrite.READ);
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- ElasticsearchTestDataSet.deleteIndex(client, ElasticsearchTestDataSet.ReadOrWrite.WRITE);
- client.close();
- }
-
- @Test
- public void testSplitsVolume() throws Exception {
- ElasticsearchIO.Read read =
- ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration);
- ElasticsearchIO.BoundedElasticsearchSource initialSource =
- new ElasticsearchIO.BoundedElasticsearchSource(read, null);
- //desiredBundleSize is ignored because in ES 2.x there is no way to split shards. So we get
- // as many bundles as ES shards and bundle size is shard size
- long desiredBundleSizeBytes = 0;
- List<? extends BoundedSource<String>> splits =
- initialSource.split(desiredBundleSizeBytes, options);
- SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
- //this is the number of ES shards
- // (By default, each index in Elasticsearch is allocated 5 primary shards)
- long expectedNumSplits = 5;
- assertEquals(expectedNumSplits, splits.size());
- int nonEmptySplits = 0;
- for (BoundedSource<String> subSource : splits) {
- if (readFromSource(subSource, options).size() > 0) {
- nonEmptySplits += 1;
- }
- }
- assertEquals(expectedNumSplits, nonEmptySplits);
- }
-
- @Test
- public void testReadVolume() throws Exception {
- PCollection<String> output =
- pipeline.apply(
- ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration));
- PAssert.thatSingleton(output.apply("Count", Count.<String>globally()))
- .isEqualTo(ElasticsearchTestDataSet.NUM_DOCS);
- pipeline.run();
- }
-
- @Test
- public void testWriteVolume() throws Exception {
- ElasticsearchIO.ConnectionConfiguration writeConnectionConfiguration =
- ElasticsearchTestDataSet.getConnectionConfiguration(
- options, ElasticsearchTestDataSet.ReadOrWrite.WRITE);
- List<String> data =
- ElasticSearchIOTestUtils.createDocuments(
- ElasticsearchTestDataSet.NUM_DOCS,
- ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
- pipeline
- .apply(Create.of(data))
- .apply(ElasticsearchIO.write().withConnectionConfiguration(writeConnectionConfiguration));
- pipeline.run();
-
- long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ElasticsearchTestDataSet.ES_INDEX, ElasticsearchTestDataSet.ES_TYPE, client);
- assertEquals(ElasticsearchTestDataSet.NUM_DOCS, currentNumDocs);
- }
-
- @Test
- public void testEstimatedSizesVolume() throws Exception {
- ElasticsearchIO.Read read =
- ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration);
- ElasticsearchIO.BoundedElasticsearchSource initialSource =
- new ElasticsearchIO.BoundedElasticsearchSource(read, null);
- // can't use equal assert as Elasticsearch indexes never have same size
- // (due to internal Elasticsearch implementation)
- long estimatedSize = initialSource.getEstimatedSizeBytes(options);
- LOGGER.info("Estimated size: {}", estimatedSize);
- assertThat(
- "Wrong estimated size bellow minimum",
- estimatedSize,
- greaterThan(ElasticsearchTestDataSet.AVERAGE_DOC_SIZE * ElasticsearchTestDataSet.NUM_DOCS));
- assertThat(
- "Wrong estimated size beyond maximum",
- estimatedSize,
- greaterThan(ElasticsearchTestDataSet.MAX_DOC_SIZE * ElasticsearchTestDataSet.NUM_DOCS));
- }
-}
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
deleted file mode 100644
index 260af79..0000000
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.elasticsearch;
-
-import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
-import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.core.Is.isA;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.net.ServerSocket;
-import java.util.List;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFnTester;
-import org.apache.beam.sdk.values.PCollection;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
-import org.hamcrest.CustomMatcher;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** Tests for {@link ElasticsearchIO}. */
-@RunWith(JUnit4.class)
-public class ElasticsearchIOTest implements Serializable {
-
- private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIOTest.class);
-
- private static final String ES_INDEX = "beam";
- private static final String ES_TYPE = "test";
- private static final String ES_IP = "127.0.0.1";
- private static final long NUM_DOCS = 400L;
- private static final int NUM_SCIENTISTS = 10;
- private static final long BATCH_SIZE = 200L;
- private static final long AVERAGE_DOC_SIZE = 25L;
- private static final long BATCH_SIZE_BYTES = 2048L;
-
- private static Node node;
- private static ElasticsearchIO.ConnectionConfiguration connectionConfiguration;
-
- @ClassRule public static TemporaryFolder folder = new TemporaryFolder();
- @Rule
- public TestPipeline pipeline = TestPipeline.create();
-
- @BeforeClass
- public static void beforeClass() throws IOException {
- ServerSocket serverSocket = new ServerSocket(0);
- int esHttpPort = serverSocket.getLocalPort();
- serverSocket.close();
- LOG.info("Starting embedded Elasticsearch instance ({})", esHttpPort);
- Settings.Builder settingsBuilder =
- Settings.settingsBuilder()
- .put("cluster.name", "beam")
- .put("http.enabled", "true")
- .put("node.data", "true")
- .put("path.data", folder.getRoot().getPath())
- .put("path.home", folder.getRoot().getPath())
- .put("node.name", "beam")
- .put("network.host", ES_IP)
- .put("http.port", esHttpPort)
- .put("index.store.stats_refresh_interval", 0)
- // had problems with some jdk, embedded ES was too slow for bulk insertion,
- // and queue of 50 was full. No pb with real ES instance (cf testWrite integration test)
- .put("threadpool.bulk.queue_size", 100);
- node = NodeBuilder.nodeBuilder().settings(settingsBuilder).build();
- LOG.info("Elasticsearch node created");
- node.start();
- connectionConfiguration =
- ElasticsearchIO.ConnectionConfiguration.create(
- new String[] {"http://" + ES_IP + ":" + esHttpPort}, ES_INDEX, ES_TYPE);
- }
-
- @AfterClass
- public static void afterClass() {
- node.close();
- }
-
- @Before
- public void before() throws Exception {
- ElasticSearchIOTestUtils.deleteIndex(ES_INDEX, node.client());
- }
-
- @Test
- public void testSizes() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
- PipelineOptions options = PipelineOptionsFactory.create();
- ElasticsearchIO.Read read =
- ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
- BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null);
- // can't use equal assert as Elasticsearch indexes never have same size
- // (due to internal Elasticsearch implementation)
- long estimatedSize = initialSource.getEstimatedSizeBytes(options);
- LOG.info("Estimated size: {}", estimatedSize);
- assertThat("Wrong estimated size", estimatedSize, greaterThan(AVERAGE_DOC_SIZE * NUM_DOCS));
- }
-
- @Test
- public void testRead() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
-
- PCollection<String> output =
- pipeline.apply(
- ElasticsearchIO.read()
- .withConnectionConfiguration(connectionConfiguration)
- //set to default value, useful just to test parameter passing.
- .withScrollKeepalive("5m")
- //set to default value, useful just to test parameter passing.
- .withBatchSize(100L));
- PAssert.thatSingleton(output.apply("Count", Count.<String>globally())).isEqualTo(NUM_DOCS);
- pipeline.run();
- }
-
- @Test
- public void testReadWithQuery() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
-
- String query =
- "{\n"
- + " \"query\": {\n"
- + " \"match\" : {\n"
- + " \"scientist\" : {\n"
- + " \"query\" : \"Einstein\",\n"
- + " \"type\" : \"boolean\"\n"
- + " }\n"
- + " }\n"
- + " }\n"
- + "}";
-
- PCollection<String> output =
- pipeline.apply(
- ElasticsearchIO.read()
- .withConnectionConfiguration(connectionConfiguration)
- .withQuery(query));
- PAssert.thatSingleton(output.apply("Count", Count.<String>globally()))
- .isEqualTo(NUM_DOCS / NUM_SCIENTISTS);
- pipeline.run();
- }
-
- @Test
- public void testWrite() throws Exception {
- List<String> data =
- ElasticSearchIOTestUtils.createDocuments(
- NUM_DOCS, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
- pipeline
- .apply(Create.of(data))
- .apply(ElasticsearchIO.write().withConnectionConfiguration(connectionConfiguration));
- pipeline.run();
-
- long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(ES_INDEX, ES_TYPE, node.client());
- assertEquals(NUM_DOCS, currentNumDocs);
-
- QueryBuilder queryBuilder = QueryBuilders.queryStringQuery("Einstein").field("scientist");
- SearchResponse searchResponse =
- node.client()
- .prepareSearch(ES_INDEX)
- .setTypes(ES_TYPE)
- .setQuery(queryBuilder)
- .execute()
- .actionGet();
- assertEquals(NUM_DOCS / NUM_SCIENTISTS, searchResponse.getHits().getTotalHits());
- }
-
- @Rule public ExpectedException exception = ExpectedException.none();
-
- @Test
- public void testWriteWithErrors() throws Exception {
- ElasticsearchIO.Write write =
- ElasticsearchIO.write()
- .withConnectionConfiguration(connectionConfiguration)
- .withMaxBatchSize(BATCH_SIZE);
- // write bundles size is the runner decision, we cannot force a bundle size,
- // so we test the Writer as a DoFn outside of a runner.
- DoFnTester<String, Void> fnTester = DoFnTester.of(new ElasticsearchIO.Write.WriteFn(write));
-
- List<String> input =
- ElasticSearchIOTestUtils.createDocuments(
- NUM_DOCS, ElasticSearchIOTestUtils.InjectionMode.INJECT_SOME_INVALID_DOCS);
- exception.expect(isA(IOException.class));
- exception.expectMessage(
- new CustomMatcher<String>("RegExp matcher") {
- @Override
- public boolean matches(Object o) {
- String message = (String) o;
- // This regexp tests that 2 malformed documents are actually in error
- // and that the message contains their IDs.
- // It also ensures that root reason, root error type,
- // caused by reason and caused by error type are present in message.
- // To avoid flakiness of the test in case of Elasticsearch error message change,
- // only "failed to parse" root reason is matched,
- // the other messages are matched using .+
- return message.matches(
- "(?is).*Error writing to Elasticsearch, some elements could not be inserted"
- + ".*Document id .+: failed to parse \\(.+\\).*Caused by: .+ \\(.+\\).*"
- + "Document id .+: failed to parse \\(.+\\).*Caused by: .+ \\(.+\\).*");
- }
- });
- // inserts into Elasticsearch
- fnTester.processBundle(input);
- }
-
- @Test
- public void testWriteWithMaxBatchSize() throws Exception {
- ElasticsearchIO.Write write =
- ElasticsearchIO.write()
- .withConnectionConfiguration(connectionConfiguration)
- .withMaxBatchSize(BATCH_SIZE);
- // write bundles size is the runner decision, we cannot force a bundle size,
- // so we test the Writer as a DoFn outside of a runner.
- DoFnTester<String, Void> fnTester = DoFnTester.of(new ElasticsearchIO.Write.WriteFn(write));
- List<String> input =
- ElasticSearchIOTestUtils.createDocuments(
- NUM_DOCS, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
- long numDocsProcessed = 0;
- long numDocsInserted = 0;
- for (String document : input) {
- fnTester.processElement(document);
- numDocsProcessed++;
- // test every 100 docs to avoid overloading ES
- if ((numDocsProcessed % 100) == 0) {
- // force the index to upgrade after inserting for the inserted docs
- // to be searchable immediately
- long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ES_INDEX, ES_TYPE, node.client());
- if ((numDocsProcessed % BATCH_SIZE) == 0) {
- /* bundle end */
- assertEquals(
- "we are at the end of a bundle, we should have inserted all processed documents",
- numDocsProcessed,
- currentNumDocs);
- numDocsInserted = currentNumDocs;
- } else {
- /* not bundle end */
- assertEquals(
- "we are not at the end of a bundle, we should have inserted no more documents",
- numDocsInserted,
- currentNumDocs);
- }
- }
- }
- }
-
- @Test
- public void testWriteWithMaxBatchSizeBytes() throws Exception {
- ElasticsearchIO.Write write =
- ElasticsearchIO.write()
- .withConnectionConfiguration(connectionConfiguration)
- .withMaxBatchSizeBytes(BATCH_SIZE_BYTES);
- // write bundles size is the runner decision, we cannot force a bundle size,
- // so we test the Writer as a DoFn outside of a runner.
- DoFnTester<String, Void> fnTester = DoFnTester.of(new ElasticsearchIO.Write.WriteFn(write));
- List<String> input =
- ElasticSearchIOTestUtils.createDocuments(
- NUM_DOCS, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
- long numDocsProcessed = 0;
- long sizeProcessed = 0;
- long numDocsInserted = 0;
- long batchInserted = 0;
- for (String document : input) {
- fnTester.processElement(document);
- numDocsProcessed++;
- sizeProcessed += document.getBytes().length;
- // test every 40 docs to avoid overloading ES
- if ((numDocsProcessed % 40) == 0) {
- // force the index to upgrade after inserting for the inserted docs
- // to be searchable immediately
- long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ES_INDEX, ES_TYPE, node.client());
- if (sizeProcessed / BATCH_SIZE_BYTES > batchInserted) {
- /* bundle end */
- assertThat(
- "we have passed a bundle size, we should have inserted some documents",
- currentNumDocs,
- greaterThan(numDocsInserted));
- numDocsInserted = currentNumDocs;
- batchInserted = (sizeProcessed / BATCH_SIZE_BYTES);
- } else {
- /* not bundle end */
- assertEquals(
- "we are not at the end of a bundle, we should have inserted no more documents",
- numDocsInserted,
- currentNumDocs);
- }
- }
- }
- }
-
- @Test
- public void testSplit() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
- PipelineOptions options = PipelineOptionsFactory.create();
- ElasticsearchIO.Read read =
- ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
- BoundedElasticsearchSource initialSource = new BoundedElasticsearchSource(read, null);
- //desiredBundleSize is ignored because in ES 2.x there is no way to split shards. So we get
- // as many bundles as ES shards and bundle size is shard size
- int desiredBundleSizeBytes = 0;
- List<? extends BoundedSource<String>> splits =
- initialSource.split(desiredBundleSizeBytes, options);
- SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
- //this is the number of ES shards
- // (By default, each index in Elasticsearch is allocated 5 primary shards)
- int expectedNumSplits = 5;
- assertEquals(expectedNumSplits, splits.size());
- int nonEmptySplits = 0;
- for (BoundedSource<String> subSource : splits) {
- if (readFromSource(subSource, options).size() > 0) {
- nonEmptySplits += 1;
- }
- }
- assertEquals("Wrong number of empty splits", expectedNumSplits, nonEmptySplits);
- }
-}
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
deleted file mode 100644
index 3a9aae6..0000000
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.elasticsearch;
-
-import static java.net.InetAddress.getByName;
-
-import java.io.IOException;
-import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-
-/**
- * Manipulates test data used by the {@link ElasticsearchIO}
- * integration tests.
- *
- * <p>This is independent from the tests so that for read tests it can be run separately after data
- * store creation rather than every time (which can be more fragile.)
- */
-public class ElasticsearchTestDataSet {
-
- public static final String ES_INDEX = "beam";
- public static final String ES_TYPE = "test";
- public static final long NUM_DOCS = 60000;
- public static final int AVERAGE_DOC_SIZE = 25;
- public static final int MAX_DOC_SIZE = 35;
- private static String writeIndex = ES_INDEX + org.joda.time.Instant.now().getMillis();
-
- /**
- * Use this to create the index for reading before IT read tests.
- *
- * <p>To invoke this class, you can use this command line from elasticsearch io module directory:
- *
- * <pre>
- * mvn test-compile exec:java \
- * -Dexec.mainClass=org.apache.beam.sdk.io.elasticsearch.ElasticsearchTestDataSet \
- * -Dexec.args="--elasticsearchServer=1.2.3.4 \
- * --elasticsearchHttpPort=9200 \
- * --elasticsearchTcpPort=9300" \
- * -Dexec.classpathScope=test
- * </pre>
- *
- * @param args Please pass options from ElasticsearchTestOptions used for connection to
- * Elasticsearch as shown above.
- */
- public static void main(String[] args) throws Exception {
- PipelineOptionsFactory.register(IOTestPipelineOptions.class);
- IOTestPipelineOptions options =
- PipelineOptionsFactory.fromArgs(args).as(IOTestPipelineOptions.class);
-
- createAndPopulateIndex(getClient(options), ReadOrWrite.READ);
- }
-
- private static void createAndPopulateIndex(TransportClient client, ReadOrWrite rOw)
- throws Exception {
- // automatically creates the index and insert docs
- ElasticSearchIOTestUtils.insertTestDocuments(
- (rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex, ES_TYPE, NUM_DOCS, client);
- }
-
- public static TransportClient getClient(IOTestPipelineOptions options) throws Exception {
- TransportClient client =
- TransportClient.builder()
- .build()
- .addTransportAddress(
- new InetSocketTransportAddress(
- getByName(options.getElasticsearchServer()),
- options.getElasticsearchTcpPort()));
- return client;
- }
-
- public static ElasticsearchIO.ConnectionConfiguration getConnectionConfiguration(
- IOTestPipelineOptions options, ReadOrWrite rOw) throws IOException {
- ElasticsearchIO.ConnectionConfiguration connectionConfiguration =
- ElasticsearchIO.ConnectionConfiguration.create(
- new String[] {
- "http://"
- + options.getElasticsearchServer()
- + ":"
- + options.getElasticsearchHttpPort()
- },
- (rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex,
- ES_TYPE);
- return connectionConfiguration;
- }
-
- public static void deleteIndex(TransportClient client, ReadOrWrite rOw) throws Exception {
- ElasticSearchIOTestUtils.deleteIndex((rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex, client);
- }
-
- /** Enum that tells whether we use the index for reading or for writing. */
- public enum ReadOrWrite {
- READ,
- WRITE
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 7594365..181df7a 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,37 +32,113 @@
<packaging>jar</packaging>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamUseDummyRunner>false</beamUseDummyRunner>
- </systemPropertyVariables>
- </configuration>
- </plugin>
-
- <!-- Ensure that the Maven jar plugin runs before the Maven
- shade plugin by listing the plugin higher within the file. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-and-repackage</id>
- <phase>none</phase>
- </execution>
- </executions>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>bundle-and-repackage</id>
+ <phase>none</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
+ <profiles>
+ <!-- This profile invokes PerfKitBenchmarker, which does benchmarking of
+ the IO ITs. The arguments passed to it allow it to invoke mvn again
+ with the desired benchmark.
+
+ To invoke this, run:
+ mvn verify -Dio-it-suite -pl sdks/java/io/google-cloud-platform
+ -DpkbLocation="path-to-pkb.py" \
+ -DintegrationTestPipelineOptions='["-tempRoot=gs://bucket/staging", "-project=your-project-id"]' \
+ -DgcpIoItClass=[your favorite IO's IT]
+ -->
+ <profile>
+ <id>io-it-suite</id>
+ <properties>
+ <!-- This is based on the location of the current pom relative to the
+ root. See discussion in BEAM-2460. -->
+ <beamRootProjectDir>
+ ${project.parent.parent.parent.parent.basedir}
+ </beamRootProjectDir>
+ </properties>
+ <activation><property><name>io-it-suite</name></property></activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.gmaven</groupId>
+ <artifactId>groovy-maven-plugin</artifactId>
+ <version>${groovy-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>find-supported-python-for-compile</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>execute</goal>
+ </goals>
+ <configuration>
+ <source>${beamRootProjectDir}/sdks/python/findSupportedPython.groovy</source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven-exec-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>${python.interpreter.bin}</executable>
+ <arguments>
+ <argument>${pkbLocation}</argument>
+ <argument>-benchmarks=beam_integration_benchmark</argument>
+ <argument>-beam_it_profile=io-it</argument>
+ <argument>-beam_location=${beamRootProjectDir}</argument>
+ <argument>-beam_prebuilt=true</argument>
+ <argument>-beam_sdk=java</argument>
+ <argument>-kubeconfig=${kubeconfig}</argument>
+ <argument>-kubectl=${kubectl}</argument>
+ <!-- runner overrides, controlled via forceDirectRunner -->
+ <argument>${pkbBeamRunnerProfile}</argument>
+ <argument>${pkbBeamRunnerOption}</argument>
+ <!-- specific to this IO -->
+ <argument>-beam_it_module=runners/google-cloud-dataflow-java</argument>
+ <!-- Most IOs have only one IT so this can be hard coded, but
+ since the GCP IO dir contains multiple IOs, we allow the
+ user to specify which particular one they want to run. -->
+ <argument>-beam_it_class=${gcpIoItClass}</argument>
+ <!-- arguments typically defined by user -->
+ <argument>-beam_it_options=${integrationTestPipelineOptions}</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire-plugin.version}</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
@@ -85,18 +161,38 @@
</dependency>
<dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
</dependency>
<dependency>
+ <groupId>com.google.api</groupId>
+ <artifactId>gax-grpc</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-core-grpc</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-pubsub</artifactId>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
- <artifactId>grpc-google-pubsub-v1</artifactId>
+ <artifactId>grpc-google-cloud-pubsub-v1</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-cloud-pubsub-v1</artifactId>
</dependency>
<dependency>
@@ -121,11 +217,6 @@
<dependency>
<groupId>io.grpc</groupId>
- <artifactId>grpc-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
</dependency>
@@ -153,6 +244,16 @@
</dependency>
<dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-spanner</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-protos</artifactId>
</dependency>
@@ -188,11 +289,6 @@
</dependency>
<dependency>
- <groupId>com.google.api.grpc</groupId>
- <artifactId>grpc-google-common-protos</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
@@ -236,6 +332,16 @@
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ </dependency>
+
<!-- Test dependencies -->
<dependency>
<groupId>org.apache.beam</groupId>
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
index 0abd469..1ccd5d6 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
@@ -19,6 +19,7 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation;
import com.google.api.services.bigquery.model.TableRow;
@@ -26,18 +27,21 @@
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import java.util.List;
-import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.coders.ShardedKeyCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
@@ -47,9 +51,15 @@
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Reshuffle;
import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.Values;
import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterFirst;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
import org.apache.beam.sdk.values.KV;
@@ -57,12 +67,19 @@
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.ShardedKey;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** PTransform that uses BigQuery batch-load jobs to write a PCollection to BigQuery. */
class BatchLoads<DestinationT>
extends PTransform<PCollection<KV<DestinationT, TableRow>>, WriteResult> {
+ static final Logger LOG = LoggerFactory.getLogger(BatchLoads.class);
+
// The maximum number of file writers to keep open in a single bundle at a time, since file
// writers default to 64mb buffers. This comes into play when writing dynamic table destinations.
// The first 20 tables from a single BatchLoads transform will write files inline in the
@@ -86,6 +103,12 @@
// The maximum size of a single file - 4TiB, just under the 5 TiB limit.
static final long DEFAULT_MAX_FILE_SIZE = 4 * (1L << 40);
+ static final int DEFAULT_NUM_FILE_SHARDS = 0;
+
+ // If user triggering is supplied, we will trigger the file write after this many records are
+ // written.
+ static final int FILE_TRIGGERING_RECORD_COUNT = 500000;
+
// The maximum number of retries to poll the status of a job.
// It sets to {@code Integer.MAX_VALUE} to block until the BigQuery job finishes.
static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
@@ -103,11 +126,15 @@
private final Coder<DestinationT> destinationCoder;
private int maxNumWritersPerBundle;
private long maxFileSize;
+ private int numFileShards;
+ private Duration triggeringFrequency;
+ private ValueProvider<String> customGcsTempLocation;
BatchLoads(WriteDisposition writeDisposition, CreateDisposition createDisposition,
boolean singletonTable,
DynamicDestinations<?, DestinationT> dynamicDestinations,
- Coder<DestinationT> destinationCoder) {
+ Coder<DestinationT> destinationCoder,
+ ValueProvider<String> customGcsTempLocation) {
bigQueryServices = new BigQueryServicesImpl();
this.writeDisposition = writeDisposition;
this.createDisposition = createDisposition;
@@ -116,6 +143,9 @@
this.destinationCoder = destinationCoder;
this.maxNumWritersPerBundle = DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE;
this.maxFileSize = DEFAULT_MAX_FILE_SIZE;
+ this.numFileShards = DEFAULT_NUM_FILE_SHARDS;
+ this.triggeringFrequency = null;
+ this.customGcsTempLocation = customGcsTempLocation;
}
void setTestServices(BigQueryServices bigQueryServices) {
@@ -132,6 +162,14 @@
this.maxNumWritersPerBundle = maxNumWritersPerBundle;
}
+ public void setTriggeringFrequency(Duration triggeringFrequency) {
+ this.triggeringFrequency = triggeringFrequency;
+ }
+
+ public void setNumFileShards(int numFileShards) {
+ this.numFileShards = numFileShards;
+ }
+
@VisibleForTesting
void setMaxFileSize(long maxFileSize) {
this.maxFileSize = maxFileSize;
@@ -140,7 +178,16 @@
@Override
public void validate(PipelineOptions options) {
// We will use a BigQuery load job -- validate the temp location.
- String tempLocation = options.getTempLocation();
+ String tempLocation;
+ if (customGcsTempLocation == null) {
+ tempLocation = options.getTempLocation();
+ } else {
+ if (!customGcsTempLocation.isAccessible()) {
+ // Can't perform verification in this case.
+ return;
+ }
+ tempLocation = customGcsTempLocation.get();
+ }
checkArgument(
!Strings.isNullOrEmpty(tempLocation),
"BigQueryIO.Write needs a GCS temp location to store temp files.");
@@ -157,182 +204,347 @@
}
}
- @Override
- public WriteResult expand(PCollection<KV<DestinationT, TableRow>> input) {
+ // Expand the pipeline when the user has requested periodically-triggered file writes.
+ private WriteResult expandTriggered(PCollection<KV<DestinationT, TableRow>> input) {
+ checkArgument(numFileShards > 0);
Pipeline p = input.getPipeline();
- final String stepUuid = BigQueryHelpers.randomUUIDString();
+ final PCollectionView<String> jobIdTokenView = createJobIdView(p);
+ final PCollectionView<String> tempFilePrefixView = createTempFilePrefixView(jobIdTokenView);
+ // The user-supplied triggeringDuration is often chosen to to control how many BigQuery load
+ // jobs are generated, to prevent going over BigQuery's daily quota for load jobs. If this
+ // is set to a large value, currently we have to buffer all the data unti the trigger fires.
+ // Instead we ensure that the files are written if a threshold number of records are ready.
+ // We use only the user-supplied trigger on the actual BigQuery load. This allows us to
+ // offload the data to the filesystem.
+ PCollection<KV<DestinationT, TableRow>> inputInGlobalWindow =
+ input.apply(
+ "rewindowIntoGlobal",
+ Window.<KV<DestinationT, TableRow>>into(new GlobalWindows())
+ .triggering(
+ Repeatedly.forever(
+ AfterFirst.of(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(triggeringFrequency),
+ AfterPane.elementCountAtLeast(FILE_TRIGGERING_RECORD_COUNT))))
+ .discardingFiredPanes());
+ PCollection<WriteBundlesToFiles.Result<DestinationT>> results =
+ writeShardedFiles(inputInGlobalWindow, tempFilePrefixView);
+ // Apply the user's trigger before we start generating BigQuery load jobs.
+ results =
+ results.apply(
+ "applyUserTrigger",
+ Window.<WriteBundlesToFiles.Result<DestinationT>>into(new GlobalWindows())
+ .triggering(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(triggeringFrequency)))
+ .discardingFiredPanes());
- PCollectionView<String> tempFilePrefix =
- p.apply("Create", Create.of((Void) null))
+ TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag =
+ new TupleTag<KV<ShardedKey<DestinationT>, List<String>>>("multiPartitionsTag");
+ TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag =
+ new TupleTag<KV<ShardedKey<DestinationT>, List<String>>>("singlePartitionTag");
+
+ // If we have non-default triggered output, we can't use the side-input technique used in
+ // expandUntriggered . Instead make the result list a main input. Apply a GroupByKey first for
+ // determinism.
+ PCollectionTuple partitions =
+ results
.apply(
- "GetTempFilePrefix",
+ "AttachSingletonKey",
+ WithKeys.<Void, WriteBundlesToFiles.Result<DestinationT>>of((Void) null))
+ .setCoder(
+ KvCoder.of(VoidCoder.of(), WriteBundlesToFiles.ResultCoder.of(destinationCoder)))
+ .apply("GroupOntoSingleton", GroupByKey.<Void, Result<DestinationT>>create())
+ .apply("ExtractResultValues", Values.<Iterable<Result<DestinationT>>>create())
+ .apply(
+ "WritePartitionTriggered",
ParDo.of(
- new DoFn<Void, String>() {
- @ProcessElement
- public void getTempFilePrefix(ProcessContext c) {
- c.output(
- resolveTempLocation(
- c.getPipelineOptions().getTempLocation(),
- "BigQueryWriteTemp",
- stepUuid));
- }
- }))
- .apply("TempFilePrefixView", View.<String>asSingleton());
+ new WritePartition<>(
+ singletonTable,
+ dynamicDestinations,
+ tempFilePrefixView,
+ multiPartitionsTag,
+ singlePartitionTag))
+ .withSideInputs(tempFilePrefixView)
+ .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
+ PCollection<KV<TableDestination, String>> tempTables =
+ writeTempTables(partitions.get(multiPartitionsTag), jobIdTokenView);
+ tempTables
+ // Now that the load job has happened, we want the rename to happen immediately.
+ .apply(
+ Window.<KV<TableDestination, String>>into(new GlobalWindows())
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))))
+ .apply(WithKeys.<Void, KV<TableDestination, String>>of((Void) null))
+ .setCoder(
+ KvCoder.of(
+ VoidCoder.of(), KvCoder.of(TableDestinationCoderV2.of(), StringUtf8Coder.of())))
+ .apply(GroupByKey.<Void, KV<TableDestination, String>>create())
+ .apply(Values.<Iterable<KV<TableDestination, String>>>create())
+ .apply(
+ "WriteRenameTriggered",
+ ParDo.of(
+ new WriteRename(
+ bigQueryServices, jobIdTokenView, writeDisposition, createDisposition))
+ .withSideInputs(jobIdTokenView));
+ writeSinglePartition(partitions.get(singlePartitionTag), jobIdTokenView);
+ return writeResult(p);
+ }
- // Create a singleton job ID token at execution time. This will be used as the base for all
- // load jobs issued from this instance of the transform.
- PCollectionView<String> jobIdTokenView =
- p.apply("TriggerIdCreation", Create.of("ignored"))
- .apply(
- "CreateJobId",
- MapElements.via(
- new SimpleFunction<String, String>() {
- @Override
- public String apply(String input) {
- return stepUuid;
- }
- }))
- .apply(View.<String>asSingleton());
-
+ // Expand the pipeline when the user has not requested periodically-triggered file writes.
+ public WriteResult expandUntriggered(PCollection<KV<DestinationT, TableRow>> input) {
+ Pipeline p = input.getPipeline();
+ final PCollectionView<String> jobIdTokenView = createJobIdView(p);
+ final PCollectionView<String> tempFilePrefixView = createTempFilePrefixView(jobIdTokenView);
PCollection<KV<DestinationT, TableRow>> inputInGlobalWindow =
input.apply(
"rewindowIntoGlobal",
Window.<KV<DestinationT, TableRow>>into(new GlobalWindows())
.triggering(DefaultTrigger.of())
.discardingFiredPanes());
- PCollectionView<Map<DestinationT, String>> schemasView =
- inputInGlobalWindow.apply(new CalculateSchemas<>(dynamicDestinations));
-
- TupleTag<WriteBundlesToFiles.Result<DestinationT>> writtenFilesTag =
- new TupleTag<WriteBundlesToFiles.Result<DestinationT>>("writtenFiles"){};
- TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag =
- new TupleTag<KV<ShardedKey<DestinationT>, TableRow>>("unwrittenRecords") {};
- PCollectionTuple writeBundlesTuple = inputInGlobalWindow
- .apply("WriteBundlesToFiles",
- ParDo.of(new WriteBundlesToFiles<>(stepUuid, unwrittedRecordsTag,
- maxNumWritersPerBundle, maxFileSize))
- .withOutputTags(writtenFilesTag, TupleTagList.of(unwrittedRecordsTag)));
- PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFiles =
- writeBundlesTuple.get(writtenFilesTag)
- .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
-
- // If the bundles contain too many output tables to be written inline to files (due to memory
- // limits), any unwritten records will be spilled to the unwrittenRecordsTag PCollection.
- // Group these records by key, and write the files after grouping. Since the record is grouped
- // by key, we can ensure that only one file is open at a time in each bundle.
- PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFilesGrouped =
- writeBundlesTuple
- .get(unwrittedRecordsTag)
- .setCoder(KvCoder.of(ShardedKeyCoder.of(destinationCoder), TableRowJsonCoder.of()))
- .apply(GroupByKey.<ShardedKey<DestinationT>, TableRow>create())
- .apply(
- ParDo.of(new WriteGroupedRecordsToFiles<DestinationT>(tempFilePrefix, maxFileSize))
- .withSideInputs(tempFilePrefix))
- .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
-
- // PCollection of filename, file byte size, and table destination.
PCollection<WriteBundlesToFiles.Result<DestinationT>> results =
- PCollectionList.of(writtenFiles).and(writtenFilesGrouped)
- .apply(Flatten.<Result<DestinationT>>pCollections());
+ (numFileShards == 0)
+ ? writeDynamicallyShardedFiles(inputInGlobalWindow, tempFilePrefixView)
+ : writeShardedFiles(inputInGlobalWindow, tempFilePrefixView);
TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag =
new TupleTag<KV<ShardedKey<DestinationT>, List<String>>>("multiPartitionsTag") {};
TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag =
new TupleTag<KV<ShardedKey<DestinationT>, List<String>>>("singlePartitionTag") {};
- // Turn the list of files and record counts in a PCollectionView that can be used as a
- // side input.
- PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> resultsView =
- results.apply("ResultsView",
- View.<WriteBundlesToFiles.Result<DestinationT>>asIterable());
// This transform will look at the set of files written for each table, and if any table has
// too many files or bytes, will partition that table's files into multiple partitions for
// loading.
- PCollection<Void> singleton = p.apply(Create.of((Void) null).withCoder(VoidCoder.of()));
PCollectionTuple partitions =
- singleton.apply(
- "WritePartition",
- ParDo.of(
- new WritePartition<>(
- singletonTable,
- tempFilePrefix,
- resultsView,
- multiPartitionsTag,
- singlePartitionTag))
- .withSideInputs(tempFilePrefix, resultsView)
- .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
+ results
+ .apply("ReifyResults", new ReifyAsIterable<WriteBundlesToFiles.Result<DestinationT>>())
+ .setCoder(IterableCoder.of(WriteBundlesToFiles.ResultCoder.of(destinationCoder)))
+ .apply(
+ "WritePartitionUntriggered",
+ ParDo.of(
+ new WritePartition<>(
+ singletonTable,
+ dynamicDestinations,
+ tempFilePrefixView,
+ multiPartitionsTag,
+ singlePartitionTag))
+ .withSideInputs(tempFilePrefixView)
+ .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
+ PCollection<KV<TableDestination, String>> tempTables =
+ writeTempTables(partitions.get(multiPartitionsTag), jobIdTokenView);
- List<PCollectionView<?>> writeTablesSideInputs =
- Lists.newArrayList(jobIdTokenView, schemasView);
- writeTablesSideInputs.addAll(dynamicDestinations.getSideInputs());
+ tempTables
+ .apply("ReifyRenameInput", new ReifyAsIterable<KV<TableDestination, String>>())
+ .setCoder(IterableCoder.of(KvCoder.of(TableDestinationCoderV2.of(), StringUtf8Coder.of())))
+ .apply(
+ "WriteRenameUntriggered",
+ ParDo.of(
+ new WriteRename(
+ bigQueryServices, jobIdTokenView, writeDisposition, createDisposition))
+ .withSideInputs(jobIdTokenView));
+ writeSinglePartition(partitions.get(singlePartitionTag), jobIdTokenView);
+ return writeResult(p);
+ }
+
+ // Generate the base job id string.
+ private PCollectionView<String> createJobIdView(Pipeline p) {
+ // Create a singleton job ID token at execution time. This will be used as the base for all
+ // load jobs issued from this instance of the transform.
+ return p.apply("JobIdCreationRoot", Create.of((Void) null))
+ .apply(
+ "CreateJobId",
+ MapElements.via(
+ new SimpleFunction<Void, String>() {
+ @Override
+ public String apply(Void input) {
+ return BigQueryHelpers.randomUUIDString();
+ }
+ }))
+ .apply(View.<String>asSingleton());
+ }
+
+ // Generate the temporary-file prefix.
+ private PCollectionView<String> createTempFilePrefixView(PCollectionView<String> jobIdView) {
+ return ((PCollection<String>) jobIdView.getPCollection())
+ .apply(
+ "GetTempFilePrefix",
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void getTempFilePrefix(ProcessContext c) {
+ String tempLocationRoot;
+ if (customGcsTempLocation != null) {
+ tempLocationRoot = customGcsTempLocation.get();
+ } else {
+ tempLocationRoot = c.getPipelineOptions().getTempLocation();
+ }
+ String tempLocation =
+ resolveTempLocation(
+ tempLocationRoot,
+ "BigQueryWriteTemp",
+ c.element());
+ LOG.info(
+ "Writing BigQuery temporary files to {} before loading them.",
+ tempLocation);
+ c.output(tempLocation);
+ }
+ }))
+ .apply("TempFilePrefixView", View.<String>asSingleton());
+ }
+
+ // Writes input data to dynamically-sharded, per-bundle files. Returns a PCollection of filename,
+ // file byte size, and table destination.
+ PCollection<WriteBundlesToFiles.Result<DestinationT>> writeDynamicallyShardedFiles(
+ PCollection<KV<DestinationT, TableRow>> input, PCollectionView<String> tempFilePrefix) {
+ TupleTag<WriteBundlesToFiles.Result<DestinationT>> writtenFilesTag =
+ new TupleTag<WriteBundlesToFiles.Result<DestinationT>>("writtenFiles") {};
+ TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag =
+ new TupleTag<KV<ShardedKey<DestinationT>, TableRow>>("unwrittenRecords") {};
+ PCollectionTuple writeBundlesTuple =
+ input.apply(
+ "WriteBundlesToFiles",
+ ParDo.of(
+ new WriteBundlesToFiles<>(
+ tempFilePrefix, unwrittedRecordsTag, maxNumWritersPerBundle, maxFileSize))
+ .withSideInputs(tempFilePrefix)
+ .withOutputTags(writtenFilesTag, TupleTagList.of(unwrittedRecordsTag)));
+ PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFiles =
+ writeBundlesTuple
+ .get(writtenFilesTag)
+ .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+ PCollection<KV<ShardedKey<DestinationT>, TableRow>> unwrittenRecords =
+ writeBundlesTuple
+ .get(unwrittedRecordsTag)
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(destinationCoder), TableRowJsonCoder.of()));
+
+ // If the bundles contain too many output tables to be written inline to files (due to memory
+ // limits), any unwritten records will be spilled to the unwrittenRecordsTag PCollection.
+ // Group these records by key, and write the files after grouping. Since the record is grouped
+ // by key, we can ensure that only one file is open at a time in each bundle.
+ PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFilesGrouped =
+ writeShardedRecords(unwrittenRecords, tempFilePrefix);
+
+ // PCollection of filename, file byte size, and table destination.
+ return PCollectionList.of(writtenFiles)
+ .and(writtenFilesGrouped)
+ .apply("FlattenFiles", Flatten.<Result<DestinationT>>pCollections())
+ .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+ }
+
+ // Writes input data to statically-sharded files. Returns a PCollection of filename,
+ // file byte size, and table destination.
+ PCollection<WriteBundlesToFiles.Result<DestinationT>> writeShardedFiles(
+ PCollection<KV<DestinationT, TableRow>> input, PCollectionView<String> tempFilePrefix) {
+ checkState(numFileShards > 0);
+ PCollection<KV<ShardedKey<DestinationT>, TableRow>> shardedRecords =
+ input
+ .apply(
+ "AddShard",
+ ParDo.of(
+ new DoFn<KV<DestinationT, TableRow>, KV<ShardedKey<DestinationT>, TableRow>>() {
+ int shardNumber;
+
+ @Setup
+ public void setup() {
+ shardNumber = ThreadLocalRandom.current().nextInt(numFileShards);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ DestinationT destination = c.element().getKey();
+ TableRow tableRow = c.element().getValue();
+ c.output(
+ KV.of(
+ ShardedKey.of(destination, ++shardNumber % numFileShards),
+ tableRow));
+ }
+ }))
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(destinationCoder), TableRowJsonCoder.of()));
+
+ return writeShardedRecords(shardedRecords, tempFilePrefix);
+ }
+
+ private PCollection<Result<DestinationT>> writeShardedRecords(
+ PCollection<KV<ShardedKey<DestinationT>, TableRow>> shardedRecords,
+ PCollectionView<String> tempFilePrefix) {
+ return shardedRecords
+ .apply("GroupByDestination", GroupByKey.<ShardedKey<DestinationT>, TableRow>create())
+ .apply(
+ "WriteGroupedRecords",
+ ParDo.of(new WriteGroupedRecordsToFiles<DestinationT>(tempFilePrefix, maxFileSize))
+ .withSideInputs(tempFilePrefix))
+ .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+ }
+
+ // Take in a list of files and write them to temporary tables.
+ private PCollection<KV<TableDestination, String>> writeTempTables(
+ PCollection<KV<ShardedKey<DestinationT>, List<String>>> input,
+ PCollectionView<String> jobIdTokenView) {
+ List<PCollectionView<?>> sideInputs = Lists.<PCollectionView<?>>newArrayList(jobIdTokenView);
+ sideInputs.addAll(dynamicDestinations.getSideInputs());
Coder<KV<ShardedKey<DestinationT>, List<String>>> partitionsCoder =
- KvCoder.of(
- ShardedKeyCoder.of(NullableCoder.of(destinationCoder)),
- ListCoder.of(StringUtf8Coder.of()));
+ KvCoder.of(
+ ShardedKeyCoder.of(NullableCoder.of(destinationCoder)),
+ ListCoder.of(StringUtf8Coder.of()));
// If WriteBundlesToFiles produced more than MAX_NUM_FILES files or MAX_SIZE_BYTES bytes, then
// the import needs to be split into multiple partitions, and those partitions will be
// specified in multiPartitionsTag.
- PCollection<KV<TableDestination, String>> tempTables =
- partitions
- .get(multiPartitionsTag)
- .setCoder(partitionsCoder)
- // Reshuffle will distribute this among multiple workers, and also guard against
- // reexecution of the WritePartitions step once WriteTables has begun.
- .apply(
- "MultiPartitionsReshuffle",
- Reshuffle.<ShardedKey<DestinationT>, List<String>>of())
- .apply(
- "MultiPartitionsWriteTables",
- ParDo.of(
- new WriteTables<>(
- false,
- bigQueryServices,
- jobIdTokenView,
- schemasView,
- WriteDisposition.WRITE_EMPTY,
- CreateDisposition.CREATE_IF_NEEDED,
- dynamicDestinations))
- .withSideInputs(writeTablesSideInputs));
-
- // This view maps each final table destination to the set of temporary partitioned tables
- // the PCollection was loaded into.
- PCollectionView<Map<TableDestination, Iterable<String>>> tempTablesView =
- tempTables.apply("TempTablesView", View.<TableDestination, String>asMultimap());
-
- singleton.apply(
- "WriteRename",
- ParDo.of(
- new WriteRename(
- bigQueryServices,
- jobIdTokenView,
- writeDisposition,
- createDisposition,
- tempTablesView))
- .withSideInputs(tempTablesView, jobIdTokenView));
-
- // Write single partition to final table
- partitions
- .get(singlePartitionTag)
+ return input
.setCoder(partitionsCoder)
// Reshuffle will distribute this among multiple workers, and also guard against
// reexecution of the WritePartitions step once WriteTables has begun.
+ .apply("MultiPartitionsReshuffle", Reshuffle.<ShardedKey<DestinationT>, List<String>>of())
.apply(
- "SinglePartitionsReshuffle", Reshuffle.<ShardedKey<DestinationT>, List<String>>of())
+ "MultiPartitionsWriteTables",
+ new WriteTables<>(
+ false,
+ bigQueryServices,
+ jobIdTokenView,
+ WriteDisposition.WRITE_EMPTY,
+ CreateDisposition.CREATE_IF_NEEDED,
+ sideInputs,
+ dynamicDestinations));
+ }
+
+ // In the case where the files fit into a single load job, there's no need to write temporary
+ // tables and rename. We can load these files directly into the target BigQuery table.
+ void writeSinglePartition(
+ PCollection<KV<ShardedKey<DestinationT>, List<String>>> input,
+ PCollectionView<String> jobIdTokenView) {
+ List<PCollectionView<?>> sideInputs = Lists.<PCollectionView<?>>newArrayList(jobIdTokenView);
+ sideInputs.addAll(dynamicDestinations.getSideInputs());
+ Coder<KV<ShardedKey<DestinationT>, List<String>>> partitionsCoder =
+ KvCoder.of(
+ ShardedKeyCoder.of(NullableCoder.of(destinationCoder)),
+ ListCoder.of(StringUtf8Coder.of()));
+ // Write single partition to final table
+ input
+ .setCoder(partitionsCoder)
+ // Reshuffle will distribute this among multiple workers, and also guard against
+ // reexecution of the WritePartitions step once WriteTables has begun.
+ .apply("SinglePartitionsReshuffle", Reshuffle.<ShardedKey<DestinationT>, List<String>>of())
.apply(
"SinglePartitionWriteTables",
- ParDo.of(
- new WriteTables<>(
+ new WriteTables<>(
true,
bigQueryServices,
jobIdTokenView,
- schemasView,
writeDisposition,
createDisposition,
- dynamicDestinations))
- .withSideInputs(writeTablesSideInputs));
+ sideInputs,
+ dynamicDestinations));
+ }
- return WriteResult.in(input.getPipeline());
+ private WriteResult writeResult(Pipeline p) {
+ PCollection<TableRow> empty =
+ p.apply("CreateEmptyFailedInserts", Create.empty(TypeDescriptor.of(TableRow.class)));
+ return WriteResult.in(p, new TupleTag<TableRow>("failedInserts"), empty);
+ }
+
+ @Override
+ public WriteResult expand(PCollection<KV<DestinationT, TableRow>> input) {
+ return (triggeringFrequency != null) ? expandTriggered(input) : expandUntriggered(input);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 09508e0..02a47c2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -24,6 +24,7 @@
import com.google.api.services.bigquery.model.JobStatus;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.api.services.bigquery.model.TimePartitioning;
import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.hash.Hashing;
@@ -111,6 +112,14 @@
return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE"));
}
+ /**
+ * Strip off any partition decorator information from a tablespec.
+ */
+ public static String stripPartitionDecorator(String tableSpec) {
+ int index = tableSpec.lastIndexOf('$');
+ return (index == -1) ? tableSpec : tableSpec.substring(0, index);
+ }
+
static String jobToPrettyString(@Nullable Job job) throws IOException {
return job == null ? "null" : job.toPrettyString();
}
@@ -225,15 +234,19 @@
}
// Create a unique job id for a table load.
- static String createJobId(String prefix, TableDestination tableDestination, int partition) {
+ static String createJobId(String prefix, TableDestination tableDestination, int partition,
+ long index) {
// Job ID must be different for each partition of each table.
String destinationHash =
Hashing.murmur3_128().hashUnencodedChars(tableDestination.toString()).toString();
+ String jobId = String.format("%s_%s", prefix, destinationHash);
if (partition >= 0) {
- return String.format("%s_%s_%05d", prefix, destinationHash, partition);
- } else {
- return String.format("%s_%s", prefix, destinationHash);
+ jobId += String.format("_%05d", partition);
}
+ if (index >= 0) {
+ jobId += String.format("_%05d", index);
+ }
+ return jobId;
}
@VisibleForTesting
@@ -287,6 +300,13 @@
}
}
+ static class TimePartitioningToJson implements SerializableFunction<TimePartitioning, String> {
+ @Override
+ public String apply(TimePartitioning partitioning) {
+ return toJsonString(partitioning);
+ }
+ }
+
static String createJobIdToken(String jobName, String stepUuid) {
return String.format("beam_job_%s_%s", stepUuid, jobName.replaceAll("-", ""));
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index cf258ca..3dfd8b8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -31,8 +31,10 @@
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.api.services.bigquery.model.TimePartitioning;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Predicates;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
@@ -43,11 +45,15 @@
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.FileSystems;
@@ -58,17 +64,26 @@
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TimePartitioningToJson;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinationsHelpers.ConstantSchemaDestinations;
+import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinationsHelpers.ConstantTimePartitioningDestinations;
import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinationsHelpers.SchemaFromViewDestinations;
import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinationsHelpers.TableFunctionDestinations;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.Transport;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
@@ -76,21 +91,26 @@
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * {@link PTransform}s for reading and writing
- * <a href="https://developers.google.com/bigquery/">BigQuery</a> tables.
+ * {@link PTransform}s for reading and writing <a
+ * href="https://developers.google.com/bigquery/">BigQuery</a> tables.
*
* <h3>Table References</h3>
*
* <p>A fully-qualified BigQuery table name consists of three components:
+ *
* <ul>
- * <li>{@code projectId}: the Cloud project id (defaults to
- * {@link GcpOptions#getProject()}).
+ * <li>{@code projectId}: the Cloud project id (defaults to {@link GcpOptions#getProject()}).
* <li>{@code datasetId}: the BigQuery dataset id, unique within a project.
* <li>{@code tableId}: a table id, unique within a dataset.
* </ul>
@@ -108,36 +128,51 @@
*
* <h3>Reading</h3>
*
- * <p>To read from a BigQuery table, apply a {@link BigQueryIO.Read} transformation.
- * This produces a {@link PCollection} of {@link TableRow TableRows} as output:
+ * <p>Reading from BigQuery is supported by {@link #read(SerializableFunction)}, which parses
+ * records in <a href="https://cloud.google.com/bigquery/data-formats#avro_format">AVRO format</a>
+ * into a custom type using a specified parse function, and by {@link #readTableRows} which parses
+ * them into {@link TableRow}, which may be more convenient but has lower performance.
*
+ * <p>Both functions support reading either from a table or from the result of a query, via
+ * {@link TypedRead#from(String)} and {@link TypedRead#fromQuery} respectively. Exactly one
+ * of these must be specified.
+ *
+ * <b>Example: Reading rows of a table as {@link TableRow}.</b>
* <pre>{@code
* PCollection<TableRow> weatherData = pipeline.apply(
- * BigQueryIO.read().from("clouddataflow-readonly:samples.weather_stations"));
+ * BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
* }</pre>
*
- * <p>See {@link TableRow} for more information on the {@link TableRow} object.
- *
- * <p>Users may provide a query to read from rather than reading all of a BigQuery table. If
- * specified, the result obtained by executing the specified query will be used as the data of the
- * input transform.
- *
+ * <b>Example: Reading rows of a table and parsing them into a custom type.</b>
* <pre>{@code
- * PCollection<TableRow> meanTemperatureData = pipeline.apply(
- * BigQueryIO.read().fromQuery("SELECT year, mean_temp FROM [samples.weather_stations]"));
+ * PCollection<WeatherRecord> weatherData = pipeline.apply(
+ * BigQueryIO
+ * .read(new SerializableFunction<SchemaAndRecord, WeatherRecord>() {
+ * public WeatherRecord apply(SchemaAndRecord schemaAndRecord) {
+ * return new WeatherRecord(...);
+ * }
+ * })
+ * .from("clouddataflow-readonly:samples.weather_stations"))
+ * .withCoder(SerializableCoder.of(WeatherRecord.class));
* }</pre>
*
- * <p>When creating a BigQuery input transform, users should provide either a query or a table.
- * Pipeline construction will fail with a validation error if neither or both are specified.
+ * <p>Note: When using {@link #read(SerializableFunction)}, you may sometimes need to use
+ * {@link TypedRead#withCoder(Coder)} to specify a {@link Coder} for the result type, if Beam
+ * fails to infer it automatically.
+ *
+ * <b>Example: Reading results of a query as {@link TableRow}.</b>
+ * <pre>{@code
+ * PCollection<TableRow> meanTemperatureData = pipeline.apply(BigQueryIO.readTableRows()
+ * .fromQuery("SELECT year, mean_temp FROM [samples.weather_stations]"));
+ * }</pre>
*
* <h3>Writing</h3>
*
- * <p>To write to a BigQuery table, apply a {@link BigQueryIO.Write} transformation.
- * This consumes either a {@link PCollection} of {@link TableRow TableRows} as input when using
- * {@link BigQueryIO#writeTableRows()} or of a user-defined type when using
- * {@link BigQueryIO#write()}. When using a user-defined type, a function must be provided to
- * turn this type into a {@link TableRow} using
- * {@link BigQueryIO.Write#withFormatFunction(SerializableFunction)}.
+ * <p>To write to a BigQuery table, apply a {@link BigQueryIO.Write} transformation. This consumes
+ * either a {@link PCollection} of {@link TableRow TableRows} as input when using {@link
+ * BigQueryIO#writeTableRows()} or of a user-defined type when using {@link BigQueryIO#write()}.
+ * When using a user-defined type, a function must be provided to turn this type into a {@link
+ * TableRow} using {@link BigQueryIO.Write#withFormatFunction(SerializableFunction)}.
*
* <pre>{@code
* PCollection<TableRow> quotes = ...
@@ -170,13 +205,16 @@
* quotes.apply(Window.<TableRow>into(CalendarWindows.days(1)))
* .apply(BigQueryIO.writeTableRows()
* .withSchema(schema)
- * .to(new SerializableFunction<ValueInSingleWindow, String>() {
- * public String apply(ValueInSingleWindow value) {
+ * .to(new SerializableFunction<ValueInSingleWindow<TableRow>, TableDestination>() {
+ * public TableDestination apply(ValueInSingleWindow<TableRow> value) {
* // The cast below is safe because CalendarWindows.days(1) produces IntervalWindows.
* String dayString = DateTimeFormat.forPattern("yyyy_MM_dd")
* .withZone(DateTimeZone.UTC)
* .print(((IntervalWindow) value.getWindow()).start());
- * return "my-project:output.output_table_" + dayString;
+ * return new TableDestination(
+ * "my-project:output.output_table_" + dayString, // Table spec
+ * "Output for day " + dayString // Table description
+ * );
* }
* }));
* }</pre>
@@ -199,8 +237,32 @@
* can also be useful when writing to a single table, as it allows a previous stage to calculate the
* schema (possibly based on the full collection of records being written to BigQuery).
*
- * <p>For the most general form of dynamic table destinations and schemas, look at
- * {@link BigQueryIO.Write#to(DynamicDestinations)}.
+ * <p>For the most general form of dynamic table destinations and schemas, look at {@link
+ * BigQueryIO.Write#to(DynamicDestinations)}.
+ *
+ * <h3>Insertion Method</h3>
+ *
+ * {@link BigQueryIO.Write} supports two methods of inserting data into BigQuery specified using
+ * {@link BigQueryIO.Write#withMethod}. If no method is supplied, then a default method will be
+ * chosen based on the input PCollection. See {@link BigQueryIO.Write.Method} for more information
+ * about the methods. The different insertion methods provide different tradeoffs of cost, quota,
+ * and data consistency; please see BigQuery documentation for more information about these
+ * tradeoffs.
+ *
+ * <h3>Usage with templates</h3>
+ *
+ * <p>When using {@link #read} or {@link #readTableRows()} in a template, it's required to specify
+ * {@link Read#withTemplateCompatibility()}. Specifying this in a non-template pipeline is not
+ * recommended because it has somewhat lower performance.
+ *
+ * <p>When using {@link #write()} or {@link #writeTableRows()} with batch loads in a template, it is
+ * recommended to specify {@link Write#withCustomGcsTempLocation}. Writing to BigQuery via batch
+ * loads involves writing temporary files to this location, so the location must be accessible at
+ * pipeline execution time. By default, this location is captured at pipeline <i>construction</i>
+ * time, may be inaccessible if the template may be reused from a different project or at a moment
+ * when the original location no longer exists.
+ * {@link Write#withCustomGcsTempLocation(ValueProvider)} allows specifying the location as an
+ * argument to the template invocation.
*
* <h3>Permissions</h3>
*
@@ -260,60 +322,116 @@
};
/**
- * A {@link PTransform} that reads from a BigQuery table and returns a
- * {@link PCollection} of {@link TableRow TableRows} containing each of the rows of the table.
+ * @deprecated Use {@link #read(SerializableFunction)} or {@link #readTableRows} instead.
+ * {@link #readTableRows()} does exactly the same as {@link #read}, however
+ * {@link #read(SerializableFunction)} performs better.
+ */
+ @Deprecated
+ public static Read read() {
+ return new Read();
+ }
+
+ /**
+ * Like {@link #read(SerializableFunction)} but represents each row as a {@link TableRow}.
*
- * <p>Each {@link TableRow} contains values indexed by column name. Here is a
- * sample processing function that processes a "line" column from rows:
+ * <p>This method is more convenient to use in some cases, but usually has significantly lower
+ * performance than using {@link #read(SerializableFunction)} directly to parse data into a
+ * domain-specific type, due to the overhead of converting the rows to {@link TableRow}.
+ */
+ public static TypedRead<TableRow> readTableRows() {
+ return read(new TableRowParser()).withCoder(TableRowJsonCoder.of());
+ }
+
+ /**
+ * Reads from a BigQuery table or query and returns a {@link PCollection} with one element per
+ * each row of the table or query result, parsed from the BigQuery AVRO format using the specified
+ * function.
+ *
+ * <p>Each {@link SchemaAndRecord} contains a BigQuery {@link TableSchema} and a
+ * {@link GenericRecord} representing the row, indexed by column name. Here is a
+ * sample parse function that parses click events from a table.
*
* <pre>{@code
- * static class ExtractWordsFn extends DoFn<TableRow, String> {
- * public void processElement(ProcessContext c) {
- * // Get the "line" field of the TableRow object, split it into words, and emit them.
- * TableRow row = c.element();
- * String[] words = row.get("line").toString().split("[^a-zA-Z']+");
- * for (String word : words) {
- * if (!word.isEmpty()) {
- * c.output(word);
- * }
- * }
+ * p.apply(BigQueryIO.read(new SerializableFunction<SchemaAndRecord, ClickEvent>() {
+ * public Event apply(SchemaAndRecord record) {
+ * GenericRecord r = record.getRecord();
+ * return new Event((Long) r.get("userId"), (String) r.get("url"));
* }
- * }
+ * }).from("...");
* }</pre>
*/
- public static Read read() {
- return new AutoValue_BigQueryIO_Read.Builder()
+ public static <T> TypedRead<T> read(
+ SerializableFunction<SchemaAndRecord, T> parseFn) {
+ return new AutoValue_BigQueryIO_TypedRead.Builder<T>()
.setValidate(true)
+ .setWithTemplateCompatibility(false)
.setBigQueryServices(new BigQueryServicesImpl())
+ .setParseFn(parseFn)
.build();
}
- /** Implementation of {@link #read}. */
- @AutoValue
- public abstract static class Read extends PTransform<PBegin, PCollection<TableRow>> {
- @Nullable abstract ValueProvider<String> getJsonTableRef();
- @Nullable abstract ValueProvider<String> getQuery();
- abstract boolean getValidate();
- @Nullable abstract Boolean getFlattenResults();
- @Nullable abstract Boolean getUseLegacySql();
- abstract BigQueryServices getBigQueryServices();
- abstract Builder toBuilder();
+ @VisibleForTesting
+ static class TableRowParser
+ implements SerializableFunction<SchemaAndRecord, TableRow> {
- @AutoValue.Builder
- abstract static class Builder {
- abstract Builder setJsonTableRef(ValueProvider<String> jsonTableRef);
- abstract Builder setQuery(ValueProvider<String> query);
- abstract Builder setValidate(boolean validate);
- abstract Builder setFlattenResults(Boolean flattenResults);
- abstract Builder setUseLegacySql(Boolean useLegacySql);
- abstract Builder setBigQueryServices(BigQueryServices bigQueryServices);
- abstract Read build();
+ public static final TableRowParser INSTANCE = new TableRowParser();
+
+ public TableRow apply(SchemaAndRecord schemaAndRecord) {
+ return BigQueryAvroUtils.convertGenericRecordToTableRow(
+ schemaAndRecord.getRecord(),
+ schemaAndRecord.getTableSchema());
+ }
+ }
+
+ /** Implementation of {@link BigQueryIO#read()}. */
+ public static class Read
+ extends PTransform<PBegin, PCollection<TableRow>> {
+ private final TypedRead<TableRow> inner;
+
+ Read() {
+ this(BigQueryIO.read(TableRowParser.INSTANCE).withCoder(TableRowJsonCoder.of()));
}
- /** Ensures that methods of the from() / fromQuery() family are called at most once. */
- private void ensureFromNotCalledYet() {
- checkState(
- getJsonTableRef() == null && getQuery() == null, "from() or fromQuery() already called");
+ Read(TypedRead<TableRow> inner) {
+ this.inner = inner;
+ }
+
+ @Override
+ public PCollection<TableRow> expand(PBegin input) {
+ return input.apply(inner);
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ this.inner.populateDisplayData(builder);
+ }
+
+ boolean getValidate() {
+ return this.inner.getValidate();
+ }
+
+ ValueProvider<String> getQuery() {
+ return this.inner.getQuery();
+ }
+
+ Read withTestServices(BigQueryServices testServices) {
+ return new Read(this.inner.withTestServices(testServices));
+ }
+
+ ///////////////////////////////////////////////////////////////////
+
+ /**
+ * Returns the table to read, or {@code null} if reading from a query instead.
+ */
+ @Nullable
+ public ValueProvider<TableReference> getTableProvider() {
+ return this.inner.getTableProvider();
+ }
+
+ /** Returns the table to read, or {@code null} if reading from a query instead. */
+ @Nullable
+ public TableReference getTable() {
+ return this.inner.getTable();
}
/**
@@ -321,18 +439,19 @@
* {@code "[dataset_id].[table_id]"} for tables within the current project.
*/
public Read from(String tableSpec) {
- return from(StaticValueProvider.of(tableSpec));
+ return new Read(this.inner.from(tableSpec));
}
/** Same as {@code from(String)}, but with a {@link ValueProvider}. */
public Read from(ValueProvider<String> tableSpec) {
- ensureFromNotCalledYet();
- return toBuilder()
- .setJsonTableRef(
- NestedValueProvider.of(
- NestedValueProvider.of(tableSpec, new TableSpecToTableRef()),
- new TableRefToJson()))
- .build();
+ return new Read(this.inner.from(tableSpec));
+ }
+
+ /**
+ * Read from table specified by a {@link TableReference}.
+ */
+ public Read from(TableReference table) {
+ return new Read(this.inner.from(table));
}
/**
@@ -340,40 +459,28 @@
*
* <p>By default, the query results will be flattened -- see "flattenResults" in the <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/jobs">Jobs documentation</a> for
- * more information. To disable flattening, use {@link BigQueryIO.Read#withoutResultFlattening}.
+ * more information. To disable flattening, use {@link Read#withoutResultFlattening}.
*
* <p>By default, the query will use BigQuery's legacy SQL dialect. To use the BigQuery Standard
- * SQL dialect, use {@link BigQueryIO.Read#usingStandardSql}.
+ * SQL dialect, use {@link Read#usingStandardSql}.
*/
public Read fromQuery(String query) {
- return fromQuery(StaticValueProvider.of(query));
+ return new Read(this.inner.fromQuery(query));
}
/**
* Same as {@code fromQuery(String)}, but with a {@link ValueProvider}.
*/
public Read fromQuery(ValueProvider<String> query) {
- ensureFromNotCalledYet();
- return toBuilder().setQuery(query).setFlattenResults(true).setUseLegacySql(true).build();
+ return new Read(this.inner.fromQuery(query));
}
/**
- * Read from table specified by a {@link TableReference}.
- */
- public Read from(TableReference table) {
- return from(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
- }
-
- private static final String QUERY_VALIDATION_FAILURE_ERROR =
- "Validation of query \"%1$s\" failed. If the query depends on an earlier stage of the"
- + " pipeline, This validation can be disabled using #withoutValidation.";
-
- /**
* Disable validation that the table exists or the query succeeds prior to pipeline submission.
* Basic validation (such as ensuring that a query or table is specified) still occurs.
*/
public Read withoutValidation() {
- return toBuilder().setValidate(false).build();
+ return new Read(this.inner.withoutValidation());
}
/**
@@ -384,7 +491,7 @@
* from a table will cause an error during validation.
*/
public Read withoutResultFlattening() {
- return toBuilder().setFlattenResults(false).build();
+ return new Read(this.inner.withoutResultFlattening());
}
/**
@@ -394,13 +501,101 @@
* from a table will cause an error during validation.
*/
public Read usingStandardSql() {
- return toBuilder().setUseLegacySql(false).build();
+ return new Read(this.inner.usingStandardSql());
}
- @VisibleForTesting
- Read withTestServices(BigQueryServices testServices) {
- return toBuilder().setBigQueryServices(testServices).build();
+ /**
+ * Use new template-compatible source implementation.
+ *
+ * <p>Use new template-compatible source implementation. This implementation is compatible with
+ * repeated template invocations. It does not support dynamic work rebalancing.
+ */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ public Read withTemplateCompatibility() {
+ return new Read(this.inner.withTemplateCompatibility());
}
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Implementation of {@link BigQueryIO#read(SerializableFunction)}.
+ */
+ @AutoValue
+ public abstract static class TypedRead<T> extends PTransform<PBegin, PCollection<T>> {
+ abstract Builder<T> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setJsonTableRef(ValueProvider<String> jsonTableRef);
+ abstract Builder<T> setQuery(ValueProvider<String> query);
+ abstract Builder<T> setValidate(boolean validate);
+ abstract Builder<T> setFlattenResults(Boolean flattenResults);
+ abstract Builder<T> setUseLegacySql(Boolean useLegacySql);
+ abstract Builder<T> setWithTemplateCompatibility(Boolean useTemplateCompatibility);
+ abstract Builder<T> setBigQueryServices(BigQueryServices bigQueryServices);
+ abstract TypedRead<T> build();
+
+ abstract Builder<T> setParseFn(
+ SerializableFunction<SchemaAndRecord, T> parseFn);
+ abstract Builder<T> setCoder(Coder<T> coder);
+ }
+
+ @Nullable abstract ValueProvider<String> getJsonTableRef();
+ @Nullable abstract ValueProvider<String> getQuery();
+ abstract boolean getValidate();
+ @Nullable abstract Boolean getFlattenResults();
+ @Nullable abstract Boolean getUseLegacySql();
+
+ abstract Boolean getWithTemplateCompatibility();
+
+ abstract BigQueryServices getBigQueryServices();
+
+ abstract SerializableFunction<SchemaAndRecord, T> getParseFn();
+
+ @Nullable abstract Coder<T> getCoder();
+
+ @VisibleForTesting
+ Coder<T> inferCoder(CoderRegistry coderRegistry) {
+ if (getCoder() != null) {
+ return getCoder();
+ }
+
+ try {
+ return coderRegistry.getCoder(TypeDescriptors.outputOf(getParseFn()));
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalArgumentException(
+ "Unable to infer coder for output of parseFn. Specify it explicitly using withCoder().",
+ e);
+ }
+ }
+
+ private BigQuerySourceBase<T> createSource(String jobUuid, Coder<T> coder) {
+ BigQuerySourceBase<T> source;
+ if (getQuery() == null) {
+ source = BigQueryTableSource.create(
+ jobUuid,
+ getTableProvider(),
+ getBigQueryServices(),
+ coder,
+ getParseFn());
+ } else {
+ source =
+ BigQueryQuerySource.create(
+ jobUuid,
+ getQuery(),
+ getFlattenResults(),
+ getUseLegacySql(),
+ getBigQueryServices(),
+ coder,
+ getParseFn());
+ }
+ return source;
+ }
+
+ private static final String QUERY_VALIDATION_FAILURE_ERROR =
+ "Validation of query \"%1$s\" failed. If the query depends on an earlier stage of the"
+ + " pipeline, This validation can be disabled using #withoutValidation.";
@Override
public void validate(PipelineOptions options) {
@@ -426,19 +621,48 @@
ValueProvider<TableReference> table = getTableProvider();
- checkState(
- table == null || getQuery() == null,
- "Invalid BigQueryIO.Read: table reference and query may not both be set");
- checkState(
- table != null || getQuery() != null,
- "Invalid BigQueryIO.Read: one of table reference and query must be set");
+ // Note that a table or query check can fail if the table or dataset are created by
+ // earlier stages of the pipeline or if a query depends on earlier stages of a pipeline.
+ // For these cases the withoutValidation method can be used to disable the check.
+ if (getValidate()) {
+ if (table != null) {
+ checkArgument(table.isAccessible(), "Cannot call validate if table is dynamically set.");
+ }
+ if (table != null && table.get().getProjectId() != null) {
+ // Check for source table presence for early failure notification.
+ DatasetService datasetService = getBigQueryServices().getDatasetService(bqOptions);
+ BigQueryHelpers.verifyDatasetPresence(datasetService, table.get());
+ BigQueryHelpers.verifyTablePresence(datasetService, table.get());
+ } else if (getQuery() != null) {
+ checkArgument(
+ getQuery().isAccessible(), "Cannot call validate if query is dynamically set.");
+ JobService jobService = getBigQueryServices().getJobService(bqOptions);
+ try {
+ jobService.dryRunQuery(
+ bqOptions.getProject(),
+ new JobConfigurationQuery()
+ .setQuery(getQuery().get())
+ .setFlattenResults(getFlattenResults())
+ .setUseLegacySql(getUseLegacySql()));
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ String.format(QUERY_VALIDATION_FAILURE_ERROR, getQuery().get()), e);
+ }
+ }
+ }
+ }
+
+ @Override
+ public PCollection<T> expand(PBegin input) {
+ ValueProvider<TableReference> table = getTableProvider();
if (table != null) {
- checkState(
+ checkArgument(getQuery() == null, "from() and fromQuery() are exclusive");
+ checkArgument(
getFlattenResults() == null,
"Invalid BigQueryIO.Read: Specifies a table with a result flattening"
+ " preference, which only applies to queries");
- checkState(
+ checkArgument(
getUseLegacySql() == null,
"Invalid BigQueryIO.Read: Specifies a table with a SQL dialect"
+ " preference, which only applies to queries");
@@ -448,76 +672,113 @@
TableReference.class.getSimpleName(),
BigQueryOptions.class.getSimpleName());
}
- } else /* query != null */ {
- checkState(
- getFlattenResults() != null, "flattenResults should not be null if query is set");
- checkState(getUseLegacySql() != null, "useLegacySql should not be null if query is set");
- }
-
- // Note that a table or query check can fail if the table or dataset are created by
- // earlier stages of the pipeline or if a query depends on earlier stages of a pipeline.
- // For these cases the withoutValidation method can be used to disable the check.
- if (getValidate() && table != null && table.isAccessible()
- && table.get().getProjectId() != null) {
- checkState(table.isAccessible(), "Cannot call validate if table is dynamically set.");
- // Check for source table presence for early failure notification.
- DatasetService datasetService = getBigQueryServices().getDatasetService(bqOptions);
- BigQueryHelpers.verifyDatasetPresence(datasetService, table.get());
- BigQueryHelpers.verifyTablePresence(datasetService, table.get());
- } else if (getValidate() && getQuery() != null) {
- checkState(getQuery().isAccessible(), "Cannot call validate if query is dynamically set.");
- JobService jobService = getBigQueryServices().getJobService(bqOptions);
- try {
- jobService.dryRunQuery(
- bqOptions.getProject(),
- new JobConfigurationQuery()
- .setQuery(getQuery().get())
- .setFlattenResults(getFlattenResults())
- .setUseLegacySql(getUseLegacySql()));
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format(QUERY_VALIDATION_FAILURE_ERROR, getQuery().get()), e);
- }
- }
- }
-
- @Override
- public PCollection<TableRow> expand(PBegin input) {
- final String stepUuid = BigQueryHelpers.randomUUIDString();
- BoundedSource<TableRow> source;
-
- if (getQuery() != null
- && (!getQuery().isAccessible() || !Strings.isNullOrEmpty(getQuery().get()))) {
- source =
- BigQueryQuerySource.create(
- stepUuid,
- getQuery(),
- getFlattenResults(),
- getUseLegacySql(),
- getBigQueryServices());
} else {
- source =
- BigQueryTableSource.create(
- stepUuid,
- getTableProvider(),
- getBigQueryServices());
+ checkArgument(getQuery() != null, "Either from() or fromQuery() is required");
+ checkArgument(
+ getFlattenResults() != null, "flattenResults should not be null if query is set");
+ checkArgument(getUseLegacySql() != null, "useLegacySql should not be null if query is set");
+ }
+ checkArgument(getParseFn() != null, "A parseFn is required");
+
+ Pipeline p = input.getPipeline();
+ final Coder<T> coder = inferCoder(p.getCoderRegistry());
+ final PCollectionView<String> jobIdTokenView;
+ PCollection<String> jobIdTokenCollection;
+ PCollection<T> rows;
+ if (!getWithTemplateCompatibility()) {
+ // Create a singleton job ID token at construction time.
+ final String staticJobUuid = BigQueryHelpers.randomUUIDString();
+ jobIdTokenView =
+ p.apply("TriggerIdCreation", Create.of(staticJobUuid))
+ .apply("ViewId", View.<String>asSingleton());
+ // Apply the traditional Source model.
+ rows = p.apply(org.apache.beam.sdk.io.Read.from(createSource(staticJobUuid, coder)));
+ } else {
+ // Create a singleton job ID token at execution time.
+ jobIdTokenCollection =
+ p.apply("TriggerIdCreation", Create.of("ignored"))
+ .apply(
+ "CreateJobId",
+ MapElements.via(
+ new SimpleFunction<String, String>() {
+ @Override
+ public String apply(String input) {
+ return BigQueryHelpers.randomUUIDString();
+ }
+ }));
+ jobIdTokenView = jobIdTokenCollection.apply("ViewId", View.<String>asSingleton());
+
+ final TupleTag<String> filesTag = new TupleTag<>();
+ final TupleTag<String> tableSchemaTag = new TupleTag<>();
+ PCollectionTuple tuple =
+ jobIdTokenCollection.apply(
+ "RunCreateJob",
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ String jobUuid = c.element();
+ BigQuerySourceBase<T> source = createSource(jobUuid, coder);
+ BigQuerySourceBase.ExtractResult res =
+ source.extractFiles(c.getPipelineOptions());
+ for (ResourceId file : res.extractedFiles) {
+ c.output(file.toString());
+ }
+ c.output(tableSchemaTag, BigQueryHelpers.toJsonString(res.schema));
+ }
+ })
+ .withOutputTags(filesTag, TupleTagList.of(tableSchemaTag)));
+ tuple.get(filesTag).setCoder(StringUtf8Coder.of());
+ tuple.get(tableSchemaTag).setCoder(StringUtf8Coder.of());
+ final PCollectionView<String> schemaView =
+ tuple.get(tableSchemaTag).apply(View.<String>asSingleton());
+ rows =
+ tuple
+ .get(filesTag)
+ .apply(Reshuffle.<String>viaRandomKey())
+ .apply(
+ "ReadFiles",
+ ParDo.of(
+ new DoFn<String, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ TableSchema schema =
+ BigQueryHelpers.fromJsonString(
+ c.sideInput(schemaView), TableSchema.class);
+ String jobUuid = c.sideInput(jobIdTokenView);
+ BigQuerySourceBase<T> source = createSource(jobUuid, coder);
+ List<BoundedSource<T>> sources =
+ source.createSources(
+ ImmutableList.of(
+ FileSystems.matchNewResource(
+ c.element(), false /* is directory */)),
+ schema);
+ checkArgument(sources.size() == 1, "Expected exactly one source.");
+ BoundedSource<T> avroSource = sources.get(0);
+ BoundedSource.BoundedReader<T> reader =
+ avroSource.createReader(c.getPipelineOptions());
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ c.output(reader.getCurrent());
+ }
+ }
+ })
+ .withSideInputs(schemaView, jobIdTokenView))
+ .setCoder(coder);
}
PassThroughThenCleanup.CleanupOperation cleanupOperation =
new PassThroughThenCleanup.CleanupOperation() {
@Override
- void cleanup(PipelineOptions options) throws Exception {
+ void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception {
+ PipelineOptions options = c.getPipelineOptions();
BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ String jobUuid = c.getJobId();
final String extractDestinationDir =
- resolveTempLocation(
- bqOptions.getTempLocation(),
- "BigQueryExtractTemp",
- stepUuid);
-
+ resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", jobUuid);
+ final String executingProject = bqOptions.getProject();
JobReference jobRef =
new JobReference()
- .setProjectId(bqOptions.getProject())
- .setJobId(
- getExtractJobId(createJobIdToken(bqOptions.getJobName(), stepUuid)));
+ .setProjectId(executingProject)
+ .setJobId(getExtractJobId(createJobIdToken(bqOptions.getJobName(), jobUuid)));
Job extractJob = getBigQueryServices().getJobService(bqOptions).getJob(jobRef);
@@ -525,21 +786,13 @@
List<ResourceId> extractFiles =
getExtractFilePaths(extractDestinationDir, extractJob);
if (extractFiles != null && !extractFiles.isEmpty()) {
- FileSystems.delete(extractFiles,
- MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
+ FileSystems.delete(
+ extractFiles, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
}
}
}
};
- return input.getPipeline()
- .apply(org.apache.beam.sdk.io.Read.from(source))
- .setCoder(getDefaultOutputCoder())
- .apply(new PassThroughThenCleanup<TableRow>(cleanupOperation));
- }
-
- @Override
- protected Coder<TableRow> getDefaultOutputCoder() {
- return TableRowJsonCoder.of();
+ return rows.apply(new PassThroughThenCleanup<T>(cleanupOperation, jobIdTokenView));
}
@Override
@@ -559,20 +812,91 @@
true);
}
- /**
- * Returns the table to read, or {@code null} if reading from a query instead.
- */
+ /** Ensures that methods of the from() / fromQuery() family are called at most once. */
+ private void ensureFromNotCalledYet() {
+ checkState(
+ getJsonTableRef() == null && getQuery() == null, "from() or fromQuery() already called");
+ }
+
+ /** See {@link Read#getTableProvider()}. */
@Nullable
public ValueProvider<TableReference> getTableProvider() {
return getJsonTableRef() == null
? null : NestedValueProvider.of(getJsonTableRef(), new JsonTableRefToTableRef());
}
- /** Returns the table to read, or {@code null} if reading from a query instead. */
+
+ /** See {@link Read#getTable()}. */
@Nullable
public TableReference getTable() {
ValueProvider<TableReference> provider = getTableProvider();
return provider == null ? null : provider.get();
}
+
+ /**
+ * Sets a {@link Coder} for the result of the parse function. This may be required if a coder
+ * can not be inferred automatically.
+ */
+ public TypedRead<T> withCoder(Coder<T> coder) {
+ return toBuilder().setCoder(coder).build();
+ }
+
+ /** See {@link Read#from(String)}. */
+ public TypedRead<T> from(String tableSpec) {
+ return from(StaticValueProvider.of(tableSpec));
+ }
+
+ /** See {@link Read#from(ValueProvider)}. */
+ public TypedRead<T> from(ValueProvider<String> tableSpec) {
+ ensureFromNotCalledYet();
+ return toBuilder()
+ .setJsonTableRef(
+ NestedValueProvider.of(
+ NestedValueProvider.of(tableSpec, new TableSpecToTableRef()),
+ new TableRefToJson()))
+ .build();
+ }
+
+ /** See {@link Read#fromQuery(String)}. */
+ public TypedRead<T> fromQuery(String query) {
+ return fromQuery(StaticValueProvider.of(query));
+ }
+
+ /** See {@link Read#fromQuery(ValueProvider)}. */
+ public TypedRead<T> fromQuery(ValueProvider<String> query) {
+ ensureFromNotCalledYet();
+ return toBuilder().setQuery(query).setFlattenResults(true).setUseLegacySql(true).build();
+ }
+
+ /** See {@link Read#from(TableReference)}. */
+ public TypedRead<T> from(TableReference table) {
+ return from(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
+ }
+
+ /** See {@link Read#withoutValidation()}. */
+ public TypedRead<T> withoutValidation() {
+ return toBuilder().setValidate(false).build();
+ }
+
+ /** See {@link Read#withoutResultFlattening()}. */
+ public TypedRead<T> withoutResultFlattening() {
+ return toBuilder().setFlattenResults(false).build();
+ }
+
+ /** See {@link Read#usingStandardSql()}. */
+ public TypedRead<T> usingStandardSql() {
+ return toBuilder().setUseLegacySql(false).build();
+ }
+
+ /** See {@link Read#withTemplateCompatibility()}. */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ public TypedRead<T> withTemplateCompatibility() {
+ return toBuilder().setWithTemplateCompatibility(true).build();
+ }
+
+ @VisibleForTesting
+ TypedRead<T> withTestServices(BigQueryServices testServices) {
+ return toBuilder().setBigQueryServices(testServices).build();
+ }
}
static String getExtractDestinationUri(String extractDestinationDir) {
@@ -644,6 +968,8 @@
.setBigQueryServices(new BigQueryServicesImpl())
.setCreateDisposition(Write.CreateDisposition.CREATE_IF_NEEDED)
.setWriteDisposition(Write.WriteDisposition.WRITE_EMPTY)
+ .setNumFileShards(0)
+ .setMethod(Write.Method.DEFAULT)
.build();
}
@@ -658,6 +984,41 @@
/** Implementation of {@link #write}. */
@AutoValue
public abstract static class Write<T> extends PTransform<PCollection<T>, WriteResult> {
+ /** Determines the method used to insert data in BigQuery. */
+ public enum Method {
+ /**
+ * The default behavior if no method is explicitly set. If the input is bounded, then file
+ * loads will be used. If the input is unbounded, then streaming inserts will be used.
+ */
+ DEFAULT,
+
+ /**
+ * Use BigQuery load jobs to insert data. Records will first be written to files, and these
+ * files will be loaded into BigQuery. This is the default method when the input is bounded.
+ * This method can be chosen for unbounded inputs as well, as long as a triggering frequency
+ * is also set using {@link #withTriggeringFrequency}. BigQuery has daily quotas on the number
+ * of load jobs allowed per day, so be careful not to set the triggering frequency too
+ * frequent. For more information, see <a
+ * href="https://cloud.google.com/bigquery/docs/loading-data-cloud-storage">Loading Data from
+ * Cloud Storage</a>.
+ */
+ FILE_LOADS,
+
+ /**
+ * Use the BigQuery streaming insert API to insert data. This provides the lowest-latency
+ * insert path into BigQuery, and therefore is the default method when the input is unbounded.
+ * BigQuery will make a strong effort to ensure no duplicates when using this path, however
+ * there are some scenarios in which BigQuery is unable to make this guarantee (see
+ * https://cloud.google.com/bigquery/streaming-data-into-bigquery). A query can be run over
+ * the output table to periodically clean these rare duplicates. Alternatively, using the
+ * {@link #FILE_LOADS} insert method does guarantee no duplicates, though the latency for the
+ * insert into BigQuery will be much higher. For more information, see <a
+ * href="https://cloud.google.com/bigquery/streaming-data-into-bigquery">Streaming Data into
+ * BigQuery</a>.
+ */
+ STREAMING_INSERTS
+ }
+
@Nullable abstract ValueProvider<String> getJsonTableRef();
@Nullable abstract SerializableFunction<ValueInSingleWindow<T>, TableDestination>
getTableFunction();
@@ -665,6 +1026,7 @@
@Nullable abstract DynamicDestinations<T, ?> getDynamicDestinations();
@Nullable abstract PCollectionView<Map<String, String>> getSchemaFromView();
@Nullable abstract ValueProvider<String> getJsonSchema();
+ @Nullable abstract ValueProvider<String> getJsonTimePartitioning();
abstract CreateDisposition getCreateDisposition();
abstract WriteDisposition getWriteDisposition();
/** Table description. Default is empty. */
@@ -675,6 +1037,17 @@
@Nullable abstract Integer getMaxFilesPerBundle();
@Nullable abstract Long getMaxFileSize();
+ abstract int getNumFileShards();
+
+ @Nullable
+ abstract Duration getTriggeringFrequency();
+
+ abstract Method getMethod();
+
+ @Nullable abstract InsertRetryPolicy getFailedInsertRetryPolicy();
+
+ @Nullable abstract ValueProvider<String> getCustomGcsTempLocation();
+
abstract Builder<T> toBuilder();
@AutoValue.Builder
@@ -686,6 +1059,7 @@
abstract Builder<T> setDynamicDestinations(DynamicDestinations<T, ?> dynamicDestinations);
abstract Builder<T> setSchemaFromView(PCollectionView<Map<String, String>> view);
abstract Builder<T> setJsonSchema(ValueProvider<String> jsonSchema);
+ abstract Builder<T> setJsonTimePartitioning(ValueProvider<String> jsonTimePartitioning);
abstract Builder<T> setCreateDisposition(CreateDisposition createDisposition);
abstract Builder<T> setWriteDisposition(WriteDisposition writeDisposition);
abstract Builder<T> setTableDescription(String tableDescription);
@@ -694,6 +1068,16 @@
abstract Builder<T> setMaxFilesPerBundle(Integer maxFilesPerBundle);
abstract Builder<T> setMaxFileSize(Long maxFileSize);
+ abstract Builder<T> setNumFileShards(int numFileShards);
+
+ abstract Builder<T> setTriggeringFrequency(Duration triggeringFrequency);
+
+ abstract Builder<T> setMethod(Method method);
+
+ abstract Builder<T> setFailedInsertRetryPolicy(InsertRetryPolicy retryPolicy);
+
+ abstract Builder<T> setCustomGcsTempLocation(ValueProvider<String> customGcsTempLocation);
+
abstract Write<T> build();
}
@@ -846,6 +1230,33 @@
return toBuilder().setSchemaFromView(view).build();
}
+ /**
+ * Allows newly created tables to include a {@link TimePartitioning} class. Can only be used
+ * when writing to a single table. If {@link #to(SerializableFunction)} or
+ * {@link #to(DynamicDestinations)} is used to write dynamic tables, time partitioning can be
+ * directly in the returned {@link TableDestination}.
+ */
+ public Write<T> withTimePartitioning(TimePartitioning partitioning) {
+ return withJsonTimePartitioning(
+ StaticValueProvider.of(BigQueryHelpers.toJsonString(partitioning)));
+ }
+
+ /**
+ * Like {@link #withTimePartitioning(TimePartitioning)} but using a deferred
+ * {@link ValueProvider}.
+ */
+ public Write<T> withTimePartitioning(ValueProvider<TimePartitioning> partition) {
+ return withJsonTimePartitioning(NestedValueProvider.of(
+ partition, new TimePartitioningToJson()));
+ }
+
+ /**
+ * The same as {@link #withTimePartitioning}, but takes a JSON-serialized object.
+ */
+ public Write<T> withJsonTimePartitioning(ValueProvider<String> partition) {
+ return toBuilder().setJsonTimePartitioning(partition).build();
+ }
+
/** Specifies whether the table should be created if it does not exist. */
public Write<T> withCreateDisposition(CreateDisposition createDisposition) {
return toBuilder().setCreateDisposition(createDisposition).build();
@@ -861,11 +1272,65 @@
return toBuilder().setTableDescription(tableDescription).build();
}
+ /** Specfies a policy for handling failed inserts.
+ *
+ * <p>Currently this only is allowed when writing an unbounded collection to BigQuery. Bounded
+ * collections are written using batch load jobs, so we don't get per-element failures.
+ * Unbounded collections are written using streaming inserts, so we have access to per-element
+ * insert results.
+ */
+ public Write<T> withFailedInsertRetryPolicy(InsertRetryPolicy retryPolicy) {
+ return toBuilder().setFailedInsertRetryPolicy(retryPolicy).build();
+ }
+
/** Disables BigQuery table validation. */
public Write<T> withoutValidation() {
return toBuilder().setValidate(false).build();
}
+ /**
+ * Choose the method used to write data to BigQuery. See the Javadoc on {@link Method} for
+ * information and restrictions of the different methods.
+ */
+ public Write<T> withMethod(Method method) {
+ return toBuilder().setMethod(method).build();
+ }
+
+ /**
+ * Choose the frequency at which file writes are triggered.
+ *
+ * <p>This is only applicable when the write method is set to {@link Method#FILE_LOADS}, and
+ * only when writing a bounded {@link PCollection}.
+ *
+ * <p>Every triggeringFrequency duration, a BigQuery load job will be generated for all the data
+ * written since the last load job. BigQuery has limits on how many load jobs can be triggered
+ * per day, so be careful not to set this duration too low, or you may exceed daily quota. Often
+ * this is set to 5 or 10 minutes to ensure that the project stays well under the BigQuery
+ * quota. See <a href="https://cloud.google.com/bigquery/quota-policy">Quota Policy</a> for more
+ * information about BigQuery quotas.
+ */
+ public Write<T> withTriggeringFrequency(Duration triggeringFrequency) {
+ return toBuilder().setTriggeringFrequency(triggeringFrequency).build();
+ }
+
+ /**
+ * Control how many file shards are written when using BigQuery load jobs. Applicable only when
+ * also setting {@link #withTriggeringFrequency}. The default value is 1000.
+ */
+ @Experimental
+ public Write<T> withNumFileShards(int numFileShards) {
+ return toBuilder().setNumFileShards(numFileShards).build();
+ }
+
+ /**
+ * Provides a custom location on GCS for storing temporary files to be loaded via BigQuery
+ * batch load jobs. See "Usage with templates" in {@link BigQueryIO} documentation for
+ * discussion.
+ */
+ public Write<T> withCustomGcsTempLocation(ValueProvider<String> customGcsTempLocation) {
+ return toBuilder().setCustomGcsTempLocation(customGcsTempLocation).build();
+ }
+
@VisibleForTesting
Write<T> withTestServices(BigQueryServices testServices) {
return toBuilder().setBigQueryServices(testServices).build();
@@ -903,10 +1368,21 @@
}
}
+ private Method resolveMethod(PCollection<T> input) {
+ if (getMethod() != Method.DEFAULT) {
+ return getMethod();
+ }
+ // By default, when writing an Unbounded PCollection, we use StreamingInserts and
+ // BigQuery's streaming import API.
+ return (input.isBounded() == IsBounded.UNBOUNDED)
+ ? Method.STREAMING_INSERTS
+ : Method.FILE_LOADS;
+ }
+
@Override
public WriteResult expand(PCollection<T> input) {
// We must have a destination to write to!
- checkState(
+ checkArgument(
getTableFunction() != null || getJsonTableRef() != null
|| getDynamicDestinations() != null,
"must set the table reference of a BigQueryIO.Write transform");
@@ -922,6 +1398,7 @@
|| getSchemaFromView() != null,
"CreateDisposition is CREATE_IF_NEEDED, however no schema was provided.");
+
List<?> allToArgs = Lists.newArrayList(getJsonTableRef(), getTableFunction(),
getDynamicDestinations());
checkArgument(1
@@ -935,6 +1412,30 @@
"No more than one of jsonSchema, schemaFromView, or dynamicDestinations may "
+ "be set");
+ Method method = resolveMethod(input);
+ if (input.isBounded() == IsBounded.UNBOUNDED && method == Method.FILE_LOADS) {
+ checkArgument(
+ getTriggeringFrequency() != null,
+ "When writing an unbounded PCollection via FILE_LOADS, "
+ + "triggering frequency must be specified");
+ } else {
+ checkArgument(
+ getTriggeringFrequency() == null && getNumFileShards() == 0,
+ "Triggering frequency or number of file shards can be specified only when writing "
+ + "an unbounded PCollection via FILE_LOADS, but: the collection was %s "
+ + "and the method was %s",
+ input.isBounded(),
+ method);
+ }
+ if (getJsonTimePartitioning() != null) {
+ checkArgument(getDynamicDestinations() == null,
+ "The supplied DynamicDestinations object can directly set TimePartitioning."
+ + " There is no need to call BigQueryIO.Write.withTimePartitioning.");
+ checkArgument(getTableFunction() == null,
+ "The supplied getTableFunction object can directly set TimePartitioning."
+ + " There is no need to call BigQueryIO.Write.withTimePartitioning.");
+ }
+
DynamicDestinations<T, ?> dynamicDestinations = getDynamicDestinations();
if (dynamicDestinations == null) {
if (getJsonTableRef() != null) {
@@ -942,17 +1443,26 @@
DynamicDestinationsHelpers.ConstantTableDestinations.fromJsonTableRef(
getJsonTableRef(), getTableDescription());
} else if (getTableFunction() != null) {
- dynamicDestinations = new TableFunctionDestinations(getTableFunction());
+ dynamicDestinations = new TableFunctionDestinations<>(getTableFunction());
}
// Wrap with a DynamicDestinations class that will provide a schema. There might be no
// schema provided if the create disposition is CREATE_NEVER.
if (getJsonSchema() != null) {
dynamicDestinations =
- new ConstantSchemaDestinations(dynamicDestinations, getJsonSchema());
+ new ConstantSchemaDestinations<>(
+ (DynamicDestinations<T, TableDestination>) dynamicDestinations, getJsonSchema());
} else if (getSchemaFromView() != null) {
dynamicDestinations =
- new SchemaFromViewDestinations(dynamicDestinations, getSchemaFromView());
+ new SchemaFromViewDestinations<>(
+ (DynamicDestinations<T, TableDestination>) dynamicDestinations,
+ getSchemaFromView());
+ }
+
+ // Wrap with a DynamicDestinations class that will provide the proper TimePartitioning.
+ if (getJsonTimePartitioning() != null) {
+ dynamicDestinations = new ConstantTimePartitioningDestinations(
+ dynamicDestinations, getJsonTimePartitioning());
}
}
return expandTyped(input, dynamicDestinations);
@@ -973,24 +1483,32 @@
.apply("PrepareWrite", new PrepareWrite<>(dynamicDestinations, getFormatFunction()))
.setCoder(KvCoder.of(destinationCoder, TableRowJsonCoder.of()));
- // When writing an Unbounded PCollection, we use StreamingInserts and BigQuery's streaming
- // import API.
- if (input.isBounded() == IsBounded.UNBOUNDED) {
+ Method method = resolveMethod(input);
+
+ if (method == Method.STREAMING_INSERTS) {
checkArgument(
getWriteDisposition() != WriteDisposition.WRITE_TRUNCATE,
"WriteDisposition.WRITE_TRUNCATE is not supported for an unbounded"
+ " PCollection.");
+ InsertRetryPolicy retryPolicy = MoreObjects.firstNonNull(
+ getFailedInsertRetryPolicy(), InsertRetryPolicy.alwaysRetry());
+
StreamingInserts<DestinationT> streamingInserts =
- new StreamingInserts<>(getCreateDisposition(), dynamicDestinations);
- streamingInserts.setTestServices(getBigQueryServices());
+ new StreamingInserts<>(getCreateDisposition(), dynamicDestinations)
+ .withInsertRetryPolicy(retryPolicy)
+ .withTestServices((getBigQueryServices()));
return rowsWithDestination.apply(streamingInserts);
} else {
+ checkArgument(getFailedInsertRetryPolicy() == null,
+ "Record-insert retry policies are not supported when using BigQuery load jobs.");
+
BatchLoads<DestinationT> batchLoads = new BatchLoads<>(
getWriteDisposition(),
getCreateDisposition(),
getJsonTableRef() != null,
dynamicDestinations,
- destinationCoder);
+ destinationCoder,
+ getCustomGcsTempLocation());
batchLoads.setTestServices(getBigQueryServices());
if (getMaxFilesPerBundle() != null) {
batchLoads.setMaxNumWritersPerBundle(getMaxFilesPerBundle());
@@ -998,16 +1516,13 @@
if (getMaxFileSize() != null) {
batchLoads.setMaxFileSize(getMaxFileSize());
}
+ batchLoads.setTriggeringFrequency(getTriggeringFrequency());
+ batchLoads.setNumFileShards(getNumFileShards());
return rowsWithDestination.apply(batchLoads);
}
}
@Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
index aee88e5..a2f8dd9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -27,38 +27,46 @@
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.JobStatistics;
import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A {@link BigQuerySourceBase} for querying BigQuery tables.
*/
@VisibleForTesting
-class BigQueryQuerySource extends BigQuerySourceBase {
+class BigQueryQuerySource<T> extends BigQuerySourceBase<T> {
+ private static final Logger LOG = LoggerFactory.getLogger(BigQueryQuerySource.class);
- static BigQueryQuerySource create(
+ static <T>BigQueryQuerySource<T> create(
String stepUuid,
ValueProvider<String> query,
Boolean flattenResults,
Boolean useLegacySql,
- BigQueryServices bqServices) {
- return new BigQueryQuerySource(
+ BigQueryServices bqServices,
+ Coder<T> coder,
+ SerializableFunction<SchemaAndRecord, T> parseFn) {
+ return new BigQueryQuerySource<T>(
stepUuid,
query,
flattenResults,
useLegacySql,
- bqServices);
+ bqServices,
+ coder,
+ parseFn);
}
private final ValueProvider<String> query;
@@ -71,8 +79,10 @@
ValueProvider<String> query,
Boolean flattenResults,
Boolean useLegacySql,
- BigQueryServices bqServices) {
- super(stepUuid, bqServices);
+ BigQueryServices bqServices,
+ Coder<T> coder,
+ SerializableFunction<SchemaAndRecord, T> parseFn) {
+ super(stepUuid, bqServices, coder, parseFn);
this.query = checkNotNull(query, "query");
this.flattenResults = checkNotNull(flattenResults, "flattenResults");
this.useLegacySql = checkNotNull(useLegacySql, "useLegacySql");
@@ -86,13 +96,6 @@
}
@Override
- public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- return new BigQueryReader(this, bqServices.getReaderFromQuery(
- bqOptions, bqOptions.getProject(), createBasicQueryConfig()));
- }
-
- @Override
protected TableReference getTableToExtract(BigQueryOptions bqOptions)
throws IOException, InterruptedException {
// 1. Find the location of the query.
@@ -109,19 +112,31 @@
TableReference tableToExtract = createTempTableReference(
bqOptions.getProject(), createJobIdToken(bqOptions.getJobName(), stepUuid));
+ LOG.info("Creating temporary dataset {} for query results", tableToExtract.getDatasetId());
tableService.createDataset(
tableToExtract.getProjectId(),
tableToExtract.getDatasetId(),
location,
- "Dataset for BigQuery query job temporary table");
+ "Temporary tables for query results of job " + bqOptions.getJobName(),
+ // Set a TTL of 1 day on the temporary tables, which ought to be enough in all cases:
+ // the temporary tables are used only to immediately extract them into files.
+ // They are normally cleaned up, but in case of job failure the cleanup step may not run,
+ // and then they'll get deleted after the TTL.
+ 24 * 3600 * 1000L /* 1 day */);
// 3. Execute the query.
String queryJobId = createJobIdToken(bqOptions.getJobName(), stepUuid) + "-query";
+ LOG.info(
+ "Exporting query results into temporary table {} using job {}",
+ tableToExtract,
+ queryJobId);
executeQuery(
bqOptions.getProject(),
queryJobId,
tableToExtract,
bqServices.getJobService(bqOptions));
+ LOG.info("Query job {} completed", queryJobId);
+
return tableToExtract;
}
@@ -131,7 +146,9 @@
bqOptions.getProject(), createJobIdToken(bqOptions.getJobName(), stepUuid));
DatasetService tableService = bqServices.getDatasetService(bqOptions);
+ LOG.info("Deleting temporary table with query results {}", tableToRemove);
tableService.deleteTable(tableToRemove);
+ LOG.info("Deleting temporary dataset with query results {}", tableToRemove.getDatasetId());
tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId());
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
index 1ae10bc..dde005d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
@@ -31,8 +31,8 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.List;
-import java.util.NoSuchElementException;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
/** An interface for real, mock, or fake implementations of Cloud BigQuery services. */
interface BigQueryServices extends Serializable {
@@ -48,17 +48,6 @@
DatasetService getDatasetService(BigQueryOptions bqOptions);
/**
- * Returns a real, mock, or fake {@link BigQueryJsonReader} to read tables.
- */
- BigQueryJsonReader getReaderFromTable(BigQueryOptions bqOptions, TableReference tableRef);
-
- /**
- * Returns a real, mock, or fake {@link BigQueryJsonReader} to query tables.
- */
- BigQueryJsonReader getReaderFromQuery(
- BigQueryOptions bqOptions, String projectId, JobConfigurationQuery queryConfig);
-
- /**
* An interface for the Cloud BigQuery load service.
*/
interface JobService {
@@ -144,10 +133,15 @@
throws IOException, InterruptedException;
/**
- * Create a {@link Dataset} with the given {@code location} and {@code description}.
+ * Create a {@link Dataset} with the given {@code location}, {@code description} and default
+ * expiration time for tables in the dataset (if {@code null}, tables don't expire).
*/
void createDataset(
- String projectId, String datasetId, @Nullable String location, @Nullable String description)
+ String projectId,
+ String datasetId,
+ @Nullable String location,
+ @Nullable String description,
+ @Nullable Long defaultTableExpirationMs)
throws IOException, InterruptedException;
/**
@@ -161,9 +155,14 @@
/**
* Inserts {@link TableRow TableRows} with the specified insertIds if not null.
*
+ * <p>If any insert fail permanently according to the retry policy, those rows are added
+ * to failedInserts.
+ *
* <p>Returns the total bytes count of {@link TableRow TableRows}.
*/
- long insertAll(TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
+ long insertAll(TableReference ref, List<ValueInSingleWindow<TableRow>> rowList,
+ @Nullable List<String> insertIdList, InsertRetryPolicy retryPolicy,
+ List<ValueInSingleWindow<TableRow>> failedInserts)
throws IOException, InterruptedException;
/** Patch BigQuery {@link Table} description. */
@@ -171,36 +170,4 @@
throws IOException, InterruptedException;
}
- /**
- * An interface to read the Cloud BigQuery directly.
- */
- interface BigQueryJsonReader {
- /**
- * Initializes the reader and advances the reader to the first record.
- */
- boolean start() throws IOException;
-
- /**
- * Advances the reader to the next valid record.
- */
- boolean advance() throws IOException;
-
- /**
- * Returns the value of the data item that was read by the last {@link #start} or
- * {@link #advance} call. The returned value must be effectively immutable and remain valid
- * indefinitely.
- *
- * <p>Multiple calls to this method without an intervening call to {@link #advance} should
- * return the same result.
- *
- * @throws java.util.NoSuchElementException if {@link #start} was never called, or if
- * the last {@link #start} or {@link #advance} returned {@code false}.
- */
- TableRow getCurrent() throws NoSuchElementException;
-
- /**
- * Closes the reader. The reader cannot be used after this method is called.
- */
- void close() throws IOException;
- }
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 5d5a519..97663bb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -54,7 +54,6 @@
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import java.util.NoSuchElementException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -69,6 +68,7 @@
import org.apache.beam.sdk.util.FluentBackoff;
import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
import org.apache.beam.sdk.util.Transport;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
@@ -107,17 +107,6 @@
return new DatasetServiceImpl(options);
}
- @Override
- public BigQueryJsonReader getReaderFromTable(BigQueryOptions bqOptions, TableReference tableRef) {
- return BigQueryJsonReaderImpl.fromTable(bqOptions, tableRef);
- }
-
- @Override
- public BigQueryJsonReader getReaderFromQuery(
- BigQueryOptions bqOptions, String projectId, JobConfigurationQuery queryConfig) {
- return BigQueryJsonReaderImpl.fromQuery(bqOptions, projectId, queryConfig);
- }
-
private static BackOff createDefaultBackoff() {
return BackOffAdapter.toGcpBackOff(DEFAULT_BACKOFF_FACTORY.backoff());
}
@@ -585,10 +574,20 @@
*/
@Override
public void createDataset(
- String projectId, String datasetId, @Nullable String location, @Nullable String description)
+ String projectId,
+ String datasetId,
+ @Nullable String location,
+ @Nullable String description,
+ @Nullable Long defaultTableExpirationMs)
throws IOException, InterruptedException {
createDataset(
- projectId, datasetId, location, description, Sleeper.DEFAULT, createDefaultBackoff());
+ projectId,
+ datasetId,
+ location,
+ description,
+ defaultTableExpirationMs,
+ Sleeper.DEFAULT,
+ createDefaultBackoff());
}
private void createDataset(
@@ -596,6 +595,7 @@
String datasetId,
@Nullable String location,
@Nullable String description,
+ @Nullable Long defaultTableExpirationMs,
Sleeper sleeper,
BackOff backoff) throws IOException, InterruptedException {
DatasetReference datasetRef = new DatasetReference()
@@ -610,6 +610,9 @@
dataset.setFriendlyName(description);
dataset.setDescription(description);
}
+ if (defaultTableExpirationMs != null) {
+ dataset.setDefaultTableExpirationMs(defaultTableExpirationMs);
+ }
Exception lastException;
do {
@@ -656,8 +659,11 @@
}
@VisibleForTesting
- long insertAll(TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList,
- BackOff backoff, final Sleeper sleeper) throws IOException, InterruptedException {
+ long insertAll(TableReference ref, List<ValueInSingleWindow<TableRow>> rowList,
+ @Nullable List<String> insertIdList,
+ BackOff backoff, final Sleeper sleeper, InsertRetryPolicy retryPolicy,
+ List<ValueInSingleWindow<TableRow>> failedInserts)
+ throws IOException, InterruptedException {
checkNotNull(ref, "ref");
if (executor == null) {
this.executor = options.as(GcsOptions.class).getExecutorService();
@@ -671,10 +677,10 @@
List<TableDataInsertAllResponse.InsertErrors> allErrors = new ArrayList<>();
// These lists contain the rows to publish. Initially the contain the entire list.
// If there are failures, they will contain only the failed rows to be retried.
- List<TableRow> rowsToPublish = rowList;
+ List<ValueInSingleWindow<TableRow>> rowsToPublish = rowList;
List<String> idsToPublish = insertIdList;
while (true) {
- List<TableRow> retryRows = new ArrayList<>();
+ List<ValueInSingleWindow<TableRow>> retryRows = new ArrayList<>();
List<String> retryIds = (idsToPublish != null) ? new ArrayList<String>() : null;
int strideIndex = 0;
@@ -686,7 +692,7 @@
List<Integer> strideIndices = new ArrayList<>();
for (int i = 0; i < rowsToPublish.size(); ++i) {
- TableRow row = rowsToPublish.get(i);
+ TableRow row = rowsToPublish.get(i).getValue();
TableDataInsertAllRequest.Rows out = new TableDataInsertAllRequest.Rows();
if (idsToPublish != null) {
out.setInsertId(idsToPublish.get(i));
@@ -743,18 +749,23 @@
try {
for (int i = 0; i < futures.size(); i++) {
List<TableDataInsertAllResponse.InsertErrors> errors = futures.get(i).get();
- if (errors != null) {
- for (TableDataInsertAllResponse.InsertErrors error : errors) {
- allErrors.add(error);
- if (error.getIndex() == null) {
- throw new IOException("Insert failed: " + allErrors);
- }
+ if (errors == null) {
+ continue;
+ }
+ for (TableDataInsertAllResponse.InsertErrors error : errors) {
+ if (error.getIndex() == null) {
+ throw new IOException("Insert failed: " + error + ", other errors: " + allErrors);
+ }
- int errorIndex = error.getIndex().intValue() + strideIndices.get(i);
+ int errorIndex = error.getIndex().intValue() + strideIndices.get(i);
+ if (retryPolicy.shouldRetry(new InsertRetryPolicy.Context(error))) {
+ allErrors.add(error);
retryRows.add(rowsToPublish.get(errorIndex));
if (retryIds != null) {
retryIds.add(idsToPublish.get(errorIndex));
}
+ } else {
+ failedInserts.add(rowsToPublish.get(errorIndex));
}
}
}
@@ -793,13 +804,15 @@
@Override
public long insertAll(
- TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
+ TableReference ref, List<ValueInSingleWindow<TableRow>> rowList,
+ @Nullable List<String> insertIdList,
+ InsertRetryPolicy retryPolicy, List<ValueInSingleWindow<TableRow>> failedInserts)
throws IOException, InterruptedException {
return insertAll(
ref, rowList, insertIdList,
BackOffAdapter.toGcpBackOff(
INSERT_BACKOFF_FACTORY.backoff()),
- Sleeper.DEFAULT);
+ Sleeper.DEFAULT, retryPolicy, failedInserts);
}
@@ -825,58 +838,6 @@
}
}
- private static class BigQueryJsonReaderImpl implements BigQueryJsonReader {
- private BigQueryTableRowIterator iterator;
-
- private BigQueryJsonReaderImpl(BigQueryTableRowIterator iterator) {
- this.iterator = iterator;
- }
-
- private static BigQueryJsonReader fromQuery(
- BigQueryOptions bqOptions, String projectId, JobConfigurationQuery queryConfig) {
- return new BigQueryJsonReaderImpl(
- BigQueryTableRowIterator.fromQuery(
- queryConfig, projectId, newBigQueryClient(bqOptions).build()));
- }
-
- private static BigQueryJsonReader fromTable(
- BigQueryOptions bqOptions, TableReference tableRef) {
- return new BigQueryJsonReaderImpl(BigQueryTableRowIterator.fromTable(
- tableRef, newBigQueryClient(bqOptions).build()));
- }
-
- @Override
- public boolean start() throws IOException {
- try {
- iterator.open();
- return iterator.advance();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException("Interrupted during start() operation", e);
- }
- }
-
- @Override
- public boolean advance() throws IOException {
- try {
- return iterator.advance();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException("Interrupted during advance() operation", e);
- }
- }
-
- @Override
- public TableRow getCurrent() throws NoSuchElementException {
- return iterator.getCurrent();
- }
-
- @Override
- public void close() throws IOException {
- iterator.close();
- }
- }
-
static final SerializableFunction<IOException, Boolean> DONT_RETRY_NOT_FOUND =
new SerializableFunction<IOException, Boolean>() {
@Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index 945c7d4..ca900d6 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -27,13 +27,16 @@
import com.google.api.services.bigquery.model.JobConfigurationExtract;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.io.IOException;
+import java.io.Serializable;
import java.util.List;
-import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
import org.apache.avro.generic.GenericRecord;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.AvroSource;
@@ -59,7 +62,7 @@
* </ul>
* ...
*/
-abstract class BigQuerySourceBase extends BoundedSource<TableRow> {
+abstract class BigQuerySourceBase<T> extends BoundedSource<T> {
private static final Logger LOG = LoggerFactory.getLogger(BigQuerySourceBase.class);
// The maximum number of retries to poll a BigQuery job.
@@ -68,37 +71,63 @@
protected final String stepUuid;
protected final BigQueryServices bqServices;
- private transient List<BoundedSource<TableRow>> cachedSplitResult;
+ private transient List<BoundedSource<T>> cachedSplitResult;
+ private SerializableFunction<SchemaAndRecord, T> parseFn;
+ private Coder<T> coder;
- BigQuerySourceBase(String stepUuid, BigQueryServices bqServices) {
+ BigQuerySourceBase(
+ String stepUuid,
+ BigQueryServices bqServices,
+ Coder<T> coder,
+ SerializableFunction<SchemaAndRecord, T> parseFn
+ ) {
this.stepUuid = checkNotNull(stepUuid, "stepUuid");
this.bqServices = checkNotNull(bqServices, "bqServices");
+ this.coder = checkNotNull(coder, "coder");
+ this.parseFn = checkNotNull(parseFn, "parseFn");
+ }
+
+ protected static class ExtractResult {
+ public final TableSchema schema;
+ public final List<ResourceId> extractedFiles;
+
+ public ExtractResult(TableSchema schema, List<ResourceId> extractedFiles) {
+ this.schema = schema;
+ this.extractedFiles = extractedFiles;
+ }
+ }
+
+ protected ExtractResult extractFiles(PipelineOptions options) throws Exception {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ TableReference tableToExtract = getTableToExtract(bqOptions);
+ TableSchema schema =
+ bqServices.getDatasetService(bqOptions).getTable(tableToExtract).getSchema();
+ JobService jobService = bqServices.getJobService(bqOptions);
+ String extractJobId = getExtractJobId(createJobIdToken(options.getJobName(), stepUuid));
+ final String extractDestinationDir =
+ resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
+ List<ResourceId> tempFiles =
+ executeExtract(
+ extractJobId,
+ tableToExtract,
+ jobService,
+ bqOptions.getProject(),
+ extractDestinationDir);
+ return new ExtractResult(schema, tempFiles);
}
@Override
- public List<BoundedSource<TableRow>> split(
+ public List<BoundedSource<T>> split(
long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
// split() can be called multiple times, e.g. Dataflow runner may call it multiple times
// with different desiredBundleSizeBytes in case the split() call produces too many sources.
// We ignore desiredBundleSizeBytes anyway, however in any case, we should not initiate
// another BigQuery extract job for the repeated split() calls.
if (cachedSplitResult == null) {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- TableReference tableToExtract = getTableToExtract(bqOptions);
- JobService jobService = bqServices.getJobService(bqOptions);
-
- final String extractDestinationDir =
- resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
-
- String extractJobId = getExtractJobId(createJobIdToken(options.getJobName(), stepUuid));
- List<ResourceId> tempFiles = executeExtract(
- extractJobId, tableToExtract, jobService, bqOptions.getProject(), extractDestinationDir);
-
- TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
- .getTable(tableToExtract).getSchema();
-
- cleanupTempResource(bqOptions);
- cachedSplitResult = checkNotNull(createSources(tempFiles, tableSchema));
+ ExtractResult res = extractFiles(options);
+ LOG.info("Extract job produced {} files", res.extractedFiles.size());
+ cleanupTempResource(options.as(BigQueryOptions.class));
+ cachedSplitResult = checkNotNull(createSources(res.extractedFiles, res.schema));
}
return cachedSplitResult;
}
@@ -108,13 +137,18 @@
protected abstract void cleanupTempResource(BigQueryOptions bqOptions) throws Exception;
@Override
+ public BoundedReader<T> createReader(PipelineOptions options) throws IOException {
+ throw new UnsupportedOperationException("BigQuery source must be split before being read");
+ }
+
+ @Override
public void validate() {
// Do nothing, validation is done in BigQuery.Read.
}
@Override
- public Coder<TableRow> getDefaultOutputCoder() {
- return TableRowJsonCoder.of();
+ public Coder<T> getOutputCoder() {
+ return coder;
}
private List<ResourceId> executeExtract(
@@ -147,59 +181,34 @@
return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
}
- private List<BoundedSource<TableRow>> createSources(
- List<ResourceId> files, TableSchema tableSchema) throws IOException, InterruptedException {
- final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(tableSchema);
-
- SerializableFunction<GenericRecord, TableRow> function =
- new SerializableFunction<GenericRecord, TableRow>() {
- @Override
- public TableRow apply(GenericRecord input) {
- return BigQueryAvroUtils.convertGenericRecordToTableRow(
- input, BigQueryHelpers.fromJsonString(jsonSchema, TableSchema.class));
- }};
-
- List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
- for (ResourceId file : files) {
- avroSources.add(new TransformingSource<>(
- AvroSource.from(file.toString()), function, getDefaultOutputCoder()));
+ private static class TableSchemaFunction
+ implements Serializable, Function<String, TableSchema> {
+ @Nullable
+ @Override
+ public TableSchema apply(@Nullable String input) {
+ return BigQueryHelpers.fromJsonString(input, TableSchema.class);
}
- return ImmutableList.copyOf(avroSources);
}
- protected static class BigQueryReader extends BoundedReader<TableRow> {
- private final BigQuerySourceBase source;
- private final BigQueryServices.BigQueryJsonReader reader;
+ List<BoundedSource<T>> createSources(List<ResourceId> files, TableSchema schema)
+ throws IOException, InterruptedException {
- BigQueryReader(
- BigQuerySourceBase source, BigQueryServices.BigQueryJsonReader reader) {
- this.source = source;
- this.reader = reader;
- }
+ final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(schema);
+ SerializableFunction<GenericRecord, T> fnWrapper =
+ new SerializableFunction<GenericRecord, T>() {
+ private Supplier<TableSchema> schema = Suppliers.memoize(
+ Suppliers.compose(new TableSchemaFunction(), Suppliers.ofInstance(jsonSchema)));
- @Override
- public BoundedSource<TableRow> getCurrentSource() {
- return source;
+ @Override
+ public T apply(GenericRecord input) {
+ return parseFn.apply(new SchemaAndRecord(input, schema.get()));
+ }
+ };
+ List<BoundedSource<T>> avroSources = Lists.newArrayList();
+ for (ResourceId file : files) {
+ avroSources.add(
+ AvroSource.from(file.toString()).withParseFn(fnWrapper, getOutputCoder()));
}
-
- @Override
- public boolean start() throws IOException {
- return reader.start();
- }
-
- @Override
- public boolean advance() throws IOException {
- return reader.advance();
- }
-
- @Override
- public TableRow getCurrent() throws NoSuchElementException {
- return reader.getCurrent();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
+ return ImmutableList.copyOf(avroSources);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
deleted file mode 100644
index ba19cf0..0000000
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
+++ /dev/null
@@ -1,501 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.api.client.googleapis.services.AbstractGoogleClientRequest;
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.ClassInfo;
-import com.google.api.client.util.Data;
-import com.google.api.client.util.Sleeper;
-import com.google.api.services.bigquery.Bigquery;
-import com.google.api.services.bigquery.model.Dataset;
-import com.google.api.services.bigquery.model.DatasetReference;
-import com.google.api.services.bigquery.model.ErrorProto;
-import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfiguration;
-import com.google.api.services.bigquery.model.JobConfigurationQuery;
-import com.google.api.services.bigquery.model.JobReference;
-import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.JobStatus;
-import com.google.api.services.bigquery.model.Table;
-import com.google.api.services.bigquery.model.TableCell;
-import com.google.api.services.bigquery.model.TableDataList;
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.Uninterruptibles;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Objects;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.util.BackOffAdapter;
-import org.apache.beam.sdk.util.FluentBackoff;
-import org.joda.time.Duration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Iterates over all rows in a table.
- */
-class BigQueryTableRowIterator implements AutoCloseable {
- private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableRowIterator.class);
-
- @Nullable private TableReference ref;
- @Nullable private final String projectId;
- @Nullable private TableSchema schema;
- @Nullable private JobConfigurationQuery queryConfig;
- private final Bigquery client;
- private String pageToken;
- private Iterator<TableRow> iteratorOverCurrentBatch;
- private TableRow current;
- // Set true when the final page is seen from the service.
- private boolean lastPage = false;
-
- // The maximum number of times a BigQuery request will be retried
- private static final int MAX_RETRIES = 3;
- // Initial wait time for the backoff implementation
- private static final Duration INITIAL_BACKOFF_TIME = Duration.standardSeconds(1);
-
- // After sending a query to BQ service we will be polling the BQ service to check the status with
- // following interval to check the status of query execution job
- private static final Duration QUERY_COMPLETION_POLL_TIME = Duration.standardSeconds(1);
-
- // Temporary dataset used to store query results.
- private String temporaryDatasetId = null;
- // Temporary table used to store query results.
- private String temporaryTableId = null;
-
- private BigQueryTableRowIterator(
- @Nullable TableReference ref, @Nullable JobConfigurationQuery queryConfig,
- @Nullable String projectId, Bigquery client) {
- this.ref = ref;
- this.queryConfig = queryConfig;
- this.projectId = projectId;
- this.client = checkNotNull(client, "client");
- }
-
- /**
- * Constructs a {@code BigQueryTableRowIterator} that reads from the specified table.
- */
- static BigQueryTableRowIterator fromTable(TableReference ref, Bigquery client) {
- checkNotNull(ref, "ref");
- checkNotNull(client, "client");
- return new BigQueryTableRowIterator(ref, /* queryConfig */null, ref.getProjectId(), client);
- }
-
- /**
- * Constructs a {@code BigQueryTableRowIterator} that reads from the results of executing the
- * specified query in the specified project.
- */
- static BigQueryTableRowIterator fromQuery(
- JobConfigurationQuery queryConfig, String projectId, Bigquery client) {
- checkNotNull(queryConfig, "queryConfig");
- checkNotNull(projectId, "projectId");
- checkNotNull(client, "client");
- return new BigQueryTableRowIterator(/* ref */null, queryConfig, projectId, client);
- }
-
- /**
- * Opens the table for read.
- * @throws IOException on failure
- */
- void open() throws IOException, InterruptedException {
- if (queryConfig != null) {
- ref = executeQueryAndWaitForCompletion();
- }
- // Get table schema.
- schema = getTable(ref).getSchema();
- }
-
- boolean advance() throws IOException, InterruptedException {
- while (true) {
- if (iteratorOverCurrentBatch != null && iteratorOverCurrentBatch.hasNext()) {
- // Embed schema information into the raw row, so that values have an
- // associated key.
- current = getTypedTableRow(schema.getFields(), iteratorOverCurrentBatch.next());
- return true;
- }
- if (lastPage) {
- return false;
- }
-
- Bigquery.Tabledata.List list =
- client.tabledata().list(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
- if (pageToken != null) {
- list.setPageToken(pageToken);
- }
-
- TableDataList result = executeWithBackOff(
- list,
- String.format(
- "Error reading from BigQuery table %s of dataset %s.",
- ref.getTableId(), ref.getDatasetId()));
-
- pageToken = result.getPageToken();
- iteratorOverCurrentBatch =
- result.getRows() != null
- ? result.getRows().iterator()
- : Collections.<TableRow>emptyIterator();
-
- // The server may return a page token indefinitely on a zero-length table.
- if (pageToken == null || result.getTotalRows() != null && result.getTotalRows() == 0) {
- lastPage = true;
- }
- }
- }
-
- TableRow getCurrent() {
- if (current == null) {
- throw new NoSuchElementException();
- }
- return current;
- }
-
- /**
- * Adjusts a field returned from the BigQuery API to match what we will receive when running
- * BigQuery's export-to-GCS and parallel read, which is the efficient parallel implementation
- * used for batch jobs executed on the Beam Runners that perform initial splitting.
- *
- * <p>The following is the relationship between BigQuery schema and Java types:
- *
- * <ul>
- * <li>Nulls are {@code null}.
- * <li>Repeated fields are {@code List} of objects.
- * <li>Record columns are {@link TableRow} objects.
- * <li>{@code BOOLEAN} columns are JSON booleans, hence Java {@code Boolean} objects.
- * <li>{@code FLOAT} columns are JSON floats, hence Java {@code Double} objects.
- * <li>{@code TIMESTAMP} columns are {@code String} objects that are of the format
- * {@code yyyy-MM-dd HH:mm:ss[.SSSSSS] UTC}, where the {@code .SSSSSS} has no trailing
- * zeros and can be 1 to 6 digits long.
- * <li>Every other atomic type is a {@code String}.
- * </ul>
- *
- * <p>Note that integers are encoded as strings to match BigQuery's exported JSON format.
- *
- * <p>Finally, values are stored in the {@link TableRow} as {"field name": value} pairs
- * and are not accessible through the {@link TableRow#getF} function.
- */
- @Nullable private Object getTypedCellValue(TableFieldSchema fieldSchema, Object v) {
- if (Data.isNull(v)) {
- return null;
- }
-
- if (Objects.equals(fieldSchema.getMode(), "REPEATED")) {
- TableFieldSchema elementSchema = fieldSchema.clone().setMode("REQUIRED");
- @SuppressWarnings("unchecked")
- List<Map<String, Object>> rawCells = (List<Map<String, Object>>) v;
- ImmutableList.Builder<Object> values = ImmutableList.builder();
- for (Map<String, Object> element : rawCells) {
- values.add(getTypedCellValue(elementSchema, element.get("v")));
- }
- return values.build();
- }
-
- if (fieldSchema.getType().equals("RECORD")) {
- @SuppressWarnings("unchecked")
- Map<String, Object> typedV = (Map<String, Object>) v;
- return getTypedTableRow(fieldSchema.getFields(), typedV);
- }
-
- if (fieldSchema.getType().equals("FLOAT")) {
- return Double.parseDouble((String) v);
- }
-
- if (fieldSchema.getType().equals("BOOLEAN")) {
- return Boolean.parseBoolean((String) v);
- }
-
- if (fieldSchema.getType().equals("TIMESTAMP")) {
- return BigQueryAvroUtils.formatTimestamp((String) v);
- }
-
- // Returns the original value for:
- // 1. String, 2. base64 encoded BYTES, 3. DATE, DATETIME, TIME strings.
- return v;
- }
-
- /**
- * A list of the field names that cannot be used in BigQuery tables processed by Apache Beam,
- * because they are reserved keywords in {@link TableRow}.
- */
- // TODO: This limitation is unfortunate. We need to give users a way to use BigQueryIO that does
- // not indirect through our broken use of {@link TableRow}.
- // See discussion: https://github.com/GoogleCloudPlatform/DataflowJavaSDK/pull/41
- private static final Collection<String> RESERVED_FIELD_NAMES =
- ClassInfo.of(TableRow.class).getNames();
-
- /**
- * Converts a row returned from the BigQuery JSON API as a {@code Map<String, Object>} into a
- * Java {@link TableRow} with nested {@link TableCell TableCells}. The {@code Object} values in
- * the cells are converted to Java types according to the provided field schemas.
- *
- * <p>See {@link #getTypedCellValue(TableFieldSchema, Object)} for details on how BigQuery
- * types are mapped to Java types.
- */
- private TableRow getTypedTableRow(List<TableFieldSchema> fields, Map<String, Object> rawRow) {
- // If rawRow is a TableRow, use it. If not, create a new one.
- TableRow row;
- List<? extends Map<String, Object>> cells;
- if (rawRow instanceof TableRow) {
- // Since rawRow is a TableRow it already has TableCell objects in setF. We do not need to do
- // any type conversion, but extract the cells for cell-wise processing below.
- row = (TableRow) rawRow;
- cells = row.getF();
- // Clear the cells from the row, so that row.getF() will return null. This matches the
- // behavior of rows produced by the BigQuery export API used on the service.
- row.setF(null);
- } else {
- row = new TableRow();
-
- // Since rawRow is a Map<String, Object> we use Map.get("f") instead of TableRow.getF() to
- // get its cells. Similarly, when rawCell is a Map<String, Object> instead of a TableCell,
- // we will use Map.get("v") instead of TableCell.getV() get its value.
- @SuppressWarnings("unchecked")
- List<? extends Map<String, Object>> rawCells =
- (List<? extends Map<String, Object>>) rawRow.get("f");
- cells = rawCells;
- }
-
- checkState(cells.size() == fields.size(),
- "Expected that the row has the same number of cells %s as fields in the schema %s",
- cells.size(), fields.size());
-
- // Loop through all the fields in the row, normalizing their types with the TableFieldSchema
- // and storing the normalized values by field name in the Map<String, Object> that
- // underlies the TableRow.
- Iterator<? extends Map<String, Object>> cellIt = cells.iterator();
- Iterator<TableFieldSchema> fieldIt = fields.iterator();
- while (cellIt.hasNext()) {
- Map<String, Object> cell = cellIt.next();
- TableFieldSchema fieldSchema = fieldIt.next();
-
- // Convert the object in this cell to the Java type corresponding to its type in the schema.
- Object convertedValue = getTypedCellValue(fieldSchema, cell.get("v"));
-
- String fieldName = fieldSchema.getName();
- checkArgument(!RESERVED_FIELD_NAMES.contains(fieldName),
- "BigQueryIO does not support records with columns named %s", fieldName);
-
- if (convertedValue == null) {
- // BigQuery does not include null values when the export operation (to JSON) is used.
- // To match that behavior, BigQueryTableRowiterator, and the DirectRunner,
- // intentionally omits columns with null values.
- continue;
- }
-
- row.set(fieldName, convertedValue);
- }
- return row;
- }
-
- // Get the BiqQuery table.
- private Table getTable(TableReference ref) throws IOException, InterruptedException {
- Bigquery.Tables.Get get =
- client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
-
- return executeWithBackOff(
- get,
- String.format(
- "Error opening BigQuery table %s of dataset %s.",
- ref.getTableId(),
- ref.getDatasetId()));
- }
-
- // Create a new BigQuery dataset
- private void createDataset(String datasetId, @Nullable String location)
- throws IOException, InterruptedException {
- Dataset dataset = new Dataset();
- DatasetReference reference = new DatasetReference();
- reference.setProjectId(projectId);
- reference.setDatasetId(datasetId);
- dataset.setDatasetReference(reference);
- if (location != null) {
- dataset.setLocation(location);
- }
-
- executeWithBackOff(
- client.datasets().insert(projectId, dataset),
- String.format(
- "Error when trying to create the temporary dataset %s in project %s.",
- datasetId, projectId));
- }
-
- // Delete the given table that is available in the given dataset.
- private void deleteTable(String datasetId, String tableId)
- throws IOException, InterruptedException {
- executeWithBackOff(
- client.tables().delete(projectId, datasetId, tableId),
- String.format(
- "Error when trying to delete the temporary table %s in dataset %s of project %s. "
- + "Manual deletion may be required.",
- tableId, datasetId, projectId));
- }
-
- // Delete the given dataset. This will fail if the given dataset has any tables.
- private void deleteDataset(String datasetId) throws IOException, InterruptedException {
- executeWithBackOff(
- client.datasets().delete(projectId, datasetId),
- String.format(
- "Error when trying to delete the temporary dataset %s in project %s. "
- + "Manual deletion may be required.",
- datasetId, projectId));
- }
-
- /**
- * Executes the specified query and returns a reference to the temporary BigQuery table created
- * to hold the results.
- *
- * @throws IOException if the query fails.
- */
- private TableReference executeQueryAndWaitForCompletion()
- throws IOException, InterruptedException {
- checkState(projectId != null, "Unable to execute a query without a configured project id");
- checkState(queryConfig != null, "Unable to execute a query without a configured query");
- // Dry run query to get source table location
- Job dryRunJob = new Job()
- .setConfiguration(new JobConfiguration()
- .setQuery(queryConfig)
- .setDryRun(true));
- JobStatistics jobStats = executeWithBackOff(
- client.jobs().insert(projectId, dryRunJob),
- String.format("Error when trying to dry run query %s.",
- queryConfig.toPrettyString())).getStatistics();
-
- // Let BigQuery to pick default location if the query does not read any tables.
- String location = null;
- @Nullable List<TableReference> tables = jobStats.getQuery().getReferencedTables();
- if (tables != null && !tables.isEmpty()) {
- Table table = getTable(tables.get(0));
- location = table.getLocation();
- }
-
- // Create a temporary dataset to store results.
- // Starting dataset name with an "_" so that it is hidden.
- Random rnd = new Random(System.currentTimeMillis());
- temporaryDatasetId = "_beam_temporary_dataset_" + rnd.nextInt(1000000);
- temporaryTableId = "beam_temporary_table_" + rnd.nextInt(1000000);
-
- createDataset(temporaryDatasetId, location);
- Job job = new Job();
- JobConfiguration config = new JobConfiguration();
- config.setQuery(queryConfig);
- job.setConfiguration(config);
-
- TableReference destinationTable = new TableReference();
- destinationTable.setProjectId(projectId);
- destinationTable.setDatasetId(temporaryDatasetId);
- destinationTable.setTableId(temporaryTableId);
- queryConfig.setDestinationTable(destinationTable);
- queryConfig.setAllowLargeResults(true);
-
- Job queryJob = executeWithBackOff(
- client.jobs().insert(projectId, job),
- String.format("Error when trying to execute the job for query %s.",
- queryConfig.toPrettyString()));
- JobReference jobId = queryJob.getJobReference();
-
- while (true) {
- Job pollJob = executeWithBackOff(
- client.jobs().get(projectId, jobId.getJobId()),
- String.format("Error when trying to get status of the job for query %s.",
- queryConfig.toPrettyString()));
- JobStatus status = pollJob.getStatus();
- if (status.getState().equals("DONE")) {
- // Job is DONE, but did not necessarily succeed.
- ErrorProto error = status.getErrorResult();
- if (error == null) {
- return pollJob.getConfiguration().getQuery().getDestinationTable();
- } else {
- // There will be no temporary table to delete, so null out the reference.
- temporaryTableId = null;
- throw new IOException(String.format(
- "Executing query %s failed: %s", queryConfig.toPrettyString(), error.getMessage()));
- }
- }
- Uninterruptibles.sleepUninterruptibly(
- QUERY_COMPLETION_POLL_TIME.getMillis(), TimeUnit.MILLISECONDS);
- }
- }
-
- // Execute a BQ request with exponential backoff and return the result.
- // client - BQ request to be executed
- // error - Formatted message to log if when a request fails. Takes exception message as a
- // formatter parameter.
- private static <T> T executeWithBackOff(AbstractGoogleClientRequest<T> client, String error)
- throws IOException, InterruptedException {
- Sleeper sleeper = Sleeper.DEFAULT;
- BackOff backOff =
- BackOffAdapter.toGcpBackOff(
- FluentBackoff.DEFAULT
- .withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff());
-
- T result = null;
- while (true) {
- try {
- result = client.execute();
- break;
- } catch (IOException e) {
- LOG.error("{}", error, e);
- if (!BackOffUtils.next(sleeper, backOff)) {
- String errorMessage = String.format(
- "%s Failing to execute job after %d attempts.", error, MAX_RETRIES + 1);
- LOG.error("{}", errorMessage, e);
- throw new IOException(errorMessage, e);
- }
- }
- }
- return result;
- }
-
- @Override
- public void close() {
- // Prevent any further requests.
- lastPage = true;
-
- try {
- // Deleting temporary table and dataset that gets generated when executing a query.
- if (temporaryDatasetId != null) {
- if (temporaryTableId != null) {
- deleteTable(temporaryDatasetId, temporaryTableId);
- }
- deleteDataset(temporaryDatasetId);
- }
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new RuntimeException(e);
- }
-
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
index 1d45641..dbac00f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
@@ -21,16 +21,18 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import com.google.api.services.bigquery.model.Table;
import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,14 +41,16 @@
* A {@link BigQuerySourceBase} for reading BigQuery tables.
*/
@VisibleForTesting
-class BigQueryTableSource extends BigQuerySourceBase {
+class BigQueryTableSource<T> extends BigQuerySourceBase<T> {
private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableSource.class);
- static BigQueryTableSource create(
+ static <T>BigQueryTableSource<T> create(
String stepUuid,
ValueProvider<TableReference> table,
- BigQueryServices bqServices) {
- return new BigQueryTableSource(stepUuid, table, bqServices);
+ BigQueryServices bqServices,
+ Coder<T> coder,
+ SerializableFunction<SchemaAndRecord, T> parseFn) {
+ return new BigQueryTableSource<>(stepUuid, table, bqServices, coder, parseFn);
}
private final ValueProvider<String> jsonTable;
@@ -55,15 +59,17 @@
private BigQueryTableSource(
String stepUuid,
ValueProvider<TableReference> table,
- BigQueryServices bqServices) {
- super(stepUuid, bqServices);
+ BigQueryServices bqServices,
+ Coder<T> coder,
+ SerializableFunction<SchemaAndRecord, T> parseFn
+ ) {
+ super(stepUuid, bqServices, coder, parseFn);
this.jsonTable = NestedValueProvider.of(checkNotNull(table, "table"), new TableRefToJson());
this.tableSizeBytes = new AtomicReference<>();
}
@Override
protected TableReference getTableToExtract(BigQueryOptions bqOptions) throws IOException {
- checkState(jsonTable.isAccessible());
TableReference tableReference =
BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
return setDefaultProjectIfAbsent(bqOptions, tableReference);
@@ -92,22 +98,18 @@
}
@Override
- public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- checkState(jsonTable.isAccessible());
- TableReference tableRef = BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(),
- TableReference.class);
- return new BigQueryReader(this, bqServices.getReaderFromTable(bqOptions, tableRef));
- }
-
- @Override
public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
if (tableSizeBytes.get() == null) {
TableReference table = setDefaultProjectIfAbsent(options.as(BigQueryOptions.class),
BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class));
- Long numBytes = bqServices.getDatasetService(options.as(BigQueryOptions.class))
- .getTable(table).getNumBytes();
+ Table tableRef = bqServices.getDatasetService(options.as(BigQueryOptions.class))
+ .getTable(table);
+ Long numBytes = tableRef.getNumBytes();
+ if (tableRef.getStreamingBuffer() != null) {
+ numBytes += tableRef.getStreamingBuffer().getEstimatedBytes().longValue();
+ }
+
tableSizeBytes.compareAndSet(null, numBytes);
}
return tableSizeBytes.get();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
deleted file mode 100644
index 1ac216f..0000000
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.common.collect.Lists;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.transforms.Distinct;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Keys;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** Compute the mapping of destinations to json-formatted schema objects. */
-class CalculateSchemas<DestinationT>
- extends PTransform<
- PCollection<KV<DestinationT, TableRow>>, PCollectionView<Map<DestinationT, String>>> {
- private static final Logger LOG = LoggerFactory.getLogger(CalculateSchemas.class);
-
- private final DynamicDestinations<?, DestinationT> dynamicDestinations;
-
- public CalculateSchemas(DynamicDestinations<?, DestinationT> dynamicDestinations) {
- this.dynamicDestinations = dynamicDestinations;
- }
-
- @Override
- public PCollectionView<Map<DestinationT, String>> expand(
- PCollection<KV<DestinationT, TableRow>> input) {
- List<PCollectionView<?>> sideInputs = Lists.newArrayList();
- sideInputs.addAll(dynamicDestinations.getSideInputs());
-
- return input
- .apply("Keys", Keys.<DestinationT>create())
- .apply("Distinct Keys", Distinct.<DestinationT>create())
- .apply(
- "GetSchemas",
- ParDo.of(
- new DoFn<DestinationT, KV<DestinationT, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- dynamicDestinations.setSideInputAccessorFromProcessContext(c);
- TableSchema tableSchema = dynamicDestinations.getSchema(c.element());
- if (tableSchema != null) {
- // If the createDisposition is CREATE_NEVER, then there's no need for a
- // schema, and getSchema might return null. In this case, we simply
- // leave it out of the map.
- c.output(KV.of(c.element(), BigQueryHelpers.toJsonString(tableSchema)));
- }
- }
- })
- .withSideInputs(sideInputs))
- .apply("asMap", View.<DestinationT, String>asMap());
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
index 3dc10b0..fedd2fe 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
@@ -73,7 +73,7 @@
}
CreateTables<DestinationT> withTestServices(BigQueryServices bqServices) {
- return new CreateTables<DestinationT>(createDisposition, bqServices, dynamicDestinations);
+ return new CreateTables<>(createDisposition, bqServices, dynamicDestinations);
}
@Override
@@ -113,9 +113,7 @@
private void possibleCreateTable(
BigQueryOptions options, TableDestination tableDestination, TableSchema tableSchema)
throws InterruptedException, IOException {
- String tableSpec = tableDestination.getTableSpec();
- TableReference tableReference = tableDestination.getTableReference();
- String tableDescription = tableDestination.getTableDescription();
+ String tableSpec = BigQueryHelpers.stripPartitionDecorator(tableDestination.getTableSpec());
if (createDisposition != createDisposition.CREATE_NEVER && !createdTables.contains(tableSpec)) {
synchronized (createdTables) {
// Another thread may have succeeded in creating the table in the meanwhile, so
@@ -123,12 +121,19 @@
// every thread from attempting a create and overwhelming our BigQuery quota.
DatasetService datasetService = bqServices.getDatasetService(options);
if (!createdTables.contains(tableSpec)) {
+ TableReference tableReference = tableDestination.getTableReference();
+ String tableDescription = tableDestination.getTableDescription();
+ tableReference.setTableId(
+ BigQueryHelpers.stripPartitionDecorator(tableReference.getTableId()));
if (datasetService.getTable(tableReference) == null) {
- datasetService.createTable(
- new Table()
- .setTableReference(tableReference)
- .setSchema(tableSchema)
- .setDescription(tableDescription));
+ Table table = new Table()
+ .setTableReference(tableReference)
+ .setSchema(tableSchema)
+ .setDescription(tableDescription);
+ if (tableDestination.getTimePartitioning() != null) {
+ table.setTimePartitioning(tableDestination.getTimePartitioning());
+ }
+ datasetService.createTable(table);
}
createdTables.add(tableSpec);
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
index edb1e0d..e351138 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
@@ -19,12 +19,11 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.sdk.values.TypeDescriptors.extractFromTypeParameters;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.common.collect.Lists;
import java.io.Serializable;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
@@ -32,6 +31,8 @@
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.sdk.values.ValueInSingleWindow;
/**
@@ -157,22 +158,19 @@
return destinationCoder;
}
// If dynamicDestinations doesn't provide a coder, try to find it in the coder registry.
- // We must first use reflection to figure out what the type parameter is.
- for (Type superclass = getClass().getGenericSuperclass();
- superclass != null;
- superclass = ((Class) superclass).getGenericSuperclass()) {
- if (superclass instanceof ParameterizedType) {
- ParameterizedType parameterized = (ParameterizedType) superclass;
- if (parameterized.getRawType() == DynamicDestinations.class) {
- // DestinationT is the second parameter.
- Type parameter = parameterized.getActualTypeArguments()[1];
- @SuppressWarnings("unchecked")
- Class<DestinationT> parameterClass = (Class<DestinationT>) parameter;
- return registry.getCoder(parameterClass);
- }
- }
+ TypeDescriptor<DestinationT> descriptor =
+ extractFromTypeParameters(
+ this,
+ DynamicDestinations.class,
+ new TypeDescriptors.TypeVariableExtractor<
+ DynamicDestinations<T, DestinationT>, DestinationT>() {});
+ try {
+ return registry.getCoder(descriptor);
+ } catch (CannotProvideCoderException e) {
+ throw new CannotProvideCoderException(
+ "Failed to infer coder for DestinationT from type "
+ + descriptor + ", please provide it explicitly by overriding getDestinationCoder()",
+ e);
}
- throw new AssertionError(
- "Couldn't find the DynamicDestinations superclass of " + this.getClass());
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
index 530e2b6..818ea34 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
@@ -108,7 +108,7 @@
@Override
public Coder<TableDestination> getDestinationCoder() {
- return TableDestinationCoder.of();
+ return TableDestinationCoderV2.of();
}
}
@@ -164,6 +164,31 @@
}
}
+ static class ConstantTimePartitioningDestinations<T>
+ extends DelegatingDynamicDestinations<T, TableDestination> {
+
+ @Nullable
+ private final ValueProvider<String> jsonTimePartitioning;
+
+ ConstantTimePartitioningDestinations(DynamicDestinations<T, TableDestination> inner,
+ ValueProvider<String> jsonTimePartitioning) {
+ super(inner);
+ this.jsonTimePartitioning = jsonTimePartitioning;
+ }
+
+ @Override
+ public TableDestination getDestination(ValueInSingleWindow<T> element) {
+ TableDestination destination = super.getDestination(element);
+ return new TableDestination(destination.getTableSpec(), destination.getTableDescription(),
+ jsonTimePartitioning.get());
+ }
+
+ @Override
+ public Coder<TableDestination> getDestinationCoder() {
+ return TableDestinationCoderV2.of();
+ }
+ }
+
/**
* Takes in a side input mapping tablespec to json table schema, and always returns the
* matching schema from the side input.
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/GenerateShardedTable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/GenerateShardedTable.java
index 90d41a0..55672ff 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/GenerateShardedTable.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/GenerateShardedTable.java
@@ -23,6 +23,7 @@
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ShardedKey;
/**
* Given a write to a specific table, assign that to one of the
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java
new file mode 100644
index 0000000..90a3d0d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ErrorProto;
+import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
+import com.google.common.collect.ImmutableSet;
+import java.io.Serializable;
+import java.util.Set;
+
+/** A retry policy for streaming BigQuery inserts. */
+public abstract class InsertRetryPolicy implements Serializable {
+ /**
+ * Contains information about a failed insert.
+ *
+ * <p>Currently only the list of errors returned from BigQuery. In the future this may contain
+ * more information - e.g. how many times this insert has been retried, and for how long.
+ */
+ public static class Context {
+ // A list of all errors corresponding to an attempted insert of a single record.
+ TableDataInsertAllResponse.InsertErrors errors;
+
+ public Context(TableDataInsertAllResponse.InsertErrors errors) {
+ this.errors = errors;
+ }
+ }
+
+ // A list of known persistent errors for which retrying never helps.
+ static final Set<String> PERSISTENT_ERRORS =
+ ImmutableSet.of("invalid", "invalidQuery", "notImplemented");
+
+ /** Return true if this failure should be retried. */
+ public abstract boolean shouldRetry(Context context);
+
+ /** Never retry any failures. */
+ public static InsertRetryPolicy neverRetry() {
+ return new InsertRetryPolicy() {
+ @Override
+ public boolean shouldRetry(Context context) {
+ return false;
+ }
+ };
+ }
+
+ /** Always retry all failures. */
+ public static InsertRetryPolicy alwaysRetry() {
+ return new InsertRetryPolicy() {
+ @Override
+ public boolean shouldRetry(Context context) {
+ return true;
+ }
+ };
+ }
+
+ /** Retry all failures except for known persistent errors. */
+ public static InsertRetryPolicy retryTransientErrors() {
+ return new InsertRetryPolicy() {
+ @Override
+ public boolean shouldRetry(Context context) {
+ if (context.errors.getErrors() != null) {
+ for (ErrorProto error : context.errors.getErrors()) {
+ if (error.getReason() != null && PERSISTENT_ERRORS.contains(error.getReason())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ };
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
index f49c4e1..2f7da08 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
@@ -41,9 +41,12 @@
class PassThroughThenCleanup<T> extends PTransform<PCollection<T>, PCollection<T>> {
private CleanupOperation cleanupOperation;
+ private PCollectionView<String> jobIdSideInput;
- PassThroughThenCleanup(CleanupOperation cleanupOperation) {
+ PassThroughThenCleanup(
+ CleanupOperation cleanupOperation, PCollectionView<String> jobIdSideInput) {
this.cleanupOperation = cleanupOperation;
+ this.jobIdSideInput = jobIdSideInput;
}
@Override
@@ -57,18 +60,21 @@
.setCoder(VoidCoder.of())
.apply(View.<Void>asIterable());
- input.getPipeline()
+ input
+ .getPipeline()
.apply("Create(CleanupOperation)", Create.of(cleanupOperation))
- .apply("Cleanup", ParDo.of(
- new DoFn<CleanupOperation, Void>() {
- @ProcessElement
- public void processElement(ProcessContext c)
- throws Exception {
- c.element().cleanup(c.getPipelineOptions());
- }
- }).withSideInputs(cleanupSignalView));
+ .apply(
+ "Cleanup",
+ ParDo.of(
+ new DoFn<CleanupOperation, Void>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ c.element().cleanup(new ContextContainer(c, jobIdSideInput));
+ }
+ })
+ .withSideInputs(jobIdSideInput, cleanupSignalView));
- return outputs.get(mainOutput);
+ return outputs.get(mainOutput).setCoder(input.getCoder());
}
private static class IdentityFn<T> extends DoFn<T, T> {
@@ -79,6 +85,24 @@
}
abstract static class CleanupOperation implements Serializable {
- abstract void cleanup(PipelineOptions options) throws Exception;
+ abstract void cleanup(ContextContainer container) throws Exception;
+ }
+
+ static class ContextContainer {
+ private PCollectionView<String> view;
+ private DoFn<?, ?>.ProcessContext context;
+
+ public ContextContainer(DoFn<?, ?>.ProcessContext context, PCollectionView<String> view) {
+ this.view = view;
+ this.context = context;
+ }
+
+ public PipelineOptions getPipelineOptions() {
+ return context.getPipelineOptions();
+ }
+
+ public String getJobId() {
+ return context.sideInput(view);
+ }
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ReifyAsIterable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ReifyAsIterable.java
new file mode 100644
index 0000000..18a359c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ReifyAsIterable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/**
+ * This transforms turns a side input into a singleton PCollection that can be used as the main
+ * input for another transform.
+ */
+public class ReifyAsIterable<T> extends PTransform<PCollection<T>, PCollection<Iterable<T>>> {
+ @Override
+ public PCollection<Iterable<T>> expand(PCollection<T> input) {
+ final PCollectionView<Iterable<T>> view = input.apply(View.<T>asIterable());
+ return input
+ .getPipeline()
+ .apply(Create.of((Void) null).withCoder(VoidCoder.of()))
+ .apply(
+ ParDo.of(
+ new DoFn<Void, Iterable<T>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.sideInput(view));
+ }
+ })
+ .withSideInputs(view));
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaAndRecord.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaAndRecord.java
new file mode 100644
index 0000000..e6811ef
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaAndRecord.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableSchema;
+import org.apache.avro.generic.GenericRecord;
+
+/**
+ * A wrapper for a {@link GenericRecord} and the {@link TableSchema} representing the schema of the
+ * table (or query) it was generated from.
+ */
+public class SchemaAndRecord {
+ private final GenericRecord record;
+ private final TableSchema tableSchema;
+
+ public SchemaAndRecord(GenericRecord record, TableSchema tableSchema) {
+ this.record = record;
+ this.tableSchema = tableSchema;
+ }
+
+ public GenericRecord getRecord() {
+ return record;
+ }
+
+ public TableSchema getTableSchema() {
+ return tableSchema;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
deleted file mode 100644
index c2b739f..0000000
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-/**
- * A key and a shard number.
- */
-class ShardedKey<K> implements Serializable {
- private static final long serialVersionUID = 1L;
- private final K key;
- private final int shardNumber;
-
- public static <K> ShardedKey<K> of(K key, int shardNumber) {
- return new ShardedKey<>(key, shardNumber);
- }
-
- ShardedKey(K key, int shardNumber) {
- this.key = key;
- this.shardNumber = shardNumber;
- }
-
- public K getKey() {
- return key;
- }
-
- public int getShardNumber() {
- return shardNumber;
- }
-
- @Override
- public String toString() {
- return "key: " + key + " shard: " + shardNumber;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof ShardedKey)) {
- return false;
- }
- ShardedKey<K> other = (ShardedKey<K>) o;
- return Objects.equals(key, other.key) && Objects.equals(shardNumber, other.shardNumber);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(key, shardNumber);
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
deleted file mode 100644
index c2b62b7..0000000
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StructuredCoder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-
-
-/**
- * A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}.
- */
-@VisibleForTesting
-class ShardedKeyCoder<KeyT>
- extends StructuredCoder<ShardedKey<KeyT>> {
- public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
- return new ShardedKeyCoder<>(keyCoder);
- }
-
- private final Coder<KeyT> keyCoder;
- private final VarIntCoder shardNumberCoder;
-
- protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
- this.keyCoder = keyCoder;
- this.shardNumberCoder = VarIntCoder.of();
- }
-
- @Override
- public List<? extends Coder<?>> getCoderArguments() {
- return Arrays.asList(keyCoder);
- }
-
- @Override
- public void encode(ShardedKey<KeyT> key, OutputStream outStream)
- throws IOException {
- keyCoder.encode(key.getKey(), outStream);
- shardNumberCoder.encode(key.getShardNumber(), outStream);
- }
-
- @Override
- public ShardedKey<KeyT> decode(InputStream inStream)
- throws IOException {
- return new ShardedKey<>(
- keyCoder.decode(inStream),
- shardNumberCoder.decode(inStream));
- }
-
- @Override
- public void verifyDeterministic() throws NonDeterministicException {
- keyCoder.verifyDeterministic();
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingInserts.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingInserts.java
index 9cb0027..747f2b0 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingInserts.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingInserts.java
@@ -19,8 +19,6 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import com.google.api.services.bigquery.model.TableRow;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
@@ -35,32 +33,49 @@
private BigQueryServices bigQueryServices;
private final CreateDisposition createDisposition;
private final DynamicDestinations<?, DestinationT> dynamicDestinations;
+ private InsertRetryPolicy retryPolicy;
/** Constructor. */
- StreamingInserts(CreateDisposition createDisposition,
+ public StreamingInserts(CreateDisposition createDisposition,
DynamicDestinations<?, DestinationT> dynamicDestinations) {
+ this(createDisposition, dynamicDestinations, new BigQueryServicesImpl(),
+ InsertRetryPolicy.alwaysRetry());
+ }
+
+ /** Constructor. */
+ private StreamingInserts(CreateDisposition createDisposition,
+ DynamicDestinations<?, DestinationT> dynamicDestinations,
+ BigQueryServices bigQueryServices,
+ InsertRetryPolicy retryPolicy) {
this.createDisposition = createDisposition;
this.dynamicDestinations = dynamicDestinations;
- this.bigQueryServices = new BigQueryServicesImpl();
- }
-
- void setTestServices(BigQueryServices bigQueryServices) {
this.bigQueryServices = bigQueryServices;
+ this.retryPolicy = retryPolicy;
}
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
+ /**
+ * Specify a retry policy for failed inserts.
+ */
+ public StreamingInserts<DestinationT> withInsertRetryPolicy(InsertRetryPolicy retryPolicy) {
+ return new StreamingInserts<>(
+ createDisposition, dynamicDestinations, bigQueryServices, retryPolicy);
}
+ StreamingInserts<DestinationT> withTestServices(BigQueryServices bigQueryServices) {
+ return new StreamingInserts<>(
+ createDisposition, dynamicDestinations, bigQueryServices, retryPolicy); }
+
@Override
public WriteResult expand(PCollection<KV<DestinationT, TableRow>> input) {
PCollection<KV<TableDestination, TableRow>> writes =
input.apply(
"CreateTables",
- new CreateTables<DestinationT>(createDisposition, dynamicDestinations)
+ new CreateTables<>(createDisposition, dynamicDestinations)
.withTestServices(bigQueryServices));
- return writes.apply(new StreamingWriteTables().withTestServices(bigQueryServices));
+ return writes.apply(
+ new StreamingWriteTables()
+ .withTestServices(bigQueryServices)
+ .withInsertRetryPolicy(retryPolicy));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
index f267976..a210858 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
@@ -21,6 +21,7 @@
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
@@ -29,8 +30,12 @@
import org.apache.beam.sdk.metrics.SinkMetrics;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.SystemDoFnInternal;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ShardedKey;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
/**
* Implementation of DoFn to perform streaming BigQuery write.
@@ -40,9 +45,12 @@
class StreamingWriteFn
extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {
private final BigQueryServices bqServices;
+ private final InsertRetryPolicy retryPolicy;
+ private final TupleTag<TableRow> failedOutputTag;
+
/** JsonTableRows to accumulate BigQuery rows in order to batch writes. */
- private transient Map<String, List<TableRow>> tableRows;
+ private transient Map<String, List<ValueInSingleWindow<TableRow>>> tableRows;
/** The list of unique ids for each BigQuery table row. */
private transient Map<String, List<String>> uniqueIdsForTableRows;
@@ -50,8 +58,11 @@
/** Tracks bytes written, exposed as "ByteCount" Counter. */
private Counter byteCounter = SinkMetrics.bytesWritten();
- StreamingWriteFn(BigQueryServices bqServices) {
+ StreamingWriteFn(BigQueryServices bqServices, InsertRetryPolicy retryPolicy,
+ TupleTag<TableRow> failedOutputTag) {
this.bqServices = bqServices;
+ this.retryPolicy = retryPolicy;
+ this.failedOutputTag = failedOutputTag;
}
/** Prepares a target BigQuery table. */
@@ -63,27 +74,39 @@
/** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */
@ProcessElement
- public void processElement(ProcessContext context) {
+ public void processElement(ProcessContext context, BoundedWindow window) {
String tableSpec = context.element().getKey().getKey();
- List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
- List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
- tableSpec);
+ List<ValueInSingleWindow<TableRow>> rows =
+ BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
+ List<String> uniqueIds =
+ BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows, tableSpec);
- rows.add(context.element().getValue().tableRow);
+ rows.add(
+ ValueInSingleWindow.of(
+ context.element().getValue().tableRow, context.timestamp(), window, context.pane()));
uniqueIds.add(context.element().getValue().uniqueId);
}
/** Writes the accumulated rows into BigQuery with streaming API. */
@FinishBundle
public void finishBundle(FinishBundleContext context) throws Exception {
+ List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
- for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
+ for (Map.Entry<String, List<ValueInSingleWindow<TableRow>>> entry : tableRows.entrySet()) {
TableReference tableReference = BigQueryHelpers.parseTableSpec(entry.getKey());
- flushRows(tableReference, entry.getValue(),
- uniqueIdsForTableRows.get(entry.getKey()), options);
+ flushRows(
+ tableReference,
+ entry.getValue(),
+ uniqueIdsForTableRows.get(entry.getKey()),
+ options,
+ failedInserts);
}
tableRows.clear();
uniqueIdsForTableRows.clear();
+
+ for (ValueInSingleWindow<TableRow> row : failedInserts) {
+ context.output(failedOutputTag, row.getValue(), row.getTimestamp(), row.getWindow());
+ }
}
@Override
@@ -95,12 +118,14 @@
* Writes the accumulated rows into BigQuery with streaming API.
*/
private void flushRows(TableReference tableReference,
- List<TableRow> tableRows, List<String> uniqueIds, BigQueryOptions options)
- throws InterruptedException {
+ List<ValueInSingleWindow<TableRow>> tableRows,
+ List<String> uniqueIds, BigQueryOptions options,
+ List<ValueInSingleWindow<TableRow>> failedInserts)
+ throws InterruptedException {
if (!tableRows.isEmpty()) {
try {
long totalBytes = bqServices.getDatasetService(options).insertAll(
- tableReference, tableRows, uniqueIds);
+ tableReference, tableRows, uniqueIds, retryPolicy, failedInserts);
byteCounter.inc(totalBytes);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
index 886236b..44563c0 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
@@ -19,6 +19,7 @@
import com.google.api.services.bigquery.model.TableRow;
import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ShardedKeyCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
@@ -28,6 +29,10 @@
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.ShardedKey;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
/**
* This transform takes in key-value pairs of {@link TableRow} entries and the
@@ -40,17 +45,23 @@
public class StreamingWriteTables extends PTransform<
PCollection<KV<TableDestination, TableRow>>, WriteResult> {
private BigQueryServices bigQueryServices;
+ private InsertRetryPolicy retryPolicy;
public StreamingWriteTables() {
- this(new BigQueryServicesImpl());
+ this(new BigQueryServicesImpl(), InsertRetryPolicy.alwaysRetry());
}
- private StreamingWriteTables(BigQueryServices bigQueryServices) {
+ private StreamingWriteTables(BigQueryServices bigQueryServices, InsertRetryPolicy retryPolicy) {
this.bigQueryServices = bigQueryServices;
+ this.retryPolicy = retryPolicy;
}
StreamingWriteTables withTestServices(BigQueryServices bigQueryServices) {
- return new StreamingWriteTables(bigQueryServices);
+ return new StreamingWriteTables(bigQueryServices, retryPolicy);
+ }
+
+ StreamingWriteTables withInsertRetryPolicy(InsertRetryPolicy retryPolicy) {
+ return new StreamingWriteTables(bigQueryServices, retryPolicy);
}
@Override
@@ -68,17 +79,19 @@
// get good batching into BigQuery's insert calls, and enough that we can max out the
// streaming insert quota.
PCollection<KV<ShardedKey<String>, TableRowInfo>> tagged =
- input.apply("ShardTableWrites", ParDo.of
- (new GenerateShardedTable(50)))
- .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowJsonCoder.of()))
- .apply("TagWithUniqueIds", ParDo.of(new TagWithUniqueIds()));
+ input
+ .apply("ShardTableWrites", ParDo.of(new GenerateShardedTable(50)))
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowJsonCoder.of()))
+ .apply("TagWithUniqueIds", ParDo.of(new TagWithUniqueIds()))
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()));
// To prevent having the same TableRow processed more than once with regenerated
// different unique ids, this implementation relies on "checkpointing", which is
// achieved as a side effect of having StreamingWriteFn immediately follow a GBK,
// performed by Reshuffle.
- tagged
- .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()))
+ TupleTag<Void> mainOutputTag = new TupleTag<>("mainOutput");
+ TupleTag<TableRow> failedInsertsTag = new TupleTag<>("failedInserts");
+ PCollectionTuple tuple = tagged
.apply(Reshuffle.<ShardedKey<String>, TableRowInfo>of())
// Put in the global window to ensure that DynamicDestinations side inputs are accessed
// correctly.
@@ -87,7 +100,10 @@
.triggering(DefaultTrigger.of()).discardingFiredPanes())
.apply("StreamingWrite",
ParDo.of(
- new StreamingWriteFn(bigQueryServices)));
- return WriteResult.in(input.getPipeline());
+ new StreamingWriteFn(bigQueryServices, retryPolicy, failedInsertsTag))
+ .withOutputTags(mainOutputTag, TupleTagList.of(failedInsertsTag)));
+ PCollection<TableRow> failedInserts = tuple.get(failedInsertsTag);
+ failedInserts.setCoder(TableRowJsonCoder.of());
+ return WriteResult.in(input.getPipeline(), failedInsertsTag, failedInserts);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
index ecf35d8..ce2e7c7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
@@ -19,6 +19,7 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TimePartitioning;
import java.io.Serializable;
import java.util.Objects;
import javax.annotation.Nullable;
@@ -31,16 +32,35 @@
private final String tableSpec;
@Nullable
private final String tableDescription;
+ @Nullable
+ private final String jsonTimePartitioning;
public TableDestination(String tableSpec, @Nullable String tableDescription) {
- this.tableSpec = tableSpec;
- this.tableDescription = tableDescription;
+ this(tableSpec, tableDescription, (String) null);
}
public TableDestination(TableReference tableReference, @Nullable String tableDescription) {
- this.tableSpec = BigQueryHelpers.toTableSpec(tableReference);
+ this(tableReference, tableDescription, null);
+ }
+
+ public TableDestination(TableReference tableReference, @Nullable String tableDescription,
+ TimePartitioning timePartitioning) {
+ this(BigQueryHelpers.toTableSpec(tableReference), tableDescription,
+ timePartitioning != null ? BigQueryHelpers.toJsonString(timePartitioning) : null);
+ }
+
+ public TableDestination(String tableSpec, @Nullable String tableDescription,
+ TimePartitioning timePartitioning) {
+ this(tableSpec, tableDescription,
+ timePartitioning != null ? BigQueryHelpers.toJsonString(timePartitioning) : null);
+ }
+
+ public TableDestination(String tableSpec, @Nullable String tableDescription,
+ @Nullable String jsonTimePartitioning) {
+ this.tableSpec = tableSpec;
this.tableDescription = tableDescription;
+ this.jsonTimePartitioning = jsonTimePartitioning;
}
public String getTableSpec() {
@@ -51,6 +71,18 @@
return BigQueryHelpers.parseTableSpec(tableSpec);
}
+ public String getJsonTimePartitioning() {
+ return jsonTimePartitioning;
+ }
+
+ public TimePartitioning getTimePartitioning() {
+ if (jsonTimePartitioning == null) {
+ return null;
+ } else {
+ return BigQueryHelpers.fromJsonString(jsonTimePartitioning, TimePartitioning.class);
+ }
+ }
+
@Nullable
public String getTableDescription() {
return tableDescription;
@@ -72,11 +104,12 @@
}
TableDestination other = (TableDestination) o;
return Objects.equals(this.tableSpec, other.tableSpec)
- && Objects.equals(this.tableDescription, other.tableDescription);
+ && Objects.equals(this.tableDescription, other.tableDescription)
+ && Objects.equals(this.jsonTimePartitioning, other.jsonTimePartitioning);
}
@Override
public int hashCode() {
- return Objects.hash(tableSpec, tableDescription);
+ return Objects.hash(tableSpec, tableDescription, jsonTimePartitioning);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
index f034a03..2bfc2ca 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
@@ -33,6 +33,8 @@
private static final Coder<String> tableSpecCoder = StringUtf8Coder.of();
private static final Coder<String> tableDescriptionCoder = NullableCoder.of(StringUtf8Coder.of());
+ private TableDestinationCoder() {}
+
public static TableDestinationCoder of() {
return INSTANCE;
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoderV2.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoderV2.java
new file mode 100644
index 0000000..5bdab0d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoderV2.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.NullableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+
+/**
+ * A {@link Coder} for {@link TableDestination} that includes time partitioning information. This
+ * is a new coder (instead of extending the old {@link TableDestinationCoder}) for compatibility
+ * reasons. The old coder is kept around for the same compatibility reasons.
+ */
+public class TableDestinationCoderV2 extends AtomicCoder<TableDestination> {
+ private static final TableDestinationCoderV2 INSTANCE = new TableDestinationCoderV2();
+ private static final Coder<String> timePartitioningCoder = NullableCoder.of(StringUtf8Coder.of());
+
+ public static TableDestinationCoderV2 of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(TableDestination value, OutputStream outStream) throws IOException {
+ TableDestinationCoder.of().encode(value, outStream);
+ timePartitioningCoder.encode(value.getJsonTimePartitioning(), outStream);
+ }
+
+ @Override
+ public TableDestination decode(InputStream inStream) throws IOException {
+ TableDestination destination = TableDestinationCoder.of().decode(inStream);
+ String jsonTimePartitioning = timePartitioningCoder.decode(inStream);
+ return new TableDestination(
+ destination.getTableSpec(), destination.getTableDescription(), jsonTimePartitioning);
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {}
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
index cd88222..51b9375 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
@@ -26,6 +26,7 @@
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ShardedKey;
/**
* Fn that tags each table row with a unique id and destination table. To avoid calling
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
deleted file mode 100644
index b8e6b39..0000000
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import java.io.IOException;
-import java.util.List;
-import java.util.NoSuchElementException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.joda.time.Instant;
-
-/**
- * A {@link BoundedSource} that reads from {@code BoundedSource<T>}
- * and transforms elements to type {@code V}.
-*/
-@VisibleForTesting
-class TransformingSource<T, V> extends BoundedSource<V> {
- private final BoundedSource<T> boundedSource;
- private final SerializableFunction<T, V> function;
- private final Coder<V> outputCoder;
-
- TransformingSource(
- BoundedSource<T> boundedSource,
- SerializableFunction<T, V> function,
- Coder<V> outputCoder) {
- this.boundedSource = checkNotNull(boundedSource, "boundedSource");
- this.function = checkNotNull(function, "function");
- this.outputCoder = checkNotNull(outputCoder, "outputCoder");
- }
-
- @Override
- public List<? extends BoundedSource<V>> split(
- long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
- return Lists.transform(
- boundedSource.split(desiredBundleSizeBytes, options),
- new Function<BoundedSource<T>, BoundedSource<V>>() {
- @Override
- public BoundedSource<V> apply(BoundedSource<T> input) {
- return new TransformingSource<>(input, function, outputCoder);
- }
- });
- }
-
- @Override
- public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
- return boundedSource.getEstimatedSizeBytes(options);
- }
-
- @Override
- public BoundedReader<V> createReader(PipelineOptions options) throws IOException {
- return new TransformingReader(boundedSource.createReader(options));
- }
-
- @Override
- public void validate() {
- boundedSource.validate();
- }
-
- @Override
- public Coder<V> getDefaultOutputCoder() {
- return outputCoder;
- }
-
- private class TransformingReader extends BoundedReader<V> {
- private final BoundedReader<T> boundedReader;
-
- private TransformingReader(BoundedReader<T> boundedReader) {
- this.boundedReader = checkNotNull(boundedReader, "boundedReader");
- }
-
- @Override
- public synchronized BoundedSource<V> getCurrentSource() {
- return new TransformingSource<>(boundedReader.getCurrentSource(), function, outputCoder);
- }
-
- @Override
- public boolean start() throws IOException {
- return boundedReader.start();
- }
-
- @Override
- public boolean advance() throws IOException {
- return boundedReader.advance();
- }
-
- @Override
- public V getCurrent() throws NoSuchElementException {
- T current = boundedReader.getCurrent();
- return function.apply(current);
- }
-
- @Override
- public void close() throws IOException {
- boundedReader.close();
- }
-
- @Override
- public synchronized BoundedSource<V> splitAtFraction(double fraction) {
- BoundedSource<T> split = boundedReader.splitAtFraction(fraction);
- return split == null ? null : new TransformingSource<>(split, function, outputCoder);
- }
-
- @Override
- public Double getFractionConsumed() {
- return boundedReader.getFractionConsumed();
- }
-
- @Override
- public Instant getCurrentTimestamp() throws NoSuchElementException {
- return boundedReader.getCurrentTimestamp();
- }
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index f014039..017d5c1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -18,7 +18,7 @@
package org.apache.beam.sdk.io.gcp.bigquery;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation;
+import static com.google.common.base.Preconditions.checkNotNull;
import com.google.api.services.bigquery.model.TableRow;
import com.google.common.collect.Lists;
@@ -30,6 +30,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
@@ -40,6 +41,8 @@
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.ShardedKey;
import org.apache.beam.sdk.values.TupleTag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,26 +66,53 @@
// Map from tablespec to a writer for that table.
private transient Map<DestinationT, TableRowWriter> writers;
private transient Map<DestinationT, BoundedWindow> writerWindows;
- private final String stepUuid;
- private final TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag;
+ private final PCollectionView<String> tempFilePrefixView;
+ private final TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittenRecordsTag;
private int maxNumWritersPerBundle;
private long maxFileSize;
+ private int spilledShardNumber;
/**
* The result of the {@link WriteBundlesToFiles} transform. Corresponds to a single output file,
* and encapsulates the table it is destined to as well as the file byte size.
*/
- public static final class Result<DestinationT> implements Serializable {
+ static final class Result<DestinationT> implements Serializable {
private static final long serialVersionUID = 1L;
public final String filename;
public final Long fileByteSize;
public final DestinationT destination;
public Result(String filename, Long fileByteSize, DestinationT destination) {
+ checkNotNull(destination);
this.filename = filename;
this.fileByteSize = fileByteSize;
this.destination = destination;
}
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof Result) {
+ Result<DestinationT> o = (Result<DestinationT>) other;
+ return Objects.equals(this.filename, o.filename)
+ && Objects.equals(this.fileByteSize, o.fileByteSize)
+ && Objects.equals(this.destination, o.destination);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(filename, fileByteSize, destination);
+ }
+
+ @Override
+ public String toString() {
+ return "Result{"
+ + "filename='" + filename + '\''
+ + ", fileByteSize=" + fileByteSize
+ + ", destination=" + destination
+ + '}';
+ }
}
/** a coder for the {@link Result} class. */
@@ -129,12 +159,12 @@
}
WriteBundlesToFiles(
- String stepUuid,
- TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag,
+ PCollectionView<String> tempFilePrefixView,
+ TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittenRecordsTag,
int maxNumWritersPerBundle,
long maxFileSize) {
- this.stepUuid = stepUuid;
- this.unwrittedRecordsTag = unwrittedRecordsTag;
+ this.tempFilePrefixView = tempFilePrefixView;
+ this.unwrittenRecordsTag = unwrittenRecordsTag;
this.maxNumWritersPerBundle = maxNumWritersPerBundle;
this.maxFileSize = maxFileSize;
}
@@ -145,6 +175,7 @@
// bundles.
this.writers = Maps.newHashMap();
this.writerWindows = Maps.newHashMap();
+ this.spilledShardNumber = ThreadLocalRandom.current().nextInt(SPILLED_RECORD_SHARDING_FACTOR);
}
TableRowWriter createAndInsertWriter(DestinationT destination, String tempFilePrefix,
@@ -157,8 +188,7 @@
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
- String tempFilePrefix = resolveTempLocation(
- c.getPipelineOptions().getTempLocation(), "BigQueryWriteTemp", stepUuid);
+ String tempFilePrefix = c.sideInput(tempFilePrefixView);
DestinationT destination = c.element().getKey();
TableRowWriter writer;
@@ -172,9 +202,10 @@
} else {
// This means that we already had too many writers open in this bundle. "spill" this record
// into the output. It will be grouped and written to a file in a subsequent stage.
- c.output(unwrittedRecordsTag,
- KV.of(ShardedKey.of(destination,
- ThreadLocalRandom.current().nextInt(SPILLED_RECORD_SHARDING_FACTOR)),
+ c.output(
+ unwrittenRecordsTag,
+ KV.of(
+ ShardedKey.of(destination, (++spilledShardNumber) % SPILLED_RECORD_SHARDING_FACTOR),
c.element().getValue()));
return;
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java
index 45dc2a8..e82b29d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java
@@ -22,6 +22,7 @@
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.ShardedKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,18 +48,21 @@
public void processElement(ProcessContext c) throws Exception {
String tempFilePrefix = c.sideInput(this.tempFilePrefix);
TableRowWriter writer = new TableRowWriter(tempFilePrefix);
- try (TableRowWriter ignored = writer) {
+ try {
for (TableRow tableRow : c.element().getValue()) {
if (writer.getByteSize() > maxFileSize) {
writer.close();
+ writer = new TableRowWriter(tempFilePrefix);
TableRowWriter.Result result = writer.getResult();
c.output(new WriteBundlesToFiles.Result<>(
result.resourceId.toString(), result.byteSize, c.element().getKey().getKey()));
- writer = new TableRowWriter(tempFilePrefix);
}
writer.write(tableRow);
}
+ } finally {
+ writer.close();
}
+
TableRowWriter.Result result = writer.getResult();
c.output(
new WriteBundlesToFiles.Result<>(
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
index 24693da..934f1ae 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
@@ -22,10 +22,12 @@
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.ShardedKey;
import org.apache.beam.sdk.values.TupleTag;
/**
@@ -33,11 +35,13 @@
* tablespec and the list of files corresponding to each partition of that table.
*/
class WritePartition<DestinationT>
- extends DoFn<Void, KV<ShardedKey<DestinationT>, List<String>>> {
+ extends DoFn<
+ Iterable<WriteBundlesToFiles.Result<DestinationT>>,
+ KV<ShardedKey<DestinationT>, List<String>>> {
private final boolean singletonTable;
+ private final DynamicDestinations<?, DestinationT> dynamicDestinations;
private final PCollectionView<String> tempFilePrefix;
- private final PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> results;
- private TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag;
+ @Nullable private TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag;
private TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag;
private static class PartitionData {
@@ -100,12 +104,12 @@
WritePartition(
boolean singletonTable,
+ DynamicDestinations<?, DestinationT> dynamicDestinations,
PCollectionView<String> tempFilePrefix,
- PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> results,
TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag,
TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag) {
this.singletonTable = singletonTable;
- this.results = results;
+ this.dynamicDestinations = dynamicDestinations;
this.tempFilePrefix = tempFilePrefix;
this.multiPartitionsTag = multiPartitionsTag;
this.singlePartitionTag = singlePartitionTag;
@@ -113,8 +117,7 @@
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
- List<WriteBundlesToFiles.Result<DestinationT>> results =
- Lists.newArrayList(c.sideInput(this.results));
+ List<WriteBundlesToFiles.Result<DestinationT>> results = Lists.newArrayList(c.element());
// If there are no elements to write _and_ the user specified a constant output table, then
// generate an empty table of that name.
@@ -126,8 +129,8 @@
// Return a null destination in this case - the constant DynamicDestinations class will
// resolve it to the singleton output table.
results.add(
- new Result<DestinationT>(
- writerResult.resourceId.toString(), writerResult.byteSize, null));
+ new Result<>(writerResult.resourceId.toString(), writerResult.byteSize,
+ dynamicDestinations.getDestination(null)));
}
Map<DestinationT, DestinationData> currentResults = Maps.newHashMap();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
index f641b32..ff69476 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
@@ -22,9 +22,11 @@
import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -35,74 +37,86 @@
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Copies temporary tables to destination table.
+ * Copies temporary tables to destination table. The input element is an {@link Iterable} that
+ * provides the list of all temporary tables created for a given {@link TableDestination}.
*/
-class WriteRename extends DoFn<Void, Void> {
+class WriteRename extends DoFn<Iterable<KV<TableDestination, String>>, Void> {
private static final Logger LOG = LoggerFactory.getLogger(WriteRename.class);
private final BigQueryServices bqServices;
private final PCollectionView<String> jobIdToken;
- private final WriteDisposition writeDisposition;
- private final CreateDisposition createDisposition;
- // Map from final destination to a list of temporary tables that need to be copied into it.
- private final PCollectionView<Map<TableDestination, Iterable<String>>> tempTablesView;
+ // In the triggered scenario, the user-supplied create and write dispositions only apply to the
+ // first trigger pane, as that's when when the table is created. Subsequent loads should always
+ // append to the table, and so use CREATE_NEVER and WRITE_APPEND dispositions respectively.
+ private final WriteDisposition firstPaneWriteDisposition;
+ private final CreateDisposition firstPaneCreateDisposition;
public WriteRename(
BigQueryServices bqServices,
PCollectionView<String> jobIdToken,
WriteDisposition writeDisposition,
- CreateDisposition createDisposition,
- PCollectionView<Map<TableDestination, Iterable<String>>> tempTablesView) {
+ CreateDisposition createDisposition) {
this.bqServices = bqServices;
this.jobIdToken = jobIdToken;
- this.writeDisposition = writeDisposition;
- this.createDisposition = createDisposition;
- this.tempTablesView = tempTablesView;
+ this.firstPaneWriteDisposition = writeDisposition;
+ this.firstPaneCreateDisposition = createDisposition;
}
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
- Map<TableDestination, Iterable<String>> tempTablesMap =
- Maps.newHashMap(c.sideInput(tempTablesView));
-
- // Process each destination table.
- for (Map.Entry<TableDestination, Iterable<String>> entry : tempTablesMap.entrySet()) {
- TableDestination finalTableDestination = entry.getKey();
- List<String> tempTablesJson = Lists.newArrayList(entry.getValue());
- // Do not copy if no temp tables are provided
- if (tempTablesJson.size() == 0) {
- return;
- }
-
- List<TableReference> tempTables = Lists.newArrayList();
- for (String table : tempTablesJson) {
- tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
- }
-
- // Make sure each destination table gets a unique job id.
- String jobIdPrefix = BigQueryHelpers.createJobId(
- c.sideInput(jobIdToken), finalTableDestination, -1);
-
- copy(
- bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
- jobIdPrefix,
- finalTableDestination.getTableReference(),
- tempTables,
- writeDisposition,
- createDisposition,
- finalTableDestination.getTableDescription());
-
- DatasetService tableService =
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
- removeTemporaryTables(tableService, tempTables);
+ Multimap<TableDestination, String> tempTables = ArrayListMultimap.create();
+ for (KV<TableDestination, String> entry : c.element()) {
+ tempTables.put(entry.getKey(), entry.getValue());
}
+ for (Map.Entry<TableDestination, Collection<String>> entry : tempTables.asMap().entrySet()) {
+ // Process each destination table.
+ writeRename(entry.getKey(), entry.getValue(), c);
+ }
+ }
+
+ private void writeRename(
+ TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c)
+ throws Exception {
+ WriteDisposition writeDisposition =
+ (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND;
+ CreateDisposition createDisposition =
+ (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER;
+ List<String> tempTablesJson = Lists.newArrayList(tempTableNames);
+ // Do not copy if no temp tables are provided
+ if (tempTablesJson.size() == 0) {
+ return;
+ }
+
+ List<TableReference> tempTables = Lists.newArrayList();
+ for (String table : tempTablesJson) {
+ tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
+ }
+
+ // Make sure each destination table gets a unique job id.
+ String jobIdPrefix =
+ BigQueryHelpers.createJobId(c.sideInput(jobIdToken), finalTableDestination, -1,
+ c.pane().getIndex());
+
+ copy(
+ bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ jobIdPrefix,
+ finalTableDestination.getTableReference(),
+ tempTables,
+ writeDisposition,
+ createDisposition,
+ finalTableDestination.getTableDescription());
+
+ DatasetService tableService =
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
+ removeTemporaryTables(tableService, tempTables);
}
private void copy(
@@ -174,9 +188,11 @@
super.populateDisplayData(builder);
builder
- .add(DisplayData.item("writeDisposition", writeDisposition.toString())
- .withLabel("Write Disposition"))
- .add(DisplayData.item("createDisposition", createDisposition.toString())
- .withLabel("Create Disposition"));
+ .add(
+ DisplayData.item("firstPaneWriteDisposition", firstPaneWriteDisposition.toString())
+ .withLabel("Write Disposition"))
+ .add(
+ DisplayData.item("firstPaneCreateDisposition", firstPaneCreateDisposition.toString())
+ .withLabel("Create Disposition"));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
index db0be3a..4f6b23e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
@@ -17,10 +17,12 @@
*/
package org.apache.beam.sdk.io.gcp.bigquery;
-import java.util.Collections;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.collect.ImmutableMap;
import java.util.Map;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
@@ -30,23 +32,30 @@
* The result of a {@link BigQueryIO.Write} transform.
*/
public final class WriteResult implements POutput {
-
private final Pipeline pipeline;
+ private final TupleTag<TableRow> failedInsertsTag;
+ private final PCollection<TableRow> failedInserts;
- /**
- * Creates a {@link WriteResult} in the given {@link Pipeline}.
- */
- static WriteResult in(Pipeline pipeline) {
- return new WriteResult(pipeline);
+ /** Creates a {@link WriteResult} in the given {@link Pipeline}. */
+ static WriteResult in(
+ Pipeline pipeline, TupleTag<TableRow> failedInsertsTag, PCollection<TableRow> failedInserts) {
+ return new WriteResult(pipeline, failedInsertsTag, failedInserts);
}
@Override
public Map<TupleTag<?>, PValue> expand() {
- return Collections.emptyMap();
+ return ImmutableMap.<TupleTag<?>, PValue>of(failedInsertsTag, failedInserts);
}
- private WriteResult(Pipeline pipeline) {
+ private WriteResult(
+ Pipeline pipeline, TupleTag<TableRow> failedInsertsTag, PCollection<TableRow> failedInserts) {
this.pipeline = pipeline;
+ this.failedInsertsTag = failedInsertsTag;
+ this.failedInserts = failedInserts;
+ }
+
+ public PCollection<TableRow> getFailedInserts() {
+ return failedInserts;
}
@Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
index c5494d8..f8ed796 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -23,16 +23,19 @@
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.api.services.bigquery.model.TimePartitioning;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import java.io.IOException;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.io.fs.MoveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
@@ -40,8 +43,22 @@
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.ShardedKey;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,79 +75,150 @@
* {@link KV} maps the final table to itself.
*/
class WriteTables<DestinationT>
- extends DoFn<KV<ShardedKey<DestinationT>, List<String>>, KV<TableDestination, String>> {
+ extends PTransform<PCollection<KV<ShardedKey<DestinationT>, List<String>>>,
+ PCollection<KV<TableDestination, String>>> {
private static final Logger LOG = LoggerFactory.getLogger(WriteTables.class);
private final boolean singlePartition;
private final BigQueryServices bqServices;
private final PCollectionView<String> jobIdToken;
- private final PCollectionView<Map<DestinationT, String>> schemasView;
- private final WriteDisposition writeDisposition;
- private final CreateDisposition createDisposition;
+ private final WriteDisposition firstPaneWriteDisposition;
+ private final CreateDisposition firstPaneCreateDisposition;
private final DynamicDestinations<?, DestinationT> dynamicDestinations;
+ private final List<PCollectionView<?>> sideInputs;
+ private final TupleTag<KV<TableDestination, String>> mainOutputTag;
+ private final TupleTag<String> temporaryFilesTag;
+
+
+ private class WriteTablesDoFn
+ extends DoFn<KV<ShardedKey<DestinationT>, List<String>>, KV<TableDestination, String>> {
+ private Map<DestinationT, String> jsonSchemas = Maps.newHashMap();
+
+ @StartBundle
+ public void startBundle(StartBundleContext c) {
+ // Clear the map on each bundle so we can notice side-input updates.
+ // (alternative is to use a cache with a TTL).
+ jsonSchemas.clear();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ dynamicDestinations.setSideInputAccessorFromProcessContext(c);
+ DestinationT destination = c.element().getKey().getKey();
+ TableSchema tableSchema;
+ String jsonSchema = jsonSchemas.get(destination);
+ if (jsonSchema != null) {
+ tableSchema = BigQueryHelpers.fromJsonString(jsonSchema, TableSchema.class);
+ } else {
+ tableSchema = dynamicDestinations.getSchema(destination);
+ if (tableSchema != null) {
+ jsonSchemas.put(destination, BigQueryHelpers.toJsonString(tableSchema));
+ }
+ }
+
+ TableDestination tableDestination = dynamicDestinations.getTable(destination);
+ TableReference tableReference = tableDestination.getTableReference();
+ if (Strings.isNullOrEmpty(tableReference.getProjectId())) {
+ tableReference.setProjectId(
+ c.getPipelineOptions().as(BigQueryOptions.class).getProject());
+ tableDestination = new TableDestination(
+ tableReference, tableDestination.getTableDescription());
+ }
+
+ Integer partition = c.element().getKey().getShardNumber();
+ List<String> partitionFiles = Lists.newArrayList(c.element().getValue());
+ String jobIdPrefix = BigQueryHelpers.createJobId(
+ c.sideInput(jobIdToken), tableDestination, partition, c.pane().getIndex());
+
+ if (!singlePartition) {
+ tableReference.setTableId(jobIdPrefix);
+ }
+
+ WriteDisposition writeDisposition =
+ (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND;
+ CreateDisposition createDisposition =
+ (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER;
+ load(
+ bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ jobIdPrefix,
+ tableReference,
+ tableDestination.getTimePartitioning(),
+ tableSchema,
+ partitionFiles,
+ writeDisposition,
+ createDisposition,
+ tableDestination.getTableDescription());
+ c.output(
+ mainOutputTag, KV.of(tableDestination, BigQueryHelpers.toJsonString(tableReference)));
+ for (String file : partitionFiles) {
+ c.output(temporaryFilesTag, file);
+ }
+ }
+ }
+
+ private class GarbageCollectTemporaryFiles extends DoFn<Iterable<String>, Void> {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ removeTemporaryFiles(c.element());
+ }
+ }
public WriteTables(
boolean singlePartition,
BigQueryServices bqServices,
PCollectionView<String> jobIdToken,
- PCollectionView<Map<DestinationT, String>> schemasView,
WriteDisposition writeDisposition,
CreateDisposition createDisposition,
+ List<PCollectionView<?>> sideInputs,
DynamicDestinations<?, DestinationT> dynamicDestinations) {
this.singlePartition = singlePartition;
this.bqServices = bqServices;
this.jobIdToken = jobIdToken;
- this.schemasView = schemasView;
- this.writeDisposition = writeDisposition;
- this.createDisposition = createDisposition;
+ this.firstPaneWriteDisposition = writeDisposition;
+ this.firstPaneCreateDisposition = createDisposition;
+ this.sideInputs = sideInputs;
this.dynamicDestinations = dynamicDestinations;
+ this.mainOutputTag = new TupleTag<>("WriteTablesMainOutput");
+ this.temporaryFilesTag = new TupleTag<>("TemporaryFiles");
}
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- dynamicDestinations.setSideInputAccessorFromProcessContext(c);
- DestinationT destination = c.element().getKey().getKey();
- TableSchema tableSchema =
- BigQueryHelpers.fromJsonString(
- c.sideInput(schemasView).get(destination), TableSchema.class);
- TableDestination tableDestination = dynamicDestinations.getTable(destination);
- TableReference tableReference = tableDestination.getTableReference();
- if (Strings.isNullOrEmpty(tableReference.getProjectId())) {
- tableReference.setProjectId(
- c.getPipelineOptions().as(BigQueryOptions.class).getProject());
- tableDestination = new TableDestination(
- tableReference, tableDestination.getTableDescription());
- }
+ @Override
+ public PCollection<KV<TableDestination, String>> expand(
+ PCollection<KV<ShardedKey<DestinationT>, List<String>>> input) {
+ PCollectionTuple writeTablesOutputs = input.apply(ParDo.of(new WriteTablesDoFn())
+ .withSideInputs(sideInputs)
+ .withOutputTags(mainOutputTag, TupleTagList.of(temporaryFilesTag)));
- Integer partition = c.element().getKey().getShardNumber();
- List<String> partitionFiles = Lists.newArrayList(c.element().getValue());
- String jobIdPrefix =
- BigQueryHelpers.createJobId(c.sideInput(jobIdToken), tableDestination, partition);
+ // Garbage collect temporary files.
+ // We mustn't start garbage collecting files until we are assured that the WriteTablesDoFn has
+ // succeeded in loading those files and won't be retried. Otherwise, we might fail part of the
+ // way through deleting temporary files, and retry WriteTablesDoFn. This will then fail due
+ // to missing files, causing either the entire workflow to fail or get stuck (depending on how
+ // the runner handles persistent failures).
+ writeTablesOutputs
+ .get(temporaryFilesTag)
+ .setCoder(StringUtf8Coder.of())
+ .apply(WithKeys.<Void, String>of((Void) null))
+ .setCoder(KvCoder.of(VoidCoder.of(), StringUtf8Coder.of()))
+ .apply(Window.<KV<Void, String>>into(new GlobalWindows())
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))
+ .discardingFiredPanes())
+ .apply(GroupByKey.<Void, String>create())
+ .apply(Values.<Iterable<String>>create())
+ .apply(ParDo.of(new GarbageCollectTemporaryFiles()));
- if (!singlePartition) {
- tableReference.setTableId(jobIdPrefix);
- }
-
- load(
- bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
- jobIdPrefix,
- tableReference,
- tableSchema,
- partitionFiles,
- writeDisposition,
- createDisposition,
- tableDestination.getTableDescription());
- c.output(KV.of(tableDestination, BigQueryHelpers.toJsonString(tableReference)));
-
- removeTemporaryFiles(partitionFiles);
+ return writeTablesOutputs.get(mainOutputTag);
}
+
+
private void load(
JobService jobService,
DatasetService datasetService,
String jobIdPrefix,
TableReference ref,
+ TimePartitioning timePartitioning,
@Nullable TableSchema schema,
List<String> gcsUris,
WriteDisposition writeDisposition,
@@ -145,7 +233,9 @@
.setWriteDisposition(writeDisposition.name())
.setCreateDisposition(createDisposition.name())
.setSourceFormat("NEWLINE_DELIMITED_JSON");
-
+ if (timePartitioning != null) {
+ loadConfig.setTimePartitioning(timePartitioning);
+ }
String projectId = ref.getProjectId();
Job lastFailedLoadJob = null;
for (int i = 0; i < BatchLoads.MAX_RETRY_JOBS; ++i) {
@@ -184,11 +274,11 @@
BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
}
- static void removeTemporaryFiles(Collection<String> files) throws IOException {
+ static void removeTemporaryFiles(Iterable<String> files) throws IOException {
ImmutableList.Builder<ResourceId> fileResources = ImmutableList.builder();
- for (String file: files) {
+ for (String file : files) {
fileResources.add(FileSystems.matchNewResource(file, false/* isDirectory */));
}
- FileSystems.delete(fileResources.build(), MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
+ FileSystems.delete(fileResources.build());
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 22e9f36..29dc269 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -33,6 +33,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.protobuf.ByteString;
@@ -59,7 +60,6 @@
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
@@ -139,6 +139,30 @@
* .withTableId("table"));
* }</pre>
*
+ * <h3>Using local emulator</h3>
+ *
+ * <p>In order to use local emulator for Bigtable you should use:
+ *
+ * <pre>{@code
+ * BigtableOptions.Builder optionsBuilder =
+ * new BigtableOptions.Builder()
+ * .setProjectId("project")
+ * .setInstanceId("instance")
+ * .setUsePlaintextNegotiation(true)
+ * .setCredentialOptions(CredentialOptions.nullCredential())
+ * .setDataHost("127.0.0.1") // network interface where Bigtable emulator is bound
+ * .setInstanceAdminHost("127.0.0.1")
+ * .setTableAdminHost("127.0.0.1")
+ * .setPort(LOCAL_EMULATOR_PORT))
+ *
+ * PCollection<KV<ByteString, Iterable<Mutation>>> data = ...;
+ *
+ * data.apply("write",
+ * BigtableIO.write()
+ * .withBigtableOptions(optionsBuilder)
+ * .withTableId("table");
+ * }</pre>
+ *
* <h3>Experimental</h3>
*
* <p>This connector for Cloud Bigtable is considered experimental and may break or receive
@@ -151,7 +175,7 @@
* pipeline. Please refer to the documentation of corresponding
* {@link PipelineRunner PipelineRunners} for more details.
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class BigtableIO {
private static final Logger LOG = LoggerFactory.getLogger(BigtableIO.class);
@@ -165,7 +189,10 @@
*/
@Experimental
public static Read read() {
- return new AutoValue_BigtableIO_Read.Builder().setKeyRange(ByteKeyRange.ALL_KEYS).setTableId("")
+ return new AutoValue_BigtableIO_Read.Builder()
+ .setKeyRange(ByteKeyRange.ALL_KEYS)
+ .setTableId("")
+ .setValidate(true)
.build();
}
@@ -178,7 +205,10 @@
*/
@Experimental
public static Write write() {
- return new AutoValue_BigtableIO_Write.Builder().setTableId("").build();
+ return new AutoValue_BigtableIO_Write.Builder()
+ .setTableId("")
+ .setValidate(true)
+ .build();
}
/**
@@ -187,7 +217,7 @@
*
* @see BigtableIO
*/
- @Experimental
+ @Experimental(Experimental.Kind.SOURCE_SINK)
@AutoValue
public abstract static class Read extends PTransform<PBegin, PCollection<Row>> {
@@ -205,11 +235,12 @@
@Nullable
abstract BigtableService getBigtableService();
-
/** Returns the Google Cloud Bigtable instance being read from, and other parameters. */
@Nullable
public abstract BigtableOptions getBigtableOptions();
+ public abstract boolean getValidate();
+
abstract Builder toBuilder();
@AutoValue.Builder
@@ -225,6 +256,8 @@
abstract Builder setBigtableService(BigtableService bigtableService);
+ abstract Builder setValidate(boolean validate);
+
abstract Read build();
}
@@ -235,7 +268,7 @@
* <p>Does not modify this object.
*/
public Read withBigtableOptions(BigtableOptions options) {
- checkNotNull(options, "options");
+ checkArgument(options != null, "options can not be null");
return withBigtableOptions(options.toBuilder());
}
@@ -249,16 +282,15 @@
* <p>Does not modify this object.
*/
public Read withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
- checkNotNull(optionsBuilder, "optionsBuilder");
+ checkArgument(optionsBuilder != null, "optionsBuilder can not be null");
// TODO: is there a better way to clone a Builder? Want it to be immune from user changes.
BigtableOptions options = optionsBuilder.build();
BigtableOptions.Builder clonedBuilder = options.toBuilder()
.setUseCachedDataPool(true);
- BigtableOptions optionsWithAgent =
- clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build();
+ BigtableOptions clonedOptions = clonedBuilder.build();
- return toBuilder().setBigtableOptions(optionsWithAgent).build();
+ return toBuilder().setBigtableOptions(clonedOptions).build();
}
/**
@@ -268,7 +300,7 @@
* <p>Does not modify this object.
*/
public Read withRowFilter(RowFilter filter) {
- checkNotNull(filter, "filter");
+ checkArgument(filter != null, "filter can not be null");
return toBuilder().setRowFilter(filter).build();
}
@@ -278,7 +310,7 @@
* <p>Does not modify this object.
*/
public Read withKeyRange(ByteKeyRange keyRange) {
- checkNotNull(keyRange, "keyRange");
+ checkArgument(keyRange != null, "keyRange can not be null");
return toBuilder().setKeyRange(keyRange).build();
}
@@ -288,12 +320,19 @@
* <p>Does not modify this object.
*/
public Read withTableId(String tableId) {
- checkNotNull(tableId, "tableId");
+ checkArgument(tableId != null, "tableId can not be null");
return toBuilder().setTableId(tableId).build();
}
+ /** Disables validation that the table being read from exists. */
+ public Read withoutValidation() {
+ return toBuilder().setValidate(false).build();
+ }
+
@Override
public PCollection<Row> expand(PBegin input) {
+ checkArgument(getBigtableOptions() != null, "withBigtableOptions() is required");
+ checkArgument(getTableId() != null && !getTableId().isEmpty(), "withTableId() is required");
BigtableSource source =
new BigtableSource(new SerializableFunction<PipelineOptions, BigtableService>() {
@Override
@@ -306,15 +345,15 @@
@Override
public void validate(PipelineOptions options) {
- checkArgument(getBigtableOptions() != null, "BigtableOptions not specified");
- checkArgument(getTableId() != null && !getTableId().isEmpty(), "Table ID not specified");
- try {
- checkArgument(
- getBigtableService(options).tableExists(getTableId()),
- "Table %s does not exist",
- getTableId());
- } catch (IOException e) {
- LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
+ if (getValidate()) {
+ try {
+ checkArgument(
+ getBigtableService(options).tableExists(getTableId()),
+ "Table %s does not exist",
+ getTableId());
+ } catch (IOException e) {
+ LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
+ }
}
}
@@ -358,7 +397,7 @@
* <p>Does not modify this object.
*/
Read withBigtableService(BigtableService bigtableService) {
- checkNotNull(bigtableService, "bigtableService");
+ checkArgument(bigtableService != null, "bigtableService can not be null");
return toBuilder().setBigtableService(bigtableService).build();
}
@@ -376,6 +415,7 @@
return getBigtableService();
}
BigtableOptions.Builder clonedOptions = getBigtableOptions().toBuilder();
+ clonedOptions.setUserAgent(pipelineOptions.getUserAgent());
if (getBigtableOptions().getCredentialOptions()
.getCredentialType() == CredentialType.DefaultCredentials) {
clonedOptions.setCredentialOptions(
@@ -392,7 +432,7 @@
*
* @see BigtableIO
*/
- @Experimental
+ @Experimental(Experimental.Kind.SOURCE_SINK)
@AutoValue
public abstract static class Write
extends PTransform<PCollection<KV<ByteString, Iterable<Mutation>>>, PDone> {
@@ -408,6 +448,8 @@
@Nullable
public abstract BigtableOptions getBigtableOptions();
+ abstract boolean getValidate();
+
abstract Builder toBuilder();
@AutoValue.Builder
@@ -419,6 +461,8 @@
abstract Builder setBigtableService(BigtableService bigtableService);
+ abstract Builder setValidate(boolean validate);
+
abstract Write build();
}
@@ -429,7 +473,6 @@
* <p>Does not modify this object.
*/
public Write withBigtableOptions(BigtableOptions options) {
- checkNotNull(options, "options");
return withBigtableOptions(options.toBuilder());
}
@@ -443,7 +486,7 @@
* <p>Does not modify this object.
*/
public Write withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
- checkNotNull(optionsBuilder, "optionsBuilder");
+ checkArgument(optionsBuilder != null, "optionsBuilder can not be null");
// TODO: is there a better way to clone a Builder? Want it to be immune from user changes.
BigtableOptions options = optionsBuilder.build();
@@ -454,9 +497,13 @@
.setUseBulkApi(true)
.build())
.setUseCachedDataPool(true);
- BigtableOptions optionsWithAgent =
- clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build();
- return toBuilder().setBigtableOptions(optionsWithAgent).build();
+ BigtableOptions clonedOptions = clonedBuilder.build();
+ return toBuilder().setBigtableOptions(clonedOptions).build();
+ }
+
+ /** Disables validation that the table being written to exists. */
+ public Write withoutValidation() {
+ return toBuilder().setValidate(false).build();
}
/**
@@ -465,12 +512,15 @@
* <p>Does not modify this object.
*/
public Write withTableId(String tableId) {
- checkNotNull(tableId, "tableId");
+ checkArgument(tableId != null, "tableId can not be null");
return toBuilder().setTableId(tableId).build();
}
@Override
public PDone expand(PCollection<KV<ByteString, Iterable<Mutation>>> input) {
+ checkArgument(getBigtableOptions() != null, "withBigtableOptions() is required");
+ checkArgument(getTableId() != null && !getTableId().isEmpty(), "withTableId() is required");
+
input.apply(ParDo.of(new BigtableWriterFn(getTableId(),
new SerializableFunction<PipelineOptions, BigtableService>() {
@Override
@@ -483,15 +533,15 @@
@Override
public void validate(PipelineOptions options) {
- checkArgument(getBigtableOptions() != null, "BigtableOptions not specified");
- checkArgument(getTableId() != null && !getTableId().isEmpty(), "Table ID not specified");
- try {
- checkArgument(
- getBigtableService(options).tableExists(getTableId()),
- "Table %s does not exist",
- getTableId());
- } catch (IOException e) {
- LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
+ if (getValidate()) {
+ try {
+ checkArgument(
+ getBigtableService(options).tableExists(getTableId()),
+ "Table %s does not exist",
+ getTableId());
+ } catch (IOException e) {
+ LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
+ }
}
}
@@ -504,7 +554,7 @@
* <p>Does not modify this object.
*/
Write withBigtableService(BigtableService bigtableService) {
- checkNotNull(bigtableService, "bigtableService");
+ checkArgument(bigtableService != null, "bigtableService can not be null");
return toBuilder().setBigtableService(bigtableService).build();
}
@@ -543,6 +593,7 @@
return getBigtableService();
}
BigtableOptions.Builder clonedOptions = getBigtableOptions().toBuilder();
+ clonedOptions.setUserAgent(pipelineOptions.getUserAgent());
if (getBigtableOptions().getCredentialOptions()
.getCredentialType() == CredentialType.DefaultCredentials) {
clonedOptions.setCredentialOptions(
@@ -588,8 +639,10 @@
@Teardown
public void tearDown() throws Exception {
- bigtableWriter.close();
- bigtableWriter = null;
+ if (bigtableWriter != null) {
+ bigtableWriter.close();
+ bigtableWriter = null;
+ }
}
@Override
@@ -616,12 +669,14 @@
StringBuilder logEntry = new StringBuilder();
int i = 0;
+ List<BigtableWriteException> suppressed = Lists.newArrayList();
for (; i < 10 && !failures.isEmpty(); ++i) {
BigtableWriteException exc = failures.remove();
logEntry.append("\n").append(exc.getMessage());
if (exc.getCause() != null) {
logEntry.append(": ").append(exc.getCause().getMessage());
}
+ suppressed.add(exc);
}
String message =
String.format(
@@ -630,7 +685,11 @@
i,
logEntry.toString());
LOG.error(message);
- throw new IOException(message);
+ IOException exception = new IOException(message);
+ for (BigtableWriteException e : suppressed) {
+ exception.addSuppressed(e);
+ }
+ throw exception;
}
private class WriteExceptionCallback implements FutureCallback<MutateRowResponse> {
@@ -692,19 +751,19 @@
@Nullable private transient List<SampleRowKeysResponse> sampleRowKeys;
protected BigtableSource withStartKey(ByteKey startKey) {
- checkNotNull(startKey, "startKey");
+ checkArgument(startKey != null, "startKey can not be null");
return new BigtableSource(
serviceFactory, tableId, filter, range.withStartKey(startKey), estimatedSizeBytes);
}
protected BigtableSource withEndKey(ByteKey endKey) {
- checkNotNull(endKey, "endKey");
+ checkArgument(endKey != null, "endKey can not be null");
return new BigtableSource(
serviceFactory, tableId, filter, range.withEndKey(endKey), estimatedSizeBytes);
}
protected BigtableSource withEstimatedSizeBytes(Long estimatedSizeBytes) {
- checkNotNull(estimatedSizeBytes, "estimatedSizeBytes");
+ checkArgument(estimatedSizeBytes != null, "estimatedSizeBytes can not be null");
return new BigtableSource(serviceFactory, tableId, filter, range, estimatedSizeBytes);
}
@@ -871,7 +930,7 @@
}
@Override
- public Coder<Row> getDefaultOutputCoder() {
+ public Coder<Row> getOutputCoder() {
return ProtoCoder.of(Row.class);
}
@@ -1005,7 +1064,7 @@
"{}: Failed to interpolate key for fraction {}.", rangeTracker.getRange(), fraction, e);
return null;
}
- LOG.debug(
+ LOG.info(
"Proposing to split {} at fraction {} (key {})", rangeTracker, fraction, splitKey);
BigtableSource primary;
BigtableSource residual;
@@ -1042,18 +1101,4 @@
cause);
}
}
-
- /**
- * A helper function to produce a Cloud Bigtable user agent string. This need only include
- * information about the Apache Beam SDK itself, because Bigtable will automatically append
- * other relevant system and Bigtable client-specific version information.
- *
- * @see com.google.cloud.bigtable.config.BigtableVersionInfo
- */
- private static String getBeamSdkPartOfUserAgent() {
- ReleaseInfo info = ReleaseInfo.getReleaseInfo();
- return
- String.format("%s/%s", info.getName(), info.getVersion())
- .replace(" ", "_");
- }
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
index d1a17fe..78f721f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
@@ -168,20 +168,25 @@
private BigtableSession session;
private AsyncExecutor executor;
private BulkMutation bulkMutation;
- private final MutateRowRequest.Builder partialBuilder;
+ private final String tableName;
public BigtableWriterImpl(BigtableSession session, BigtableTableName tableName) {
this.session = session;
executor = session.createAsyncExecutor();
bulkMutation = session.createBulkMutation(tableName, executor);
-
- partialBuilder = MutateRowRequest.newBuilder().setTableName(tableName.toString());
+ this.tableName = tableName.toString();
}
@Override
public void flush() throws IOException {
if (bulkMutation != null) {
- bulkMutation.flush();
+ try {
+ bulkMutation.flush();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // We fail since flush() operation was interrupted.
+ throw new IOException(e);
+ }
executor.flush();
}
}
@@ -190,7 +195,13 @@
public void close() throws IOException {
try {
if (bulkMutation != null) {
- bulkMutation.flush();
+ try {
+ bulkMutation.flush();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // We fail since flush() operation was interrupted.
+ throw new IOException(e);
+ }
bulkMutation = null;
executor.flush();
executor = null;
@@ -208,8 +219,8 @@
KV<ByteString, Iterable<Mutation>> record)
throws IOException {
MutateRowRequest r =
- partialBuilder
- .clone()
+ MutateRowRequest.newBuilder()
+ .setTableName(tableName)
.setRowKey(record.getKey())
.addAllMutations(record.getValue())
.build();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java
new file mode 100644
index 0000000..ce6ebe6
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.datastore;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.Random;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.util.MovingFunction;
+
+
+/**
+ * An implementation of client-side adaptive throttling. See
+ * https://landing.google.com/sre/book/chapters/handling-overload.html#client-side-throttling-a7sYUg
+ * for a full discussion of the use case and algorithm applied.
+ */
+class AdaptiveThrottler {
+ private final MovingFunction successfulRequests;
+ private final MovingFunction allRequests;
+
+ /** The target ratio between requests sent and successful requests. This is "K" in the formula in
+ * https://landing.google.com/sre/book/chapters/handling-overload.html */
+ private final double overloadRatio;
+
+ /** The target minimum number of requests per samplePeriodMs, even if no requests succeed. Must be
+ * greater than 0, else we could throttle to zero. Because every decision is probabilistic, there
+ * is no guarantee that the request rate in any given interval will not be zero. (This is the +1
+ * from the formula in https://landing.google.com/sre/book/chapters/handling-overload.html */
+ private static final double MIN_REQUESTS = 1;
+ private final Random random;
+
+ /**
+ * @param samplePeriodMs the time window to keep of request history to inform throttling
+ * decisions.
+ * @param sampleUpdateMs the length of buckets within this time window.
+ * @param overloadRatio the target ratio between requests sent and successful requests. You should
+ * always set this to more than 1, otherwise the client would never try to send more requests than
+ * succeeded in the past - so it could never recover from temporary setbacks.
+ */
+ public AdaptiveThrottler(long samplePeriodMs, long sampleUpdateMs,
+ double overloadRatio) {
+ this(samplePeriodMs, sampleUpdateMs, overloadRatio, new Random());
+ }
+
+ @VisibleForTesting
+ AdaptiveThrottler(long samplePeriodMs, long sampleUpdateMs,
+ double overloadRatio, Random random) {
+ allRequests =
+ new MovingFunction(samplePeriodMs, sampleUpdateMs,
+ 1 /* numSignificantBuckets */, 1 /* numSignificantSamples */, Sum.ofLongs());
+ successfulRequests =
+ new MovingFunction(samplePeriodMs, sampleUpdateMs,
+ 1 /* numSignificantBuckets */, 1 /* numSignificantSamples */, Sum.ofLongs());
+ this.overloadRatio = overloadRatio;
+ this.random = random;
+ }
+
+ @VisibleForTesting
+ double throttlingProbability(long nowMsSinceEpoch) {
+ if (!allRequests.isSignificant()) {
+ return 0;
+ }
+ long allRequestsNow = allRequests.get(nowMsSinceEpoch);
+ long successfulRequestsNow = successfulRequests.get(nowMsSinceEpoch);
+ return Math.max(0,
+ (allRequestsNow - overloadRatio * successfulRequestsNow) / (allRequestsNow + MIN_REQUESTS));
+ }
+
+ /**
+ * Call this before sending a request to the remote service; if this returns true, drop the
+ * request (treating it as a failure or trying it again at a later time).
+ */
+ public boolean throttleRequest(long nowMsSinceEpoch) {
+ double delayProbability = throttlingProbability(nowMsSinceEpoch);
+ // Note that we increment the count of all requests here, even if we return true - so even if we
+ // tell the client not to send a request at all, it still counts as a failed request.
+ allRequests.add(nowMsSinceEpoch, 1);
+
+ return (random.nextDouble() < delayProbability);
+ }
+
+ /**
+ * Call this after {@link throttleRequest} if your request was successful.
+ */
+ public void successfulRequest(long nowMsSinceEpoch) {
+ successfulRequests.add(nowMsSinceEpoch, 1);
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 16bb1b4..9b20c0d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -40,6 +40,7 @@
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.datastore.v1.CommitRequest;
import com.google.datastore.v1.Entity;
import com.google.datastore.v1.EntityResult;
@@ -65,24 +66,25 @@
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
+import java.util.Set;
import javax.annotation.Nullable;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.transforms.Values;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
@@ -95,7 +97,6 @@
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -201,11 +202,47 @@
DatastoreV1() {}
/**
- * Cloud Datastore has a limit of 500 mutations per batch operation, so we flush
- * changes to Datastore every 500 entities.
+ * The number of entity updates written per RPC, initially. We buffer updates in the connector and
+ * write a batch to Datastore once we have collected a certain number. This is the initial batch
+ * size; it is adjusted at runtime based on the performance of previous writes (see {@link
+ * DatastoreV1.WriteBatcher}).
+ *
+ * <p>Testing has found that a batch of 200 entities will generally finish within the timeout even
+ * in adverse conditions.
*/
@VisibleForTesting
- static final int DATASTORE_BATCH_UPDATE_LIMIT = 500;
+ static final int DATASTORE_BATCH_UPDATE_ENTITIES_START = 200;
+
+ /**
+ * When choosing the number of updates in a single RPC, never exceed the maximum allowed by the
+ * API.
+ */
+ @VisibleForTesting
+ static final int DATASTORE_BATCH_UPDATE_ENTITIES_LIMIT = 500;
+
+ /**
+ * When choosing the number of updates in a single RPC, do not go below this value. The actual
+ * number of entities per request may be lower when we flush for the end of a bundle or if we hit
+ * {@link DatastoreV1.DATASTORE_BATCH_UPDATE_BYTES_LIMIT}.
+ */
+ @VisibleForTesting
+ static final int DATASTORE_BATCH_UPDATE_ENTITIES_MIN = 10;
+
+ /**
+ * Cloud Datastore has a limit of 10MB per RPC, so we also flush if the total size of mutations
+ * exceeds this limit. This is set lower than the 10MB limit on the RPC, as this only accounts for
+ * the mutations themselves and not the CommitRequest wrapper around them.
+ */
+ @VisibleForTesting
+ static final int DATASTORE_BATCH_UPDATE_BYTES_LIMIT = 9_000_000;
+
+ /**
+ * Non-retryable errors.
+ * See https://cloud.google.com/datastore/docs/concepts/errors#Error_Codes .
+ */
+ private static final Set<Code> NON_RETRYABLE_ERRORS =
+ ImmutableSet.of(Code.FAILED_PRECONDITION, Code.INVALID_ARGUMENT, Code.PERMISSION_DENIED,
+ Code.UNAUTHENTICATED);
/**
* Returns an empty {@link DatastoreV1.Read} builder. Configure the source {@code projectId},
@@ -441,7 +478,7 @@
* project.
*/
public DatastoreV1.Read withProjectId(String projectId) {
- checkNotNull(projectId, "projectId");
+ checkArgument(projectId != null, "projectId can not be null");
return toBuilder().setProjectId(StaticValueProvider.of(projectId)).build();
}
@@ -449,7 +486,7 @@
* Same as {@link Read#withProjectId(String)} but with a {@link ValueProvider}.
*/
public DatastoreV1.Read withProjectId(ValueProvider<String> projectId) {
- checkNotNull(projectId, "projectId");
+ checkArgument(projectId != null, "projectId can not be null");
return toBuilder().setProjectId(projectId).build();
}
@@ -462,7 +499,7 @@
* to ensure correct results.
*/
public DatastoreV1.Read withQuery(Query query) {
- checkNotNull(query, "query");
+ checkArgument(query != null, "query can not be null");
checkArgument(!query.hasLimit() || query.getLimit().getValue() > 0,
"Invalid query limit %s: must be positive", query.getLimit().getValue());
return toBuilder().setQuery(query).build();
@@ -484,7 +521,7 @@
*/
@Experimental(Kind.SOURCE_SINK)
public DatastoreV1.Read withLiteralGqlQuery(String gqlQuery) {
- checkNotNull(gqlQuery, "gqlQuery");
+ checkArgument(gqlQuery != null, "gqlQuery can not be null");
return toBuilder().setLiteralGqlQuery(StaticValueProvider.of(gqlQuery)).build();
}
@@ -493,7 +530,10 @@
*/
@Experimental(Kind.SOURCE_SINK)
public DatastoreV1.Read withLiteralGqlQuery(ValueProvider<String> gqlQuery) {
- checkNotNull(gqlQuery, "gqlQuery");
+ checkArgument(gqlQuery != null, "gqlQuery can not be null");
+ if (gqlQuery.isAccessible()) {
+ checkArgument(gqlQuery.get() != null, "gqlQuery can not be null");
+ }
return toBuilder().setLiteralGqlQuery(gqlQuery).build();
}
@@ -545,6 +585,18 @@
@Override
public PCollection<Entity> expand(PBegin input) {
+ checkArgument(getProjectId() != null, "projectId provider cannot be null");
+ if (getProjectId().isAccessible()) {
+ checkArgument(getProjectId().get() != null, "projectId cannot be null");
+ }
+
+ checkArgument(
+ getQuery() != null || getLiteralGqlQuery() != null,
+ "Either withQuery() or withLiteralGqlQuery() is required");
+ checkArgument(
+ getQuery() == null || getLiteralGqlQuery() == null,
+ "withQuery() and withLiteralGqlQuery() are exclusive");
+
V1Options v1Options = V1Options.from(getProjectId(), getNamespace(), getLocalhost());
/*
@@ -571,47 +623,16 @@
if (getQuery() != null) {
inputQuery = input.apply(Create.of(getQuery()));
} else {
- inputQuery = input
- .apply(Create.of(getLiteralGqlQuery())
- .withCoder(SerializableCoder.of(new TypeDescriptor<ValueProvider<String>>() {})))
- .apply(ParDo.of(new GqlQueryTranslateFn(v1Options)));
+ inputQuery =
+ input
+ .apply(Create.ofProvider(getLiteralGqlQuery(), StringUtf8Coder.of()))
+ .apply(ParDo.of(new GqlQueryTranslateFn(v1Options)));
}
- PCollection<KV<Integer, Query>> splitQueries = inputQuery
- .apply(ParDo.of(new SplitQueryFn(v1Options, getNumQuerySplits())));
-
- PCollection<Query> shardedQueries = splitQueries
- .apply(GroupByKey.<Integer, Query>create())
- .apply(Values.<Iterable<Query>>create())
- .apply(Flatten.<Query>iterables());
-
- PCollection<Entity> entities = shardedQueries
- .apply(ParDo.of(new ReadFn(v1Options)));
-
- return entities;
- }
-
- @Override
- public void validate(PipelineOptions options) {
- checkNotNull(getProjectId(), "projectId");
-
- if (getProjectId().isAccessible() && getProjectId().get() == null) {
- throw new IllegalArgumentException("Project id cannot be null");
- }
-
- if (getQuery() == null && getLiteralGqlQuery() == null) {
- throw new IllegalArgumentException(
- "Either query or gql query ValueProvider should be provided");
- }
-
- if (getQuery() != null && getLiteralGqlQuery() != null) {
- throw new IllegalArgumentException(
- "Only one of query or gql query ValueProvider should be provided");
- }
-
- if (getLiteralGqlQuery() != null && getLiteralGqlQuery().isAccessible()) {
- checkNotNull(getLiteralGqlQuery().get(), "gqlQuery");
- }
+ return inputQuery
+ .apply("Split", ParDo.of(new SplitQueryFn(v1Options, getNumQuerySplits())))
+ .apply("Reshuffle", Reshuffle.<Query>viaRandomKey())
+ .apply("Read", ParDo.of(new ReadFn(v1Options)));
}
@Override
@@ -690,7 +711,7 @@
/**
* A DoFn that translates a Cloud Datastore gql query string to {@code Query}.
*/
- static class GqlQueryTranslateFn extends DoFn<ValueProvider<String>, Query> {
+ static class GqlQueryTranslateFn extends DoFn<String, Query> {
private final V1Options v1Options;
private transient Datastore datastore;
private final V1DatastoreFactory datastoreFactory;
@@ -706,14 +727,15 @@
@StartBundle
public void startBundle(StartBundleContext c) throws Exception {
- datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), v1Options.getProjectId());
+ datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), v1Options.getProjectId(),
+ v1Options.getLocalhost());
}
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
- ValueProvider<String> gqlQuery = c.element();
- LOG.info("User query: '{}'", gqlQuery.get());
- Query query = translateGqlQueryWithLimitCheck(gqlQuery.get(), datastore,
+ String gqlQuery = c.element();
+ LOG.info("User query: '{}'", gqlQuery);
+ Query query = translateGqlQueryWithLimitCheck(gqlQuery, datastore,
v1Options.getNamespace());
LOG.info("User gql query translated to Query({})", query);
c.output(query);
@@ -725,7 +747,7 @@
* keys and outputs them as {@link KV}.
*/
@VisibleForTesting
- static class SplitQueryFn extends DoFn<Query, KV<Integer, Query>> {
+ static class SplitQueryFn extends DoFn<Query, Query> {
private final V1Options options;
// number of splits to make for a given query
private final int numSplits;
@@ -757,12 +779,11 @@
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
- int key = 1;
Query query = c.element();
// If query has a user set limit, then do not split.
if (query.hasLimit()) {
- c.output(KV.of(key, query));
+ c.output(query);
return;
}
@@ -786,7 +807,7 @@
// assign unique keys to query splits.
for (Query subquery : querySplits) {
- c.output(KV.of(key++, subquery));
+ c.output(subquery);
}
}
@@ -810,6 +831,14 @@
private final V1DatastoreFactory datastoreFactory;
// Datastore client
private transient Datastore datastore;
+ private final Counter rpcErrors =
+ Metrics.counter(DatastoreWriterFn.class, "datastoreRpcErrors");
+ private final Counter rpcSuccesses =
+ Metrics.counter(DatastoreWriterFn.class, "datastoreRpcSuccesses");
+ private static final int MAX_RETRIES = 5;
+ private static final FluentBackoff RUNQUERY_BACKOFF =
+ FluentBackoff.DEFAULT
+ .withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
public ReadFn(V1Options options) {
this(options, new V1DatastoreFactory());
@@ -827,6 +856,28 @@
options.getLocalhost());
}
+ private RunQueryResponse runQueryWithRetries(RunQueryRequest request) throws Exception {
+ Sleeper sleeper = Sleeper.DEFAULT;
+ BackOff backoff = RUNQUERY_BACKOFF.backoff();
+ while (true) {
+ try {
+ RunQueryResponse response = datastore.runQuery(request);
+ rpcSuccesses.inc();
+ return response;
+ } catch (DatastoreException exception) {
+ rpcErrors.inc();
+
+ if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) {
+ throw exception;
+ }
+ if (!BackOffUtils.next(sleeper, backoff)) {
+ LOG.error("Aborting after {} retries.", MAX_RETRIES);
+ throw exception;
+ }
+ }
+ }
+ }
+
/** Read and output entities for the given query. */
@ProcessElement
public void processElement(ProcessContext context) throws Exception {
@@ -848,7 +899,7 @@
}
RunQueryRequest request = makeRequest(queryBuilder.build(), namespace);
- RunQueryResponse response = datastore.runQuery(request);
+ RunQueryResponse response = runQueryWithRetries(request);
currentBatch = response.getBatch();
@@ -928,7 +979,7 @@
* Returns a new {@link Write} that writes to the Cloud Datastore for the specified project.
*/
public Write withProjectId(String projectId) {
- checkNotNull(projectId, "projectId");
+ checkArgument(projectId != null, "projectId can not be null");
return withProjectId(StaticValueProvider.of(projectId));
}
@@ -936,7 +987,7 @@
* Same as {@link Write#withProjectId(String)} but with a {@link ValueProvider}.
*/
public Write withProjectId(ValueProvider<String> projectId) {
- checkNotNull(projectId, "projectId ValueProvider");
+ checkArgument(projectId != null, "projectId can not be null");
return new Write(projectId, localhost);
}
@@ -945,7 +996,7 @@
* the specified host port.
*/
public Write withLocalhost(String localhost) {
- checkNotNull(localhost, "localhost");
+ checkArgument(localhost != null, "localhost can not be null");
return new Write(projectId, localhost);
}
}
@@ -969,7 +1020,7 @@
* specified project.
*/
public DeleteEntity withProjectId(String projectId) {
- checkNotNull(projectId, "projectId");
+ checkArgument(projectId != null, "projectId can not be null");
return withProjectId(StaticValueProvider.of(projectId));
}
@@ -977,7 +1028,7 @@
* Same as {@link DeleteEntity#withProjectId(String)} but with a {@link ValueProvider}.
*/
public DeleteEntity withProjectId(ValueProvider<String> projectId) {
- checkNotNull(projectId, "projectId ValueProvider");
+ checkArgument(projectId != null, "projectId can not be null");
return new DeleteEntity(projectId, localhost);
}
@@ -986,7 +1037,7 @@
* running locally on the specified host port.
*/
public DeleteEntity withLocalhost(String localhost) {
- checkNotNull(localhost, "localhost");
+ checkArgument(localhost != null, "localhost can not be null");
return new DeleteEntity(projectId, localhost);
}
}
@@ -1011,7 +1062,7 @@
* specified project.
*/
public DeleteKey withProjectId(String projectId) {
- checkNotNull(projectId, "projectId");
+ checkArgument(projectId != null, "projectId can not be null");
return withProjectId(StaticValueProvider.of(projectId));
}
@@ -1020,7 +1071,7 @@
* running locally on the specified host port.
*/
public DeleteKey withLocalhost(String localhost) {
- checkNotNull(localhost, "localhost");
+ checkArgument(localhost != null, "localhost can not be null");
return new DeleteKey(projectId, localhost);
}
@@ -1028,7 +1079,7 @@
* Same as {@link DeleteKey#withProjectId(String)} but with a {@link ValueProvider}.
*/
public DeleteKey withProjectId(ValueProvider<String> projectId) {
- checkNotNull(projectId, "projectId ValueProvider");
+ checkArgument(projectId != null, "projectId can not be null");
return new DeleteKey(projectId, localhost);
}
}
@@ -1061,6 +1112,12 @@
@Override
public PDone expand(PCollection<T> input) {
+ checkArgument(projectId != null, "withProjectId() is required");
+ if (projectId.isAccessible()) {
+ checkArgument(projectId.get() != null, "projectId can not be null");
+ }
+ checkArgument(mutationFn != null, "mutationFn can not be null");
+
input.apply("Convert to Mutation", MapElements.via(mutationFn))
.apply("Write Mutation to Datastore", ParDo.of(
new DatastoreWriterFn(projectId, localhost)));
@@ -1069,15 +1126,6 @@
}
@Override
- public void validate(PipelineOptions options) {
- checkNotNull(projectId, "projectId ValueProvider");
- if (projectId.isAccessible()) {
- checkNotNull(projectId.get(), "projectId");
- }
- checkNotNull(mutationFn, "mutationFn");
- }
-
- @Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
.add("projectId", projectId)
@@ -1099,18 +1147,74 @@
}
}
+ /** Determines batch sizes for commit RPCs. */
+ @VisibleForTesting
+ interface WriteBatcher {
+ /** Call before using this WriteBatcher. */
+ void start();
+
+ /**
+ * Reports the latency of a previous commit RPC, and the number of mutations that it contained.
+ */
+ void addRequestLatency(long timeSinceEpochMillis, long latencyMillis, int numMutations);
+
+ /** Returns the number of entities to include in the next CommitRequest. */
+ int nextBatchSize(long timeSinceEpochMillis);
+ }
+
+ /**
+ * Determines batch sizes for commit RPCs based on past performance.
+ *
+ * <p>It aims for a target response time per RPC: it uses the response times for previous RPCs
+ * and the number of entities contained in them, calculates a rolling average time-per-entity, and
+ * chooses the number of entities for future writes to hit the target time.
+ *
+ * <p>This enables us to send large batches without sending over-large requests in the case of
+ * expensive entity writes that may timeout before the server can apply them all.
+ */
+ @VisibleForTesting
+ static class WriteBatcherImpl implements WriteBatcher, Serializable {
+ /** Target time per RPC for writes. */
+ static final int DATASTORE_BATCH_TARGET_LATENCY_MS = 5000;
+
+ @Override
+ public void start() {
+ meanLatencyPerEntityMs = new MovingAverage(
+ 120000 /* sample period 2 minutes */, 10000 /* sample interval 10s */,
+ 1 /* numSignificantBuckets */, 1 /* numSignificantSamples */);
+ }
+
+ @Override
+ public void addRequestLatency(long timeSinceEpochMillis, long latencyMillis, int numMutations) {
+ meanLatencyPerEntityMs.add(timeSinceEpochMillis, latencyMillis / numMutations);
+ }
+
+ @Override
+ public int nextBatchSize(long timeSinceEpochMillis) {
+ if (!meanLatencyPerEntityMs.hasValue(timeSinceEpochMillis)) {
+ return DATASTORE_BATCH_UPDATE_ENTITIES_START;
+ }
+ long recentMeanLatency = Math.max(meanLatencyPerEntityMs.get(timeSinceEpochMillis), 1);
+ return (int) Math.max(DATASTORE_BATCH_UPDATE_ENTITIES_MIN,
+ Math.min(DATASTORE_BATCH_UPDATE_ENTITIES_LIMIT,
+ DATASTORE_BATCH_TARGET_LATENCY_MS / recentMeanLatency));
+ }
+
+ private transient MovingAverage meanLatencyPerEntityMs;
+ }
+
/**
* {@link DoFn} that writes {@link Mutation}s to Cloud Datastore. Mutations are written in
- * batches, where the maximum batch size is {@link DatastoreV1#DATASTORE_BATCH_UPDATE_LIMIT}.
+ * batches; see {@link DatastoreV1.WriteBatcherImpl}.
*
* <p>See <a
* href="https://cloud.google.com/datastore/docs/concepts/entities">
* Datastore: Entities, Properties, and Keys</a> for information about entity keys and mutations.
*
* <p>Commits are non-transactional. If a commit fails because of a conflict over an entity
- * group, the commit will be retried (up to {@link DatastoreV1#DATASTORE_BATCH_UPDATE_LIMIT}
+ * group, the commit will be retried (up to {@link DatastoreV1.DatastoreWriterFn#MAX_RETRIES}
* times). This means that the mutation operation should be idempotent. Thus, the writer should
- * only be used for {code upsert} and {@code delete} mutation operations, as these are the only
+ * only be used for {@code upsert} and {@code delete} mutation operations, as these are the only
* two Cloud Datastore mutations that are idempotent.
*/
@VisibleForTesting
@@ -1123,6 +1227,15 @@
private final V1DatastoreFactory datastoreFactory;
// Current batch of mutations to be written.
private final List<Mutation> mutations = new ArrayList<>();
+ private int mutationsSize = 0; // Accumulated size of protos in mutations.
+ private WriteBatcher writeBatcher;
+ private transient AdaptiveThrottler throttler;
+ private final Counter throttledSeconds =
+ Metrics.counter(DatastoreWriterFn.class, "cumulativeThrottlingSeconds");
+ private final Counter rpcErrors =
+ Metrics.counter(DatastoreWriterFn.class, "datastoreRpcErrors");
+ private final Counter rpcSuccesses =
+ Metrics.counter(DatastoreWriterFn.class, "datastoreRpcSuccesses");
private static final int MAX_RETRIES = 5;
private static final FluentBackoff BUNDLE_WRITE_BACKOFF =
@@ -1130,30 +1243,44 @@
.withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
DatastoreWriterFn(String projectId, @Nullable String localhost) {
- this(StaticValueProvider.of(projectId), localhost, new V1DatastoreFactory());
+ this(StaticValueProvider.of(projectId), localhost, new V1DatastoreFactory(),
+ new WriteBatcherImpl());
}
DatastoreWriterFn(ValueProvider<String> projectId, @Nullable String localhost) {
- this(projectId, localhost, new V1DatastoreFactory());
+ this(projectId, localhost, new V1DatastoreFactory(), new WriteBatcherImpl());
}
@VisibleForTesting
DatastoreWriterFn(ValueProvider<String> projectId, @Nullable String localhost,
- V1DatastoreFactory datastoreFactory) {
+ V1DatastoreFactory datastoreFactory, WriteBatcher writeBatcher) {
this.projectId = checkNotNull(projectId, "projectId");
this.localhost = localhost;
this.datastoreFactory = datastoreFactory;
+ this.writeBatcher = writeBatcher;
}
@StartBundle
public void startBundle(StartBundleContext c) {
datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), projectId.get(), localhost);
+ writeBatcher.start();
+ if (throttler == null) {
+ // Initialize throttler at first use, because it is not serializable.
+ throttler = new AdaptiveThrottler(120000, 10000, 1.25);
+ }
}
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
+ Mutation write = c.element();
+ int size = write.getSerializedSize();
+ if (mutations.size() > 0
+ && mutationsSize + size >= DatastoreV1.DATASTORE_BATCH_UPDATE_BYTES_LIMIT) {
+ flushBatch();
+ }
mutations.add(c.element());
- if (mutations.size() >= DatastoreV1.DATASTORE_BATCH_UPDATE_LIMIT) {
+ mutationsSize += size;
+ if (mutations.size() >= writeBatcher.nextBatchSize(System.currentTimeMillis())) {
flushBatch();
}
}
@@ -1183,18 +1310,45 @@
while (true) {
// Batch upsert entities.
+ CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+ commitRequest.addAllMutations(mutations);
+ commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
+ long startTime = System.currentTimeMillis(), endTime;
+
+ if (throttler.throttleRequest(startTime)) {
+ LOG.info("Delaying request due to previous failures");
+ throttledSeconds.inc(WriteBatcherImpl.DATASTORE_BATCH_TARGET_LATENCY_MS / 1000);
+ sleeper.sleep(WriteBatcherImpl.DATASTORE_BATCH_TARGET_LATENCY_MS);
+ continue;
+ }
+
try {
- CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
- commitRequest.addAllMutations(mutations);
- commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
datastore.commit(commitRequest.build());
+ endTime = System.currentTimeMillis();
+
+ writeBatcher.addRequestLatency(endTime, endTime - startTime, mutations.size());
+ throttler.successfulRequest(startTime);
+ rpcSuccesses.inc();
+
// Break if the commit threw no exception.
break;
} catch (DatastoreException exception) {
+ if (exception.getCode() == Code.DEADLINE_EXCEEDED) {
+ /* Most errors are not related to request size, and should not change our expectation of
+ * the latency of successful requests. DEADLINE_EXCEEDED can be taken into
+ * consideration, though. */
+ endTime = System.currentTimeMillis();
+ writeBatcher.addRequestLatency(endTime, endTime - startTime, mutations.size());
+ }
// Only log the code and message for potentially-transient errors. The entire exception
// will be propagated upon the last retry.
- LOG.error("Error writing to the Datastore ({}): {}", exception.getCode(),
- exception.getMessage());
+ LOG.error("Error writing batch of {} mutations to Datastore ({}): {}", mutations.size(),
+ exception.getCode(), exception.getMessage());
+ rpcErrors.inc();
+
+ if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) {
+ throw exception;
+ }
if (!BackOffUtils.next(sleeper, backoff)) {
LOG.error("Aborting after {} retries.", MAX_RETRIES);
throw exception;
@@ -1203,6 +1357,7 @@
}
LOG.debug("Successfully wrote {} mutations", mutations.size());
mutations.clear();
+ mutationsSize = 0;
}
@Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/MovingAverage.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/MovingAverage.java
new file mode 100644
index 0000000..0890e79
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/MovingAverage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.datastore;
+
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.util.MovingFunction;
+
+
+class MovingAverage {
+ private final MovingFunction sum;
+ private final MovingFunction count;
+
+ public MovingAverage(long samplePeriodMs, long sampleUpdateMs,
+ int numSignificantBuckets, int numSignificantSamples) {
+ sum = new MovingFunction(samplePeriodMs, sampleUpdateMs,
+ numSignificantBuckets, numSignificantSamples, Sum.ofLongs());
+ count = new MovingFunction(samplePeriodMs, sampleUpdateMs,
+ numSignificantBuckets, numSignificantSamples, Sum.ofLongs());
+ }
+
+ public void add(long nowMsSinceEpoch, long value) {
+ sum.add(nowMsSinceEpoch, value);
+ count.add(nowMsSinceEpoch, 1);
+ }
+
+ public long get(long nowMsSinceEpoch) {
+ return sum.get(nowMsSinceEpoch) / count.get(nowMsSinceEpoch);
+ }
+
+ public boolean hasValue(long nowMsSinceEpoch) {
+ return sum.isSignificant() && count.isSignificant()
+ && count.get(nowMsSinceEpoch) > 0;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index e293b95..e3780b4 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -36,7 +36,6 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.extensions.protobuf.ProtoCoder;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.ProjectPath;
@@ -147,17 +146,11 @@
private static void populateCommonDisplayData(DisplayData.Builder builder,
String timestampAttribute, String idAttribute, ValueProvider<PubsubTopic> topic) {
builder
- .addIfNotNull(DisplayData.item("timestampAttribute", timestampAttribute)
- .withLabel("Timestamp Attribute"))
- .addIfNotNull(DisplayData.item("idAttribute", idAttribute)
- .withLabel("ID Attribute"));
-
- if (topic != null) {
- String topicString = topic.isAccessible() ? topic.get().asPath()
- : topic.toString();
- builder.add(DisplayData.item("topic", topicString)
- .withLabel("Pubsub Topic"));
- }
+ .addIfNotNull(
+ DisplayData.item("timestampAttribute", timestampAttribute)
+ .withLabel("Timestamp Attribute"))
+ .addIfNotNull(DisplayData.item("idAttribute", idAttribute).withLabel("ID Attribute"))
+ .addIfNotNull(DisplayData.item("topic", topic).withLabel("Pubsub Topic"));
}
/**
@@ -264,6 +257,11 @@
return subscription;
}
}
+
+ @Override
+ public String toString() {
+ return asPath();
+ }
}
/**
@@ -429,6 +427,11 @@
return topic;
}
}
+
+ @Override
+ public String toString() {
+ return asPath();
+ }
}
/** Returns A {@link PTransform} that continuously reads from a Google Cloud Pub/Sub stream. */
@@ -527,7 +530,7 @@
* Returns A {@link PTransform} that writes binary encoded Avro messages of a given type
* to a Google Cloud Pub/Sub stream.
*/
- public static <T extends Message> Write<T> writeAvros(Class<T> clazz) {
+ public static <T> Write<T> writeAvros(Class<T> clazz) {
// TODO: Like in readAvros(), stop using AvroCoder and instead format the payload directly.
return PubsubIO.<T>write().withFormatFn(new FormatPayloadUsingCoder<>(AvroCoder.of(clazz)));
}
@@ -727,7 +730,7 @@
getTimestampAttribute(),
getIdAttribute(),
getNeedsAttributes());
- return input.apply(source).apply(MapElements.via(getParseFn()));
+ return input.apply(source).apply(MapElements.via(getParseFn())).setCoder(getCoder());
}
@Override
@@ -735,18 +738,8 @@
super.populateDisplayData(builder);
populateCommonDisplayData(
builder, getTimestampAttribute(), getIdAttribute(), getTopicProvider());
-
- if (getSubscriptionProvider() != null) {
- String subscriptionString = getSubscriptionProvider().isAccessible()
- ? getSubscriptionProvider().get().asPath() : getSubscriptionProvider().toString();
- builder.add(DisplayData.item("subscription", subscriptionString)
- .withLabel("Pubsub Subscription"));
- }
- }
-
- @Override
- protected Coder<T> getDefaultOutputCoder() {
- return getCoder();
+ builder.addIfNotNull(DisplayData.item("subscription", getSubscriptionProvider())
+ .withLabel("Pubsub Subscription"));
}
}
@@ -870,11 +863,6 @@
builder, getTimestampAttribute(), getIdAttribute(), getTopicProvider());
}
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
-
/**
* Writer to Pubsub which batches messages from bounded collections.
*
@@ -970,8 +958,7 @@
}
}
- private static class FormatPayloadUsingCoder<T extends Message>
- extends SimpleFunction<T, PubsubMessage> {
+ private static class FormatPayloadUsingCoder<T> extends SimpleFunction<T, PubsubMessage> {
private Coder<T> coder;
public FormatPayloadUsingCoder(Coder<T> coder) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
index ad38e28..a8f6fa2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
@@ -295,11 +295,7 @@
@Override
public void populateDisplayData(Builder builder) {
super.populateDisplayData(builder);
- String topicString =
- topic == null ? null
- : topic.isAccessible() ? topic.get().getPath()
- : topic.toString();
- builder.add(DisplayData.item("topic", topicString));
+ builder.add(DisplayData.item("topic", topic));
builder.add(DisplayData.item("transport", pubsubFactory.getKind()));
builder.addIfNotNull(DisplayData.item("timestampAttribute", timestampAttribute));
builder.addIfNotNull(DisplayData.item("idAttribute", idAttribute));
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index b7df804..2271786 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -61,6 +61,7 @@
import org.apache.beam.sdk.metrics.SourceMetrics;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
@@ -1097,13 +1098,14 @@
public final PubsubUnboundedSource outer;
// The subscription to read from.
@VisibleForTesting
- final SubscriptionPath subscriptionPath;
+ final ValueProvider<SubscriptionPath> subscriptionPath;
public PubsubSource(PubsubUnboundedSource outer) {
- this(outer, outer.getSubscription());
+ this(outer, outer.getSubscriptionProvider());
}
- private PubsubSource(PubsubUnboundedSource outer, SubscriptionPath subscriptionPath) {
+ private PubsubSource(
+ PubsubUnboundedSource outer, ValueProvider<SubscriptionPath> subscriptionPath) {
this.outer = outer;
this.subscriptionPath = subscriptionPath;
}
@@ -1114,7 +1116,9 @@
List<PubsubSource> result = new ArrayList<>(desiredNumSplits);
PubsubSource splitSource = this;
if (subscriptionPath == null) {
- splitSource = new PubsubSource(outer, outer.createRandomSubscription(options));
+ splitSource =
+ new PubsubSource(
+ outer, StaticValueProvider.of(outer.createRandomSubscription(options)));
}
for (int i = 0; i < desiredNumSplits * SCALE_OUT; i++) {
// Since the source is immutable and Pubsub automatically shards we simply
@@ -1129,8 +1133,8 @@
PipelineOptions options,
@Nullable PubsubCheckpoint checkpoint) {
PubsubReader reader;
- SubscriptionPath subscription = subscriptionPath;
- if (subscription == null) {
+ SubscriptionPath subscription;
+ if (subscriptionPath == null || subscriptionPath.get() == null) {
if (checkpoint == null) {
// This reader has never been started and there was no call to #split;
// create a single random subscription, which will be kept in the checkpoint.
@@ -1138,6 +1142,8 @@
} else {
subscription = checkpoint.getSubscription();
}
+ } else {
+ subscription = subscriptionPath.get();
}
try {
reader = new PubsubReader(options.as(PubsubOptions.class), this, subscription);
@@ -1164,7 +1170,7 @@
}
@Override
- public Coder<PubsubMessage> getDefaultOutputCoder() {
+ public Coder<PubsubMessage> getOutputCoder() {
return outer.getNeedsAttributes()
? PubsubMessageWithAttributesCoder.of()
: PubsubMessagePayloadOnlyCoder.of();
@@ -1222,21 +1228,12 @@
@Override
public void populateDisplayData(Builder builder) {
super.populateDisplayData(builder);
- if (subscription != null) {
- String subscriptionString = subscription.isAccessible()
- ? subscription.get().getPath()
- : subscription.toString();
- builder.add(DisplayData.item("subscription", subscriptionString));
- }
- if (topic != null) {
- String topicString = topic.isAccessible()
- ? topic.get().getPath()
- : topic.toString();
- builder.add(DisplayData.item("topic", topicString));
- }
- builder.add(DisplayData.item("transport", pubsubFactory.getKind()));
- builder.addIfNotNull(DisplayData.item("timestampAttribute", timestampAttribute));
- builder.addIfNotNull(DisplayData.item("idAttribute", idAttribute));
+ builder
+ .addIfNotNull(DisplayData.item("subscription", subscription))
+ .addIfNotNull(DisplayData.item("topic", topic))
+ .add(DisplayData.item("transport", pubsubFactory.getKind()))
+ .addIfNotNull(DisplayData.item("timestampAttribute", timestampAttribute))
+ .addIfNotNull(DisplayData.item("idAttribute", idAttribute));
}
}
@@ -1416,8 +1413,6 @@
try (PubsubClient pubsubClient =
pubsubFactory.newClient(
timestampAttribute, idAttribute, options.as(PubsubOptions.class))) {
- checkState(project.isAccessible(), "createRandomSubscription must be called at runtime.");
- checkState(topic.isAccessible(), "createRandomSubscription must be called at runtime.");
SubscriptionPath subscriptionPath =
pubsubClient.createRandomSubscription(
project.get(), topic.get(), DEAULT_ACK_TIMEOUT_SEC);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/CreateTransactionFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/CreateTransactionFn.java
new file mode 100644
index 0000000..5574ae1
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/CreateTransactionFn.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Statement;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/** Creates a batch transaction. */
+class CreateTransactionFn extends DoFn<Object, Transaction> {
+
+ private final SpannerIO.CreateTransaction config;
+
+ CreateTransactionFn(SpannerIO.CreateTransaction config) {
+ this.config = config;
+ }
+
+ private transient SpannerAccessor spannerAccessor;
+
+ @Setup
+ public void setup() throws Exception {
+ spannerAccessor = config.getSpannerConfig().connectToSpanner();
+ }
+ @Teardown
+ public void teardown() throws Exception {
+ spannerAccessor.close();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ DatabaseClient databaseClient = spannerAccessor.getDatabaseClient();
+ try (ReadOnlyTransaction readOnlyTransaction =
+ databaseClient.readOnlyTransaction(config.getTimestampBound())) {
+ // Run a dummy sql statement to force the RPC and obtain the timestamp from the server.
+ ResultSet resultSet = readOnlyTransaction.executeQuery(Statement.of("SELECT 1"));
+ while (resultSet.next()) {
+ // do nothing
+ }
+ Transaction tx = Transaction.create(readOnlyTransaction.getReadTimestamp());
+ c.output(tx);
+ }
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java
new file mode 100644
index 0000000..5b08da2
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.Mutation;
+import com.google.common.collect.ImmutableList;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A bundle of mutations that must be submitted atomically.
+ *
+ * <p>One of the mutations is chosen to be "primary", and can be used to determine partitions.
+ */
+public final class MutationGroup implements Serializable, Iterable<Mutation> {
+ private final ImmutableList<Mutation> mutations;
+
+ /**
+ * Creates a new group.
+ *
+ * @param primary a primary mutation.
+ * @param other other mutations, usually interleaved in parent.
+ * @return new mutation group.
+ */
+ public static MutationGroup create(Mutation primary, Mutation... other) {
+ return create(primary, Arrays.asList(other));
+ }
+
+ public static MutationGroup create(Mutation primary, Iterable<Mutation> other) {
+ return new MutationGroup(ImmutableList.<Mutation>builder().add(primary).addAll(other).build());
+ }
+
+ @Override
+ public Iterator<Mutation> iterator() {
+ return mutations.iterator();
+ }
+
+ private MutationGroup(ImmutableList<Mutation> mutations) {
+ this.mutations = mutations;
+ }
+
+ public Mutation primary() {
+ return mutations.get(0);
+ }
+
+ public List<Mutation> attached() {
+ return mutations.subList(1, mutations.size());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoder.java
new file mode 100644
index 0000000..ba0b4eb
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoder.java
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.Date;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Type;
+import com.google.cloud.spanner.Value;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.util.VarInt;
+import org.joda.time.DateTime;
+import org.joda.time.Days;
+import org.joda.time.MutableDateTime;
+
+/**
+ * Given the Spanner Schema, efficiently encodes the mutation group.
+ */
+class MutationGroupEncoder {
+ private static final DateTime MIN_DATE = new DateTime(1, 1, 1, 0, 0);
+
+ private final SpannerSchema schema;
+ private final List<String> tables;
+ private final Map<String, Integer> tablesIndexes = new HashMap<>();
+
+ public MutationGroupEncoder(SpannerSchema schema) {
+ this.schema = schema;
+ tables = schema.getTables();
+
+ for (int i = 0; i < tables.size(); i++) {
+ tablesIndexes.put(tables.get(i), i);
+ }
+ }
+
+ public byte[] encode(MutationGroup g) {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+
+ try {
+ VarInt.encode(g.attached().size(), bos);
+ for (Mutation m : g) {
+ encodeMutation(bos, m);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return bos.toByteArray();
+ }
+
+ private static void setBit(byte[] bytes, int i) {
+ int word = i / 8;
+ int bit = 7 - i % 8;
+ bytes[word] |= 1 << bit;
+ }
+
+ private static boolean getBit(byte[] bytes, int i) {
+ int word = i / 8;
+ int bit = 7 - i % 8;
+ return (bytes[word] & 1 << (bit)) != 0;
+ }
+
+ private void encodeMutation(ByteArrayOutputStream bos, Mutation m) throws IOException {
+ Mutation.Op op = m.getOperation();
+ bos.write(op.ordinal());
+ if (op == Mutation.Op.DELETE) {
+ encodeDelete(bos, m);
+ } else {
+ encodeModification(bos, m);
+ }
+ }
+
+ private void encodeDelete(ByteArrayOutputStream bos, Mutation m) throws IOException {
+ String table = m.getTable().toLowerCase();
+ int tableIndex = getTableIndex(table);
+ VarInt.encode(tableIndex, bos);
+ ObjectOutput out = new ObjectOutputStream(bos);
+ out.writeObject(m.getKeySet());
+ }
+
+ private Integer getTableIndex(String table) {
+ Integer result = tablesIndexes.get(table);
+ checkArgument(result != null, "Unknown table '%s'", table);
+ return result;
+ }
+
+ private Mutation decodeDelete(ByteArrayInputStream bis)
+ throws IOException {
+ int tableIndex = VarInt.decodeInt(bis);
+ String tableName = tables.get(tableIndex);
+
+ ObjectInputStream in = new ObjectInputStream(bis);
+ KeySet keySet;
+ try {
+ keySet = (KeySet) in.readObject();
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ return Mutation.delete(tableName, keySet);
+ }
+
+ // Encodes a mutation that is not a delete one, using the following format
+ // [bitset of modified columns][value of column1][value of column2][value of column3]...
+ private void encodeModification(ByteArrayOutputStream bos, Mutation m) throws IOException {
+ String tableName = m.getTable().toLowerCase();
+ int tableIndex = getTableIndex(tableName);
+ VarInt.encode(tableIndex, bos);
+ List<SpannerSchema.Column> columns = schema.getColumns(tableName);
+ checkArgument(columns != null, "Schema for table " + tableName + " not " + "found");
+ Map<String, Value> map = mutationAsMap(m);
+ // java.util.BitSet#toByteArray returns array of unpredictable length. Using byte arrays
+ // instead.
+ int bitsetSize = (columns.size() + 7) / 8;
+ byte[] exists = new byte[bitsetSize];
+ byte[] nulls = new byte[bitsetSize];
+ for (int i = 0; i < columns.size(); i++) {
+ String columnName = columns.get(i).getName();
+ boolean columnExists = map.containsKey(columnName);
+ boolean columnNull = columnExists && map.get(columnName).isNull();
+ if (columnExists) {
+ setBit(exists, i);
+ }
+ if (columnNull) {
+ setBit(nulls, i);
+ map.remove(columnName);
+ }
+ }
+ bos.write(exists);
+ bos.write(nulls);
+ for (int i = 0; i < columns.size(); i++) {
+ if (!getBit(exists, i) || getBit(nulls, i)) {
+ continue;
+ }
+ SpannerSchema.Column column = columns.get(i);
+ Value value = map.remove(column.getName());
+ encodeValue(bos, value);
+ }
+ checkArgument(map.isEmpty(), "Columns %s were not defined in table %s", map.keySet(),
+ m.getTable());
+ }
+
+ private void encodeValue(ByteArrayOutputStream bos, Value value) throws IOException {
+ switch (value.getType().getCode()) {
+ case ARRAY:
+ encodeArray(bos, value);
+ break;
+ default:
+ encodePrimitive(bos, value);
+ }
+ }
+
+ private void encodeArray(ByteArrayOutputStream bos, Value value) throws IOException {
+ // TODO: avoid using Java serialization here.
+ ObjectOutputStream out = new ObjectOutputStream(bos);
+ switch (value.getType().getArrayElementType().getCode()) {
+ case BOOL: {
+ out.writeObject(new ArrayList<>(value.getBoolArray()));
+ break;
+ }
+ case INT64: {
+ out.writeObject(new ArrayList<>(value.getInt64Array()));
+ break;
+ }
+ case FLOAT64: {
+ out.writeObject(new ArrayList<>(value.getFloat64Array()));
+ break;
+ }
+ case STRING: {
+ out.writeObject(new ArrayList<>(value.getStringArray()));
+ break;
+ }
+ case BYTES: {
+ out.writeObject(new ArrayList<>(value.getBytesArray()));
+ break;
+ }
+ case TIMESTAMP: {
+ out.writeObject(new ArrayList<>(value.getTimestampArray()));
+ break;
+ }
+ case DATE: {
+ out.writeObject(new ArrayList<>(value.getDateArray()));
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("Unknown type " + value.getType());
+ }
+ }
+
+ private void encodePrimitive(ByteArrayOutputStream bos, Value value) throws IOException {
+ switch (value.getType().getCode()) {
+ case BOOL:
+ bos.write(value.getBool() ? 1 : 0);
+ break;
+ case INT64:
+ VarInt.encode(value.getInt64(), bos);
+ break;
+ case FLOAT64:
+ new DataOutputStream(bos).writeDouble(value.getFloat64());
+ break;
+ case STRING: {
+ String str = value.getString();
+ VarInt.encode(str.length(), bos);
+ bos.write(str.getBytes(StandardCharsets.UTF_8));
+ break;
+ }
+ case BYTES: {
+ ByteArray bytes = value.getBytes();
+ VarInt.encode(bytes.length(), bos);
+ bos.write(bytes.toByteArray());
+ break;
+ }
+ case TIMESTAMP: {
+ Timestamp timestamp = value.getTimestamp();
+ VarInt.encode(timestamp.getSeconds(), bos);
+ VarInt.encode(timestamp.getNanos(), bos);
+ break;
+ }
+ case DATE: {
+ Date date = value.getDate();
+ VarInt.encode(encodeDate(date), bos);
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("Unknown type " + value.getType());
+ }
+ }
+
+ public MutationGroup decode(byte[] bytes) {
+ ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+
+ try {
+ int numMutations = VarInt.decodeInt(bis);
+ Mutation primary = decodeMutation(bis);
+ List<Mutation> attached = new ArrayList<>(numMutations);
+ for (int i = 0; i < numMutations; i++) {
+ attached.add(decodeMutation(bis));
+ }
+ return MutationGroup.create(primary, attached);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Mutation decodeMutation(ByteArrayInputStream bis) throws IOException {
+ Mutation.Op op = Mutation.Op.values()[bis.read()];
+ if (op == Mutation.Op.DELETE) {
+ return decodeDelete(bis);
+ }
+ return decodeModification(bis, op);
+ }
+
+ private Mutation decodeModification(ByteArrayInputStream bis, Mutation.Op op) throws IOException {
+ int tableIndex = VarInt.decodeInt(bis);
+ String tableName = tables.get(tableIndex);
+
+ Mutation.WriteBuilder m;
+ switch (op) {
+ case INSERT:
+ m = Mutation.newInsertBuilder(tableName);
+ break;
+ case INSERT_OR_UPDATE:
+ m = Mutation.newInsertOrUpdateBuilder(tableName);
+ break;
+ case REPLACE:
+ m = Mutation.newReplaceBuilder(tableName);
+ break;
+ case UPDATE:
+ m = Mutation.newUpdateBuilder(tableName);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown operation " + op);
+ }
+ List<SpannerSchema.Column> columns = schema.getColumns(tableName);
+ int bitsetSize = (columns.size() + 7) / 8;
+ byte[] exists = readBytes(bis, bitsetSize);
+ byte[] nulls = readBytes(bis, bitsetSize);
+
+ for (int i = 0; i < columns.size(); i++) {
+ if (!getBit(exists, i)) {
+ continue;
+ }
+ SpannerSchema.Column column = columns.get(i);
+ boolean isNull = getBit(nulls, i);
+ Type type = column.getType();
+ String fieldName = column.getName();
+ switch (type.getCode()) {
+ case ARRAY:
+ try {
+ decodeArray(bis, fieldName, type, isNull, m);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ break;
+ default:
+ decodePrimitive(bis, fieldName, type, isNull, m);
+ }
+
+ }
+ return m.build();
+ }
+
+ private void decodeArray(ByteArrayInputStream bis, String fieldName, Type type, boolean isNull,
+ Mutation.WriteBuilder m) throws IOException, ClassNotFoundException {
+ // TODO: avoid using Java serialization here.
+ switch (type.getArrayElementType().getCode()) {
+ case BOOL: {
+ if (isNull) {
+ m.set(fieldName).toBoolArray((Iterable<Boolean>) null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toBoolArray((List<Boolean>) out.readObject());
+ }
+ break;
+ }
+ case INT64: {
+ if (isNull) {
+ m.set(fieldName).toInt64Array((Iterable<Long>) null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toInt64Array((List<Long>) out.readObject());
+ }
+ break;
+ }
+ case FLOAT64: {
+ if (isNull) {
+ m.set(fieldName).toFloat64Array((Iterable<Double>) null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toFloat64Array((List<Double>) out.readObject());
+ }
+ break;
+ }
+ case STRING: {
+ if (isNull) {
+ m.set(fieldName).toStringArray(null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toStringArray((List<String>) out.readObject());
+ }
+ break;
+ }
+ case BYTES: {
+ if (isNull) {
+ m.set(fieldName).toBytesArray(null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toBytesArray((List<ByteArray>) out.readObject());
+ }
+ break;
+ }
+ case TIMESTAMP: {
+ if (isNull) {
+ m.set(fieldName).toTimestampArray(null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toTimestampArray((List<Timestamp>) out.readObject());
+ }
+ break;
+ }
+ case DATE: {
+ if (isNull) {
+ m.set(fieldName).toDateArray(null);
+ } else {
+ ObjectInputStream out = new ObjectInputStream(bis);
+ m.set(fieldName).toDateArray((List<Date>) out.readObject());
+ }
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("Unknown type " + type);
+ }
+ }
+
+ private void decodePrimitive(ByteArrayInputStream bis, String fieldName, Type type,
+ boolean isNull, Mutation.WriteBuilder m) throws IOException {
+ switch (type.getCode()) {
+ case BOOL:
+ if (isNull) {
+ m.set(fieldName).to((Boolean) null);
+ } else {
+ m.set(fieldName).to(bis.read() != 0);
+ }
+ break;
+ case INT64:
+ if (isNull) {
+ m.set(fieldName).to((Long) null);
+ } else {
+ m.set(fieldName).to(VarInt.decodeLong(bis));
+ }
+ break;
+ case FLOAT64:
+ if (isNull) {
+ m.set(fieldName).to((Double) null);
+ } else {
+ m.set(fieldName).to(new DataInputStream(bis).readDouble());
+ }
+ break;
+ case STRING: {
+ if (isNull) {
+ m.set(fieldName).to((String) null);
+ } else {
+ int len = VarInt.decodeInt(bis);
+ byte[] bytes = readBytes(bis, len);
+ m.set(fieldName).to(new String(bytes, StandardCharsets.UTF_8));
+ }
+ break;
+ }
+ case BYTES: {
+ if (isNull) {
+ m.set(fieldName).to((ByteArray) null);
+ } else {
+ int len = VarInt.decodeInt(bis);
+ byte[] bytes = readBytes(bis, len);
+ m.set(fieldName).to(ByteArray.copyFrom(bytes));
+ }
+ break;
+ }
+ case TIMESTAMP: {
+ if (isNull) {
+ m.set(fieldName).to((Timestamp) null);
+ } else {
+ int seconds = VarInt.decodeInt(bis);
+ int nanoseconds = VarInt.decodeInt(bis);
+ m.set(fieldName).to(Timestamp.ofTimeSecondsAndNanos(seconds, nanoseconds));
+ }
+ break;
+ }
+ case DATE: {
+ if (isNull) {
+ m.set(fieldName).to((Date) null);
+ } else {
+ int days = VarInt.decodeInt(bis);
+ m.set(fieldName).to(decodeDate(days));
+ }
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("Unknown type " + type);
+ }
+ }
+
+ private byte[] readBytes(ByteArrayInputStream bis, int len) throws IOException {
+ byte[] tmp = new byte[len];
+ new DataInputStream(bis).readFully(tmp);
+ return tmp;
+ }
+
+ /**
+ * Builds a lexicographically sortable binary key based on a primary key descriptor.
+ * @param m a spanner mutation.
+ * @return a binary string that preserves the ordering of the primary key.
+ */
+ public byte[] encodeKey(Mutation m) {
+ Map<String, Value> mutationMap = mutationAsMap(m);
+ OrderedCode orderedCode = new OrderedCode();
+ for (SpannerSchema.KeyPart part : schema.getKeyParts(m.getTable())) {
+ Value val = mutationMap.get(part.getField());
+ if (val.isNull()) {
+ if (part.isDesc()) {
+ orderedCode.writeInfinityDecreasing();
+ } else {
+ orderedCode.writeInfinity();
+ }
+ } else {
+ Type.Code code = val.getType().getCode();
+ switch (code) {
+ case BOOL:
+ long v = val.getBool() ? 0 : 1;
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(v);
+ } else {
+ orderedCode.writeSignedNumIncreasing(v);
+ }
+ break;
+ case INT64:
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(val.getInt64());
+ } else {
+ orderedCode.writeSignedNumIncreasing(val.getInt64());
+ }
+ break;
+ case FLOAT64:
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(Double.doubleToLongBits(val.getFloat64()));
+ } else {
+ orderedCode.writeSignedNumIncreasing(Double.doubleToLongBits(val.getFloat64()));
+ }
+ break;
+ case STRING:
+ if (part.isDesc()) {
+ orderedCode.writeBytesDecreasing(val.getString().getBytes());
+ } else {
+ orderedCode.writeBytes(val.getString().getBytes());
+ }
+ break;
+ case BYTES:
+ if (part.isDesc()) {
+ orderedCode.writeBytesDecreasing(val.getBytes().toByteArray());
+ } else {
+ orderedCode.writeBytes(val.getBytes().toByteArray());
+ }
+ break;
+ case TIMESTAMP: {
+ Timestamp value = val.getTimestamp();
+ if (part.isDesc()) {
+ orderedCode.writeNumDecreasing(value.getSeconds());
+ orderedCode.writeNumDecreasing(value.getNanos());
+ } else {
+ orderedCode.writeNumIncreasing(value.getSeconds());
+ orderedCode.writeNumIncreasing(value.getNanos());
+ }
+ break;
+ }
+ case DATE:
+ Date value = val.getDate();
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(encodeDate(value));
+ } else {
+ orderedCode.writeSignedNumIncreasing(encodeDate(value));
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown type " + val.getType());
+ }
+ }
+ }
+ return orderedCode.getEncodedBytes();
+ }
+
+ public byte[] encodeKey(String table, Key key) {
+ OrderedCode orderedCode = new OrderedCode();
+ List<SpannerSchema.KeyPart> parts = schema.getKeyParts(table);
+ Iterator<Object> it = key.getParts().iterator();
+ for (SpannerSchema.KeyPart part : parts) {
+ Object value = it.next();
+ if (value == null) {
+ if (part.isDesc()) {
+ orderedCode.writeInfinityDecreasing();
+ } else {
+ orderedCode.writeInfinity();
+ }
+ } else {
+ if (value instanceof Boolean) {
+ long v = (Boolean) value ? 0 : 1;
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(v);
+ } else {
+ orderedCode.writeSignedNumIncreasing(v);
+ }
+ } else if (value instanceof Long) {
+ long v = (long) value;
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(v);
+ } else {
+ orderedCode.writeSignedNumIncreasing(v);
+ }
+ } else if (value instanceof Double) {
+ long v = Double.doubleToLongBits((double) value);
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(v);
+ } else {
+ orderedCode.writeSignedNumIncreasing(v);
+ }
+ } else if (value instanceof String) {
+ String v = (String) value;
+ if (part.isDesc()) {
+ orderedCode.writeBytesDecreasing(v.getBytes());
+ } else {
+ orderedCode.writeBytes(v.getBytes());
+ }
+ } else if (value instanceof ByteArray) {
+ ByteArray v = (ByteArray) value;
+ if (part.isDesc()) {
+ orderedCode.writeBytesDecreasing(v.toByteArray());
+ } else {
+ orderedCode.writeBytes(v.toByteArray());
+ }
+ } else if (value instanceof Timestamp) {
+ Timestamp v = (Timestamp) value;
+ if (part.isDesc()) {
+ orderedCode.writeNumDecreasing(v.getSeconds());
+ orderedCode.writeNumDecreasing(v.getNanos());
+ } else {
+ orderedCode.writeNumIncreasing(v.getSeconds());
+ orderedCode.writeNumIncreasing(v.getNanos());
+ }
+ } else if (value instanceof Date) {
+ Date v = (Date) value;
+ if (part.isDesc()) {
+ orderedCode.writeSignedNumDecreasing(encodeDate(v));
+ } else {
+ orderedCode.writeSignedNumIncreasing(encodeDate(v));
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown key part " + value);
+ }
+ }
+ }
+ return orderedCode.getEncodedBytes();
+ }
+
+ private static Map<String, Value> mutationAsMap(Mutation m) {
+ Map<String, Value> result = new HashMap<>();
+ Iterator<String> coli = m.getColumns().iterator();
+ Iterator<Value> vali = m.getValues().iterator();
+ while (coli.hasNext()) {
+ String column = coli.next();
+ Value val = vali.next();
+ result.put(column.toLowerCase(), val);
+ }
+ return result;
+ }
+
+ private static int encodeDate(Date date) {
+
+ MutableDateTime jodaDate = new MutableDateTime();
+ jodaDate.setDate(date.getYear(), date.getMonth(), date.getDayOfMonth());
+
+ return Days.daysBetween(MIN_DATE, jodaDate).getDays();
+ }
+
+ private static Date decodeDate(int daysSinceEpoch) {
+
+ DateTime jodaDate = MIN_DATE.plusDays(daysSinceEpoch);
+
+ return Date
+ .fromYearMonthDay(jodaDate.getYear(), jodaDate.getMonthOfYear(), jodaDate.getDayOfMonth());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
new file mode 100644
index 0000000..c483af9
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.Date;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeyRange;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Value;
+
+/** Estimates the logical size of {@link com.google.cloud.spanner.Mutation}. */
+class MutationSizeEstimator {
+
+ // Prevent construction.
+ private MutationSizeEstimator() {}
+
+ /** Estimates a size of mutation in bytes. */
+ static long sizeOf(Mutation m) {
+ if (m.getOperation() == Mutation.Op.DELETE) {
+ return sizeOf(m.getKeySet());
+ }
+ long result = 0;
+ for (Value v : m.getValues()) {
+ switch (v.getType().getCode()) {
+ case ARRAY:
+ result += estimateArrayValue(v);
+ break;
+ case STRUCT:
+ throw new IllegalArgumentException("Structs are not supported in mutation.");
+ default:
+ result += estimatePrimitiveValue(v);
+ }
+ }
+ return result;
+ }
+
+ private static long sizeOf(KeySet keySet) {
+ long result = 0;
+ for (Key k : keySet.getKeys()) {
+ result += sizeOf(k);
+ }
+ for (KeyRange kr : keySet.getRanges()) {
+ result += sizeOf(kr);
+ }
+ return result;
+ }
+
+ private static long sizeOf(KeyRange kr) {
+ return sizeOf(kr.getStart()) + sizeOf(kr.getEnd());
+ }
+
+ private static long sizeOf(Key k) {
+ long result = 0;
+ for (Object part : k.getParts()) {
+ if (part == null) {
+ continue;
+ }
+ if (part instanceof Boolean) {
+ result += 1;
+ } else if (part instanceof Long) {
+ result += 8;
+ } else if (part instanceof Double) {
+ result += 8;
+ } else if (part instanceof String) {
+ result += ((String) part).length();
+ } else if (part instanceof ByteArray) {
+ result += ((ByteArray) part).length();
+ } else if (part instanceof Timestamp) {
+ result += 12;
+ } else if (part instanceof Date) {
+ result += 12;
+ }
+ }
+ return result;
+ }
+
+ /** Estimates a size of the mutation group in bytes. */
+ public static long sizeOf(MutationGroup group) {
+ long result = 0;
+ for (Mutation m : group) {
+ result += sizeOf(m);
+ }
+ return result;
+ }
+
+ private static long estimatePrimitiveValue(Value v) {
+ switch (v.getType().getCode()) {
+ case BOOL:
+ return 1;
+ case INT64:
+ case FLOAT64:
+ return 8;
+ case DATE:
+ case TIMESTAMP:
+ return 12;
+ case STRING:
+ return v.isNull() ? 0 : v.getString().length();
+ case BYTES:
+ return v.isNull() ? 0 : v.getBytes().length();
+ }
+ throw new IllegalArgumentException("Unsupported type " + v.getType());
+ }
+
+ private static long estimateArrayValue(Value v) {
+ switch (v.getType().getArrayElementType().getCode()) {
+ case BOOL:
+ return v.getBoolArray().size();
+ case INT64:
+ return 8 * v.getInt64Array().size();
+ case FLOAT64:
+ return 8 * v.getFloat64Array().size();
+ case STRING:
+ long totalLength = 0;
+ for (String s : v.getStringArray()) {
+ if (s == null) {
+ continue;
+ }
+ totalLength += s.length();
+ }
+ return totalLength;
+ case BYTES:
+ totalLength = 0;
+ for (ByteArray bytes : v.getBytesArray()) {
+ if (bytes == null) {
+ continue;
+ }
+ totalLength += bytes.length();
+ }
+ return totalLength;
+ case DATE:
+ return 12 * v.getDateArray().size();
+ case TIMESTAMP:
+ return 12 * v.getTimestampArray().size();
+ }
+ throw new IllegalArgumentException("Unsupported type " + v.getType());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerReadFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerReadFn.java
new file mode 100644
index 0000000..34996f1
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerReadFn.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.TimestampBound;
+import com.google.common.annotations.VisibleForTesting;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.PCollectionView;
+
+/** A simplest read function implementation. Parallelism support is coming. */
+@VisibleForTesting
+class NaiveSpannerReadFn extends DoFn<ReadOperation, Struct> {
+ private final SpannerConfig config;
+ @Nullable private final PCollectionView<Transaction> transaction;
+ private transient SpannerAccessor spannerAccessor;
+
+ NaiveSpannerReadFn(SpannerConfig config, @Nullable PCollectionView<Transaction> transaction) {
+ this.config = config;
+ this.transaction = transaction;
+ }
+
+ NaiveSpannerReadFn(SpannerConfig config) {
+ this(config, null);
+ }
+
+
+ @Setup
+ public void setup() throws Exception {
+ spannerAccessor = config.connectToSpanner();
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ spannerAccessor.close();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ TimestampBound timestampBound = TimestampBound.strong();
+ if (transaction != null) {
+ Transaction transaction = c.sideInput(this.transaction);
+ timestampBound = TimestampBound.ofReadTimestamp(transaction.timestamp());
+ }
+ ReadOperation op = c.element();
+ DatabaseClient databaseClient = spannerAccessor.getDatabaseClient();
+ try (ReadOnlyTransaction readOnlyTransaction =
+ databaseClient.readOnlyTransaction(timestampBound)) {
+ ResultSet resultSet = execute(op, readOnlyTransaction);
+ while (resultSet.next()) {
+ c.output(resultSet.getCurrentRowAsStruct());
+ }
+ }
+ }
+
+ private ResultSet execute(ReadOperation op, ReadOnlyTransaction readOnlyTransaction) {
+ if (op.getQuery() != null) {
+ return readOnlyTransaction.executeQuery(op.getQuery());
+ }
+ if (op.getIndex() != null) {
+ return readOnlyTransaction.readUsingIndex(
+ op.getTable(), op.getIndex(), op.getKeySet(), op.getColumns());
+ }
+ return readOnlyTransaction.read(op.getTable(), op.getKeySet(), op.getColumns());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java
new file mode 100644
index 0000000..80290d6
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java
@@ -0,0 +1,764 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.math.LongMath;
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.UnsignedInteger;
+
+import java.math.RoundingMode;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/**
+ * This module provides routines for encoding a sequence of typed
+ * entities into a byte array. The resulting byte arrays can be
+ * lexicographically compared to yield the same comparison value that
+ * would have been generated if the encoded items had been compared
+ * one by one according to their type.
+ *
+ * <p>More precisely, suppose:
+ * 1. byte array A is generated by encoding the sequence of items [A_1..A_n]
+ * 2. byte array B is generated by encoding the sequence of items [B_1..B_n]
+ * 3. The types match; i.e., for all i: A_i was encoded using
+ * the same routine as B_i
+ *
+ * <p>Then:
+ * Comparing A vs. B lexicographically is the same as comparing
+ * the vectors [A_1..A_n] and [B_1..B_n] lexicographically.
+ *
+ * <p><b>This class is NOT thread safe.</b>
+ */
+class OrderedCode {
+ // We want to encode a few extra symbols in strings:
+ // <sep> Separator between items
+ // <infinity> Infinite string
+ //
+ // Therefore we need an alphabet with at least 258 characters. We
+ // achieve this by using two-letter sequences starting with '\0' and '\xff'
+ // as extra symbols:
+ // <sep> encoded as => \0\1
+ // \0 encoded as => \0\xff
+ // \xff encoded as => \xff\x00
+ // <infinity> encoded as => \xff\xff
+ //
+ // The remaining two letter sequences starting with '\0' and '\xff'
+ // are currently unused.
+
+ static final byte ESCAPE1 = 0x00;
+ static final byte NULL_CHARACTER =
+ (byte) 0xff; // Combined with ESCAPE1
+ static final byte SEPARATOR = 0x01; // Combined with ESCAPE1
+
+ static final byte ESCAPE2 = (byte) 0xff;
+ static final byte INFINITY =
+ (byte) 0xff; // Combined with ESCAPE2
+ static final byte FF_CHARACTER = 0x00; // Combined with ESCAPE2
+
+ static final byte[] ESCAPE1_SEPARATOR = { ESCAPE1, SEPARATOR };
+
+ static final byte[] INFINITY_ENCODED = { ESCAPE2, INFINITY };
+
+ static final byte[] INFINITY_ENCODED_DECREASING = {invert(ESCAPE2), invert(INFINITY)};
+
+ /**
+ * This array maps encoding length to header bits in the first two bytes for
+ * SignedNumIncreasing encoding.
+ */
+ private static final byte[][] LENGTH_TO_HEADER_BITS = {
+ { 0, 0 },
+ { (byte) 0x80, 0 },
+ { (byte) 0xc0, 0 },
+ { (byte) 0xe0, 0 },
+ { (byte) 0xf0, 0 },
+ { (byte) 0xf8, 0 },
+ { (byte) 0xfc, 0 },
+ { (byte) 0xfe, 0 },
+ { (byte) 0xff, 0 },
+ { (byte) 0xff, (byte) 0x80 },
+ { (byte) 0xff, (byte) 0xc0 }
+ };
+
+ /**
+ * This array maps encoding lengths to the header bits that overlap with
+ * the payload and need fixing during readSignedNumIncreasing.
+ */
+ private static final long[] LENGTH_TO_MASK = {
+ 0L,
+ 0x80L,
+ 0xc000L,
+ 0xe00000L,
+ 0xf0000000L,
+ 0xf800000000L,
+ 0xfc0000000000L,
+ 0xfe000000000000L,
+ 0xff00000000000000L,
+ 0x8000000000000000L,
+ 0L
+ };
+
+ /**
+ * This array maps the number of bits in a number to the encoding
+ * length produced by WriteSignedNumIncreasing.
+ * For positive numbers, the number of bits is 1 plus the most significant
+ * bit position (the highest bit position in a positive long is 63).
+ * For a negative number n, we count the bits in ~n.
+ * That is, length = BITS_TO_LENGTH[log2Floor(n < 0 ? ~n : n) + 1].
+ */
+ private static final short[] BITS_TO_LENGTH = {
+ 1, 1, 1, 1, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2,
+ 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4,
+ 5, 5, 5, 5, 5, 5, 5,
+ 6, 6, 6, 6, 6, 6, 6,
+ 7, 7, 7, 7, 7, 7, 7,
+ 8, 8, 8, 8, 8, 8, 8,
+ 9, 9, 9, 9, 9, 9, 9,
+ 10
+ };
+
+ // stores the current encoded value as a list of byte arrays. Note that this
+ // is manipulated as we read/write items.
+ // Note that every item will fit on at most one array. One array may
+ // have more than one item (eg when used for decoding). While encoding,
+ // one array will have exactly one item. While returning the encoded array
+ // we will merge all the arrays in this list.
+ private final ArrayList<byte[]> encodedArrays = new ArrayList<byte[]>();
+
+ // This is the current position on the first array. Will be non-zero
+ // only if the ordered code was created using encoded byte array.
+ private int firstArrayPosition = 0;
+
+ /**
+ * Creates OrderedCode from scratch. Typically used at encoding time.
+ */
+ public OrderedCode(){
+ }
+
+ /**
+ * Creates OrderedCode from a given encoded byte array. Typically used at
+ * decoding time.
+ *
+ * <p><b> For better performance, it uses the input array provided (not a copy).
+ * Therefore the input array should not be modified.</b>
+ */
+ public OrderedCode(byte[] encodedByteArray) {
+ encodedArrays.add(encodedByteArray);
+ }
+
+ /**
+ * Adds the given byte array item to the OrderedCode. It encodes the input
+ * byte array, followed by a separator and appends the result to its
+ * internal encoded byte array store.
+ *
+ * <p>It works with the input array,
+ * so the input array 'value' should not be modified till the method returns.
+ *
+ * @param value bytes to be written.
+ * @see #readBytes()
+ */
+ public void writeBytes(byte[] value) {
+ writeBytes(value, false);
+ }
+
+ public void writeBytesDecreasing(byte[] value) {
+ writeBytes(value, true);
+ }
+
+ private void writeBytes(byte[] value, boolean invert) {
+ // Determine the length of the encoded array
+ int encodedLength = 2; // for separator
+ for (byte b : value) {
+ if ((b == ESCAPE1) || (b == ESCAPE2)) {
+ encodedLength += 2;
+ } else {
+ encodedLength++;
+ }
+ }
+
+ byte[] encodedArray = new byte[encodedLength];
+ int copyStart = 0;
+ int outIndex = 0;
+ for (int i = 0; i < value.length; i++) {
+ byte b = value[i];
+ if (b == ESCAPE1) {
+ arraycopy(invert, value, copyStart, encodedArray, outIndex,
+ i - copyStart);
+ outIndex += i - copyStart;
+ encodedArray[outIndex++] = convert(invert, ESCAPE1);
+ encodedArray[outIndex++] = convert(invert, NULL_CHARACTER);
+ copyStart = i + 1;
+ } else if (b == ESCAPE2) {
+ arraycopy(invert, value, copyStart, encodedArray, outIndex,
+ i - copyStart);
+ outIndex += i - copyStart;
+ encodedArray[outIndex++] = convert(invert, ESCAPE2);
+ encodedArray[outIndex++] = convert(invert, FF_CHARACTER);
+ copyStart = i + 1;
+ }
+ }
+ if (copyStart < value.length) {
+ arraycopy(invert, value, copyStart, encodedArray, outIndex,
+ value.length - copyStart);
+ outIndex += value.length - copyStart;
+ }
+ encodedArray[outIndex++] = convert(invert, ESCAPE1);
+ encodedArray[outIndex] = convert(invert, SEPARATOR);
+
+ encodedArrays.add(encodedArray);
+ }
+
+ private static byte convert(boolean invert, byte val) {
+ return invert ? (byte) ~val : val;
+ }
+
+ private static byte invert(byte val) {
+ return convert(true, val);
+ }
+
+ private void arraycopy(
+ boolean invert, byte[] src, int srcPos, byte[] dest, int destPos, int length) {
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ if (invert) {
+ for (int i = destPos; i < destPos + length; i++) {
+ dest[i] = (byte) ~dest[i];
+ }
+ }
+ }
+
+ /**
+ * Encodes the long item, in big-endian format, and appends the result to its
+ * internal encoded byte array store.
+ *
+ * @see #readNumIncreasing()
+ */
+ public void writeNumIncreasing(long value) {
+ // Values are encoded with a single byte length prefix, followed
+ // by the actual value in big-endian format with leading 0 bytes
+ // dropped.
+ byte[] bufer = new byte[9]; // 8 bytes for value plus one byte for length
+ int len = 0;
+ while (value != 0) {
+ len++;
+ bufer[9 - len] = (byte) (value & 0xff);
+ value >>>= 8;
+ }
+ bufer[9 - len - 1] = (byte) len;
+ len++;
+ byte[] encodedArray = new byte[len];
+ System.arraycopy(bufer, 9 - len, encodedArray, 0, len);
+ encodedArrays.add(encodedArray);
+ }
+
+ public void writeNumIncreasing(UnsignedInteger unsignedInt) {
+ writeNumIncreasing(unsignedInt.longValue());
+ }
+
+ /**
+ * Encodes the long item, in big-endian format, and appends the result to its
+ * internal encoded byte array store.
+ *
+ * @see #readNumIncreasing()
+ */
+ public void writeNumDecreasing(long value) {
+ // Values are encoded with a complemented single byte length prefix,
+ // followed by the complement of the actual value in big-endian format with
+ // leading 0xff bytes dropped.
+ byte[] bufer = new byte[9]; // 8 bytes for value plus one byte for length
+ int len = 0;
+ while (value != 0) {
+ len++;
+ bufer[9 - len] = (byte) ~(value & 0xff);
+ value >>>= 8;
+ }
+ bufer[9 - len - 1] = (byte) ~len;
+ len++;
+ byte[] encodedArray = new byte[len];
+ System.arraycopy(bufer, 9 - len, encodedArray, 0, len);
+ encodedArrays.add(encodedArray);
+ }
+
+ public void writeNumDecreasing(UnsignedInteger unsignedInt) {
+ writeNumDecreasing(unsignedInt.longValue());
+ }
+
+ /**
+ * Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0.
+ *
+ */
+ @VisibleForTesting
+ int log2Floor(long n) {
+ checkArgument(n >= 0);
+ return n == 0 ? -1 : LongMath.log2(n, RoundingMode.FLOOR);
+ }
+
+ /**
+ * Calculates the encoding length in bytes of the signed number n.
+ */
+ @VisibleForTesting
+ int getSignedEncodingLength(long n) {
+ return BITS_TO_LENGTH[log2Floor(n < 0 ? ~n : n) + 1];
+ }
+
+ /**
+ * @see #readSignedNumIncreasing()
+ */
+ public void writeSignedNumIncreasing(long val) {
+ long x = val < 0 ? ~val : val;
+ if (x < 64) { // Fast path for encoding length == 1.
+ byte[] encodedArray = new byte[] { (byte) (LENGTH_TO_HEADER_BITS[1][0] ^ val) };
+ encodedArrays.add(encodedArray);
+ return;
+ }
+ // buf = val in network byte order, sign extended to 10 bytes.
+ byte signByte = val < 0 ? (byte) 0xff : 0;
+ byte[] buf = new byte[2 + Longs.BYTES];
+ buf[0] = buf[1] = signByte;
+ System.arraycopy(Longs.toByteArray(val), 0, buf, 2, Longs.BYTES);
+ int len = getSignedEncodingLength(x);
+ if (len < 2) {
+ throw new IllegalStateException(
+ "Invalid length (" + len + ")" + " returned by getSignedEncodingLength(" + x + ")");
+ }
+ int beginIndex = buf.length - len;
+ buf[beginIndex] ^= LENGTH_TO_HEADER_BITS[len][0];
+ buf[beginIndex + 1] ^= LENGTH_TO_HEADER_BITS[len][1];
+
+ byte[] encodedArray = new byte[len];
+ System.arraycopy(buf, beginIndex, encodedArray, 0, len);
+ encodedArrays.add(encodedArray);
+ }
+
+ public void writeSignedNumDecreasing(long val) {
+ writeSignedNumIncreasing(~val);
+ }
+
+ /**
+ * Encodes and appends INFINITY item to its internal encoded byte array
+ * store.
+ *
+ * @see #readInfinity()
+ */
+ public void writeInfinity() {
+ writeTrailingBytes(INFINITY_ENCODED);
+ }
+
+ /**
+ * Encodes and appends INFINITY item which would come before any real string.
+ *
+ * @see #readInfinityDecreasing()
+ */
+ public void writeInfinityDecreasing() {
+ writeTrailingBytes(INFINITY_ENCODED_DECREASING);
+ }
+
+ /**
+ * Appends the byte array item to its internal encoded byte array
+ * store. This is used for the last item and is not encoded.
+ *
+ * <p>It stores the input array in the store,
+ * so the input array 'value' should not be modified.
+ *
+ * @param value bytes to be written.
+ * @see #readTrailingBytes()
+ */
+ public void writeTrailingBytes(byte[] value) {
+ if ((value == null) || (value.length == 0)) {
+ throw new IllegalArgumentException(
+ "Value cannot be null or have 0 elements");
+ }
+
+ encodedArrays.add(value);
+ }
+
+ /**
+ * Returns the next byte array item from its encoded byte array store and
+ * removes the item from the store.
+ *
+ * @see #writeBytes(byte[])
+ */
+ public byte[] readBytes() {
+ return readBytes(false);
+ }
+
+ public byte[] readBytesDecreasing() {
+ return readBytes(true);
+ }
+
+ private byte[] readBytes(boolean invert) {
+ if ((encodedArrays == null) || (encodedArrays.size() == 0) || (
+ (encodedArrays.get(0)).length - firstArrayPosition <= 0)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ // Determine the length of the decoded array
+ // We only scan up to "length-2" since a valid string must end with
+ // a two character terminator: 'ESCAPE1 SEPARATOR'
+ byte[] store = encodedArrays.get(0);
+ int decodedLength = 0;
+ boolean valid = false;
+ int i = firstArrayPosition;
+ while (i < store.length - 1) {
+ byte b = store[i++];
+ if (b == convert(invert, ESCAPE1)) {
+ b = store[i++];
+ if (b == convert(invert, SEPARATOR)) {
+ valid = true;
+ break;
+ } else if (b == convert(invert, NULL_CHARACTER)) {
+ decodedLength++;
+ } else {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ } else if (b == convert(invert, ESCAPE2)) {
+ b = store[i++];
+ if (b == convert(invert, FF_CHARACTER)) {
+ decodedLength++;
+ } else {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ } else {
+ decodedLength++;
+ }
+ }
+ if (!valid) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ byte[] decodedArray = new byte[decodedLength];
+ int copyStart = firstArrayPosition;
+ int outIndex = 0;
+ int j = firstArrayPosition;
+ while (j < store.length - 1) {
+ byte b = store[j++]; // note that j has been incremented
+ if (b == convert(invert, ESCAPE1)) {
+ arraycopy(invert, store, copyStart, decodedArray, outIndex, j - copyStart - 1);
+ outIndex += j - copyStart - 1;
+ // ESCAPE1 SEPARATOR ends component
+ // ESCAPE1 NULL_CHARACTER represents '\0'
+ b = store[j++];
+ if (b == convert(invert, SEPARATOR)) {
+ if ((store.length - j) == 0) {
+ // we are done with the first array
+ encodedArrays.remove(0);
+ firstArrayPosition = 0;
+ } else {
+ firstArrayPosition = j;
+ }
+ return decodedArray;
+ } else if (b == convert(invert, NULL_CHARACTER)) {
+ decodedArray[outIndex++] = 0x00;
+ } // else not required - handled during length determination
+ copyStart = j;
+ } else if (b == convert(invert, ESCAPE2)) {
+ arraycopy(invert, store, copyStart, decodedArray, outIndex, j - copyStart - 1);
+ outIndex += j - copyStart - 1;
+ // ESCAPE2 FF_CHARACTER represents '\xff'
+ // ESCAPE2 INFINITY is an error
+ b = store[j++];
+ if (b == convert(invert, FF_CHARACTER)) {
+ decodedArray[outIndex++] = (byte) 0xff;
+ } // else not required - handled during length determination
+ copyStart = j;
+ }
+ }
+ // not required due to the first phase, but need to entertain the compiler
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ /**
+ * Returns the next long item (encoded in big-endian format via
+ * {@code writeNumIncreasing(long)}) from its internal encoded byte array
+ * store and removes the item from the store.
+ *
+ * @see #writeNumIncreasing(long)
+ */
+ public long readNumIncreasing() {
+ if ((encodedArrays == null) || (encodedArrays.size() == 0) || (
+ (encodedArrays.get(0)).length - firstArrayPosition < 1)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ byte[] store = encodedArrays.get(0);
+ // Decode length byte
+ int len = store[firstArrayPosition];
+ if ((firstArrayPosition + len + 1 > store.length) || len > 8) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ long result = 0;
+ for (int i = 0; i < len; i++) {
+ result <<= 8;
+ result |= (store[firstArrayPosition + i + 1] & 0xff);
+ }
+
+ if ((store.length - firstArrayPosition - len - 1) == 0) {
+ // we are done with the first array
+ encodedArrays.remove(0);
+ firstArrayPosition = 0;
+ } else {
+ firstArrayPosition = firstArrayPosition + len + 1;
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the next long item (encoded in big-endian format via
+ * {@code writeNumDecreasing(long)}) from its internal encoded byte array
+ * store and removes the item from the store.
+ *
+ * @see #writeNumDecreasing(long)
+ */
+ public long readNumDecreasing() {
+ if ((encodedArrays == null) || (encodedArrays.size() == 0)
+ || ((encodedArrays.get(0)).length - firstArrayPosition < 1)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ byte[] store = encodedArrays.get(0);
+ // Decode length byte
+ int len = ~store[firstArrayPosition] & 0xff;
+ if ((firstArrayPosition + len + 1 > store.length) || len > 8) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ long result = 0;
+ for (int i = 0; i < len; i++) {
+ result <<= 8;
+ result |= (~store[firstArrayPosition + i + 1] & 0xff);
+ }
+
+ if ((store.length - firstArrayPosition - len - 1) == 0) {
+ // we are done with the first array
+ encodedArrays.remove(0);
+ firstArrayPosition = 0;
+ } else {
+ firstArrayPosition = firstArrayPosition + len + 1;
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the next long item (encoded via
+ * {@code writeSignedNumIncreasing(long)}) from its internal encoded byte
+ * array store and removes the item from the store.
+ *
+ * @see #writeSignedNumIncreasing(long)
+ */
+ public long readSignedNumIncreasing() {
+ if ((encodedArrays == null) || (encodedArrays.size() == 0) || (
+ (encodedArrays.get(0)).length - firstArrayPosition < 1)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ byte[] store = encodedArrays.get(0);
+
+ long xorMask = ((store[firstArrayPosition] & 0x80) == 0) ? ~0L : 0L;
+ // Store first byte as an int rather than a (signed) byte -- to avoid
+ // accidental byte-to-int promotion later which would extend the byte's
+ // sign bit (if any).
+ int firstByte = (store[firstArrayPosition] & 0xff) ^ (int) (xorMask & 0xff);
+
+ // Now calculate and test length, and set x to raw (unmasked) result.
+ int len;
+ long x;
+ if (firstByte != 0xff) {
+ len = 7 - log2Floor(firstByte ^ 0xff);
+ if (store.length - firstArrayPosition < len) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ x = xorMask; // Sign extend using xorMask.
+ for (int i = firstArrayPosition; i < firstArrayPosition + len; i++) {
+ x = (x << 8) | (store[i] & 0xff);
+ }
+ } else {
+ len = 8;
+ if (store.length - firstArrayPosition < len) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ int secondByte = (store[firstArrayPosition + 1] & 0xff) ^ (int) (xorMask & 0xff);
+ if (secondByte >= 0x80) {
+ if (secondByte < 0xc0) {
+ len = 9;
+ } else {
+ int thirdByte = (store[firstArrayPosition + 2] & 0xff) ^ (int) (xorMask & 0xff);
+ if (secondByte == 0xc0 && thirdByte < 0x80) {
+ len = 10;
+ } else {
+ // Either len > 10 or len == 10 and #bits > 63.
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ }
+ if (store.length - firstArrayPosition < len) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ }
+ x = Longs.fromByteArray(
+ Arrays.copyOfRange(store, firstArrayPosition + len - 8, firstArrayPosition + len));
+ }
+
+ x ^= LENGTH_TO_MASK[len]; // Remove spurious header bits.
+
+ if (len != getSignedEncodingLength(x)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ if ((store.length - firstArrayPosition - len) == 0) {
+ // We are done with the first array.
+ encodedArrays.remove(0);
+ firstArrayPosition = 0;
+ } else {
+ firstArrayPosition = firstArrayPosition + len;
+ }
+
+ return x;
+ }
+
+ public long readSignedNumDecreasing() {
+ return ~readSignedNumIncreasing();
+ }
+
+ /**
+ * Removes INFINITY item from its internal encoded byte array store
+ * if present. Returns whether INFINITY was present.
+ *
+ * @see #writeInfinity()
+ */
+ public boolean readInfinity() {
+ return readInfinityInternal(INFINITY_ENCODED);
+ }
+
+ /**
+ * Removes INFINITY item from its internal encoded byte array store if present. Returns whether
+ * INFINITY was present.
+ *
+ * @see #writeInfinityDecreasing()
+ */
+ public boolean readInfinityDecreasing() {
+ return readInfinityInternal(INFINITY_ENCODED_DECREASING);
+ }
+
+ private boolean readInfinityInternal(byte[] codes) {
+ if ((encodedArrays == null) || (encodedArrays.size() == 0)
+ || ((encodedArrays.get(0)).length - firstArrayPosition < 1)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+ byte[] store = encodedArrays.get(0);
+ if (store.length - firstArrayPosition < 2) {
+ return false;
+ }
+ if ((store[firstArrayPosition] == codes[0]) && (store[firstArrayPosition + 1] == codes[1])) {
+ if ((store.length - firstArrayPosition - 2) == 0) {
+ // we are done with the first array
+ encodedArrays.remove(0);
+ firstArrayPosition = 0;
+ } else {
+ firstArrayPosition = firstArrayPosition + 2;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the trailing byte array item from its internal encoded byte array
+ * store and removes the item from the store.
+ *
+ * @see #writeTrailingBytes(byte[])
+ */
+ public byte[] readTrailingBytes() {
+ // one item is contained within one byte array
+ if ((encodedArrays == null) || (encodedArrays.size() != 1)) {
+ throw new IllegalArgumentException("Invalid encoded byte array");
+ }
+
+ byte[] store = encodedArrays.get(0);
+ encodedArrays.remove(0);
+ assert encodedArrays.size() == 0;
+ return Arrays.copyOfRange(store, firstArrayPosition, store.length);
+ }
+
+ /**
+ * Returns the encoded bytes that represents the current state of the
+ * OrderedCode.
+ *
+ * <p><b> NOTE: This method returns OrederedCode's internal array (not a
+ * copy) for better performance. Therefore the returned array should not be
+ * modified.</b>
+ */
+ public byte[] getEncodedBytes() {
+ if (encodedArrays.size() == 0) {
+ return new byte[0];
+ }
+ if ((encodedArrays.size() == 1) && (firstArrayPosition == 0)) {
+ return encodedArrays.get(0);
+ }
+
+ int totalLength = 0;
+
+ for (int i = 0; i < encodedArrays.size(); i++) {
+ byte[] bytes = encodedArrays.get(i);
+ if (i == 0) {
+ totalLength += bytes.length - firstArrayPosition;
+ } else {
+ totalLength += bytes.length;
+ }
+ }
+
+ byte[] encodedBytes = new byte[totalLength];
+ int destPos = 0;
+ for (int i = 0; i < encodedArrays.size(); i++) {
+ byte[] bytes = encodedArrays.get(i);
+ if (i == 0) {
+ System.arraycopy(bytes, firstArrayPosition, encodedBytes, destPos,
+ bytes.length - firstArrayPosition);
+ destPos += bytes.length - firstArrayPosition;
+ } else {
+ System.arraycopy(bytes, 0, encodedBytes, destPos, bytes.length);
+ destPos += bytes.length;
+ }
+ }
+
+ // replace the store with merged array, so that repeated calls
+ // don't need to merge. The reads can handle both the versions.
+ encodedArrays.clear();
+ encodedArrays.add(encodedBytes);
+ firstArrayPosition = 0;
+
+ return encodedBytes;
+ }
+
+ /**
+ * Returns true if it has more encoded bytes that haven't been read,
+ * false otherwise. Return value of true doesn't imply anything about
+ * validity of remaining data.
+ * @return true if it has more encoded bytes that haven't been read,
+ * false otherwise.
+ */
+ public boolean hasRemainingEncodedBytes() {
+ // We delete an array after fully consuming it.
+ return encodedArrays != null && encodedArrays.size() != 0;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
new file mode 100644
index 0000000..3b2bb6b
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Statement;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/** Encapsulates a spanner read operation. */
+@AutoValue
+public abstract class ReadOperation implements Serializable {
+
+ public static ReadOperation create() {
+ return new AutoValue_ReadOperation.Builder().setKeySet(KeySet.all()).build();
+ }
+
+ @Nullable
+ public abstract Statement getQuery();
+
+ @Nullable
+ public abstract String getTable();
+
+ @Nullable
+ public abstract String getIndex();
+
+ @Nullable
+ public abstract List<String> getColumns();
+
+ @Nullable
+ public abstract KeySet getKeySet();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ abstract Builder setQuery(Statement statement);
+
+ abstract Builder setTable(String table);
+
+ abstract Builder setIndex(String index);
+
+ abstract Builder setColumns(List<String> columns);
+
+ abstract Builder setKeySet(KeySet keySet);
+
+ abstract ReadOperation build();
+ }
+
+ abstract Builder toBuilder();
+
+ public ReadOperation withTable(String table) {
+ return toBuilder().setTable(table).build();
+ }
+
+ public ReadOperation withColumns(String... columns) {
+ return withColumns(Arrays.asList(columns));
+ }
+
+ public ReadOperation withColumns(List<String> columns) {
+ return toBuilder().setColumns(columns).build();
+ }
+
+ public ReadOperation withQuery(Statement statement) {
+ return toBuilder().setQuery(statement).build();
+ }
+
+ public ReadOperation withQuery(String sql) {
+ return withQuery(Statement.of(sql));
+ }
+
+ public ReadOperation withKeySet(KeySet keySet) {
+ return toBuilder().setKeySet(keySet).build();
+ }
+
+ public ReadOperation withIndex(String index) {
+ return toBuilder().setIndex(index).build();
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchema.java
new file mode 100644
index 0000000..1c16fa8
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchema.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Statement;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/**
+ * This {@link DoFn} reads Cloud Spanner 'information_schema.*' tables to build the
+ * {@link SpannerSchema}.
+ */
+class ReadSpannerSchema extends DoFn<Void, SpannerSchema> {
+
+ private final SpannerConfig config;
+
+ private transient SpannerAccessor spannerAccessor;
+
+ public ReadSpannerSchema(SpannerConfig config) {
+ this.config = config;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ spannerAccessor = config.connectToSpanner();
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ spannerAccessor.close();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+ DatabaseClient databaseClient = spannerAccessor.getDatabaseClient();
+ try (ReadOnlyTransaction tx =
+ databaseClient.readOnlyTransaction()) {
+ ResultSet resultSet = readTableInfo(tx);
+
+ while (resultSet.next()) {
+ String tableName = resultSet.getString(0);
+ String columnName = resultSet.getString(1);
+ String type = resultSet.getString(2);
+
+ builder.addColumn(tableName, columnName, type);
+ }
+
+ resultSet = readPrimaryKeyInfo(tx);
+ while (resultSet.next()) {
+ String tableName = resultSet.getString(0);
+ String columnName = resultSet.getString(1);
+ String ordering = resultSet.getString(2);
+
+ builder.addKeyPart(tableName, columnName, ordering.toUpperCase().equals("DESC"));
+ }
+ }
+ c.output(builder.build());
+ }
+
+ private ResultSet readTableInfo(ReadOnlyTransaction tx) {
+ return tx.executeQuery(Statement.of(
+ "SELECT c.table_name, c.column_name, c.spanner_type"
+ + " FROM information_schema.columns as c"
+ + " WHERE c.table_catalog = '' AND c.table_schema = ''"
+ + " ORDER BY c.table_name, c.ordinal_position"));
+ }
+
+ private ResultSet readPrimaryKeyInfo(ReadOnlyTransaction tx) {
+ return tx.executeQuery(Statement
+ .of("SELECT t.table_name, t.column_name, t.column_ordering"
+ + " FROM information_schema.index_columns AS t "
+ + " WHERE t.index_name = 'PRIMARY_KEY' AND t.table_catalog = ''"
+ + " AND t.table_schema = ''"
+ + " ORDER BY t.table_name, t.ordinal_position"));
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutation.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutation.java
new file mode 100644
index 0000000..a5bebce
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutation.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+abstract class SerializedMutation {
+ static SerializedMutation create(String tableName, byte[] key,
+ byte[] bytes) {
+ return new AutoValue_SerializedMutation(tableName, key, bytes);
+ }
+
+ abstract String getTableName();
+
+ abstract byte[] getEncodedKey();
+
+ abstract byte[] getMutationGroupBytes();
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutationCoder.java
new file mode 100644
index 0000000..33ec1ed
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutationCoder.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+
+class SerializedMutationCoder extends AtomicCoder<SerializedMutation> {
+
+ private static final SerializedMutationCoder INSTANCE = new SerializedMutationCoder();
+
+ public static SerializedMutationCoder of() {
+ return INSTANCE;
+ }
+
+ private final ByteArrayCoder byteArrayCoder;
+ private final StringUtf8Coder stringCoder;
+
+ private SerializedMutationCoder() {
+ byteArrayCoder = ByteArrayCoder.of();
+ stringCoder = StringUtf8Coder.of();
+ }
+
+ @Override
+ public void encode(SerializedMutation value, OutputStream out)
+ throws IOException {
+ stringCoder.encode(value.getTableName(), out);
+ byteArrayCoder.encode(value.getEncodedKey(), out);
+ byteArrayCoder.encode(value.getMutationGroupBytes(), out);
+ }
+
+ @Override
+ public SerializedMutation decode(InputStream in)
+ throws IOException {
+ String tableName = stringCoder.decode(in);
+ byte[] encodedKey = byteArrayCoder.decode(in);
+ byte[] mutationBytes = byteArrayCoder.decode(in);
+ return SerializedMutation.create(tableName, encodedKey, mutationBytes);
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
new file mode 100644
index 0000000..f32e661
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.Spanner;
+
+/**
+ * Manages lifecycle of {@link DatabaseClient} and {@link Spanner} instances.
+ */
+public class SpannerAccessor implements AutoCloseable {
+ private final Spanner spanner;
+ private final DatabaseClient databaseClient;
+
+ SpannerAccessor(Spanner spanner, DatabaseClient databaseClient) {
+ this.spanner = spanner;
+ this.databaseClient = databaseClient;
+ }
+
+ public DatabaseClient getDatabaseClient() {
+ return databaseClient;
+ }
+
+ @Override
+ public void close() {
+ spanner.close();
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java
new file mode 100644
index 0000000..9be641f
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.ServiceFactory;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.ReleaseInfo;
+
+/** Configuration for a Cloud Spanner client. */
+@AutoValue
+public abstract class SpannerConfig implements Serializable {
+ // A common user agent token that indicates that this request was originated from Apache Beam.
+ private static final String USER_AGENT_PREFIX = "Apache_Beam_Java";
+ // A default host name for batch traffic.
+ private static final String DEFAULT_HOST = "https://batch-spanner.googleapis.com/";
+
+ @Nullable
+ abstract ValueProvider<String> getProjectId();
+
+ @Nullable
+ abstract ValueProvider<String> getInstanceId();
+
+ @Nullable
+ abstract ValueProvider<String> getDatabaseId();
+
+ @Nullable
+ abstract String getHost();
+
+ @Nullable
+ @VisibleForTesting
+ abstract ServiceFactory<Spanner, SpannerOptions> getServiceFactory();
+
+ abstract Builder toBuilder();
+
+ public static SpannerConfig create() {
+ return builder().setHost(DEFAULT_HOST).build();
+ }
+
+ static Builder builder() {
+ return new AutoValue_SpannerConfig.Builder();
+ }
+
+ public void validate() {
+ checkNotNull(
+ getInstanceId(),
+ "SpannerIO.read() requires instance id to be set with withInstanceId method");
+ checkNotNull(
+ getDatabaseId(),
+ "SpannerIO.read() requires database id to be set with withDatabaseId method");
+ }
+
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder
+ .addIfNotNull(DisplayData.item("projectId", getProjectId()).withLabel("Output Project"))
+ .addIfNotNull(DisplayData.item("instanceId", getInstanceId()).withLabel("Output Instance"))
+ .addIfNotNull(DisplayData.item("databaseId", getDatabaseId()).withLabel("Output Database"));
+
+ if (getServiceFactory() != null) {
+ builder.addIfNotNull(
+ DisplayData.item("serviceFactory", getServiceFactory().getClass().getName())
+ .withLabel("Service Factory"));
+ }
+ }
+
+ /** Builder for {@link SpannerConfig}. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+
+ abstract Builder setProjectId(ValueProvider<String> projectId);
+
+ abstract Builder setInstanceId(ValueProvider<String> instanceId);
+
+ abstract Builder setDatabaseId(ValueProvider<String> databaseId);
+
+ abstract Builder setHost(String host);
+
+ abstract Builder setServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory);
+
+ public abstract SpannerConfig build();
+ }
+
+ public SpannerConfig withProjectId(ValueProvider<String> projectId) {
+ return toBuilder().setProjectId(projectId).build();
+ }
+
+ public SpannerConfig withProjectId(String projectId) {
+ return withProjectId(ValueProvider.StaticValueProvider.of(projectId));
+ }
+
+ public SpannerConfig withInstanceId(ValueProvider<String> instanceId) {
+ return toBuilder().setInstanceId(instanceId).build();
+ }
+
+ public SpannerConfig withInstanceId(String instanceId) {
+ return withInstanceId(ValueProvider.StaticValueProvider.of(instanceId));
+ }
+
+ public SpannerConfig withDatabaseId(ValueProvider<String> databaseId) {
+ return toBuilder().setDatabaseId(databaseId).build();
+ }
+
+ public SpannerConfig withDatabaseId(String databaseId) {
+ return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId));
+ }
+
+ public SpannerConfig withHost(String host) {
+ return toBuilder().setHost(host).build();
+ }
+
+ @VisibleForTesting
+ SpannerConfig withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+ return toBuilder().setServiceFactory(serviceFactory).build();
+ }
+
+ public SpannerAccessor connectToSpanner() {
+ SpannerOptions.Builder builder = SpannerOptions.newBuilder();
+ if (getProjectId() != null) {
+ builder.setProjectId(getProjectId().get());
+ }
+ if (getServiceFactory() != null) {
+ builder.setServiceFactory(this.getServiceFactory());
+ }
+ if (getHost() != null) {
+ builder.setHost(getHost());
+ }
+ ReleaseInfo releaseInfo = ReleaseInfo.getReleaseInfo();
+ builder.setUserAgentPrefix(USER_AGENT_PREFIX + "/" + releaseInfo.getVersion());
+ SpannerOptions options = builder.build();
+ Spanner spanner = options.getService();
+ DatabaseClient databaseClient = spanner.getDatabaseClient(
+ DatabaseId.of(options.getProjectId(), getInstanceId().get(), getDatabaseId().get()));
+ return new SpannerAccessor(spanner, databaseClient);
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
new file mode 100644
index 0000000..530c466
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -0,0 +1,991 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.ServiceFactory;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.AbortedException;
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.TimestampBound;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
+import com.google.common.primitives.UnsignedBytes;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.ApproximateQuantiles;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
+import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.Sleeper;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+
+/**
+ * Experimental {@link PTransform Transforms} for reading from and writing to <a
+ * href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.
+ *
+ * <h3>Reading from Cloud Spanner</h3>
+ *
+ * <p>To read from Cloud Spanner, apply {@link SpannerIO.Read} transformation. It will return a
+ * {@link PCollection} of {@link Struct Structs}, where each element represents
+ * an individual row returned from the read operation. Both Query and Read APIs are supported.
+ * See more information about <a href="https://cloud.google.com/spanner/docs/reads">reading from
+ * Cloud Spanner</a>
+ *
+ * <p>To execute a <strong>query</strong>, specify a {@link SpannerIO.Read#withQuery(Statement)} or
+ * {@link SpannerIO.Read#withQuery(String)} during the construction of the transform.
+ *
+ * <pre>{@code
+ * PCollection<Struct> rows = p.apply(
+ * SpannerIO.read()
+ * .withInstanceId(instanceId)
+ * .withDatabaseId(dbId)
+ * .withQuery("SELECT id, name, email FROM users"));
+ * }</pre>
+ *
+ * <p>To use the Read API, specify a {@link SpannerIO.Read#withTable(String) table name} and
+ * a {@link SpannerIO.Read#withColumns(List) list of columns}.
+ *
+ * <pre>{@code
+ * PCollection<Struct> rows = p.apply(
+ * SpannerIO.read()
+ * .withInstanceId(instanceId)
+ * .withDatabaseId(dbId)
+ * .withTable("users")
+ * .withColumns("id", "name", "email"));
+ * }</pre>
+ *
+ * <p>To optimally read using index, specify the index name using {@link SpannerIO.Read#withIndex}.
+ *
+ * <p>The transform is guaranteed to be executed on a consistent snapshot of data, utilizing the
+ * power of read only transactions. Staleness of data can be controlled using
+ * {@link SpannerIO.Read#withTimestampBound} or {@link SpannerIO.Read#withTimestamp(Timestamp)}
+ * methods. <a href="https://cloud.google.com/spanner/docs/transactions">Read more</a> about
+ * transactions in Cloud Spanner.
+ *
+ * <p>It is possible to read several {@link PCollection PCollections} within a single transaction.
+ * Apply {@link SpannerIO#createTransaction()} transform, that lazily creates a transaction. The
+ * result of this transformation can be passed to read operation using
+ * {@link SpannerIO.Read#withTransaction(PCollectionView)}.
+ *
+ * <pre>{@code
+ * SpannerConfig spannerConfig = ...
+ *
+ * PCollectionView<Transaction> tx =
+ * p.apply(
+ * SpannerIO.createTransaction()
+ * .withSpannerConfig(spannerConfig)
+ * .withTimestampBound(TimestampBound.strong()));
+ *
+ * PCollection<Struct> users = p.apply(
+ * SpannerIO.read()
+ * .withSpannerConfig(spannerConfig)
+ * .withQuery("SELECT name, email FROM users")
+ * .withTransaction(tx));
+ *
+ * PCollection<Struct> tweets = p.apply(
+ * SpannerIO.read()
+ * .withSpannerConfig(spannerConfig)
+ * .withQuery("SELECT user, tweet, date FROM tweets")
+ * .withTransaction(tx));
+ * }</pre>
+ *
+ * <h3>Writing to Cloud Spanner</h3>
+ *
+ * <p>The Cloud Spanner {@link SpannerIO.Write} transform writes to Cloud Spanner by executing a
+ * collection of input row {@link Mutation Mutations}. The mutations grouped into batches for
+ * efficiency.
+ *
+ * <p>To configure the write transform, create an instance using {@link #write()} and then specify
+ * the destination Cloud Spanner instance ({@link Write#withInstanceId(String)} and destination
+ * database ({@link Write#withDatabaseId(String)}). For example:
+ *
+ * <pre>{@code
+ * // Earlier in the pipeline, create a PCollection of Mutations to be written to Cloud Spanner.
+ * PCollection<Mutation> mutations = ...;
+ * // Write mutations.
+ * mutations.apply(
+ * "Write", SpannerIO.write().withInstanceId("instance").withDatabaseId("database"));
+ * }</pre>
+ *
+ * <p>The default size of the batch is set to 1MB, to override this use {@link
+ * Write#withBatchSizeBytes(long)}. Setting batch size to a small value or zero practically disables
+ * batching.
+ *
+ * <p>The transform does not provide same transactional guarantees as Cloud Spanner. In particular,
+ *
+ * <ul>
+ * <li>Mutations are not submitted atomically;
+ * <li>A mutation is applied at least once;
+ * <li>If the pipeline was unexpectedly stopped, mutations that were already applied will not get
+ * rolled back.
+ * </ul>
+ *
+ * <p>Use {@link MutationGroup} to ensure that a small set mutations is bundled together. It is
+ * guaranteed that mutations in a group are submitted in the same transaction. Build
+ * {@link SpannerIO.Write} transform, and call {@link Write#grouped()} method. It will return a
+ * transformation that can be applied to a PCollection of MutationGroup.
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class SpannerIO {
+
+ private static final long DEFAULT_BATCH_SIZE_BYTES = 1024 * 1024; // 1 MB
+ // Max number of mutations to batch together.
+ private static final int MAX_NUM_MUTATIONS = 10000;
+ // The maximum number of keys to fit in memory when computing approximate quantiles.
+ private static final long MAX_NUM_KEYS = (long) 1e6;
+ // TODO calculate number of samples based on the size of the input.
+ private static final int DEFAULT_NUM_SAMPLES = 1000;
+
+ /**
+ * Creates an uninitialized instance of {@link Read}. Before use, the {@link Read} must be
+ * configured with a {@link Read#withInstanceId} and {@link Read#withDatabaseId} that identify the
+ * Cloud Spanner database.
+ */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ public static Read read() {
+ return new AutoValue_SpannerIO_Read.Builder()
+ .setSpannerConfig(SpannerConfig.create())
+ .setTimestampBound(TimestampBound.strong())
+ .setReadOperation(ReadOperation.create())
+ .build();
+ }
+
+ /**
+ * A {@link PTransform} that works like {@link #read}, but executes read operations coming from a
+ * {@link PCollection}.
+ */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ public static ReadAll readAll() {
+ return new AutoValue_SpannerIO_ReadAll.Builder()
+ .setSpannerConfig(SpannerConfig.create())
+ .build();
+ }
+
+ /**
+ * Returns a transform that creates a batch transaction. By default,
+ * {@link TimestampBound#strong()} transaction is created, to override this use
+ * {@link CreateTransaction#withTimestampBound(TimestampBound)}.
+ */
+ @Experimental
+ public static CreateTransaction createTransaction() {
+ return new AutoValue_SpannerIO_CreateTransaction.Builder()
+ .setSpannerConfig(SpannerConfig.create())
+ .setTimestampBound(TimestampBound.strong())
+ .build();
+ }
+
+ /**
+ * Creates an uninitialized instance of {@link Write}. Before use, the {@link Write} must be
+ * configured with a {@link Write#withInstanceId} and {@link Write#withDatabaseId} that identify
+ * the Cloud Spanner database being written.
+ */
+ @Experimental
+ public static Write write() {
+ return new AutoValue_SpannerIO_Write.Builder()
+ .setSpannerConfig(SpannerConfig.create())
+ .setBatchSizeBytes(DEFAULT_BATCH_SIZE_BYTES)
+ .setNumSamples(DEFAULT_NUM_SAMPLES)
+ .build();
+ }
+
+ /** Implementation of {@link #readAll}. */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ @AutoValue
+ public abstract static class ReadAll
+ extends PTransform<PCollection<ReadOperation>, PCollection<Struct>> {
+
+ abstract SpannerConfig getSpannerConfig();
+
+ @Nullable
+ abstract PCollectionView<Transaction> getTransaction();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setSpannerConfig(SpannerConfig spannerConfig);
+
+ abstract Builder setTransaction(PCollectionView<Transaction> transaction);
+
+ abstract ReadAll build();
+ }
+
+ /** Specifies the Cloud Spanner configuration. */
+ public ReadAll withSpannerConfig(SpannerConfig spannerConfig) {
+ return toBuilder().setSpannerConfig(spannerConfig).build();
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public ReadAll withProjectId(String projectId) {
+ return withProjectId(ValueProvider.StaticValueProvider.of(projectId));
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public ReadAll withProjectId(ValueProvider<String> projectId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withProjectId(projectId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public ReadAll withInstanceId(String instanceId) {
+ return withInstanceId(ValueProvider.StaticValueProvider.of(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public ReadAll withInstanceId(ValueProvider<String> instanceId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withInstanceId(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public ReadAll withDatabaseId(String databaseId) {
+ return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner host. */
+ public ReadAll witHost(String host) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withHost(host));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public ReadAll withDatabaseId(ValueProvider<String> databaseId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withDatabaseId(databaseId));
+ }
+
+ @VisibleForTesting
+ ReadAll withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withServiceFactory(serviceFactory));
+ }
+
+ public ReadAll withTransaction(PCollectionView<Transaction> transaction) {
+ return toBuilder().setTransaction(transaction).build();
+ }
+
+ @Override
+ public PCollection<Struct> expand(PCollection<ReadOperation> input) {
+ List<PCollectionView<Transaction>> sideInputs =
+ getTransaction() == null
+ ? Collections.<PCollectionView<Transaction>>emptyList()
+ : Collections.singletonList(getTransaction());
+ return input
+ .apply(Reshuffle.<ReadOperation>viaRandomKey())
+ .apply(
+ "Execute queries",
+ ParDo.of(new NaiveSpannerReadFn(getSpannerConfig(), getTransaction()))
+ .withSideInputs(sideInputs));
+ }
+ }
+
+ /** Implementation of {@link #read}. */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<Struct>> {
+
+ abstract SpannerConfig getSpannerConfig();
+
+ abstract ReadOperation getReadOperation();
+
+ @Nullable
+ abstract TimestampBound getTimestampBound();
+
+ @Nullable
+ abstract PCollectionView<Transaction> getTransaction();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ abstract Builder setSpannerConfig(SpannerConfig spannerConfig);
+
+ abstract Builder setReadOperation(ReadOperation readOperation);
+
+ abstract Builder setTimestampBound(TimestampBound timestampBound);
+
+ abstract Builder setTransaction(PCollectionView<Transaction> transaction);
+
+ abstract Read build();
+ }
+
+ /** Specifies the Cloud Spanner configuration. */
+ public Read withSpannerConfig(SpannerConfig spannerConfig) {
+ return toBuilder().setSpannerConfig(spannerConfig).build();
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public Read withProjectId(String projectId) {
+ return withProjectId(ValueProvider.StaticValueProvider.of(projectId));
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public Read withProjectId(ValueProvider<String> projectId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withProjectId(projectId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public Read withInstanceId(String instanceId) {
+ return withInstanceId(ValueProvider.StaticValueProvider.of(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public Read withInstanceId(ValueProvider<String> instanceId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withInstanceId(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public Read withDatabaseId(String databaseId) {
+ return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public Read withDatabaseId(ValueProvider<String> databaseId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withDatabaseId(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner host. */
+ public Read witHost(String host) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withHost(host));
+ }
+
+ @VisibleForTesting
+ Read withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withServiceFactory(serviceFactory));
+ }
+
+ public Read withTransaction(PCollectionView<Transaction> transaction) {
+ return toBuilder().setTransaction(transaction).build();
+ }
+
+ public Read withTimestamp(Timestamp timestamp) {
+ return withTimestampBound(TimestampBound.ofReadTimestamp(timestamp));
+ }
+
+ public Read withTimestampBound(TimestampBound timestampBound) {
+ return toBuilder().setTimestampBound(timestampBound).build();
+ }
+
+ public Read withTable(String table) {
+ return withReadOperation(getReadOperation().withTable(table));
+ }
+
+ public Read withReadOperation(ReadOperation operation) {
+ return toBuilder().setReadOperation(operation).build();
+ }
+
+ public Read withColumns(String... columns) {
+ return withColumns(Arrays.asList(columns));
+ }
+
+ public Read withColumns(List<String> columns) {
+ return withReadOperation(getReadOperation().withColumns(columns));
+ }
+
+ public Read withQuery(Statement statement) {
+ return withReadOperation(getReadOperation().withQuery(statement));
+ }
+
+ public Read withQuery(String sql) {
+ return withQuery(Statement.of(sql));
+ }
+
+ public Read withKeySet(KeySet keySet) {
+ return withReadOperation(getReadOperation().withKeySet(keySet));
+ }
+
+ public Read withIndex(String index) {
+ return withReadOperation(getReadOperation().withIndex(index));
+ }
+
+ @Override
+ public PCollection<Struct> expand(PBegin input) {
+ getSpannerConfig().validate();
+ checkArgument(
+ getTimestampBound() != null,
+ "SpannerIO.read() runs in a read only transaction and requires timestamp to be set "
+ + "with withTimestampBound or withTimestamp method");
+
+ if (getReadOperation().getQuery() != null) {
+ // TODO: validate query?
+ } else if (getReadOperation().getTable() != null) {
+ // Assume read
+ checkNotNull(
+ getReadOperation().getColumns(),
+ "For a read operation SpannerIO.read() requires a list of "
+ + "columns to set with withColumns method");
+ checkArgument(
+ !getReadOperation().getColumns().isEmpty(),
+ "For a read operation SpannerIO.read() requires a"
+ + " list of columns to set with withColumns method");
+ } else {
+ throw new IllegalArgumentException(
+ "SpannerIO.read() requires configuring query or read operation.");
+ }
+
+ PCollectionView<Transaction> transaction = getTransaction();
+ if (transaction == null && getTimestampBound() != null) {
+ transaction =
+ input.apply(
+ createTransaction()
+ .withTimestampBound(getTimestampBound())
+ .withSpannerConfig(getSpannerConfig()));
+ }
+ ReadAll readAll =
+ readAll().withSpannerConfig(getSpannerConfig()).withTransaction(transaction);
+ return input.apply(Create.of(getReadOperation())).apply("Execute query", readAll);
+ }
+ }
+
+ /**
+ * A {@link PTransform} that create a transaction.
+ *
+ * @see SpannerIO
+ */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ @AutoValue
+ public abstract static class CreateTransaction
+ extends PTransform<PBegin, PCollectionView<Transaction>> {
+
+ abstract SpannerConfig getSpannerConfig();
+
+ @Nullable
+ abstract TimestampBound getTimestampBound();
+
+ abstract Builder toBuilder();
+
+ @Override
+ public PCollectionView<Transaction> expand(PBegin input) {
+ getSpannerConfig().validate();
+
+ return input.apply(Create.of(1))
+ .apply("Create transaction", ParDo.of(new CreateTransactionFn(this)))
+ .apply("As PCollectionView", View.<Transaction>asSingleton());
+ }
+
+ /** Specifies the Cloud Spanner configuration. */
+ public CreateTransaction withSpannerConfig(SpannerConfig spannerConfig) {
+ return toBuilder().setSpannerConfig(spannerConfig).build();
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public CreateTransaction withProjectId(String projectId) {
+ return withProjectId(ValueProvider.StaticValueProvider.of(projectId));
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public CreateTransaction withProjectId(ValueProvider<String> projectId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withProjectId(projectId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public CreateTransaction withInstanceId(String instanceId) {
+ return withInstanceId(ValueProvider.StaticValueProvider.of(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public CreateTransaction withInstanceId(ValueProvider<String> instanceId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withInstanceId(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public CreateTransaction withDatabaseId(String databaseId) {
+ return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public CreateTransaction withDatabaseId(ValueProvider<String> databaseId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withDatabaseId(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner host. */
+ public CreateTransaction witHost(String host) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withHost(host));
+ }
+
+ @VisibleForTesting
+ CreateTransaction withServiceFactory(
+ ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withServiceFactory(serviceFactory));
+ }
+
+ public CreateTransaction withTimestampBound(TimestampBound timestampBound) {
+ return toBuilder().setTimestampBound(timestampBound).build();
+ }
+
+ /** A builder for {@link CreateTransaction}. */
+ @AutoValue.Builder public abstract static class Builder {
+
+ public abstract Builder setSpannerConfig(SpannerConfig spannerConfig);
+
+ public abstract Builder setTimestampBound(TimestampBound newTimestampBound);
+
+ public abstract CreateTransaction build();
+ }
+ }
+
+
+ /**
+ * A {@link PTransform} that writes {@link Mutation} objects to Google Cloud Spanner.
+ *
+ * @see SpannerIO
+ */
+ @Experimental(Experimental.Kind.SOURCE_SINK)
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<Mutation>, PDone> {
+
+ abstract SpannerConfig getSpannerConfig();
+
+ abstract long getBatchSizeBytes();
+
+ abstract int getNumSamples();
+
+ @Nullable
+ abstract PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>>
+ getSampler();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ abstract Builder setSpannerConfig(SpannerConfig spannerConfig);
+
+ abstract Builder setBatchSizeBytes(long batchSizeBytes);
+
+ abstract Builder setNumSamples(int numSamples);
+
+ abstract Builder setSampler(
+ PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>>
+ sampler);
+
+ abstract Write build();
+ }
+
+ /** Specifies the Cloud Spanner configuration. */
+ public Write withSpannerConfig(SpannerConfig spannerConfig) {
+ return toBuilder().setSpannerConfig(spannerConfig).build();
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public Write withProjectId(String projectId) {
+ return withProjectId(ValueProvider.StaticValueProvider.of(projectId));
+ }
+
+ /** Specifies the Cloud Spanner project. */
+ public Write withProjectId(ValueProvider<String> projectId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withProjectId(projectId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public Write withInstanceId(String instanceId) {
+ return withInstanceId(ValueProvider.StaticValueProvider.of(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner instance. */
+ public Write withInstanceId(ValueProvider<String> instanceId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withInstanceId(instanceId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public Write withDatabaseId(String databaseId) {
+ return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner database. */
+ public Write withDatabaseId(ValueProvider<String> databaseId) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withDatabaseId(databaseId));
+ }
+
+ /** Specifies the Cloud Spanner host. */
+ public Write witHost(String host) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withHost(host));
+ }
+
+ @VisibleForTesting
+ Write withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+ SpannerConfig config = getSpannerConfig();
+ return withSpannerConfig(config.withServiceFactory(serviceFactory));
+ }
+
+ @VisibleForTesting
+ Write withSampler(
+ PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>>
+ sampler) {
+ return toBuilder().setSampler(sampler).build();
+ }
+
+ /**
+ * Same transform but can be applied to {@link PCollection} of {@link MutationGroup}.
+ */
+ public WriteGrouped grouped() {
+ return new WriteGrouped(this);
+ }
+
+ /** Specifies the batch size limit. */
+ public Write withBatchSizeBytes(long batchSizeBytes) {
+ return toBuilder().setBatchSizeBytes(batchSizeBytes).build();
+ }
+
+ @Override
+ public PDone expand(PCollection<Mutation> input) {
+ getSpannerConfig().validate();
+
+ input
+ .apply("To mutation group", ParDo.of(new ToMutationGroupFn()))
+ .apply("Write mutations to Cloud Spanner", new WriteGrouped(this));
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ getSpannerConfig().populateDisplayData(builder);
+ builder.add(
+ DisplayData.item("batchSizeBytes", getBatchSizeBytes()).withLabel("Batch Size in Bytes"));
+ }
+ }
+
+ /**
+ * A singleton that wraps {@code UnsignedBytes#lexicographicalComparator} which unfortunately
+ * is not serializable.
+ */
+ @VisibleForTesting
+ enum SerializableBytesComparator implements Comparator<byte[]>, Serializable {
+ INSTANCE {
+ @Override public int compare(byte[] a, byte[] b) {
+ return UnsignedBytes.lexicographicalComparator().compare(a, b);
+ }
+ }
+ }
+
+ /** Same as {@link Write} but supports grouped mutations. */
+ public static class WriteGrouped extends PTransform<PCollection<MutationGroup>, PDone> {
+ private final Write spec;
+
+ public WriteGrouped(Write spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ public PDone expand(PCollection<MutationGroup> input) {
+ PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>>
+ sampler = spec.getSampler();
+ if (sampler == null) {
+ sampler = createDefaultSampler();
+ }
+ // First, read the Cloud Spanner schema.
+ final PCollectionView<SpannerSchema> schemaView = input.getPipeline()
+ .apply(Create.of((Void) null))
+ .apply("Read information schema",
+ ParDo.of(new ReadSpannerSchema(spec.getSpannerConfig())))
+ .apply("Schema View", View.<SpannerSchema>asSingleton());
+
+ // Serialize mutations, we don't need to encode/decode them while reshuffling.
+ // The primary key is encoded via OrderedCode so we can calculate quantiles.
+ PCollection<SerializedMutation> serialized = input
+ .apply("Serialize mutations",
+ ParDo.of(new SerializeMutationsFn(schemaView)).withSideInputs(schemaView))
+ .setCoder(SerializedMutationCoder.of());
+
+ // Sample primary keys using ApproximateQuantiles.
+ PCollectionView<Map<String, List<byte[]>>> keySample = serialized
+ .apply("Extract keys", ParDo.of(new ExtractKeys()))
+ .apply("Sample keys", sampler)
+ .apply("Keys sample as view", View.<String, List<byte[]>>asMap());
+
+ // Assign partition based on the closest element in the sample and group mutations.
+ AssignPartitionFn assignPartitionFn = new AssignPartitionFn(keySample);
+ serialized
+ .apply("Partition input", ParDo.of(assignPartitionFn).withSideInputs(keySample))
+ .setCoder(KvCoder.of(StringUtf8Coder.of(), SerializedMutationCoder.of()))
+ .apply("Group by partition", GroupByKey.<String, SerializedMutation>create())
+ .apply("Batch mutations together",
+ ParDo.of(new BatchFn(spec.getBatchSizeBytes(), spec.getSpannerConfig(), schemaView))
+ .withSideInputs(schemaView))
+ .apply("Write mutations to Spanner",
+ ParDo.of(new WriteToSpannerFn(spec.getSpannerConfig())));
+ return PDone.in(input.getPipeline());
+
+ }
+
+ private PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>>
+ createDefaultSampler() {
+ return Combine.perKey(ApproximateQuantiles.ApproximateQuantilesCombineFn
+ .create(spec.getNumSamples(), SerializableBytesComparator.INSTANCE, MAX_NUM_KEYS,
+ 1. / spec.getNumSamples()));
+ }
+ }
+
+ private static class ToMutationGroupFn extends DoFn<Mutation, MutationGroup> {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Mutation value = c.element();
+ c.output(MutationGroup.create(value));
+ }
+ }
+
+ /**
+ * Serializes mutations to ((table name, serialized key), serialized value) tuple.
+ */
+ private static class SerializeMutationsFn
+ extends DoFn<MutationGroup, SerializedMutation> {
+
+ final PCollectionView<SpannerSchema> schemaView;
+
+ private SerializeMutationsFn(PCollectionView<SpannerSchema> schemaView) {
+ this.schemaView = schemaView;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ MutationGroup g = c.element();
+ Mutation m = g.primary();
+ SpannerSchema schema = c.sideInput(schemaView);
+ String table = m.getTable();
+ MutationGroupEncoder mutationGroupEncoder = new MutationGroupEncoder(schema);
+
+ byte[] key;
+ if (m.getOperation() != Mutation.Op.DELETE) {
+ key = mutationGroupEncoder.encodeKey(m);
+ } else if (isPointDelete(m)) {
+ Key next = m.getKeySet().getKeys().iterator().next();
+ key = mutationGroupEncoder.encodeKey(m.getTable(), next);
+ } else {
+ // The key is left empty for non-point deletes, since there is no general way to batch them.
+ key = new byte[] {};
+ }
+ byte[] value = mutationGroupEncoder.encode(g);
+ c.output(SerializedMutation.create(table, key, value));
+ }
+ }
+
+ private static class ExtractKeys
+ extends DoFn<SerializedMutation, KV<String, byte[]>> {
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ SerializedMutation m = c.element();
+ c.output(KV.of(m.getTableName(), m.getEncodedKey()));
+ }
+ }
+
+
+
+ private static boolean isPointDelete(Mutation m) {
+ return m.getOperation() == Mutation.Op.DELETE && Iterables.isEmpty(m.getKeySet().getRanges())
+ && Iterables.size(m.getKeySet().getKeys()) == 1;
+ }
+
+ /**
+ * Assigns a partition to the mutation group token based on the sampled data.
+ */
+ private static class AssignPartitionFn
+ extends DoFn<SerializedMutation, KV<String, SerializedMutation>> {
+
+ final PCollectionView<Map<String, List<byte[]>>> sampleView;
+
+ public AssignPartitionFn(PCollectionView<Map<String, List<byte[]>>> sampleView) {
+ this.sampleView = sampleView;
+ }
+
+ @ProcessElement public void processElement(ProcessContext c) {
+ Map<String, List<byte[]>> sample = c.sideInput(sampleView);
+ SerializedMutation g = c.element();
+ String table = g.getTableName();
+ byte[] key = g.getEncodedKey();
+ String groupKey;
+ if (key.length == 0) {
+ // This is a range or multi-key delete mutation. We cannot group it with other mutations
+ // so we assign a random group key to it so it is applied independently.
+ groupKey = UUID.randomUUID().toString();
+ } else {
+ int partition = Collections
+ .binarySearch(sample.get(table), key, SerializableBytesComparator.INSTANCE);
+ if (partition < 0) {
+ partition = -partition - 1;
+ }
+ groupKey = table + "%" + partition;
+ }
+ c.output(KV.of(groupKey, g));
+ }
+ }
+
+ /**
+ * Batches mutations together.
+ */
+ private static class BatchFn
+ extends DoFn<KV<String, Iterable<SerializedMutation>>, Iterable<Mutation>> {
+
+ private static final int MAX_RETRIES = 5;
+ private static final FluentBackoff BUNDLE_WRITE_BACKOFF = FluentBackoff.DEFAULT
+ .withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
+
+ private final long maxBatchSizeBytes;
+ private final SpannerConfig spannerConfig;
+ private final PCollectionView<SpannerSchema> schemaView;
+
+ private transient SpannerAccessor spannerAccessor;
+ // Current batch of mutations to be written.
+ private List<Mutation> mutations;
+ // total size of the current batch.
+ private long batchSizeBytes;
+
+ private BatchFn(long maxBatchSizeBytes, SpannerConfig spannerConfig,
+ PCollectionView<SpannerSchema> schemaView) {
+ this.maxBatchSizeBytes = maxBatchSizeBytes;
+ this.spannerConfig = spannerConfig;
+ this.schemaView = schemaView;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ mutations = new ArrayList<>();
+ batchSizeBytes = 0;
+ spannerAccessor = spannerConfig.connectToSpanner();
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ spannerAccessor.close();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ MutationGroupEncoder mutationGroupEncoder = new MutationGroupEncoder(c.sideInput(schemaView));
+ KV<String, Iterable<SerializedMutation>> element = c.element();
+ for (SerializedMutation kv : element.getValue()) {
+ byte[] value = kv.getMutationGroupBytes();
+ MutationGroup mg = mutationGroupEncoder.decode(value);
+ Iterables.addAll(mutations, mg);
+ batchSizeBytes += MutationSizeEstimator.sizeOf(mg);
+ if (batchSizeBytes >= maxBatchSizeBytes || mutations.size() > MAX_NUM_MUTATIONS) {
+ c.output(mutations);
+ mutations = new ArrayList<>();
+ batchSizeBytes = 0;
+ }
+ }
+ if (!mutations.isEmpty()) {
+ c.output(mutations);
+ mutations = new ArrayList<>();
+ batchSizeBytes = 0;
+ }
+ }
+ }
+
+ private static class WriteToSpannerFn
+ extends DoFn<Iterable<Mutation>, Void> {
+ private static final int MAX_RETRIES = 5;
+ private static final FluentBackoff BUNDLE_WRITE_BACKOFF = FluentBackoff.DEFAULT
+ .withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
+
+ private transient SpannerAccessor spannerAccessor;
+ private final SpannerConfig spannerConfig;
+
+ public WriteToSpannerFn(SpannerConfig spannerConfig) {
+ this.spannerConfig = spannerConfig;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ spannerAccessor = spannerConfig.connectToSpanner();
+ }
+
+ @Teardown
+ public void teardown() throws Exception {
+ spannerAccessor.close();
+ }
+
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Sleeper sleeper = Sleeper.DEFAULT;
+ BackOff backoff = BUNDLE_WRITE_BACKOFF.backoff();
+
+ Iterable<Mutation> mutations = c.element();
+
+ while (true) {
+ // Batch upsert rows.
+ try {
+ spannerAccessor.getDatabaseClient().writeAtLeastOnce(mutations);
+ // Break if the commit threw no exception.
+ break;
+ } catch (AbortedException exception) {
+ // Only log the code and message for potentially-transient errors. The entire exception
+ // will be propagated upon the last retry.
+ if (!BackOffUtils.next(sleeper, backoff)) {
+ throw exception;
+ }
+ }
+ }
+ }
+
+ }
+
+ private SpannerIO() {} // Prevent construction.
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java
new file mode 100644
index 0000000..4c12b8d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.spanner.Type;
+import com.google.common.collect.ArrayListMultimap;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates Cloud Spanner Schema.
+ */
+class SpannerSchema implements Serializable {
+ private final List<String> tables;
+ private final ArrayListMultimap<String, Column> columns;
+ private final ArrayListMultimap<String, KeyPart> keyParts;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Builder for {@link SpannerSchema}.
+ */
+ static class Builder {
+ private final ArrayListMultimap<String, Column> columns = ArrayListMultimap.create();
+ private final ArrayListMultimap<String, KeyPart> keyParts = ArrayListMultimap.create();
+
+ public Builder addColumn(String table, String name, String type) {
+ addColumn(table, Column.create(name.toLowerCase(), type));
+ return this;
+ }
+
+ private Builder addColumn(String table, Column column) {
+ columns.put(table.toLowerCase(), column);
+ return this;
+ }
+
+ public Builder addKeyPart(String table, String column, boolean desc) {
+ keyParts.put(table, KeyPart.create(column.toLowerCase(), desc));
+ return this;
+ }
+
+ public SpannerSchema build() {
+ return new SpannerSchema(columns, keyParts);
+ }
+ }
+
+ private SpannerSchema(ArrayListMultimap<String, Column> columns,
+ ArrayListMultimap<String, KeyPart> keyParts) {
+ this.columns = columns;
+ this.keyParts = keyParts;
+ tables = new ArrayList<>(columns.keySet());
+ }
+
+ public List<String> getTables() {
+ return tables;
+ }
+
+ public List<Column> getColumns(String table) {
+ return columns.get(table);
+ }
+
+ public List<KeyPart> getKeyParts(String table) {
+ return keyParts.get(table);
+ }
+
+ @AutoValue
+ abstract static class KeyPart implements Serializable {
+ static KeyPart create(String field, boolean desc) {
+ return new AutoValue_SpannerSchema_KeyPart(field, desc);
+ }
+
+ abstract String getField();
+
+ abstract boolean isDesc();
+ }
+
+ @AutoValue
+ abstract static class Column implements Serializable {
+
+ static Column create(String name, Type type) {
+ return new AutoValue_SpannerSchema_Column(name, type);
+ }
+
+ static Column create(String name, String spannerType) {
+ return create(name, parseSpannerType(spannerType));
+ }
+
+ public abstract String getName();
+
+ public abstract Type getType();
+
+ private static Type parseSpannerType(String spannerType) {
+ spannerType = spannerType.toUpperCase();
+ if (spannerType.equals("BOOL")) {
+ return Type.bool();
+ }
+ if (spannerType.equals("INT64")) {
+ return Type.int64();
+ }
+ if (spannerType.equals("FLOAT64")) {
+ return Type.float64();
+ }
+ if (spannerType.startsWith("STRING")) {
+ return Type.string();
+ }
+ if (spannerType.startsWith("BYTES")) {
+ return Type.bytes();
+ }
+ if (spannerType.equals("TIMESTAMP")) {
+ return Type.timestamp();
+ }
+ if (spannerType.equals("DATE")) {
+ return Type.date();
+ }
+
+ if (spannerType.startsWith("ARRAY")) {
+ // Substring "ARRAY<xxx>"
+ String spannerArrayType = spannerType.substring(6, spannerType.length() - 1);
+ Type itemType = parseSpannerType(spannerArrayType);
+ return Type.array(itemType);
+ }
+ throw new IllegalArgumentException("Unknown spanner type " + spannerType);
+ }
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/Transaction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/Transaction.java
new file mode 100644
index 0000000..22af3b8
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/Transaction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.Timestamp;
+import java.io.Serializable;
+
+/** A transaction object. */
+@AutoValue
+public abstract class Transaction implements Serializable {
+
+ abstract Timestamp timestamp();
+
+ public static Transaction create(Timestamp timestamp) {
+ return new AutoValue_Transaction(timestamp);
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java
new file mode 100644
index 0000000..19e468c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <p>Provides an API for reading from and writing to
+ * <a href="https://developers.google.com/spanner/">Google Cloud Spanner</a>.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
index 7025004..748d87f 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
@@ -52,6 +52,7 @@
@SuppressWarnings("unchecked")
final Set<Matcher<Class<?>>> allowedClasses =
ImmutableSet.of(
+ classesInPackage("com.google.api.core"),
classesInPackage("com.google.api.client.googleapis"),
classesInPackage("com.google.api.client.http"),
classesInPackage("com.google.api.client.json"),
@@ -60,10 +61,22 @@
classesInPackage("com.google.auth"),
classesInPackage("com.google.bigtable.v2"),
classesInPackage("com.google.cloud.bigtable.config"),
+ classesInPackage("com.google.spanner.v1"),
+ Matchers.<Class<?>>equalTo(com.google.api.gax.grpc.ApiException.class),
Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableClusterName.class),
Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableInstanceName.class),
Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableTableName.class),
- // via Bigtable, PR above out to fix.
+ Matchers.<Class<?>>equalTo(com.google.cloud.BaseServiceException.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.BaseServiceException.Error.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.BaseServiceException.ExceptionData.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.BaseServiceException.ExceptionData.Builder
+ .class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.RetryHelper.RetryHelperException.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.grpc.BaseGrpcServiceException.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.ByteArray.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.Date.class),
+ Matchers.<Class<?>>equalTo(com.google.cloud.Timestamp.class),
+ classesInPackage("com.google.cloud.spanner"),
classesInPackage("com.google.datastore.v1"),
classesInPackage("com.google.protobuf"),
classesInPackage("com.google.type"),
@@ -73,9 +86,10 @@
classesInPackage("io.grpc"),
classesInPackage("java"),
classesInPackage("javax"),
+ classesInPackage("org.apache.avro"),
classesInPackage("org.apache.beam"),
classesInPackage("org.apache.commons.logging"),
- // via Bigtable
+ classesInPackage("org.codehaus.jackson"),
classesInPackage("org.joda.time"));
assertThat(apiSurface, containsOnlyClassesMatching(allowedClasses));
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 70d5377..5b4b7e6 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -35,32 +35,37 @@
import static org.junit.Assert.assertTrue;
import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.model.ErrorProto;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobStatistics;
import com.google.api.services.bigquery.model.JobStatistics2;
import com.google.api.services.bigquery.model.JobStatistics4;
import com.google.api.services.bigquery.model.JobStatus;
+import com.google.api.services.bigquery.model.Streamingbuffer;
import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.api.services.bigquery.model.TimePartitioning;
+import com.google.bigtable.v2.Mutation;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.protobuf.ByteString;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
-import java.math.BigDecimal;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.nio.charset.StandardCharsets;
+import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -76,19 +81,19 @@
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ShardedKeyCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.extensions.protobuf.ByteStringCoder;
+import org.apache.beam.sdk.extensions.protobuf.ProtoCoder;
import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.CountingSource;
-import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.GenerateSequence;
-import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.PassThroughThenCleanup.CleanupOperation;
import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
@@ -101,12 +106,15 @@
import org.apache.beam.sdk.testing.ExpectedLogs;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.testing.SourceTestUtils.ExpectedSplitOutcome;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.testing.UsesTestStream;
+import org.apache.beam.sdk.testing.ValidatesRunner;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFnTester;
import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.SimpleFunction;
@@ -115,30 +123,30 @@
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
+import org.apache.beam.sdk.values.ShardedKey;
import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.ValueInSingleWindow;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.hamcrest.CoreMatchers;
import org.hamcrest.Matchers;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
@@ -296,7 +304,7 @@
bqOptions.setTempLocation(baseDir.toString());
FakeDatasetService fakeDatasetService = new FakeDatasetService();
- fakeDatasetService.createDataset(projectId, datasetId, "", "");
+ fakeDatasetService.createDataset(projectId, datasetId, "", "", null);
TableReference tableReference =
new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId);
fakeDatasetService.createTable(new Table()
@@ -337,7 +345,7 @@
}));
PAssert.that(output).containsInAnyOrder(ImmutableList.of(KV.of("a", 1L), KV.of("b", 2L),
KV.of("c", 3L), KV.of("d", 4L), KV.of("e", 5L), KV.of("f", 6L)));
- p.run();
+ p.run();
}
@Test
@@ -347,7 +355,7 @@
bqOptions.setTempLocation("gs://testbucket/testdir");
Pipeline p = TestPipeline.create(bqOptions);
- thrown.expect(IllegalStateException.class);
+ thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(
"Invalid BigQueryIO.Read: Specifies a table with a result flattening preference,"
+ " which only applies to queries");
@@ -365,7 +373,7 @@
bqOptions.setTempLocation("gs://testbucket/testdir");
Pipeline p = TestPipeline.create(bqOptions);
- thrown.expect(IllegalStateException.class);
+ thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(
"Invalid BigQueryIO.Read: Specifies a table with a result flattening preference,"
+ " which only applies to queries");
@@ -384,7 +392,7 @@
bqOptions.setTempLocation("gs://testbucket/testdir");
Pipeline p = TestPipeline.create(bqOptions);
- thrown.expect(IllegalStateException.class);
+ thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(
"Invalid BigQueryIO.Read: Specifies a table with a SQL dialect preference,"
+ " which only applies to queries");
@@ -396,20 +404,35 @@
}
@Test
- public void testReadFromTable() throws IOException, InterruptedException {
+ public void testReadFromTableWithoutTemplateCompatibility()
+ throws IOException, InterruptedException {
+ testReadFromTable(false, false);
+ }
+
+ @Test
+ public void testReadFromTableWithTemplateCompatibility()
+ throws IOException, InterruptedException {
+ testReadFromTable(true, false);
+ }
+
+ @Test
+ public void testReadTableRowsFromTableWithoutTemplateCompatibility()
+ throws IOException, InterruptedException {
+ testReadFromTable(false, true);
+ }
+
+ @Test
+ public void testReadTableRowsFromTableWithTemplateCompatibility()
+ throws IOException, InterruptedException {
+ testReadFromTable(true, true);
+ }
+
+ private void testReadFromTable(boolean useTemplateCompatibility, boolean useReadTableRows)
+ throws IOException, InterruptedException {
BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
bqOptions.setProject("defaultproject");
bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
- Job job = new Job();
- JobStatus status = new JobStatus();
- job.setStatus(status);
- JobStatistics jobStats = new JobStatistics();
- job.setStatistics(jobStats);
- JobStatistics4 extract = new JobStatistics4();
- jobStats.setExtract(extract);
- extract.setDestinationUriFileCounts(ImmutableList.of(1L));
-
Table sometable = new Table();
sometable.setSchema(
new TableSchema()
@@ -424,7 +447,7 @@
.setTableId("sometable"));
sometable.setNumBytes(1024L * 1024L);
FakeDatasetService fakeDatasetService = new FakeDatasetService();
- fakeDatasetService.createDataset("non-executing-project", "somedataset", "", "");
+ fakeDatasetService.createDataset("non-executing-project", "somedataset", "", "", null);
fakeDatasetService.createTable(sometable);
List<TableRow> records = Lists.newArrayList(
@@ -438,17 +461,35 @@
.withDatasetService(fakeDatasetService);
Pipeline p = TestPipeline.create(bqOptions);
- PCollection<KV<String, Long>> output = p
- .apply(BigQueryIO.read().from("non-executing-project:somedataset.sometable")
- .withTestServices(fakeBqServices)
- .withoutValidation())
- .apply(ParDo.of(new DoFn<TableRow, KV<String, Long>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- c.output(KV.of((String) c.element().get("name"),
- Long.valueOf((String) c.element().get("number"))));
- }
- }));
+ PTransform<PBegin, PCollection<TableRow>> readTransform;
+ if (useReadTableRows) {
+ BigQueryIO.Read read =
+ BigQueryIO.read()
+ .from("non-executing-project:somedataset.sometable")
+ .withTestServices(fakeBqServices)
+ .withoutValidation();
+ readTransform = useTemplateCompatibility ? read.withTemplateCompatibility() : read;
+ } else {
+ BigQueryIO.TypedRead<TableRow> read =
+ BigQueryIO.readTableRows()
+ .from("non-executing-project:somedataset.sometable")
+ .withTestServices(fakeBqServices)
+ .withoutValidation();
+ readTransform = useTemplateCompatibility ? read.withTemplateCompatibility() : read;
+ }
+ PCollection<KV<String, Long>> output =
+ p.apply(readTransform)
+ .apply(
+ ParDo.of(
+ new DoFn<TableRow, KV<String, Long>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ c.output(
+ KV.of(
+ (String) c.element().get("name"),
+ Long.valueOf((String) c.element().get("number"))));
+ }
+ }));
PAssert.that(output)
.containsInAnyOrder(ImmutableList.of(KV.of("a", 1L), KV.of("b", 2L), KV.of("c", 3L)));
@@ -460,6 +501,40 @@
}
@Test
+ public void testWriteEmptyPCollection() throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ Pipeline p = TestPipeline.create(bqOptions);
+
+ TableSchema schema = new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("number").setType("INTEGER")));
+
+ p.apply(Create.empty(TableRowJsonCoder.of()))
+ .apply(BigQueryIO.writeTableRows()
+ .to("project-id:dataset-id.table-id")
+ .withTestServices(fakeBqServices)
+ .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND)
+ .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
+ .withSchema(schema)
+ .withoutValidation());
+ p.run();
+ checkNotNull(datasetService.getTable(
+ BigQueryHelpers.parseTableSpec("project-id:dataset-id.table-id")));
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
+ }
+
+ @Test
public void testWriteDynamicDestinationsBatch() throws Exception {
writeDynamicDestinations(false);
}
@@ -479,7 +554,7 @@
.withJobService(new FakeJobService())
.withDatasetService(datasetService);
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
final Pattern userPattern = Pattern.compile("([a-z]+)([0-9]+)");
Pipeline p = TestPipeline.create(bqOptions);
@@ -515,65 +590,73 @@
if (streaming) {
users = users.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
}
- users.apply("WriteBigQuery", BigQueryIO.<String>write()
+ users.apply(
+ "WriteBigQuery",
+ BigQueryIO.<String>write()
.withTestServices(fakeBqServices)
.withMaxFilesPerBundle(5)
.withMaxFileSize(10)
.withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
- .withFormatFunction(new SerializableFunction<String, TableRow>() {
- @Override
- public TableRow apply(String user) {
- Matcher matcher = userPattern.matcher(user);
- if (matcher.matches()) {
- return new TableRow().set("name", matcher.group(1))
- .set("id", Integer.valueOf(matcher.group(2)));
- }
- throw new RuntimeException("Unmatching element " + user);
- }
- })
- .to(new StringIntegerDestinations() {
- @Override
- public Integer getDestination(ValueInSingleWindow<String> element) {
- assertThat(element.getWindow(), Matchers.instanceOf(PartitionedGlobalWindow.class));
- Matcher matcher = userPattern.matcher(element.getValue());
- if (matcher.matches()) {
- // Since we name tables by userid, we can simply store an Integer to represent
- // a table.
- return Integer.valueOf(matcher.group(2));
- }
- throw new RuntimeException("Unmatching destination " + element.getValue());
- }
+ .withFormatFunction(
+ new SerializableFunction<String, TableRow>() {
+ @Override
+ public TableRow apply(String user) {
+ Matcher matcher = userPattern.matcher(user);
+ if (matcher.matches()) {
+ return new TableRow()
+ .set("name", matcher.group(1))
+ .set("id", Integer.valueOf(matcher.group(2)));
+ }
+ throw new RuntimeException("Unmatching element " + user);
+ }
+ })
+ .to(
+ new StringIntegerDestinations() {
+ @Override
+ public Integer getDestination(ValueInSingleWindow<String> element) {
+ assertThat(
+ element.getWindow(), Matchers.instanceOf(PartitionedGlobalWindow.class));
+ Matcher matcher = userPattern.matcher(element.getValue());
+ if (matcher.matches()) {
+ // Since we name tables by userid, we can simply store an Integer to represent
+ // a table.
+ return Integer.valueOf(matcher.group(2));
+ }
+ throw new RuntimeException("Unmatching destination " + element.getValue());
+ }
- @Override
- public TableDestination getTable(Integer userId) {
- verifySideInputs();
- // Each user in it's own table.
- return new TableDestination("dataset-id.userid-" + userId,
- "table for userid " + userId);
- }
+ @Override
+ public TableDestination getTable(Integer userId) {
+ verifySideInputs();
+ // Each user in it's own table.
+ return new TableDestination(
+ "dataset-id.userid-" + userId, "table for userid " + userId);
+ }
- @Override
- public TableSchema getSchema(Integer userId) {
- verifySideInputs();
- return new TableSchema().setFields(
- ImmutableList.of(
- new TableFieldSchema().setName("name").setType("STRING"),
- new TableFieldSchema().setName("id").setType("INTEGER")));
- }
+ @Override
+ public TableSchema getSchema(Integer userId) {
+ verifySideInputs();
+ return new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("id").setType("INTEGER")));
+ }
- @Override
- public List<PCollectionView<?>> getSideInputs() {
- return ImmutableList.of(sideInput1, sideInput2);
- }
+ @Override
+ public List<PCollectionView<?>> getSideInputs() {
+ return ImmutableList.of(sideInput1, sideInput2);
+ }
- private void verifySideInputs() {
- assertThat(sideInput(sideInput1), containsInAnyOrder("a", "b", "c"));
- Map<String, String> mapSideInput = sideInput(sideInput2);
- assertEquals(3, mapSideInput.size());
- assertThat(mapSideInput,
- allOf(hasEntry("a", "a"), hasEntry("b", "b"), hasEntry("c", "c")));
- }
- })
+ private void verifySideInputs() {
+ assertThat(sideInput(sideInput1), containsInAnyOrder("a", "b", "c"));
+ Map<String, String> mapSideInput = sideInput(sideInput2);
+ assertEquals(3, mapSideInput.size());
+ assertThat(
+ mapSideInput,
+ allOf(hasEntry("a", "a"), hasEntry("b", "b"), hasEntry("c", "c")));
+ }
+ })
.withoutValidation());
p.run();
@@ -598,6 +681,211 @@
assertThat(datasetService.getAllRows("project-id", "dataset-id", "userid-" + entry.getKey()),
containsInAnyOrder(Iterables.toArray(entry.getValue(), TableRow.class)));
}
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
+ }
+
+ @Test
+ public void testTimePartitioningStreamingInserts() throws Exception {
+ testTimePartitioning(Method.STREAMING_INSERTS);
+ }
+
+ @Test
+ public void testTimePartitioningBatchLoads() throws Exception {
+ testTimePartitioning(Method.FILE_LOADS);
+ }
+
+ public void testTimePartitioning(BigQueryIO.Write.Method insertMethod) throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices =
+ new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ TableRow row1 = new TableRow().set("name", "a").set("number", "1");
+ TableRow row2 = new TableRow().set("name", "b").set("number", "2");
+
+ TimePartitioning timePartitioning = new TimePartitioning()
+ .setType("DAY")
+ .setExpirationMs(1000L);
+ TableSchema schema = new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("number").setType("INTEGER")));
+ p.apply(Create.of(row1, row2))
+ .apply(
+ BigQueryIO.writeTableRows()
+ .to("project-id:dataset-id.table-id")
+ .withTestServices(fakeBqServices)
+ .withMethod(insertMethod)
+ .withSchema(schema)
+ .withTimePartitioning(timePartitioning)
+ .withoutValidation());
+ p.run();
+ Table table = datasetService.getTable(
+ BigQueryHelpers.parseTableSpec("project-id:dataset-id.table-id"));
+ assertEquals(schema, table.getSchema());
+ assertEquals(timePartitioning, table.getTimePartitioning());
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
+ }
+
+ @Test
+ @Category({ValidatesRunner.class, UsesTestStream.class})
+ public void testTriggeredFileLoads() throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices =
+ new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+
+ List<TableRow> elements = Lists.newArrayList();
+ for (int i = 0; i < 30; ++i) {
+ elements.add(new TableRow().set("number", i));
+ }
+
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+ TestStream<TableRow> testStream =
+ TestStream.create(TableRowJsonCoder.of())
+ .addElements(
+ elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class))
+ .advanceProcessingTime(Duration.standardMinutes(1))
+ .addElements(
+ elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class))
+ .advanceProcessingTime(Duration.standardMinutes(1))
+ .addElements(
+ elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class))
+ .advanceWatermarkToInfinity();
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ p.apply(testStream)
+ .apply(
+ BigQueryIO.writeTableRows()
+ .to("project-id:dataset-id.table-id")
+ .withSchema(
+ new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("number").setType("INTEGER"))))
+ .withTestServices(fakeBqServices)
+ .withTriggeringFrequency(Duration.standardSeconds(30))
+ .withNumFileShards(2)
+ .withMethod(Method.FILE_LOADS)
+ .withoutValidation());
+ p.run();
+
+ assertThat(
+ datasetService.getAllRows("project-id", "dataset-id", "table-id"),
+ containsInAnyOrder(Iterables.toArray(elements, TableRow.class)));
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
+ }
+
+ @Test
+ public void testFailuresNoRetryPolicy() throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ TableRow row1 = new TableRow().set("name", "a").set("number", "1");
+ TableRow row2 = new TableRow().set("name", "b").set("number", "2");
+ TableRow row3 = new TableRow().set("name", "c").set("number", "3");
+
+ TableDataInsertAllResponse.InsertErrors ephemeralError =
+ new TableDataInsertAllResponse.InsertErrors().setErrors(
+ ImmutableList.of(new ErrorProto().setReason("timeout")));
+
+ datasetService.failOnInsert(
+ ImmutableMap.<TableRow, List<TableDataInsertAllResponse.InsertErrors>>of(
+ row1, ImmutableList.of(ephemeralError, ephemeralError),
+ row2, ImmutableList.of(ephemeralError, ephemeralError)));
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ p.apply(Create.of(row1, row2, row3))
+ .apply(
+ BigQueryIO.writeTableRows()
+ .to("project-id:dataset-id.table-id")
+ .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
+ .withMethod(Method.STREAMING_INSERTS)
+ .withSchema(
+ new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("number").setType("INTEGER"))))
+ .withTestServices(fakeBqServices)
+ .withoutValidation());
+ p.run();
+
+ assertThat(datasetService.getAllRows("project-id", "dataset-id", "table-id"),
+ containsInAnyOrder(row1, row2, row3));
+ }
+
+ @Test
+ public void testRetryPolicy() throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ TableRow row1 = new TableRow().set("name", "a").set("number", "1");
+ TableRow row2 = new TableRow().set("name", "b").set("number", "2");
+ TableRow row3 = new TableRow().set("name", "c").set("number", "3");
+
+ TableDataInsertAllResponse.InsertErrors ephemeralError =
+ new TableDataInsertAllResponse.InsertErrors().setErrors(
+ ImmutableList.of(new ErrorProto().setReason("timeout")));
+ TableDataInsertAllResponse.InsertErrors persistentError =
+ new TableDataInsertAllResponse.InsertErrors().setErrors(
+ ImmutableList.of(new ErrorProto().setReason("invalidQuery")));
+
+ datasetService.failOnInsert(
+ ImmutableMap.<TableRow, List<TableDataInsertAllResponse.InsertErrors>>of(
+ row1, ImmutableList.of(ephemeralError, ephemeralError),
+ row2, ImmutableList.of(ephemeralError, ephemeralError, persistentError)));
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ PCollection<TableRow> failedRows =
+ p.apply(Create.of(row1, row2, row3))
+ .apply(BigQueryIO.writeTableRows().to("project-id:dataset-id.table-id")
+ .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
+ .withMethod(Method.STREAMING_INSERTS)
+ .withSchema(new TableSchema().setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("number").setType("INTEGER"))))
+ .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors())
+ .withTestServices(fakeBqServices)
+ .withoutValidation()).getFailedInserts();
+ // row2 finally fails with a non-retryable error, so we expect to see it in the collection of
+ // failed rows.
+ PAssert.that(failedRows).containsInAnyOrder(row2);
+ p.run();
+
+ // Only row1 and row3 were successfully inserted.
+ assertThat(datasetService.getAllRows("project-id", "dataset-id", "table-id"),
+ containsInAnyOrder(row1, row3));
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
}
@Test
@@ -611,7 +899,7 @@
.withJobService(new FakeJobService())
.withDatasetService(datasetService);
- datasetService.createDataset("defaultproject", "dataset-id", "", "");
+ datasetService.createDataset("defaultproject", "dataset-id", "", "", null);
Pipeline p = TestPipeline.create(bqOptions);
p.apply(Create.of(
@@ -641,7 +929,7 @@
bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
FakeDatasetService datasetService = new FakeDatasetService();
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
.withDatasetService(datasetService);
@@ -670,6 +958,7 @@
new TableRow().set("name", "b").set("number", 2),
new TableRow().set("name", "c").set("number", 3),
new TableRow().set("name", "d").set("number", 4)));
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
}
/**
@@ -698,6 +987,18 @@
}
@Override
+ public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+ if (!this.isCompatible(other)) {
+ throw new IncompatibleWindowException(
+ other,
+ String.format(
+ "%s is only compatible with %s.",
+ PartitionedGlobalWindows.class.getSimpleName(),
+ PartitionedGlobalWindows.class.getSimpleName()));
+ }
+ }
+
+ @Override
public Coder<PartitionedGlobalWindow> windowCoder() {
return new PartitionedGlobalWindowCoder();
}
@@ -790,7 +1091,7 @@
bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
FakeDatasetService datasetService = new FakeDatasetService();
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
.withDatasetService(datasetService)
.withJobService(new FakeJobService());
@@ -878,6 +1179,7 @@
new TableRow().set("name", String.format("number%d", i)).set("number", i),
new TableRow().set("name", String.format("number%d", i + 5)).set("number", i + 5)));
}
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
}
@Test
@@ -890,8 +1192,9 @@
FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
.withJobService(new FakeJobService())
.withDatasetService(datasetService);
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
Pipeline p = TestPipeline.create(bqOptions);
+
p.apply(Create.of(
new TableRow().set("name", "a").set("number", 1),
new TableRow().set("name", "b").set("number", 2),
@@ -910,6 +1213,7 @@
File tempDir = new File(bqOptions.getTempLocation());
testNumFiles(tempDir, 0);
}
+ testNumFiles(new File(bqOptions.getTempLocation()), 0);
}
@Test
@@ -1339,40 +1643,6 @@
}
@Test
- public void testBigQueryTableSourceThroughJsonAPI() throws Exception {
- FakeDatasetService datasetService = new FakeDatasetService();
- FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
- .withJobService(new FakeJobService())
- .withDatasetService(datasetService);
-
- List<TableRow> expected = ImmutableList.of(
- new TableRow().set("name", "a").set("number", "1"),
- new TableRow().set("name", "b").set("number", "2"),
- new TableRow().set("name", "c").set("number", "3"),
- new TableRow().set("name", "d").set("number", "4"),
- new TableRow().set("name", "e").set("number", "5"),
- new TableRow().set("name", "f").set("number", "6"));
-
- TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
- datasetService.createDataset(table.getProjectId(), table.getDatasetId(), "", "");
- datasetService.createTable(new Table().setTableReference(table));
- datasetService.insertAll(table, expected, null);
-
- Path baseDir = Files.createTempDirectory(tempFolder, "testBigQueryTableSourceThroughJsonAPI");
- String stepUuid = "testStepUuid";
- BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
- stepUuid, StaticValueProvider.of(table), fakeBqServices);
-
- PipelineOptions options = PipelineOptionsFactory.create();
- options.setTempLocation(baseDir.toString());
- Assert.assertThat(
- SourceTestUtils.readFromSource(bqSource, options),
- CoreMatchers.is(expected));
- SourceTestUtils.assertSplitAtFractionBehavior(
- bqSource, 2, 0.3, ExpectedSplitOutcome.MUST_BE_CONSISTENT_IF_SUCCEEDS, options);
- }
-
- @Test
public void testBigQueryTableSourceInitSplit() throws Exception {
FakeDatasetService fakeDatasetService = new FakeDatasetService();
FakeJobService fakeJobService = new FakeJobService();
@@ -1389,7 +1659,7 @@
new TableRow().set("name", "f").set("number", 6L));
TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
- fakeDatasetService.createDataset("project", "data_set", "", "");
+ fakeDatasetService.createDataset("project", "data_set", "", "", null);
fakeDatasetService.createTable(new Table().setTableReference(table)
.setSchema(new TableSchema()
.setFields(
@@ -1402,31 +1672,110 @@
String stepUuid = "testStepUuid";
BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
- stepUuid, StaticValueProvider.of(table), fakeBqServices);
+ stepUuid,
+ StaticValueProvider.of(table),
+ fakeBqServices,
+ TableRowJsonCoder.of(),
+ BigQueryIO.TableRowParser.INSTANCE);
PipelineOptions options = PipelineOptionsFactory.create();
options.setTempLocation(baseDir.toString());
BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
bqOptions.setProject("project");
- List<TableRow> read = SourceTestUtils.readFromSource(bqSource, options);
+ List<TableRow> read =
+ convertStringsToLong(
+ SourceTestUtils.readFromSplitsOfSource(bqSource, 0L /* ignored */, options));
assertThat(read, containsInAnyOrder(Iterables.toArray(expected, TableRow.class)));
- SourceTestUtils.assertSplitAtFractionBehavior(
- bqSource, 2, 0.3, ExpectedSplitOutcome.MUST_BE_CONSISTENT_IF_SUCCEEDS, options);
List<? extends BoundedSource<TableRow>> sources = bqSource.split(100, options);
assertEquals(2, sources.size());
// Simulate a repeated call to split(), like a Dataflow worker will sometimes do.
sources = bqSource.split(200, options);
assertEquals(2, sources.size());
- BoundedSource<TableRow> actual = sources.get(0);
- assertThat(actual, CoreMatchers.instanceOf(TransformingSource.class));
// A repeated call to split() should not have caused a duplicate extract job.
assertEquals(1, fakeJobService.getNumExtractJobCalls());
}
@Test
+ public void testEstimatedSizeWithoutStreamingBuffer() throws Exception {
+ FakeDatasetService fakeDatasetService = new FakeDatasetService();
+ FakeJobService fakeJobService = new FakeJobService();
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .withJobService(fakeJobService)
+ .withDatasetService(fakeDatasetService);
+
+ List<TableRow> data = ImmutableList.of(
+ new TableRow().set("name", "a").set("number", 1L),
+ new TableRow().set("name", "b").set("number", 2L),
+ new TableRow().set("name", "c").set("number", 3L),
+ new TableRow().set("name", "d").set("number", 4L),
+ new TableRow().set("name", "e").set("number", 5L),
+ new TableRow().set("name", "f").set("number", 6L));
+
+ TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
+ fakeDatasetService.createDataset("project", "data_set", "", "", null);
+ fakeDatasetService.createTable(new Table().setTableReference(table)
+ .setSchema(new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("number").setType("INTEGER")))));
+ fakeDatasetService.insertAll(table, data, null);
+
+ String stepUuid = "testStepUuid";
+ BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
+ stepUuid,
+ StaticValueProvider.of(table),
+ fakeBqServices,
+ TableRowJsonCoder.of(),
+ BigQueryIO.TableRowParser.INSTANCE);
+
+ PipelineOptions options = PipelineOptionsFactory.create();
+ assertEquals(108, bqSource.getEstimatedSizeBytes(options));
+ }
+
+ @Test
+ public void testEstimatedSizeWithStreamingBuffer() throws Exception {
+ FakeDatasetService fakeDatasetService = new FakeDatasetService();
+ FakeJobService fakeJobService = new FakeJobService();
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .withJobService(fakeJobService)
+ .withDatasetService(fakeDatasetService);
+
+ List<TableRow> data = ImmutableList.of(
+ new TableRow().set("name", "a").set("number", 1L),
+ new TableRow().set("name", "b").set("number", 2L),
+ new TableRow().set("name", "c").set("number", 3L),
+ new TableRow().set("name", "d").set("number", 4L),
+ new TableRow().set("name", "e").set("number", 5L),
+ new TableRow().set("name", "f").set("number", 6L));
+
+ TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
+ fakeDatasetService.createDataset("project", "data_set", "", "", null);
+ fakeDatasetService.createTable(new Table().setTableReference(table)
+ .setSchema(new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("number").setType("INTEGER"))))
+ .setStreamingBuffer(new Streamingbuffer().setEstimatedBytes(BigInteger.valueOf(10))));
+ fakeDatasetService.insertAll(table, data, null);
+
+ String stepUuid = "testStepUuid";
+ BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
+ stepUuid,
+ StaticValueProvider.of(table),
+ fakeBqServices,
+ TableRowJsonCoder.of(),
+ BigQueryIO.TableRowParser.INSTANCE);
+
+ PipelineOptions options = PipelineOptionsFactory.create();
+ assertEquals(118, bqSource.getEstimatedSizeBytes(options));
+ }
+
+ @Test
public void testBigQueryQuerySourceInitSplit() throws Exception {
TableReference dryRunTable = new TableReference();
@@ -1468,7 +1817,7 @@
TableReference tempTableReference = createTempTableReference(
bqOptions.getProject(), createJobIdToken(bqOptions.getJobName(), stepUuid));
fakeDatasetService.createDataset(
- bqOptions.getProject(), tempTableReference.getDatasetId(), "", "");
+ bqOptions.getProject(), tempTableReference.getDatasetId(), "", "", null);
fakeDatasetService.createTable(new Table()
.setTableReference(tempTableReference)
.setSchema(new TableSchema()
@@ -1481,8 +1830,13 @@
String query = FakeBigQueryServices.encodeQuery(expected);
BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
- stepUuid, StaticValueProvider.of(query),
- true /* flattenResults */, true /* useLegacySql */, fakeBqServices);
+ stepUuid,
+ StaticValueProvider.of(query),
+ true /* flattenResults */,
+ true /* useLegacySql */,
+ fakeBqServices,
+ TableRowJsonCoder.of(),
+ BigQueryIO.TableRowParser.INSTANCE);
options.setTempLocation(baseDir.toString());
TableReference queryTable = new TableReference()
@@ -1496,15 +1850,12 @@
.setTotalBytesProcessed(100L)
.setReferencedTables(ImmutableList.of(queryTable))));
- List<TableRow> read = SourceTestUtils.readFromSource(bqSource, options);
+ List<TableRow> read = convertStringsToLong(
+ SourceTestUtils.readFromSplitsOfSource(bqSource, 0L /* ignored */, options));
assertThat(read, containsInAnyOrder(Iterables.toArray(expected, TableRow.class)));
- SourceTestUtils.assertSplitAtFractionBehavior(
- bqSource, 2, 0.3, ExpectedSplitOutcome.MUST_BE_CONSISTENT_IF_SUCCEEDS, options);
List<? extends BoundedSource<TableRow>> sources = bqSource.split(100, options);
assertEquals(2, sources.size());
- BoundedSource<TableRow> actual = sources.get(0);
- assertThat(actual, CoreMatchers.instanceOf(TransformingSource.class));
}
@Test
@@ -1548,7 +1899,7 @@
new TableRow().set("name", "e").set("number", 5L),
new TableRow().set("name", "f").set("number", 6L));
datasetService.createDataset(
- tempTableReference.getProjectId(), tempTableReference.getDatasetId(), "", "");
+ tempTableReference.getProjectId(), tempTableReference.getDatasetId(), "", "", null);
Table table = new Table()
.setTableReference(tempTableReference)
.setSchema(new TableSchema()
@@ -1570,93 +1921,36 @@
BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
stepUuid,
StaticValueProvider.of(query),
- true /* flattenResults */, true /* useLegacySql */, fakeBqServices);
+ true /* flattenResults */,
+ true /* useLegacySql */,
+ fakeBqServices,
+ TableRowJsonCoder.of(),
+ BigQueryIO.TableRowParser.INSTANCE);
options.setTempLocation(baseDir.toString());
- List<TableRow> read = convertBigDecimaslToLong(
- SourceTestUtils.readFromSource(bqSource, options));
+ List<TableRow> read = convertStringsToLong(
+ SourceTestUtils.readFromSplitsOfSource(bqSource, 0L /* ignored */, options));
assertThat(read, containsInAnyOrder(Iterables.toArray(expected, TableRow.class)));
- SourceTestUtils.assertSplitAtFractionBehavior(
- bqSource, 2, 0.3, ExpectedSplitOutcome.MUST_BE_CONSISTENT_IF_SUCCEEDS, options);
List<? extends BoundedSource<TableRow>> sources = bqSource.split(100, options);
assertEquals(2, sources.size());
- BoundedSource<TableRow> actual = sources.get(0);
- assertThat(actual, CoreMatchers.instanceOf(TransformingSource.class));
- }
-
- @Test
- public void testTransformingSource() throws Exception {
- int numElements = 10000;
- @SuppressWarnings("deprecation")
- BoundedSource<Long> longSource = CountingSource.upTo(numElements);
- SerializableFunction<Long, String> toStringFn =
- new SerializableFunction<Long, String>() {
- @Override
- public String apply(Long input) {
- return input.toString();
- }};
- BoundedSource<String> stringSource = new TransformingSource<>(
- longSource, toStringFn, StringUtf8Coder.of());
-
- List<String> expected = Lists.newArrayList();
- for (int i = 0; i < numElements; i++) {
- expected.add(String.valueOf(i));
- }
-
- PipelineOptions options = PipelineOptionsFactory.create();
- Assert.assertThat(
- SourceTestUtils.readFromSource(stringSource, options),
- CoreMatchers.is(expected));
- SourceTestUtils.assertSplitAtFractionBehavior(
- stringSource, 100, 0.3, ExpectedSplitOutcome.MUST_SUCCEED_AND_BE_CONSISTENT, options);
-
- SourceTestUtils.assertSourcesEqualReferenceSource(
- stringSource, stringSource.split(100, options), options);
- }
-
- @Test
- public void testTransformingSourceUnsplittable() throws Exception {
- int numElements = 10000;
- @SuppressWarnings("deprecation")
- BoundedSource<Long> longSource =
- SourceTestUtils.toUnsplittableSource(CountingSource.upTo(numElements));
- SerializableFunction<Long, String> toStringFn =
- new SerializableFunction<Long, String>() {
- @Override
- public String apply(Long input) {
- return input.toString();
- }
- };
- BoundedSource<String> stringSource =
- new TransformingSource<>(longSource, toStringFn, StringUtf8Coder.of());
-
- List<String> expected = Lists.newArrayList();
- for (int i = 0; i < numElements; i++) {
- expected.add(String.valueOf(i));
- }
-
- PipelineOptions options = PipelineOptionsFactory.create();
- Assert.assertThat(
- SourceTestUtils.readFromSource(stringSource, options), CoreMatchers.is(expected));
- SourceTestUtils.assertSplitAtFractionBehavior(
- stringSource, 100, 0.3, ExpectedSplitOutcome.MUST_BE_CONSISTENT_IF_SUCCEEDS, options);
-
- SourceTestUtils.assertSourcesEqualReferenceSource(
- stringSource, stringSource.split(100, options), options);
}
@Test
public void testPassThroughThenCleanup() throws Exception {
- PCollection<Integer> output = p
- .apply(Create.of(1, 2, 3))
- .apply(new PassThroughThenCleanup<Integer>(new CleanupOperation() {
- @Override
- void cleanup(PipelineOptions options) throws Exception {
- // no-op
- }}));
+ PCollection<Integer> output =
+ p.apply(Create.of(1, 2, 3))
+ .apply(
+ new PassThroughThenCleanup<Integer>(
+ new CleanupOperation() {
+ @Override
+ void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception {
+ // no-op
+ }
+ },
+ p.apply("Create1", Create.of("")).apply(View.<String>asSingleton())));
PAssert.that(output).containsInAnyOrder(1, 2, 3);
@@ -1667,11 +1961,15 @@
public void testPassThroughThenCleanupExecuted() throws Exception {
p.apply(Create.empty(VarIntCoder.of()))
- .apply(new PassThroughThenCleanup<Integer>(new CleanupOperation() {
- @Override
- void cleanup(PipelineOptions options) throws Exception {
- throw new RuntimeException("cleanup executed");
- }}));
+ .apply(
+ new PassThroughThenCleanup<Integer>(
+ new CleanupOperation() {
+ @Override
+ void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception {
+ throw new RuntimeException("cleanup executed");
+ }
+ },
+ p.apply("Create1", Create.of("")).apply(View.<String>asSingleton())));
thrown.expect(RuntimeException.class);
thrown.expectMessage("cleanup executed");
@@ -1728,20 +2026,23 @@
// function) and there is no input data, WritePartition will generate an empty table. This
// code is to test that path.
boolean isSingleton = numTables == 1 && numFilesPerTable == 0;
-
- List<ShardedKey<String>> expectedPartitions = Lists.newArrayList();
+ DynamicDestinations<String, TableDestination> dynamicDestinations =
+ new DynamicDestinationsHelpers.ConstantTableDestinations<>(
+ StaticValueProvider.of("SINGLETON"), "");
+ List<ShardedKey<TableDestination>> expectedPartitions = Lists.newArrayList();
if (isSingleton) {
- expectedPartitions.add(ShardedKey.<String>of(null, 1));
+ expectedPartitions.add(ShardedKey.<TableDestination>of(
+ new TableDestination("SINGLETON", ""), 1));
} else {
for (int i = 0; i < numTables; ++i) {
for (int j = 1; j <= expectedNumPartitionsPerTable; ++j) {
String tableName = String.format("project-id:dataset-id.tables%05d", i);
- expectedPartitions.add(ShardedKey.of(tableName, j));
+ expectedPartitions.add(ShardedKey.of(new TableDestination(tableName, ""), j));
}
}
}
- List<WriteBundlesToFiles.Result<String>> files = Lists.newArrayList();
+ List<WriteBundlesToFiles.Result<TableDestination>> files = Lists.newArrayList();
Map<String, List<String>> filenamesPerTable = Maps.newHashMap();
for (int i = 0; i < numTables; ++i) {
String tableName = String.format("project-id:dataset-id.tables%05d", i);
@@ -1753,36 +2054,35 @@
for (int j = 0; j < numFilesPerTable; ++j) {
String fileName = String.format("%s_files%05d", tableName, j);
filenames.add(fileName);
- files.add(new Result<>(fileName, fileSize, tableName));
+ files.add(new Result<>(fileName, fileSize, new TableDestination(tableName, "")));
}
}
- TupleTag<KV<ShardedKey<String>, List<String>>> multiPartitionsTag =
- new TupleTag<KV<ShardedKey<String>, List<String>>>("multiPartitionsTag") {};
- TupleTag<KV<ShardedKey<String>, List<String>>> singlePartitionTag =
- new TupleTag<KV<ShardedKey<String>, List<String>>>("singlePartitionTag") {};
-
- PCollectionView<Iterable<WriteBundlesToFiles.Result<String>>> resultsView =
- p.apply(
- Create.of(files)
- .withCoder(WriteBundlesToFiles.ResultCoder.of(StringUtf8Coder.of())))
- .apply(View.<WriteBundlesToFiles.Result<String>>asIterable());
+ TupleTag<KV<ShardedKey<TableDestination>, List<String>>> multiPartitionsTag =
+ new TupleTag<KV<ShardedKey<TableDestination>, List<String>>>("multiPartitionsTag") {};
+ TupleTag<KV<ShardedKey<TableDestination>, List<String>>> singlePartitionTag =
+ new TupleTag<KV<ShardedKey<TableDestination>, List<String>>>("singlePartitionTag") {};
String tempFilePrefix = testFolder.newFolder("BigQueryIOTest").getAbsolutePath();
PCollectionView<String> tempFilePrefixView =
p.apply(Create.of(tempFilePrefix)).apply(View.<String>asSingleton());
- WritePartition<String> writePartition =
+ WritePartition<TableDestination> writePartition =
new WritePartition<>(
- isSingleton, tempFilePrefixView, resultsView, multiPartitionsTag, singlePartitionTag);
+ isSingleton,
+ dynamicDestinations,
+ tempFilePrefixView,
+ multiPartitionsTag,
+ singlePartitionTag);
- DoFnTester<Void, KV<ShardedKey<String>, List<String>>> tester =
- DoFnTester.of(writePartition);
- tester.setSideInput(resultsView, GlobalWindow.INSTANCE, files);
+ DoFnTester<
+ Iterable<WriteBundlesToFiles.Result<TableDestination>>,
+ KV<ShardedKey<TableDestination>, List<String>>>
+ tester = DoFnTester.of(writePartition);
tester.setSideInput(tempFilePrefixView, GlobalWindow.INSTANCE, tempFilePrefix);
- tester.processElement(null);
+ tester.processElement(files);
- List<KV<ShardedKey<String>, List<String>>> partitions;
+ List<KV<ShardedKey<TableDestination>, List<String>>> partitions;
if (expectedNumPartitionsPerTable > 1) {
partitions = tester.takeOutputElements(multiPartitionsTag);
} else {
@@ -1790,10 +2090,10 @@
}
- List<ShardedKey<String>> partitionsResult = Lists.newArrayList();
+ List<ShardedKey<TableDestination>> partitionsResult = Lists.newArrayList();
Map<String, List<String>> filesPerTableResult = Maps.newHashMap();
- for (KV<ShardedKey<String>, List<String>> partition : partitions) {
- String table = partition.getKey().getKey();
+ for (KV<ShardedKey<TableDestination>, List<String>> partition : partitions) {
+ String table = partition.getKey().getKey().getTableSpec();
partitionsResult.add(partition.getKey());
List<String> tableFilesResult = filesPerTableResult.get(table);
if (tableFilesResult == null) {
@@ -1830,25 +2130,28 @@
@Override
public TableSchema getSchema(String destination) {
- throw new UnsupportedOperationException("getSchema not expected in this test.");
+ return null;
}
}
@Test
public void testWriteTables() throws Exception {
- p.enableAbandonedNodeEnforcement(false);
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
FakeDatasetService datasetService = new FakeDatasetService();
FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
.withJobService(new FakeJobService())
.withDatasetService(datasetService);
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ Pipeline p = TestPipeline.create(bqOptions);
long numTables = 3;
long numPartitions = 3;
long numFilesPerPartition = 10;
- String jobIdToken = "jobIdToken";
- String stepUuid = "stepUuid";
- Map<TableDestination, List<String>> expectedTempTables = Maps.newHashMap();
+ String jobIdToken = "jobId";
+ final Multimap<TableDestination, String> expectedTempTables = ArrayListMultimap.create();
Path baseDir = Files.createTempDirectory(tempFolder, "testWriteTables");
@@ -1857,74 +2160,65 @@
String tableName = String.format("project-id:dataset-id.table%05d", i);
TableDestination tableDestination = new TableDestination(tableName, tableName);
for (int j = 0; j < numPartitions; ++j) {
- String tempTableId = BigQueryHelpers.createJobId(jobIdToken, tableDestination, j);
+ String tempTableId = BigQueryHelpers.createJobId(jobIdToken, tableDestination, j, 0);
List<String> filesPerPartition = Lists.newArrayList();
for (int k = 0; k < numFilesPerPartition; ++k) {
String filename = Paths.get(baseDir.toString(),
String.format("files0x%08x_%05d", tempTableId.hashCode(), k)).toString();
- ResourceId fileResource =
- FileSystems.matchNewResource(filename, false /* isDirectory */);
- try (WritableByteChannel channel = FileSystems.create(fileResource, MimeTypes.TEXT)) {
- try (OutputStream output = Channels.newOutputStream(channel)) {
- TableRow tableRow = new TableRow().set("name", tableName);
- TableRowJsonCoder.of().encode(tableRow, output, Context.OUTER);
- output.write("\n".getBytes(StandardCharsets.UTF_8));
- }
+ TableRowWriter writer = new TableRowWriter(filename);
+ try (TableRowWriter ignored = writer) {
+ TableRow tableRow = new TableRow().set("name", tableName);
+ writer.write(tableRow);
}
- filesPerPartition.add(filename);
+ filesPerPartition.add(writer.getResult().resourceId.toString());
}
partitions.add(KV.of(ShardedKey.of(tableDestination.getTableSpec(), j),
filesPerPartition));
- List<String> expectedTables = expectedTempTables.get(tableDestination);
- if (expectedTables == null) {
- expectedTables = Lists.newArrayList();
- expectedTempTables.put(tableDestination, expectedTables);
- }
String json = String.format(
"{\"datasetId\":\"dataset-id\",\"projectId\":\"project-id\",\"tableId\":\"%s\"}",
tempTableId);
- expectedTables.add(json);
+ expectedTempTables.put(tableDestination, json);
}
}
+ PCollection<KV<ShardedKey<String>, List<String>>> writeTablesInput =
+ p.apply(Create.of(partitions));
PCollectionView<String> jobIdTokenView = p
.apply("CreateJobId", Create.of("jobId"))
.apply(View.<String>asSingleton());
+ List<PCollectionView<?>> sideInputs = ImmutableList.<PCollectionView<?>>of(jobIdTokenView);
- PCollectionView<Map<String, String>> schemaMapView =
- p.apply("CreateEmptySchema",
- Create.empty(new TypeDescriptor<KV<String, String>>() {}))
- .apply(View.<String, String>asMap());
WriteTables<String> writeTables =
new WriteTables<>(
false,
fakeBqServices,
jobIdTokenView,
- schemaMapView,
WriteDisposition.WRITE_EMPTY,
CreateDisposition.CREATE_IF_NEEDED,
+ sideInputs,
new IdentityDynamicTables());
- DoFnTester<KV<ShardedKey<String>, List<String>>,
- KV<TableDestination, String>> tester = DoFnTester.of(writeTables);
- tester.setSideInput(jobIdTokenView, GlobalWindow.INSTANCE, jobIdToken);
- tester.setSideInput(schemaMapView, GlobalWindow.INSTANCE, ImmutableMap.<String, String>of());
- tester.getPipelineOptions().setTempLocation("tempLocation");
- for (KV<ShardedKey<String>, List<String>> partition : partitions) {
- tester.processElement(partition);
- }
+ PCollection<KV<TableDestination, String>> writeTablesOutput =
+ writeTablesInput.apply(writeTables);
- Map<TableDestination, List<String>> tempTablesResult = Maps.newHashMap();
- for (KV<TableDestination, String> element : tester.takeOutputElements()) {
- List<String> tables = tempTablesResult.get(element.getKey());
- if (tables == null) {
- tables = Lists.newArrayList();
- tempTablesResult.put(element.getKey(), tables);
- }
- tables.add(element.getValue());
- }
- assertEquals(expectedTempTables, tempTablesResult);
+ PAssert.thatMultimap(writeTablesOutput)
+ .satisfies(
+ new SerializableFunction<Map<TableDestination, Iterable<String>>, Void>() {
+ @Override
+ public Void apply(Map<TableDestination, Iterable<String>> input) {
+ assertEquals(input.keySet(), expectedTempTables.keySet());
+ for (Map.Entry<TableDestination, Iterable<String>> entry : input.entrySet()) {
+ @SuppressWarnings("unchecked")
+ String[] expectedValues = Iterables.toArray(
+ expectedTempTables.get(entry.getKey()), String.class);
+ assertThat(entry.getValue(), containsInAnyOrder(expectedValues));
+ }
+ return null;
+ }
+ });
+ p.run();
+ testNumFiles(baseDir.toFile(), 0);
}
@Test
@@ -1959,27 +2253,20 @@
FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
.withJobService(new FakeJobService())
.withDatasetService(datasetService);
- datasetService.createDataset("project-id", "dataset-id", "", "");
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
final int numFinalTables = 3;
final int numTempTablesPerFinalTable = 3;
final int numRecordsPerTempTable = 10;
- Map<TableDestination, List<TableRow>> expectedRowsPerTable = Maps.newHashMap();
+ Multimap<TableDestination, TableRow> expectedRowsPerTable = ArrayListMultimap.create();
String jobIdToken = "jobIdToken";
- Map<TableDestination, Iterable<String>> tempTables = Maps.newHashMap();
+ Multimap<TableDestination, String> tempTables = ArrayListMultimap.create();
+ List<KV<TableDestination, String>> tempTablesElement = Lists.newArrayList();
for (int i = 0; i < numFinalTables; ++i) {
String tableName = "project-id:dataset-id.table_" + i;
TableDestination tableDestination = new TableDestination(
tableName, "table_" + i + "_desc");
- List<String> tables = Lists.newArrayList();
- tempTables.put(tableDestination, tables);
-
- List<TableRow> expectedRows = expectedRowsPerTable.get(tableDestination);
- if (expectedRows == null) {
- expectedRows = Lists.newArrayList();
- expectedRowsPerTable.put(tableDestination, expectedRows);
- }
for (int j = 0; i < numTempTablesPerFinalTable; ++i) {
TableReference tempTable = new TableReference()
.setProjectId("project-id")
@@ -1992,56 +2279,36 @@
rows.add(new TableRow().set("number", j * numTempTablesPerFinalTable + k));
}
datasetService.insertAll(tempTable, rows, null);
- expectedRows.addAll(rows);
- tables.add(BigQueryHelpers.toJsonString(tempTable));
+ expectedRowsPerTable.putAll(tableDestination, rows);
+ String tableJson = BigQueryHelpers.toJsonString(tempTable);
+ tempTables.put(tableDestination, tableJson);
+ tempTablesElement.add(KV.of(tableDestination, tableJson));
}
}
- PCollection<KV<TableDestination, String>> tempTablesPCollection =
- p.apply(Create.of(tempTables)
- .withCoder(KvCoder.of(TableDestinationCoder.of(),
- IterableCoder.of(StringUtf8Coder.of()))))
- .apply(ParDo.of(new DoFn<KV<TableDestination, Iterable<String>>,
- KV<TableDestination, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) {
- TableDestination tableDestination = c.element().getKey();
- for (String tempTable : c.element().getValue()) {
- c.output(KV.of(tableDestination, tempTable));
- }
- }
- }));
-
- PCollectionView<Map<TableDestination, Iterable<String>>> tempTablesView =
- PCollectionViews.multimapView(
- tempTablesPCollection,
- WindowingStrategy.globalDefault(),
- KvCoder.of(TableDestinationCoder.of(),
- StringUtf8Coder.of()));
PCollectionView<String> jobIdTokenView = p
.apply("CreateJobId", Create.of("jobId"))
.apply(View.<String>asSingleton());
- WriteRename writeRename = new WriteRename(
- fakeBqServices,
- jobIdTokenView,
- WriteDisposition.WRITE_EMPTY,
- CreateDisposition.CREATE_IF_NEEDED,
- tempTablesView);
+ WriteRename writeRename =
+ new WriteRename(
+ fakeBqServices,
+ jobIdTokenView,
+ WriteDisposition.WRITE_EMPTY,
+ CreateDisposition.CREATE_IF_NEEDED);
- DoFnTester<Void, Void> tester = DoFnTester.of(writeRename);
- tester.setSideInput(tempTablesView, GlobalWindow.INSTANCE, tempTables);
+ DoFnTester<Iterable<KV<TableDestination, String>>, Void> tester = DoFnTester.of(writeRename);
tester.setSideInput(jobIdTokenView, GlobalWindow.INSTANCE, jobIdToken);
- tester.processElement(null);
+ tester.processElement(tempTablesElement);
- for (Map.Entry<TableDestination, Iterable<String>> entry : tempTables.entrySet()) {
+ for (Map.Entry<TableDestination, Collection<String>> entry : tempTables.asMap().entrySet()) {
TableDestination tableDestination = entry.getKey();
TableReference tableReference = tableDestination.getTableReference();
Table table = checkNotNull(datasetService.getTable(tableReference));
assertEquals(tableReference.getTableId() + "_desc", tableDestination.getTableDescription());
- List<TableRow> expectedRows = expectedRowsPerTable.get(tableDestination);
+ Collection<TableRow> expectedRows = expectedRowsPerTable.get(tableDestination);
assertThat(datasetService.getAllRows(tableReference.getProjectId(),
tableReference.getDatasetId(), tableReference.getTableId()),
containsInAnyOrder(Iterables.toArray(expectedRows, TableRow.class)));
@@ -2060,7 +2327,7 @@
FakeDatasetService datasetService = new FakeDatasetService();
String projectId = "project";
String datasetId = "dataset";
- datasetService.createDataset(projectId, datasetId, "", "");
+ datasetService.createDataset(projectId, datasetId, "", "", null);
List<TableReference> tableRefs = Lists.newArrayList(
BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table1")),
BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table2")),
@@ -2169,18 +2436,70 @@
IntervalWindow.getCoder()));
}
- List<TableRow> convertBigDecimaslToLong(List<TableRow> toConvert) {
- // The numbers come back as BigDecimal objects after JSON serialization. Change them back to
+ List<TableRow> convertStringsToLong(List<TableRow> toConvert) {
+ // The numbers come back as String after JSON serialization. Change them back to
// longs so that we can assert the output.
List<TableRow> converted = Lists.newArrayList();
for (TableRow entry : toConvert) {
TableRow convertedEntry = entry.clone();
- Object num = convertedEntry.get("number");
- if (num instanceof BigDecimal) {
- convertedEntry.set("number", ((BigDecimal) num).longValue());
- }
+ convertedEntry.set("number", Long.parseLong((String) convertedEntry.get("number")));
converted.add(convertedEntry);
}
return converted;
}
+
+ @Test
+ public void testWriteToTableDecorator() throws Exception {
+ BigQueryOptions bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class);
+ bqOptions.setProject("project-id");
+ bqOptions.setTempLocation(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+
+ FakeDatasetService datasetService = new FakeDatasetService();
+ FakeBigQueryServices fakeBqServices =
+ new FakeBigQueryServices()
+ .withJobService(new FakeJobService())
+ .withDatasetService(datasetService);
+ datasetService.createDataset("project-id", "dataset-id", "", "", null);
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ TableRow row1 = new TableRow().set("name", "a").set("number", "1");
+ TableRow row2 = new TableRow().set("name", "b").set("number", "2");
+
+ TableSchema schema = new TableSchema()
+ .setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("number").setType("INTEGER")));
+ p.apply(Create.of(row1, row2))
+ .apply(
+ BigQueryIO.writeTableRows()
+ .to("project-id:dataset-id.table-id$decorator")
+ .withTestServices(fakeBqServices)
+ .withMethod(Method.STREAMING_INSERTS)
+ .withSchema(schema)
+ .withoutValidation());
+ p.run();
+ }
+
+ @Test
+ public void testTableDecoratorStripping() {
+ assertEquals("project:dataset.table",
+ BigQueryHelpers.stripPartitionDecorator("project:dataset.table$decorator"));
+ assertEquals("project:dataset.table",
+ BigQueryHelpers.stripPartitionDecorator("project:dataset.table"));
+ }
+
+ @Test
+ public void testCoderInference() {
+ SerializableFunction<SchemaAndRecord, KV<ByteString, Mutation>> parseFn =
+ new SerializableFunction<SchemaAndRecord, KV<ByteString, Mutation>>() {
+ @Override
+ public KV<ByteString, Mutation> apply(SchemaAndRecord input) {
+ return null;
+ }
+ };
+
+ assertEquals(
+ KvCoder.of(ByteStringCoder.of(), ProtoCoder.of(Mutation.class)),
+ BigQueryIO.read(parseFn).inferCoder(CoderRegistry.createDefault()));
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
index b41490f..f602038 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
@@ -58,6 +58,7 @@
import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.cloud.hadoop.util.RetryBoundedBackOff;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -67,11 +68,14 @@
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl.JobServiceImpl;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.ExpectedLogs;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.BackOffAdapter;
import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
import org.apache.beam.sdk.util.FluentBackoff;
import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
import org.apache.beam.sdk.util.Transport;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
@@ -485,6 +489,11 @@
verify(response, times(1)).getContentType();
}
+ private ValueInSingleWindow<TableRow> wrapTableRow(TableRow row) {
+ return ValueInSingleWindow.of(row, GlobalWindow.TIMESTAMP_MAX_VALUE,
+ GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ }
+
/**
* Tests that {@link DatasetServiceImpl#insertAll} retries quota rate limited attempts.
*/
@@ -492,8 +501,8 @@
public void testInsertRetry() throws Exception {
TableReference ref =
new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
- List<TableRow> rows = new ArrayList<>();
- rows.add(new TableRow());
+ List<ValueInSingleWindow<TableRow>> rows = new ArrayList<>();
+ rows.add(wrapTableRow(new TableRow()));
// First response is 403 rate limited, second response has valid payload.
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
@@ -505,7 +514,8 @@
DatasetServiceImpl dataService =
new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
dataService.insertAll(ref, rows, null,
- BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
+ BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(),
+ InsertRetryPolicy.alwaysRetry(), null);
verify(response, times(2)).getStatusCode();
verify(response, times(2)).getContent();
verify(response, times(2)).getContentType();
@@ -524,8 +534,9 @@
public void testInsertRetrySelectRows() throws Exception {
TableReference ref =
new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
- List<TableRow> rows = ImmutableList.of(
- new TableRow().set("row", "a"), new TableRow().set("row", "b"));
+ List<ValueInSingleWindow<TableRow>> rows = ImmutableList.of(
+ wrapTableRow(new TableRow().set("row", "a")),
+ wrapTableRow(new TableRow().set("row", "b")));
List<String> insertIds = ImmutableList.of("a", "b");
final TableDataInsertAllResponse bFailed = new TableDataInsertAllResponse()
@@ -542,11 +553,11 @@
DatasetServiceImpl dataService =
new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
dataService.insertAll(ref, rows, insertIds,
- BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
+ BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(),
+ InsertRetryPolicy.alwaysRetry(), null);
verify(response, times(2)).getStatusCode();
verify(response, times(2)).getContent();
verify(response, times(2)).getContentType();
- expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");
}
/**
@@ -556,7 +567,8 @@
public void testInsertFailsGracefully() throws Exception {
TableReference ref =
new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
- List<TableRow> rows = ImmutableList.of(new TableRow(), new TableRow());
+ List<ValueInSingleWindow<TableRow>> rows = ImmutableList.of(
+ wrapTableRow(new TableRow()), wrapTableRow(new TableRow()));
final TableDataInsertAllResponse row1Failed = new TableDataInsertAllResponse()
.setInsertErrors(ImmutableList.of(new InsertErrors().setIndex(1L)));
@@ -584,7 +596,8 @@
// Expect it to fail.
try {
dataService.insertAll(ref, rows, null,
- BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
+ BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(),
+ InsertRetryPolicy.alwaysRetry(), null);
fail();
} catch (IOException e) {
assertThat(e, instanceOf(IOException.class));
@@ -606,8 +619,8 @@
public void testInsertDoesNotRetry() throws Throwable {
TableReference ref =
new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
- List<TableRow> rows = new ArrayList<>();
- rows.add(new TableRow());
+ List<ValueInSingleWindow<TableRow>> rows = new ArrayList<>();
+ rows.add(wrapTableRow(new TableRow()));
// First response is 403 not-rate-limited, second response has valid payload but should not
// be invoked.
@@ -625,7 +638,8 @@
try {
dataService.insertAll(ref, rows, null,
- BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
+ BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(),
+ InsertRetryPolicy.alwaysRetry(), null);
fail();
} catch (RuntimeException e) {
verify(response, times(1)).getStatusCode();
@@ -635,6 +649,56 @@
}
}
+ /**
+ * Tests that {@link DatasetServiceImpl#insertAll} uses the supplied {@link InsertRetryPolicy},
+ * and returns the list of rows not retried.
+ */
+ @Test
+ public void testInsertRetryPolicy() throws InterruptedException, IOException {
+ TableReference ref =
+ new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
+ List<ValueInSingleWindow<TableRow>> rows = ImmutableList.of(
+ wrapTableRow(new TableRow()), wrapTableRow(new TableRow()));
+
+ // First time row0 fails with a retryable error, and row1 fails with a persistent error.
+ final TableDataInsertAllResponse firstFailure = new TableDataInsertAllResponse()
+ .setInsertErrors(ImmutableList.of(
+ new InsertErrors().setIndex(0L).setErrors(
+ ImmutableList.of(new ErrorProto().setReason("timeout"))),
+ new InsertErrors().setIndex(1L).setErrors(
+ ImmutableList.of(new ErrorProto().setReason("invalid")))));
+
+ // Second time there is only one row, which fails with a retryable error.
+ final TableDataInsertAllResponse secondFialure = new TableDataInsertAllResponse()
+ .setInsertErrors(ImmutableList.of(new InsertErrors().setIndex(0L).setErrors(
+ ImmutableList.of(new ErrorProto().setReason("timeout")))));
+
+ // On the final attempt, no failures are returned.
+ final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();
+
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ // Always return 200.
+ when(response.getStatusCode()).thenReturn(200);
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(200).thenReturn(200);
+
+ // First fail
+ when(response.getContent())
+ .thenReturn(toStream(firstFailure))
+ .thenReturn(toStream(secondFialure))
+ .thenReturn(toStream(allRowsSucceeded));
+
+ DatasetServiceImpl dataService =
+ new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
+
+ List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
+ dataService.insertAll(ref, rows, null,
+ BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(),
+ InsertRetryPolicy.retryTransientErrors(), failedInserts);
+ assertEquals(1, failedInserts.size());
+ expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");
+ }
+
/** A helper to wrap a {@link GenericJson} object in a content stream. */
private static InputStream toStream(GenericJson content) throws IOException {
return new ByteArrayInputStream(JacksonFactory.getDefaultInstance().toByteArray(content));
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIteratorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIteratorTest.java
deleted file mode 100644
index f84d412..0000000
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIteratorTest.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.gcp.bigquery;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-import com.google.api.services.bigquery.Bigquery;
-import com.google.api.services.bigquery.model.Dataset;
-import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfiguration;
-import com.google.api.services.bigquery.model.JobConfigurationQuery;
-import com.google.api.services.bigquery.model.JobReference;
-import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.JobStatistics2;
-import com.google.api.services.bigquery.model.JobStatus;
-import com.google.api.services.bigquery.model.Table;
-import com.google.api.services.bigquery.model.TableCell;
-import com.google.api.services.bigquery.model.TableDataList;
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.BaseEncoding;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-/**
- * Tests for {@link BigQueryTableRowIterator}.
- */
-@RunWith(JUnit4.class)
-public class BigQueryTableRowIteratorTest {
-
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Mock private Bigquery mockClient;
- @Mock private Bigquery.Datasets mockDatasets;
- @Mock private Bigquery.Datasets.Delete mockDatasetsDelete;
- @Mock private Bigquery.Datasets.Insert mockDatasetsInsert;
- @Mock private Bigquery.Jobs mockJobs;
- @Mock private Bigquery.Jobs.Get mockJobsGet;
- @Mock private Bigquery.Jobs.Insert mockJobsInsert;
- @Mock private Bigquery.Tables mockTables;
- @Mock private Bigquery.Tables.Get mockTablesGet;
- @Mock private Bigquery.Tables.Delete mockTablesDelete;
- @Mock private Bigquery.Tabledata mockTabledata;
- @Mock private Bigquery.Tabledata.List mockTabledataList;
-
- @Before
- public void setUp() throws IOException {
- MockitoAnnotations.initMocks(this);
- when(mockClient.tabledata()).thenReturn(mockTabledata);
- when(mockTabledata.list(anyString(), anyString(), anyString())).thenReturn(mockTabledataList);
-
- when(mockClient.tables()).thenReturn(mockTables);
- when(mockTables.delete(anyString(), anyString(), anyString())).thenReturn(mockTablesDelete);
- when(mockTables.get(anyString(), anyString(), anyString())).thenReturn(mockTablesGet);
-
- when(mockClient.datasets()).thenReturn(mockDatasets);
- when(mockDatasets.delete(anyString(), anyString())).thenReturn(mockDatasetsDelete);
- when(mockDatasets.insert(anyString(), any(Dataset.class))).thenReturn(mockDatasetsInsert);
-
- when(mockClient.jobs()).thenReturn(mockJobs);
- when(mockJobs.insert(anyString(), any(Job.class))).thenReturn(mockJobsInsert);
- when(mockJobs.get(anyString(), anyString())).thenReturn(mockJobsGet);
- }
-
- @After
- public void tearDown() {
- verifyNoMoreInteractions(mockClient);
- verifyNoMoreInteractions(mockDatasets);
- verifyNoMoreInteractions(mockDatasetsDelete);
- verifyNoMoreInteractions(mockDatasetsInsert);
- verifyNoMoreInteractions(mockJobs);
- verifyNoMoreInteractions(mockJobsGet);
- verifyNoMoreInteractions(mockJobsInsert);
- verifyNoMoreInteractions(mockTables);
- verifyNoMoreInteractions(mockTablesDelete);
- verifyNoMoreInteractions(mockTablesGet);
- verifyNoMoreInteractions(mockTabledata);
- verifyNoMoreInteractions(mockTabledataList);
- }
-
- private static Table tableWithBasicSchema() {
- return new Table()
- .setSchema(
- new TableSchema()
- .setFields(
- Arrays.asList(
- new TableFieldSchema().setName("name").setType("STRING"),
- new TableFieldSchema().setName("answer").setType("INTEGER"),
- new TableFieldSchema().setName("photo").setType("BYTES"),
- new TableFieldSchema().setName("anniversary_date").setType("DATE"),
- new TableFieldSchema().setName("anniversary_datetime").setType("DATETIME"),
- new TableFieldSchema().setName("anniversary_time").setType("TIME"))));
- }
-
- private static Table noTableQuerySchema() {
- return new Table()
- .setSchema(
- new TableSchema()
- .setFields(
- Arrays.asList(
- new TableFieldSchema().setName("name").setType("STRING"),
- new TableFieldSchema().setName("count").setType("INTEGER"),
- new TableFieldSchema().setName("photo").setType("BYTES"))));
- }
-
- private static Table tableWithLocation() {
- return new Table()
- .setLocation("EU");
- }
-
- private TableRow rawRow(Object... args) {
- List<TableCell> cells = new LinkedList<>();
- for (Object a : args) {
- cells.add(new TableCell().setV(a));
- }
- return new TableRow().setF(cells);
- }
-
- private TableDataList rawDataList(TableRow... rows) {
- return new TableDataList().setRows(Arrays.asList(rows));
- }
-
- /**
- * Verifies that when the query runs, the correct data is returned and the temporary dataset and
- * table are both cleaned up.
- */
- @Test
- public void testReadFromQuery() throws IOException, InterruptedException {
- // Mock job inserting.
- Job dryRunJob = new Job().setStatistics(
- new JobStatistics().setQuery(new JobStatistics2().setReferencedTables(
- ImmutableList.of(new TableReference()))));
- Job insertedJob = new Job().setJobReference(new JobReference());
- when(mockJobsInsert.execute()).thenReturn(dryRunJob, insertedJob);
-
- // Mock job polling.
- JobStatus status = new JobStatus().setState("DONE");
- JobConfigurationQuery resultQueryConfig = new JobConfigurationQuery()
- .setDestinationTable(new TableReference()
- .setProjectId("project")
- .setDatasetId("tempdataset")
- .setTableId("temptable"));
- Job getJob =
- new Job()
- .setJobReference(new JobReference())
- .setStatus(status)
- .setConfiguration(new JobConfiguration().setQuery(resultQueryConfig));
- when(mockJobsGet.execute()).thenReturn(getJob);
-
- // Mock table schema fetch.
- when(mockTablesGet.execute()).thenReturn(tableWithLocation(), tableWithBasicSchema());
-
- byte[] photoBytes = "photograph".getBytes();
- String photoBytesEncoded = BaseEncoding.base64().encode(photoBytes);
- // Mock table data fetch.
- when(mockTabledataList.execute()).thenReturn(
- rawDataList(rawRow("Arthur", 42, photoBytesEncoded,
- "2000-01-01", "2000-01-01 00:00:00.000005", "00:00:00.000005")));
-
- // Run query and verify
- String query = "SELECT name, count, photo, anniversary_date, "
- + "anniversary_datetime, anniversary_time from table";
- JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(query);
- try (BigQueryTableRowIterator iterator =
- BigQueryTableRowIterator.fromQuery(queryConfig, "project", mockClient)) {
- iterator.open();
- assertTrue(iterator.advance());
- TableRow row = iterator.getCurrent();
-
- assertTrue(row.containsKey("name"));
- assertTrue(row.containsKey("answer"));
- assertTrue(row.containsKey("photo"));
- assertTrue(row.containsKey("anniversary_date"));
- assertTrue(row.containsKey("anniversary_datetime"));
- assertTrue(row.containsKey("anniversary_time"));
- assertEquals("Arthur", row.get("name"));
- assertEquals(42, row.get("answer"));
- assertEquals(photoBytesEncoded, row.get("photo"));
- assertEquals("2000-01-01", row.get("anniversary_date"));
- assertEquals("2000-01-01 00:00:00.000005", row.get("anniversary_datetime"));
- assertEquals("00:00:00.000005", row.get("anniversary_time"));
-
- assertFalse(iterator.advance());
- }
-
- // Temp dataset created and later deleted.
- verify(mockClient, times(2)).datasets();
- verify(mockDatasets).insert(anyString(), any(Dataset.class));
- verify(mockDatasetsInsert).execute();
- verify(mockDatasets).delete(anyString(), anyString());
- verify(mockDatasetsDelete).execute();
- // Job inserted to run the query, polled once.
- verify(mockClient, times(3)).jobs();
- verify(mockJobs, times(2)).insert(anyString(), any(Job.class));
- verify(mockJobsInsert, times(2)).execute();
- verify(mockJobs).get(anyString(), anyString());
- verify(mockJobsGet).execute();
- // Temp table get after query finish, deleted after reading.
- verify(mockClient, times(3)).tables();
- verify(mockTables, times(2)).get(anyString(), anyString(), anyString());
- verify(mockTablesGet, times(2)).execute();
- verify(mockTables).delete(anyString(), anyString(), anyString());
- verify(mockTablesDelete).execute();
- // Table data read.
- verify(mockClient).tabledata();
- verify(mockTabledata).list("project", "tempdataset", "temptable");
- verify(mockTabledataList).execute();
- }
-
- /**
- * Verifies that queries that reference no data can be read.
- */
- @Test
- public void testReadFromQueryNoTables() throws IOException, InterruptedException {
- // Mock job inserting.
- Job dryRunJob = new Job().setStatistics(
- new JobStatistics().setQuery(new JobStatistics2()));
- Job insertedJob = new Job().setJobReference(new JobReference());
- when(mockJobsInsert.execute()).thenReturn(dryRunJob, insertedJob);
-
- // Mock job polling.
- JobStatus status = new JobStatus().setState("DONE");
- JobConfigurationQuery resultQueryConfig = new JobConfigurationQuery()
- .setDestinationTable(new TableReference()
- .setProjectId("project")
- .setDatasetId("tempdataset")
- .setTableId("temptable"));
- Job getJob =
- new Job()
- .setJobReference(new JobReference())
- .setStatus(status)
- .setConfiguration(new JobConfiguration().setQuery(resultQueryConfig));
- when(mockJobsGet.execute()).thenReturn(getJob);
-
- // Mock table schema fetch.
- when(mockTablesGet.execute()).thenReturn(noTableQuerySchema());
-
- byte[] photoBytes = "photograph".getBytes();
- String photoBytesEncoded = BaseEncoding.base64().encode(photoBytes);
- // Mock table data fetch.
- when(mockTabledataList.execute()).thenReturn(
- rawDataList(rawRow("Arthur", 42, photoBytesEncoded)));
-
- // Run query and verify
- String query = String.format(
- "SELECT \"Arthur\" as name, 42 as count, \"%s\" as photo",
- photoBytesEncoded);
- JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(query);
- try (BigQueryTableRowIterator iterator =
- BigQueryTableRowIterator.fromQuery(queryConfig, "project", mockClient)) {
- iterator.open();
- assertTrue(iterator.advance());
- TableRow row = iterator.getCurrent();
-
- assertTrue(row.containsKey("name"));
- assertTrue(row.containsKey("count"));
- assertTrue(row.containsKey("photo"));
- assertEquals("Arthur", row.get("name"));
- assertEquals(42, row.get("count"));
- assertEquals(photoBytesEncoded, row.get("photo"));
-
- assertFalse(iterator.advance());
- }
-
- // Temp dataset created and later deleted.
- verify(mockClient, times(2)).datasets();
- verify(mockDatasets).insert(anyString(), any(Dataset.class));
- verify(mockDatasetsInsert).execute();
- verify(mockDatasets).delete(anyString(), anyString());
- verify(mockDatasetsDelete).execute();
- // Job inserted to run the query, polled once.
- verify(mockClient, times(3)).jobs();
- verify(mockJobs, times(2)).insert(anyString(), any(Job.class));
- verify(mockJobsInsert, times(2)).execute();
- verify(mockJobs).get(anyString(), anyString());
- verify(mockJobsGet).execute();
- // Temp table get after query finish, deleted after reading.
- verify(mockClient, times(2)).tables();
- verify(mockTables, times(1)).get(anyString(), anyString(), anyString());
- verify(mockTablesGet, times(1)).execute();
- verify(mockTables).delete(anyString(), anyString(), anyString());
- verify(mockTablesDelete).execute();
- // Table data read.
- verify(mockClient).tabledata();
- verify(mockTabledata).list("project", "tempdataset", "temptable");
- verify(mockTabledataList).execute();
- }
-
- /**
- * Verifies that when the query fails, the user gets a useful exception and the temporary dataset
- * is cleaned up. Also verifies that the temporary table (which is never created) is not
- * erroneously attempted to be deleted.
- */
- @Test
- public void testQueryFailed() throws IOException {
- // Job state polled with an error.
- String errorReason = "bad query";
- Exception exception = new IOException(errorReason);
- when(mockJobsInsert.execute()).thenThrow(exception, exception, exception, exception);
-
- JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery("NOT A QUERY");
- try (BigQueryTableRowIterator iterator =
- BigQueryTableRowIterator.fromQuery(queryConfig, "project", mockClient)) {
- iterator.open();
- fail();
- } catch (Exception expected) {
- // Verify message explains cause and reports the query.
- assertThat(expected.getMessage(), containsString("Error"));
- assertThat(expected.getMessage(), containsString("NOT A QUERY"));
- assertThat(expected.getCause().getMessage(), containsString(errorReason));
- }
-
- // Job inserted to run the query, then polled once.
- verify(mockClient, times(1)).jobs();
- verify(mockJobs).insert(anyString(), any(Job.class));
- verify(mockJobsInsert, times(4)).execute();
- }
-}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
index fa84119..ea19166 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
@@ -19,9 +19,7 @@
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@ -40,7 +38,6 @@
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
-import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -49,9 +46,10 @@
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl.DatasetServiceImpl;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.hamcrest.Matchers;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -157,14 +155,6 @@
.thenReturn(result);
}
- private void verifyTabledataList() throws IOException {
- verify(mockClient, atLeastOnce()).tabledata();
- verify(mockTabledata, atLeastOnce()).list("project", "dataset", "table");
- verify(mockTabledataList, atLeastOnce()).execute();
- // Max results may be set when testing for an empty table.
- verify(mockTabledataList, atLeast(0)).setMaxResults(anyLong());
- }
-
private Table basicTableSchema() {
return new Table()
.setSchema(new TableSchema()
@@ -178,68 +168,6 @@
)));
}
- private Table basicTableSchemaWithTime() {
- return new Table()
- .setSchema(new TableSchema()
- .setFields(Arrays.asList(
- new TableFieldSchema()
- .setName("time")
- .setType("TIMESTAMP")
- )));
- }
-
- @Test
- public void testReadWithTime() throws IOException, InterruptedException {
- // The BigQuery JSON API returns timestamps in the following format: floating-point seconds
- // since epoch (UTC) with microsecond precision. Test that we faithfully preserve a set of
- // known values.
- TableDataList input = rawDataList(
- rawRow("1.430397296789E9"),
- rawRow("1.45206228E9"),
- rawRow("1.452062291E9"),
- rawRow("1.4520622911E9"),
- rawRow("1.45206229112E9"),
- rawRow("1.452062291123E9"),
- rawRow("1.4520622911234E9"),
- rawRow("1.45206229112345E9"),
- rawRow("1.452062291123456E9"));
- onTableGet(basicTableSchemaWithTime());
- onTableList(input);
-
- // Known results verified from BigQuery's export to JSON on GCS API.
- List<String> expected = ImmutableList.of(
- "2015-04-30 12:34:56.789 UTC",
- "2016-01-06 06:38:00 UTC",
- "2016-01-06 06:38:11 UTC",
- "2016-01-06 06:38:11.1 UTC",
- "2016-01-06 06:38:11.12 UTC",
- "2016-01-06 06:38:11.123 UTC",
- "2016-01-06 06:38:11.1234 UTC",
- "2016-01-06 06:38:11.12345 UTC",
- "2016-01-06 06:38:11.123456 UTC");
-
- // Download the rows, verify the interactions.
- List<TableRow> rows = new ArrayList<>();
- try (BigQueryTableRowIterator iterator =
- BigQueryTableRowIterator.fromTable(
- BigQueryHelpers.parseTableSpec("project:dataset.table"), mockClient)) {
- iterator.open();
- while (iterator.advance()) {
- rows.add(iterator.getCurrent());
- }
- }
- verifyTableGet();
- verifyTabledataList();
-
- // Verify the timestamp converted as desired.
- assertEquals("Expected input and output rows to have the same size",
- expected.size(), rows.size());
- for (int i = 0; i < expected.size(); ++i) {
- assertEquals("i=" + i, expected.get(i), rows.get(i).get("time"));
- }
-
- }
-
private TableRow rawRow(Object...args) {
List<TableCell> cells = new LinkedList<>();
for (Object a : args) {
@@ -248,118 +176,6 @@
return new TableRow().setF(cells);
}
- private TableDataList rawDataList(TableRow...rows) {
- return new TableDataList()
- .setRows(Arrays.asList(rows));
- }
-
- @Test
- public void testRead() throws IOException, InterruptedException {
- onTableGet(basicTableSchema());
-
- TableDataList dataList = rawDataList(rawRow("Arthur", 42));
- onTableList(dataList);
-
- try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryHelpers.parseTableSpec("project:dataset.table"),
- mockClient)) {
- iterator.open();
- Assert.assertTrue(iterator.advance());
- TableRow row = iterator.getCurrent();
-
- Assert.assertTrue(row.containsKey("name"));
- Assert.assertTrue(row.containsKey("answer"));
- Assert.assertEquals("Arthur", row.get("name"));
- Assert.assertEquals(42, row.get("answer"));
-
- Assert.assertFalse(iterator.advance());
-
- verifyTableGet();
- verifyTabledataList();
- }
- }
-
- @Test
- public void testReadEmpty() throws IOException, InterruptedException {
- onTableGet(basicTableSchema());
-
- // BigQuery may respond with a page token for an empty table, ensure we
- // handle it.
- TableDataList dataList = new TableDataList()
- .setPageToken("FEED==")
- .setTotalRows(0L);
- onTableList(dataList);
-
- try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryHelpers.parseTableSpec("project:dataset.table"),
- mockClient)) {
- iterator.open();
-
- Assert.assertFalse(iterator.advance());
-
- verifyTableGet();
- verifyTabledataList();
- }
- }
-
- @Test
- public void testReadMultiPage() throws IOException, InterruptedException {
- onTableGet(basicTableSchema());
-
- TableDataList page1 = rawDataList(rawRow("Row1", 1))
- .setPageToken("page2");
- TableDataList page2 = rawDataList(rawRow("Row2", 2))
- .setTotalRows(2L);
-
- when(mockClient.tabledata())
- .thenReturn(mockTabledata);
- when(mockTabledata.list(anyString(), anyString(), anyString()))
- .thenReturn(mockTabledataList);
- when(mockTabledataList.execute())
- .thenReturn(page1)
- .thenReturn(page2);
-
- try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryHelpers.parseTableSpec("project:dataset.table"),
- mockClient)) {
- iterator.open();
-
- List<String> names = new LinkedList<>();
- while (iterator.advance()) {
- names.add((String) iterator.getCurrent().get("name"));
- }
-
- Assert.assertThat(names, Matchers.hasItems("Row1", "Row2"));
-
- verifyTableGet();
- verifyTabledataList();
- // The second call should have used a page token.
- verify(mockTabledataList).setPageToken("page2");
- }
- }
-
- @Test
- public void testReadOpenFailure() throws IOException, InterruptedException {
- thrown.expect(IOException.class);
-
- when(mockClient.tables())
- .thenReturn(mockTables);
- when(mockTables.get(anyString(), anyString(), anyString()))
- .thenReturn(mockTablesGet);
- when(mockTablesGet.execute())
- .thenThrow(new IOException("No such table"));
-
- try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryHelpers.parseTableSpec("project:dataset.table"),
- mockClient)) {
- try {
- iterator.open(); // throws.
- } finally {
- verifyTableGet();
- }
- }
- }
-
@Test
public void testTableGet() throws InterruptedException, IOException {
onTableGet(basicTableSchema());
@@ -391,16 +207,18 @@
.parseTableSpec("project:dataset.table");
DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, options, 5);
- List<TableRow> rows = new ArrayList<>();
+ List<ValueInSingleWindow<TableRow>> rows = new ArrayList<>();
List<String> ids = new ArrayList<>();
for (int i = 0; i < 25; ++i) {
- rows.add(rawRow("foo", 1234));
+ rows.add(ValueInSingleWindow.of(rawRow("foo", 1234), GlobalWindow.TIMESTAMP_MAX_VALUE,
+ GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING));
ids.add(new String());
}
long totalBytes = 0;
try {
- totalBytes = datasetService.insertAll(ref, rows, ids);
+ totalBytes = datasetService.insertAll(ref, rows, ids, InsertRetryPolicy.alwaysRetry(),
+ null);
} finally {
verifyInsertAll(5);
// Each of the 25 rows is 23 bytes: "{f=[{v=foo}, {v=1234}]}"
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
index 18ff688..7506cde 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
@@ -17,18 +17,12 @@
*/
package org.apache.beam.sdk.io.gcp.bigquery;
-import static org.junit.Assert.assertEquals;
-
import com.google.api.client.util.Base64;
-import com.google.api.services.bigquery.model.JobConfigurationQuery;
-import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
-import com.google.common.collect.Lists;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
-import java.util.NoSuchElementException;
import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.ListCoder;
@@ -60,28 +54,6 @@
return datasetService;
}
- @Override
- public BigQueryJsonReader getReaderFromTable(BigQueryOptions bqOptions, TableReference tableRef) {
- try {
- List<TableRow> rows = datasetService.getAllRows(
- tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId());
- return new FakeBigQueryReader(rows);
- } catch (Exception e) {
- return null;
- }
- }
-
- @Override
- public BigQueryJsonReader getReaderFromQuery(
- BigQueryOptions bqOptions, String projectId, JobConfigurationQuery queryConfig) {
- try {
- List<TableRow> rows = rowsFromEncodedQuery(queryConfig.getQuery());
- return new FakeBigQueryReader(rows);
- } catch (IOException e) {
- return null;
- }
- }
-
static List<TableRow> rowsFromEncodedQuery(String query) throws IOException {
ListCoder<TableRow> listCoder = ListCoder.of(TableRowJsonCoder.of());
ByteArrayInputStream input = new ByteArrayInputStream(Base64.decodeBase64(query));
@@ -99,56 +71,6 @@
return Base64.encodeBase64String(output.toByteArray());
}
- private static class FakeBigQueryReader implements BigQueryJsonReader {
- private static final int UNSTARTED = -1;
- private static final int CLOSED = Integer.MAX_VALUE;
-
- private List<byte[]> serializedTableRowReturns;
- private int currIndex;
-
- FakeBigQueryReader(List<TableRow> tableRowReturns) throws IOException {
- this.serializedTableRowReturns = Lists.newArrayListWithExpectedSize(tableRowReturns.size());
- for (TableRow tableRow : tableRowReturns) {
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- TableRowJsonCoder.of().encode(tableRow, output, Context.OUTER);
- serializedTableRowReturns.add(output.toByteArray());
- }
- this.currIndex = UNSTARTED;
- }
-
- @Override
- public boolean start() throws IOException {
- assertEquals(UNSTARTED, currIndex);
- currIndex = 0;
- return currIndex < serializedTableRowReturns.size();
- }
-
- @Override
- public boolean advance() throws IOException {
- return ++currIndex < serializedTableRowReturns.size();
- }
-
- @Override
- public TableRow getCurrent() throws NoSuchElementException {
- if (currIndex >= serializedTableRowReturns.size()) {
- throw new NoSuchElementException();
- }
-
- ByteArrayInputStream input = new ByteArrayInputStream(
- serializedTableRowReturns.get(currIndex));
- try {
- return convertNumbers(TableRowJsonCoder.of().decode(input, Context.OUTER));
- } catch (IOException e) {
- return null;
- }
- }
-
- @Override
- public void close() throws IOException {
- currIndex = CLOSED;
- }
- }
-
// Longs tend to get converted back to Integers due to JSON serialization. Convert them back.
static TableRow convertNumbers(TableRow tableRow) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
index 5103adb..4c67a9c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeDatasetService.java
@@ -25,20 +25,29 @@
import com.google.api.services.bigquery.model.Dataset;
import com.google.api.services.bigquery.model.DatasetReference;
import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy.Context;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
/** A fake dataset service that can be serialized, for use in testReadFromTable. */
class FakeDatasetService implements DatasetService, Serializable {
+ Map<String, List<String>> insertErrors = Maps.newHashMap();
+
@Override
public Table getTable(TableReference tableRef)
throws InterruptedException, IOException {
@@ -103,7 +112,17 @@
@Override
public void createTable(Table table) throws IOException {
+ final Pattern tableRegexp = Pattern.compile("[-\\w]{1,1024}");
+
TableReference tableReference = table.getTableReference();
+ if (!tableRegexp.matcher(tableReference.getTableId()).matches()) {
+ throw new IOException(
+ String.format(
+ "invalid table ID %s. Table IDs must be alphanumeric "
+ + "(plus underscores) and must be at most 1024 characters long. Also, table"
+ + " decorators cannot be used.",
+ tableReference.getTableId()));
+ }
synchronized (BigQueryIOTest.tables) {
Map<String, TableContainer> dataset =
BigQueryIOTest.tables.get(tableReference.getProjectId(), tableReference.getDatasetId());
@@ -145,7 +164,11 @@
@Override
public void createDataset(
- String projectId, String datasetId, String location, String description)
+ String projectId,
+ String datasetId,
+ String location,
+ String description,
+ Long defaultTableExpirationMs /* ignored */)
throws IOException, InterruptedException {
synchronized (BigQueryIOTest.tables) {
Map<String, TableContainer> dataset = BigQueryIOTest.tables.get(projectId, datasetId);
@@ -164,10 +187,24 @@
}
}
+ public long insertAll(TableReference ref, List<TableRow> rowList,
+ @Nullable List<String> insertIdList)
+ throws IOException, InterruptedException {
+ List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList();
+ for (TableRow row : rowList) {
+ windowedRows.add(ValueInSingleWindow.of(row, GlobalWindow.TIMESTAMP_MAX_VALUE,
+ GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING));
+ }
+ return insertAll(ref, windowedRows, insertIdList, InsertRetryPolicy.alwaysRetry(), null);
+ }
+
@Override
public long insertAll(
- TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
+ TableReference ref, List<ValueInSingleWindow<TableRow>> rowList,
+ @Nullable List<String> insertIdList,
+ InsertRetryPolicy retryPolicy, List<ValueInSingleWindow<TableRow>> failedInserts)
throws IOException, InterruptedException {
+ Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> insertErrors = getInsertErrors();
synchronized (BigQueryIOTest.tables) {
if (insertIdList != null) {
assertEquals(rowList.size(), insertIdList.size());
@@ -180,9 +217,25 @@
long dataSize = 0;
TableContainer tableContainer = getTableContainer(
- ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
+ ref.getProjectId(),
+ ref.getDatasetId(),
+ BigQueryHelpers.stripPartitionDecorator(ref.getTableId()));
for (int i = 0; i < rowList.size(); ++i) {
- dataSize += tableContainer.addRow(rowList.get(i), insertIdList.get(i));
+ TableRow row = rowList.get(i).getValue();
+ List<TableDataInsertAllResponse.InsertErrors> allErrors = insertErrors.get(row);
+ boolean shouldInsert = true;
+ if (allErrors != null) {
+ for (TableDataInsertAllResponse.InsertErrors errors : allErrors) {
+ if (!retryPolicy.shouldRetry(new Context(errors))) {
+ shouldInsert = false;
+ }
+ }
+ }
+ if (shouldInsert) {
+ dataSize += tableContainer.addRow(row, insertIdList.get(i));
+ } else {
+ failedInserts.add(rowList.get(i));
+ }
}
return dataSize;
}
@@ -200,9 +253,45 @@
}
}
+ /**
+ * Cause a given {@link TableRow} object to fail when it's inserted. The errors link the list
+ * will be returned on subsequent retries, and the insert will succeed when the errors run out.
+ */
+ public void failOnInsert(
+ Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> insertErrors) {
+ synchronized (BigQueryIOTest.tables) {
+ for (Map.Entry<TableRow, List<TableDataInsertAllResponse.InsertErrors>> entry
+ : insertErrors.entrySet()) {
+ List<String> errorStrings = Lists.newArrayList();
+ for (TableDataInsertAllResponse.InsertErrors errors : entry.getValue()) {
+ errorStrings.add(BigQueryHelpers.toJsonString(errors));
+ }
+ this.insertErrors.put(BigQueryHelpers.toJsonString(entry.getKey()), errorStrings);
+ }
+ }
+ }
+
+ Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> getInsertErrors() {
+ Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> parsedInsertErrors =
+ Maps.newHashMap();
+ synchronized (BigQueryIOTest.tables) {
+ for (Map.Entry<String, List<String>> entry : this.insertErrors.entrySet()) {
+ TableRow tableRow = BigQueryHelpers.fromJsonString(entry.getKey(), TableRow.class);
+ List<TableDataInsertAllResponse.InsertErrors> allErrors = Lists.newArrayList();
+ for (String errorsString : entry.getValue()) {
+ allErrors.add(BigQueryHelpers.fromJsonString(
+ errorsString, TableDataInsertAllResponse.InsertErrors.class));
+ }
+ parsedInsertErrors.put(tableRow, allErrors);
+ }
+ }
+ return parsedInsertErrors;
+ }
+
void throwNotFound(String format, Object... args) throws IOException {
throw new IOException(
- new GoogleJsonResponseException.Builder(404,
- String.format(format, args), new HttpHeaders()).build());
+ String.format(format, args),
+ new GoogleJsonResponseException.Builder(404, String.format(format, args), new HttpHeaders())
+ .build());
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
index 2045bb7..f13a7ab 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
@@ -19,6 +19,7 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.util.BackOff;
@@ -39,6 +40,7 @@
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.api.services.bigquery.model.TimePartitioning;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -61,7 +63,6 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.io.fs.MoveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
@@ -109,6 +110,7 @@
public void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig)
throws InterruptedException, IOException {
synchronized (allJobs) {
+ verifyUniqueJobId(jobRef.getJobId());
Job job = new Job();
job.setJobReference(jobRef);
job.setConfiguration(new JobConfiguration().setLoad(loadConfig));
@@ -126,8 +128,7 @@
filename + ThreadLocalRandom.current().nextInt(), false /* isDirectory */));
}
- FileSystems.copy(sourceFiles.build(), loadFiles.build(),
- MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
+ FileSystems.copy(sourceFiles.build(), loadFiles.build());
filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles.build());
}
@@ -141,6 +142,7 @@
checkArgument(extractConfig.getDestinationFormat().equals("AVRO"),
"Only extract to AVRO is supported");
synchronized (allJobs) {
+ verifyUniqueJobId(jobRef.getJobId());
++numExtractJobCalls;
Job job = new Job();
@@ -175,6 +177,7 @@
public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig)
throws IOException, InterruptedException {
synchronized (allJobs) {
+ verifyUniqueJobId(jobRef.getJobId());
Job job = new Job();
job.setJobReference(jobRef);
job.setConfiguration(new JobConfiguration().setCopy(copyConfig));
@@ -257,6 +260,12 @@
}
}
+ private void verifyUniqueJobId(String jobId) throws IOException {
+ if (allJobs.containsColumn(jobId)) {
+ throw new IOException("Duplicate job id " + jobId);
+ }
+ }
+
private JobStatus runJob(Job job) throws InterruptedException, IOException {
if (job.getConfiguration().getLoad() != null) {
return runLoadJob(job.getJobReference(), job.getConfiguration().getLoad());
@@ -301,14 +310,20 @@
if (!validateDispositions(existingTable, createDisposition, writeDisposition)) {
return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
}
-
- datasetService.createTable(new Table().setTableReference(destination).setSchema(schema));
+ if (existingTable == null) {
+ existingTable = new Table().setTableReference(destination).setSchema(schema);
+ if (load.getTimePartitioning() != null) {
+ existingTable = existingTable.setTimePartitioning(load.getTimePartitioning());
+ }
+ datasetService.createTable(existingTable);
+ }
List<TableRow> rows = Lists.newArrayList();
for (ResourceId filename : sourceFiles) {
rows.addAll(readRows(filename.toString()));
}
datasetService.insertAll(destination, rows, null);
+ FileSystems.delete(sourceFiles);
return new JobStatus().setState("DONE");
}
@@ -322,13 +337,30 @@
if (!validateDispositions(existingTable, createDisposition, writeDisposition)) {
return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
}
-
+ TimePartitioning partitioning = null;
+ TableSchema schema = null;
+ boolean first = true;
List<TableRow> allRows = Lists.newArrayList();
for (TableReference source : sources) {
+ Table table = checkNotNull(datasetService.getTable(source));
+ if (!first) {
+ if (partitioning != table.getTimePartitioning()) {
+ return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
+ }
+ if (schema != table.getSchema()) {
+ return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
+ }
+ }
+ partitioning = table.getTimePartitioning();
+ schema = table.getSchema();
+ first = false;
allRows.addAll(datasetService.getAllRows(
source.getProjectId(), source.getDatasetId(), source.getTableId()));
}
- datasetService.createTable(new Table().setTableReference(destination));
+ datasetService.createTable(new Table()
+ .setTableReference(destination)
+ .setSchema(schema)
+ .setTimePartitioning(partitioning));
datasetService.insertAll(destination, allRows, null);
return new JobStatus().setState("DONE");
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java
new file mode 100644
index 0000000..b19835d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.api.services.bigquery.model.ErrorProto;
+import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
+import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy.Context;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link InsertRetryPolicy}.
+ */
+@RunWith(JUnit4.class)
+public class InsertRetryPolicyTest {
+ @Test
+ public void testNeverRetry() {
+ assertFalse(InsertRetryPolicy.neverRetry().shouldRetry(
+ new Context(new TableDataInsertAllResponse.InsertErrors())));
+ }
+
+ @Test
+ public void testAlwaysRetry() {
+ assertTrue(InsertRetryPolicy.alwaysRetry().shouldRetry(
+ new Context(new TableDataInsertAllResponse.InsertErrors())));
+ }
+
+ @Test
+ public void testDontRetryPersistentErrors() {
+ InsertRetryPolicy policy = InsertRetryPolicy.retryTransientErrors();
+ assertTrue(policy.shouldRetry(new Context(generateErrorAmongMany(
+ 5, "timeout", "unavailable"))));
+ assertFalse(policy.shouldRetry(new Context(generateErrorAmongMany(
+ 5, "timeout", "invalid"))));
+ assertFalse(policy.shouldRetry(new Context(generateErrorAmongMany(
+ 5, "timeout", "invalidQuery"))));
+ assertFalse(policy.shouldRetry(new Context(generateErrorAmongMany(
+ 5, "timeout", "notImplemented"))));
+ }
+
+ private TableDataInsertAllResponse.InsertErrors generateErrorAmongMany(
+ int numErrors, String baseReason, String exceptionalReason) {
+ // The retry policies are expected to search through the entire list of ErrorProtos to determine
+ // whether to retry. Stick the exceptionalReason in a random position to exercise this.
+ List<ErrorProto> errorProtos = Lists.newArrayListWithExpectedSize(numErrors);
+ int exceptionalPosition = ThreadLocalRandom.current().nextInt(numErrors);
+ for (int i = 0; i < numErrors; ++i) {
+ ErrorProto error = new ErrorProto();
+ error.setReason((i == exceptionalPosition) ? exceptionalReason : baseReason);
+ errorProtos.add(error);
+ }
+ TableDataInsertAllResponse.InsertErrors errors = new TableDataInsertAllResponse.InsertErrors();
+ errors.setErrors(errorProtos);
+ return errors;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
index 8915069..e016c98 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
@@ -32,6 +32,7 @@
Long sizeBytes;
TableContainer(Table table) {
this.table = table;
+
this.rows = new ArrayList<>();
this.ids = new ArrayList<>();
this.sizeBytes = 0L;
@@ -54,6 +55,7 @@
return table;
}
+
List<TableRow> getRows() {
return rows;
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
index 0b94ded..af3354b 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
@@ -194,7 +194,7 @@
thrown.expect(IllegalArgumentException.class);
- write.validate(null);
+ write.expand(null);
}
@Test
@@ -203,7 +203,7 @@
thrown.expect(IllegalArgumentException.class);
- write.validate(null);
+ write.expand(null);
}
/** Helper function to make a single row mutation to be written. */
@@ -589,6 +589,32 @@
+ "display data", displayData, Matchers.hasItem(hasDisplayItem("rowFilter")));
}
+ @Test
+ public void testReadWithoutValidate() {
+ final String table = "fooTable";
+ BigtableIO.Read read = BigtableIO.read()
+ .withBigtableOptions(BIGTABLE_OPTIONS)
+ .withTableId(table)
+ .withBigtableService(service)
+ .withoutValidation();
+
+ // validate() will throw if withoutValidation() isn't working
+ read.validate(TestPipeline.testingPipelineOptions());
+ }
+
+ @Test
+ public void testWriteWithoutValidate() {
+ final String table = "fooTable";
+ BigtableIO.Write write = BigtableIO.write()
+ .withBigtableOptions(BIGTABLE_OPTIONS)
+ .withTableId(table)
+ .withBigtableService(service)
+ .withoutValidation();
+
+ // validate() will throw if withoutValidation() isn't working
+ write.validate(TestPipeline.testingPipelineOptions());
+ }
+
/** Tests that a record gets written to the service and messages are logged. */
@Test
public void testWriting() throws Exception {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java
index a064bd6..91f0bae 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java
@@ -20,6 +20,7 @@
import com.google.bigtable.v2.Row;
import com.google.cloud.bigtable.config.BigtableOptions;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -41,8 +42,10 @@
BigtableTestOptions options = TestPipeline.testingPipelineOptions()
.as(BigtableTestOptions.class);
+ String project = options.as(GcpOptions.class).getProject();
+
BigtableOptions.Builder bigtableOptionsBuilder = new BigtableOptions.Builder()
- .setProjectId(options.getProjectId())
+ .setProjectId(project)
.setInstanceId(options.getInstanceId());
final String tableId = "BigtableReadTest";
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java
index 0ab7576..03cb697 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java
@@ -25,11 +25,6 @@
* Properties needed when using Bigtable with the Beam SDK.
*/
public interface BigtableTestOptions extends TestPipelineOptions {
- @Description("Project ID for Bigtable")
- @Default.String("apache-beam-testing")
- String getProjectId();
- void setProjectId(String value);
-
@Description("Instance ID for Bigtable")
@Default.String("beam-test")
String getInstanceId();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java
index 1d168f1..010bcc4 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java
@@ -73,15 +73,17 @@
private static BigtableTableAdminClient tableAdminClient;
private final String tableId =
String.format("BigtableWriteIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date());
+ private String project;
@Before
public void setup() throws Exception {
PipelineOptionsFactory.register(BigtableTestOptions.class);
options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class);
+ project = options.as(GcpOptions.class).getProject();
bigtableOptions =
new Builder()
- .setProjectId(options.getProjectId())
+ .setProjectId(project)
.setInstanceId(options.getInstanceId())
.setUserAgent("apache-beam-test")
.build();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottlerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottlerTest.java
new file mode 100644
index 0000000..c12cf55
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottlerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.datastore;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+
+import java.util.Random;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+/**
+ * Tests for {@link AdaptiveThrottler}.
+ */
+@RunWith(JUnit4.class)
+public class AdaptiveThrottlerTest {
+
+ static final long START_TIME_MS = 0;
+ static final long SAMPLE_PERIOD_MS = 60000;
+ static final long SAMPLE_BUCKET_MS = 1000;
+ static final double OVERLOAD_RATIO = 2;
+
+ /** Returns a throttler configured with the standard parameters above. */
+ AdaptiveThrottler getThrottler() {
+ return new AdaptiveThrottler(SAMPLE_PERIOD_MS, SAMPLE_BUCKET_MS, OVERLOAD_RATIO);
+ }
+
+ @Test
+ public void testNoInitialThrottling() throws Exception {
+ AdaptiveThrottler throttler = getThrottler();
+ assertThat(throttler.throttlingProbability(START_TIME_MS), equalTo(0.0));
+ assertThat("first request is not throttled",
+ throttler.throttleRequest(START_TIME_MS), equalTo(false));
+ }
+
+ @Test
+ public void testNoThrottlingIfNoErrors() throws Exception {
+ AdaptiveThrottler throttler = getThrottler();
+ long t = START_TIME_MS;
+ for (; t < START_TIME_MS + 20; t++) {
+ assertFalse(throttler.throttleRequest(t));
+ throttler.successfulRequest(t);
+ }
+ assertThat(throttler.throttlingProbability(t), equalTo(0.0));
+ }
+
+ @Test
+ public void testNoThrottlingAfterErrorsExpire() throws Exception {
+ AdaptiveThrottler throttler = getThrottler();
+ long t = START_TIME_MS;
+ for (; t < START_TIME_MS + SAMPLE_PERIOD_MS; t++) {
+ throttler.throttleRequest(t);
+ // and no successfulRequest.
+ }
+ assertThat("check that we set up a non-zero probability of throttling",
+ throttler.throttlingProbability(t), greaterThan(0.0));
+ for (; t < START_TIME_MS + 2 * SAMPLE_PERIOD_MS; t++) {
+ throttler.throttleRequest(t);
+ throttler.successfulRequest(t);
+ }
+ assertThat(throttler.throttlingProbability(t), equalTo(0.0));
+ }
+
+ @Test
+ public void testThrottlingAfterErrors() throws Exception {
+ Random mockRandom = Mockito.mock(Random.class);
+ Mockito.when(mockRandom.nextDouble()).thenReturn(
+ 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,
+ 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9);
+ AdaptiveThrottler throttler = new AdaptiveThrottler(
+ SAMPLE_PERIOD_MS, SAMPLE_BUCKET_MS, OVERLOAD_RATIO, mockRandom);
+ for (int i = 0; i < 20; i++) {
+ boolean throttled = throttler.throttleRequest(START_TIME_MS + i);
+ // 1/3rd of requests succeeding.
+ if (i % 3 == 1) {
+ throttler.successfulRequest(START_TIME_MS + i);
+ }
+
+ // Once we have some history in place, check what throttling happens.
+ if (i >= 10) {
+ // Expect 1/3rd of requests to be throttled. (So 1/3rd throttled, 1/3rd succeeding, 1/3rd
+ // tried and failing).
+ assertThat(String.format("for i=%d", i),
+ throttler.throttlingProbability(START_TIME_MS + i), closeTo(0.33, /*error=*/ 0.1));
+ // Requests 10..13 should be throttled, 14..19 not throttled given the mocked random numbers
+ // that we fed to throttler.
+ assertThat(String.format("for i=%d", i), throttled, equalTo(i < 14));
+ }
+ }
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
index ba8ac84..550b6b9 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
@@ -26,7 +26,8 @@
import static com.google.datastore.v1.client.DatastoreHelper.makeOrder;
import static com.google.datastore.v1.client.DatastoreHelper.makeUpsert;
import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
-import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DATASTORE_BATCH_UPDATE_LIMIT;
+import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DATASTORE_BATCH_UPDATE_BYTES_LIMIT;
+import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_START;
import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.DEFAULT_BUNDLE_SIZE_BYTES;
import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.QUERY_BATCH_LIMIT;
import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.getEstimatedSizeBytes;
@@ -39,6 +40,7 @@
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
@@ -50,6 +52,7 @@
import static org.mockito.Mockito.when;
import com.google.datastore.v1.CommitRequest;
+import com.google.datastore.v1.CommitResponse;
import com.google.datastore.v1.Entity;
import com.google.datastore.v1.EntityResult;
import com.google.datastore.v1.GqlQuery;
@@ -186,22 +189,6 @@
}
@Test
- public void testReadValidationFailsProject() throws Exception {
- DatastoreV1.Read read = DatastoreIO.v1().read().withQuery(QUERY);
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- read.validate(null);
- }
-
- @Test
- public void testReadValidationFailsQuery() throws Exception {
- DatastoreV1.Read read = DatastoreIO.v1().read().withProjectId(PROJECT_ID);
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("Either query or gql query ValueProvider should be provided");
- read.validate(null);
- }
-
- @Test
public void testReadValidationFailsQueryAndGqlQuery() throws Exception {
DatastoreV1.Read read = DatastoreIO.v1().read()
.withProjectId(PROJECT_ID)
@@ -210,8 +197,8 @@
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(
- "Only one of query or gql query ValueProvider should be provided");
- read.validate(null);
+ "withQuery() and withLiteralGqlQuery() are exclusive");
+ read.expand(null);
}
@Test
@@ -233,13 +220,6 @@
}
@Test
- public void testReadValidationSucceedsNamespace() throws Exception {
- DatastoreV1.Read read = DatastoreIO.v1().read().withProjectId(PROJECT_ID).withQuery(QUERY);
- /* Should succeed, as a null namespace is fine. */
- read.validate(null);
- }
-
- @Test
public void testReadDisplayData() {
DatastoreV1.Read read = DatastoreIO.v1().read()
.withProjectId(PROJECT_ID)
@@ -286,42 +266,6 @@
}
@Test
- public void testWriteDoesNotAllowNullProject() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- DatastoreIO.v1().write().withProjectId((String) null);
- }
-
- @Test
- public void testWriteDoesNotAllowNullProjectValueProvider() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- DatastoreIO.v1().write().withProjectId((ValueProvider<String>) null);
- }
-
- @Test
- public void testWriteValidationFailsWithNoProject() throws Exception {
- Write write = DatastoreIO.v1().write();
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- write.validate(null);
- }
-
- @Test
- public void testWriteValidationFailsWithNoProjectInStaticValueProvider() throws Exception {
- Write write = DatastoreIO.v1().write().withProjectId(StaticValueProvider.<String>of(null));
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- write.validate(null);
- }
-
- @Test
- public void testWriteValidationSucceedsWithProject() throws Exception {
- Write write = DatastoreIO.v1().write().withProjectId(PROJECT_ID);
- write.validate(null);
- }
-
- @Test
public void testWriteDisplayData() {
Write write = DatastoreIO.v1().write().withProjectId(PROJECT_ID);
@@ -331,43 +275,6 @@
}
@Test
- public void testDeleteEntityDoesNotAllowNullProject() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- DatastoreIO.v1().deleteEntity().withProjectId((String) null);
- }
-
- @Test
- public void testDeleteEntityDoesNotAllowNullProjectValueProvider() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- DatastoreIO.v1().deleteEntity().withProjectId((ValueProvider<String>) null);
- }
-
- @Test
- public void testDeleteEntityValidationFailsWithNoProject() throws Exception {
- DeleteEntity deleteEntity = DatastoreIO.v1().deleteEntity();
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- deleteEntity.validate(null);
- }
-
- @Test
- public void testDeleteEntityValidationFailsWithNoProjectInStaticValueProvider() throws Exception {
- DeleteEntity deleteEntity = DatastoreIO.v1().deleteEntity()
- .withProjectId(StaticValueProvider.<String>of(null));
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- deleteEntity.validate(null);
- }
-
- @Test
- public void testDeleteEntityValidationSucceedsWithProject() throws Exception {
- DeleteEntity deleteEntity = DatastoreIO.v1().deleteEntity().withProjectId(PROJECT_ID);
- deleteEntity.validate(null);
- }
-
- @Test
public void testDeleteEntityDisplayData() {
DeleteEntity deleteEntity = DatastoreIO.v1().deleteEntity().withProjectId(PROJECT_ID);
@@ -377,43 +284,6 @@
}
@Test
- public void testDeleteKeyDoesNotAllowNullProject() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- DatastoreIO.v1().deleteKey().withProjectId((String) null);
- }
-
- @Test
- public void testDeleteKeyDoesNotAllowNullProjectValueProvider() throws Exception {
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- DatastoreIO.v1().deleteKey().withProjectId((ValueProvider<String>) null);
- }
-
- @Test
- public void testDeleteKeyValidationFailsWithNoProject() throws Exception {
- DeleteKey deleteKey = DatastoreIO.v1().deleteKey();
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId ValueProvider");
- deleteKey.validate(null);
- }
-
- @Test
- public void testDeleteKeyValidationFailsWithNoProjectInStaticValueProvider() throws Exception {
- DeleteKey deleteKey = DatastoreIO.v1().deleteKey().withProjectId(
- StaticValueProvider.<String>of(null));
- thrown.expect(NullPointerException.class);
- thrown.expectMessage("projectId");
- deleteKey.validate(null);
- }
-
- @Test
- public void testDeleteKeyValidationSucceedsWithProject() throws Exception {
- DeleteKey deleteKey = DatastoreIO.v1().deleteKey().withProjectId(PROJECT_ID);
- deleteKey.validate(null);
- }
-
- @Test
public void testDeleteKeyDisplayData() {
DeleteKey deleteKey = DatastoreIO.v1().deleteKey().withProjectId(PROJECT_ID);
@@ -605,7 +475,7 @@
/** Tests {@link DatastoreWriterFn} with entities of more than one batches, but not a multiple. */
@Test
public void testDatatoreWriterFnWithMultipleBatches() throws Exception {
- datastoreWriterFnTest(DATASTORE_BATCH_UPDATE_LIMIT * 3 + 100);
+ datastoreWriterFnTest(DATASTORE_BATCH_UPDATE_ENTITIES_START * 3 + 100);
}
/**
@@ -614,7 +484,7 @@
*/
@Test
public void testDatatoreWriterFnWithBatchesExactMultiple() throws Exception {
- datastoreWriterFnTest(DATASTORE_BATCH_UPDATE_LIMIT * 2);
+ datastoreWriterFnTest(DATASTORE_BATCH_UPDATE_ENTITIES_START * 2);
}
// A helper method to test DatastoreWriterFn for various batch sizes.
@@ -627,14 +497,14 @@
}
DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID),
- null, mockDatastoreFactory);
+ null, mockDatastoreFactory, new FakeWriteBatcher());
DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
doFnTester.processBundle(mutations);
int start = 0;
while (start < numMutations) {
- int end = Math.min(numMutations, start + DATASTORE_BATCH_UPDATE_LIMIT);
+ int end = Math.min(numMutations, start + DATASTORE_BATCH_UPDATE_ENTITIES_START);
CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
commitRequest.addAllMutations(mutations.subList(start, end));
@@ -645,6 +515,66 @@
}
/**
+ * Tests {@link DatastoreWriterFn} with large entities that need to be split into more batches.
+ */
+ @Test
+ public void testDatatoreWriterFnWithLargeEntities() throws Exception {
+ List<Mutation> mutations = new ArrayList<>();
+ int entitySize = 0;
+ for (int i = 0; i < 12; ++i) {
+ Entity entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1))
+ .putProperties("long", makeValue(new String(new char[900_000])
+ ).setExcludeFromIndexes(true).build())
+ .build();
+ entitySize = entity.getSerializedSize(); // Take the size of any one entity.
+ mutations.add(makeUpsert(entity).build());
+ }
+
+ DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID),
+ null, mockDatastoreFactory, new FakeWriteBatcher());
+ DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
+ doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
+ doFnTester.processBundle(mutations);
+
+ // This test is over-specific currently; it requires that we split the 12 entity writes into 3
+ // requests, but we only need each CommitRequest to be less than 10MB in size.
+ int entitiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / entitySize;
+ int start = 0;
+ while (start < mutations.size()) {
+ int end = Math.min(mutations.size(), start + entitiesPerRpc);
+ CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+ commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
+ commitRequest.addAllMutations(mutations.subList(start, end));
+ // Verify all the batch requests were made with the expected mutations.
+ verify(mockDatastore).commit(commitRequest.build());
+ start = end;
+ }
+ }
+
+ /** Tests {@link DatastoreWriterFn} with a failed request which is retried. */
+ @Test
+ public void testDatatoreWriterFnRetriesErrors() throws Exception {
+ List<Mutation> mutations = new ArrayList<>();
+ int numRpcs = 2;
+ for (int i = 0; i < DATASTORE_BATCH_UPDATE_ENTITIES_START * numRpcs; ++i) {
+ mutations.add(
+ makeUpsert(Entity.newBuilder().setKey(makeKey("key" + i, i + 1)).build()).build());
+ }
+
+ CommitResponse successfulCommit = CommitResponse.getDefaultInstance();
+ when(mockDatastore.commit(any(CommitRequest.class))).thenReturn(successfulCommit)
+ .thenThrow(
+ new DatastoreException("commit", Code.DEADLINE_EXCEEDED, "", null))
+ .thenReturn(successfulCommit);
+
+ DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID),
+ null, mockDatastoreFactory, new FakeWriteBatcher());
+ DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
+ doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
+ doFnTester.processBundle(mutations);
+ }
+
+ /**
* Tests {@link DatastoreV1.Read#getEstimatedSizeBytes} to fetch and return estimated size for a
* query.
*/
@@ -682,7 +612,7 @@
.thenReturn(splitQuery(QUERY, numSplits));
SplitQueryFn splitQueryFn = new SplitQueryFn(V_1_OPTIONS, numSplits, mockDatastoreFactory);
- DoFnTester<Query, KV<Integer, Query>> doFnTester = DoFnTester.of(splitQueryFn);
+ DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn);
/**
* Although Datastore client is marked transient in {@link SplitQueryFn}, when injected through
* mock factory using a when clause for unit testing purposes, it is not serializable
@@ -690,10 +620,15 @@
* doFn from being serialized.
*/
doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
- List<KV<Integer, Query>> queries = doFnTester.processBundle(QUERY);
+ List<Query> queries = doFnTester.processBundle(QUERY);
assertEquals(queries.size(), numSplits);
- verifyUniqueKeys(queries);
+
+ // Confirms that sub-queries are not equal to original when there is more than one split.
+ for (Query subQuery : queries) {
+ assertNotEquals(subQuery, QUERY);
+ }
+
verify(mockQuerySplitter, times(1)).getSplits(
eq(QUERY), any(PartitionId.class), eq(numSplits), any(Datastore.class));
verifyZeroInteractions(mockDatastore);
@@ -728,12 +663,11 @@
.thenReturn(splitQuery(QUERY, expectedNumSplits));
SplitQueryFn splitQueryFn = new SplitQueryFn(V_1_OPTIONS, numSplits, mockDatastoreFactory);
- DoFnTester<Query, KV<Integer, Query>> doFnTester = DoFnTester.of(splitQueryFn);
+ DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn);
doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
- List<KV<Integer, Query>> queries = doFnTester.processBundle(QUERY);
+ List<Query> queries = doFnTester.processBundle(QUERY);
assertEquals(queries.size(), expectedNumSplits);
- verifyUniqueKeys(queries);
verify(mockQuerySplitter, times(1)).getSplits(
eq(QUERY), any(PartitionId.class), eq(expectedNumSplits), any(Datastore.class));
verify(mockDatastore, times(1)).runQuery(latestTimestampRequest);
@@ -745,17 +679,16 @@
*/
@Test
public void testSplitQueryFnWithQueryLimit() throws Exception {
- Query queryWithLimit = QUERY.toBuilder().clone()
+ Query queryWithLimit = QUERY.toBuilder()
.setLimit(Int32Value.newBuilder().setValue(1))
.build();
SplitQueryFn splitQueryFn = new SplitQueryFn(V_1_OPTIONS, 10, mockDatastoreFactory);
- DoFnTester<Query, KV<Integer, Query>> doFnTester = DoFnTester.of(splitQueryFn);
+ DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn);
doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
- List<KV<Integer, Query>> queries = doFnTester.processBundle(queryWithLimit);
+ List<Query> queries = doFnTester.processBundle(queryWithLimit);
assertEquals(queries.size(), 1);
- verifyUniqueKeys(queries);
verifyNoMoreInteractions(mockDatastore);
verifyNoMoreInteractions(mockQuerySplitter);
}
@@ -778,6 +711,31 @@
readFnTest(5 * QUERY_BATCH_LIMIT);
}
+ /** Tests that {@link ReadFn} retries after an error. */
+ @Test
+ public void testReadFnRetriesErrors() throws Exception {
+ // An empty query to read entities.
+ Query query = Query.newBuilder().setLimit(
+ Int32Value.newBuilder().setValue(1)).build();
+
+ // Use mockResponseForQuery to generate results.
+ when(mockDatastore.runQuery(any(RunQueryRequest.class)))
+ .thenThrow(
+ new DatastoreException("RunQuery", Code.DEADLINE_EXCEEDED, "", null))
+ .thenAnswer(new Answer<RunQueryResponse>() {
+ @Override
+ public RunQueryResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
+ Query q = ((RunQueryRequest) invocationOnMock.getArguments()[0]).getQuery();
+ return mockResponseForQuery(q);
+ }
+ });
+
+ ReadFn readFn = new ReadFn(V_1_OPTIONS, mockDatastoreFactory);
+ DoFnTester<Query, Entity> doFnTester = DoFnTester.of(readFn);
+ doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
+ List<Entity> entities = doFnTester.processBundle(query);
+ }
+
@Test
public void testTranslateGqlQueryWithLimit() throws Exception {
String gql = "SELECT * from DummyKind LIMIT 10";
@@ -858,6 +816,50 @@
.apply(DatastoreIO.v1().write().withProjectId(options.getDatastoreProject()));
}
+ @Test
+ public void testWriteBatcherWithoutData() {
+ DatastoreV1.WriteBatcher writeBatcher = new DatastoreV1.WriteBatcherImpl();
+ writeBatcher.start();
+ assertEquals(DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_START, writeBatcher.nextBatchSize(0));
+ }
+
+ @Test
+ public void testWriteBatcherFastQueries() {
+ DatastoreV1.WriteBatcher writeBatcher = new DatastoreV1.WriteBatcherImpl();
+ writeBatcher.start();
+ writeBatcher.addRequestLatency(0, 1000, 200);
+ writeBatcher.addRequestLatency(0, 1000, 200);
+ assertEquals(DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_LIMIT, writeBatcher.nextBatchSize(0));
+ }
+
+ @Test
+ public void testWriteBatcherSlowQueries() {
+ DatastoreV1.WriteBatcher writeBatcher = new DatastoreV1.WriteBatcherImpl();
+ writeBatcher.start();
+ writeBatcher.addRequestLatency(0, 10000, 200);
+ writeBatcher.addRequestLatency(0, 10000, 200);
+ assertEquals(100, writeBatcher.nextBatchSize(0));
+ }
+
+ @Test
+ public void testWriteBatcherSizeNotBelowMinimum() {
+ DatastoreV1.WriteBatcher writeBatcher = new DatastoreV1.WriteBatcherImpl();
+ writeBatcher.start();
+ writeBatcher.addRequestLatency(0, 30000, 50);
+ writeBatcher.addRequestLatency(0, 30000, 50);
+ assertEquals(DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_MIN, writeBatcher.nextBatchSize(0));
+ }
+
+ @Test
+ public void testWriteBatcherSlidingWindow() {
+ DatastoreV1.WriteBatcher writeBatcher = new DatastoreV1.WriteBatcherImpl();
+ writeBatcher.start();
+ writeBatcher.addRequestLatency(0, 30000, 50);
+ writeBatcher.addRequestLatency(50000, 5000, 200);
+ writeBatcher.addRequestLatency(100000, 5000, 200);
+ assertEquals(200, writeBatcher.nextBatchSize(150000));
+ }
+
/** Helper Methods */
/** A helper function that verifies if all the queries have unique keys. */
@@ -996,9 +998,31 @@
/** Generate dummy query splits. */
private List<Query> splitQuery(Query query, int numSplits) {
List<Query> queries = new LinkedList<>();
+ int offsetOfOriginal = query.getOffset();
for (int i = 0; i < numSplits; i++) {
- queries.add(query.toBuilder().clone().build());
+ Query.Builder q = Query.newBuilder();
+ q.addKindBuilder().setName(KIND);
+ // Making sub-queries unique (and not equal to the original query) by setting different
+ // offsets.
+ q.setOffset(++offsetOfOriginal);
+ queries.add(q.build());
}
return queries;
}
+
+ /**
+ * A WriteBatcher for unit tests, which does no timing-based adjustments (so unit tests have
+ * consistent results).
+ */
+ static class FakeWriteBatcher implements DatastoreV1.WriteBatcher {
+ @Override
+ public void start() {}
+ @Override
+ public void addRequestLatency(long timeSinceEpochMillis, long latencyMillis, int numMutations) {
+ }
+ @Override
+ public int nextBatchSize(long timeSinceEpochMillis) {
+ return DATASTORE_BATCH_UPDATE_ENTITIES_START;
+ }
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
index 5b1066a..fa391cc 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
@@ -27,7 +27,6 @@
import org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.SplitQueryFn;
import org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.V1Options;
import org.apache.beam.sdk.transforms.DoFnTester;
-import org.apache.beam.sdk.values.KV;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -87,9 +86,9 @@
SplitQueryFn splitQueryFn = new SplitQueryFn(
V1Options.from(projectId, namespace, null), 0);
- DoFnTester<Query, KV<Integer, Query>> doFnTester = DoFnTester.of(splitQueryFn);
+ DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn);
- List<KV<Integer, Query>> queries = doFnTester.processBundle(query.build());
+ List<Query> queries = doFnTester.processBundle(query.build());
assertEquals(queries.size(), expectedNumSplits);
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
index ec7fa8f..22945f5 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
@@ -148,7 +148,7 @@
Key ancestorKey = makeAncestorKey(options.getNamespace(), options.getKind(), ancestor);
for (long i = 0; i < numEntities; i++) {
- Entity entity = makeEntity(i, ancestorKey, options.getKind(), options.getNamespace());
+ Entity entity = makeEntity(i, ancestorKey, options.getKind(), options.getNamespace(), 0);
writer.write(entity);
}
writer.close();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
index dc91638..cd61229 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
@@ -92,8 +92,10 @@
/**
* Build an entity for the given ancestorKey, kind, namespace and value.
+ * @param largePropertySize if greater than 0, add an unindexed property of the given size.
*/
- static Entity makeEntity(Long value, Key ancestorKey, String kind, @Nullable String namespace) {
+ static Entity makeEntity(Long value, Key ancestorKey, String kind, @Nullable String namespace,
+ int largePropertySize) {
Entity.Builder entityBuilder = Entity.newBuilder();
Key.Builder keyBuilder = makeKey(ancestorKey, kind, UUID.randomUUID().toString());
// NOTE: Namespace is not inherited between keys created with DatastoreHelper.makeKey, so
@@ -105,6 +107,10 @@
entityBuilder.setKey(keyBuilder.build());
entityBuilder.putProperties("value", makeValue(value).build());
+ if (largePropertySize > 0) {
+ entityBuilder.putProperties("unindexed_value", makeValue(new String(
+ new char[largePropertySize]).replace("\0", "A")).setExcludeFromIndexes(true).build());
+ }
return entityBuilder.build();
}
@@ -115,18 +121,21 @@
private final String kind;
@Nullable
private final String namespace;
+ private final int largePropertySize;
private Key ancestorKey;
- CreateEntityFn(String kind, @Nullable String namespace, String ancestor) {
+ CreateEntityFn(String kind, @Nullable String namespace, String ancestor,
+ int largePropertySize) {
this.kind = kind;
this.namespace = namespace;
+ this.largePropertySize = largePropertySize;
// Build the ancestor key for all created entities once, including the namespace.
ancestorKey = makeAncestorKey(namespace, kind, ancestor);
}
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
- c.output(makeEntity(c.element(), ancestorKey, kind, namespace));
+ c.output(makeEntity(c.element(), ancestorKey, kind, namespace, largePropertySize));
}
}
@@ -365,7 +374,7 @@
// Read the next batch of query results.
private Iterator<EntityResult> getIteratorAndMoveCursor() throws DatastoreException {
- Query.Builder query = this.query.toBuilder().clone();
+ Query.Builder query = this.query.toBuilder();
query.setLimit(Int32Value.newBuilder().setValue(QUERY_BATCH_LIMIT));
if (currentBatch != null && !currentBatch.getEndCursor().isEmpty()) {
query.setStartCursor(currentBatch.getEndCursor());
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
index 82e4d64..4a874fd 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
@@ -67,8 +67,7 @@
// Write to datastore
p.apply(GenerateSequence.from(0).to(numEntities))
- .apply(ParDo.of(new CreateEntityFn(
- options.getKind(), options.getNamespace(), ancestor)))
+ .apply(ParDo.of(new CreateEntityFn(options.getKind(), options.getNamespace(), ancestor, 0)))
.apply(DatastoreIO.v1().write().withProjectId(project));
p.run();
@@ -79,6 +78,39 @@
assertEquals(numEntitiesWritten, numEntities);
}
+ /**
+ * An end-to-end test for {@link DatastoreV1.Write}.
+ *
+ * <p>Write some large test entities to Cloud Datastore, to test that a batch is flushed when
+ * the byte size limit is reached. Read and count all the entities. Verify that the count matches
+ * the number of entities written.
+ */
+ @Test
+ public void testE2EV1WriteWithLargeEntities() throws Exception {
+ Pipeline p = Pipeline.create(options);
+
+ /*
+ * Datastore has a limit of 1MB per entity, and 10MB per write RPC. If each entity is around
+ * 1MB in size, then we hit the limit on the size of the write long before we hit the limit on
+ * the number of entities per writes.
+ */
+ final int rawPropertySize = 900_000;
+ final int numLargeEntities = 100;
+
+ // Write to datastore
+ p.apply(GenerateSequence.from(0).to(numLargeEntities))
+ .apply(ParDo.of(new CreateEntityFn(
+ options.getKind(), options.getNamespace(), ancestor, rawPropertySize)))
+ .apply(DatastoreIO.v1().write().withProjectId(project));
+
+ p.run();
+
+ // Count number of entities written to datastore.
+ long numEntitiesWritten = countEntities(options, project, ancestor);
+
+ assertEquals(numEntitiesWritten, numLargeEntities);
+ }
+
@After
public void tearDown() throws Exception {
deleteAllEntities(options, project, ancestor);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
index 8f5d1ea..6d92861 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
@@ -18,14 +18,22 @@
package org.apache.beam.sdk.io.gcp.pubsub;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import java.util.Set;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO.Read;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.UsesUnboundedPCollections;
import org.apache.beam.sdk.testing.ValidatesRunner;
import org.apache.beam.sdk.transforms.display.DisplayData;
@@ -141,6 +149,53 @@
}
@Test
+ public void testValueProviderSubscription() {
+ StaticValueProvider<String> provider =
+ StaticValueProvider.of("projects/project/subscriptions/subscription");
+ Read<String> pubsubRead =
+ PubsubIO.readStrings()
+ .fromSubscription(provider);
+ Pipeline.create().apply(pubsubRead);
+ assertThat(pubsubRead.getSubscriptionProvider(), not(nullValue()));
+ assertThat(pubsubRead.getSubscriptionProvider().isAccessible(), is(true));
+ assertThat(pubsubRead.getSubscriptionProvider().get().asPath(), equalTo(provider.get()));
+ }
+
+ @Test
+ public void testRuntimeValueProviderSubscription() {
+ TestPipeline pipeline = TestPipeline.create();
+ ValueProvider<String> subscription =
+ pipeline.newProvider("projects/project/subscriptions/subscription");
+ Read<String> pubsubRead = PubsubIO.readStrings().fromSubscription(subscription);
+ pipeline.apply(pubsubRead);
+ assertThat(pubsubRead.getSubscriptionProvider(), not(nullValue()));
+ assertThat(pubsubRead.getSubscriptionProvider().isAccessible(), is(false));
+ }
+
+ @Test
+ public void testValueProviderTopic() {
+ StaticValueProvider<String> provider = StaticValueProvider.of("projects/project/topics/topic");
+ Read<String> pubsubRead =
+ PubsubIO.readStrings().fromTopic(provider);
+ Pipeline.create().apply(pubsubRead);
+ assertThat(pubsubRead.getTopicProvider(), not(nullValue()));
+ assertThat(pubsubRead.getTopicProvider().isAccessible(), is(true));
+ assertThat(
+ pubsubRead.getTopicProvider().get().asPath(),
+ equalTo(provider.get()));
+ }
+
+ @Test
+ public void testRuntimeValueProviderTopic() {
+ TestPipeline pipeline = TestPipeline.create();
+ ValueProvider<String> topic = pipeline.newProvider("projects/project/topics/topic");
+ Read<String> pubsubRead = PubsubIO.readStrings().fromTopic(topic);
+ pipeline.apply(pubsubRead);
+ assertThat(pubsubRead.getTopicProvider(), not(nullValue()));
+ assertThat(pubsubRead.getTopicProvider().isAccessible(), is(false));
+ }
+
+ @Test
@Category({ValidatesRunner.class, UsesUnboundedPCollections.class})
public void testPrimitiveReadDisplayData() {
DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/FakeServiceFactory.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/FakeServiceFactory.java
new file mode 100644
index 0000000..753d807
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/FakeServiceFactory.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+import com.google.cloud.ServiceFactory;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.concurrent.GuardedBy;
+import org.mockito.Matchers;
+
+/**
+ * A serialization friendly type service factory that maintains a mock {@link Spanner} and
+ * {@link DatabaseClient}.
+ * */
+class FakeServiceFactory
+ implements ServiceFactory<Spanner, SpannerOptions>, Serializable {
+
+ // Marked as static so they could be returned by serviceFactory, which is serializable.
+ private static final Object lock = new Object();
+
+ @GuardedBy("lock")
+ private static final List<Spanner> mockSpanners = new ArrayList<>();
+
+ @GuardedBy("lock")
+ private static final List<DatabaseClient> mockDatabaseClients = new ArrayList<>();
+
+ @GuardedBy("lock")
+ private static int count = 0;
+
+ private final int index;
+
+ public FakeServiceFactory() {
+ synchronized (lock) {
+ index = count++;
+ mockSpanners.add(mock(Spanner.class, withSettings().serializable()));
+ mockDatabaseClients.add(mock(DatabaseClient.class, withSettings().serializable()));
+ }
+ when(mockSpanner().getDatabaseClient(Matchers.any(DatabaseId.class)))
+ .thenReturn(mockDatabaseClient());
+ }
+
+ DatabaseClient mockDatabaseClient() {
+ synchronized (lock) {
+ return mockDatabaseClients.get(index);
+ }
+ }
+
+ Spanner mockSpanner() {
+ synchronized (lock) {
+ return mockSpanners.get(index);
+ }
+ }
+
+ @Override
+ public Spanner create(SpannerOptions serviceOptions) {
+ return mockSpanner();
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoderTest.java
new file mode 100644
index 0000000..d40e356
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoderTest.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.Date;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeyRange;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Mutation;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.primitives.UnsignedBytes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Tests for {@link MutationGroupEncoder}.
+ */
+public class MutationGroupEncoderTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ private SpannerSchema allTypesSchema;
+
+ @Before
+ public void setUp() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "intkey", "INT64");
+ builder.addKeyPart("test", "intkey", false);
+
+ builder.addColumn("test", "bool", "BOOL");
+ builder.addColumn("test", "int64", "INT64");
+ builder.addColumn("test", "float64", "FLOAT64");
+ builder.addColumn("test", "string", "STRING");
+ builder.addColumn("test", "bytes", "BYTES");
+ builder.addColumn("test", "timestamp", "TIMESTAMP");
+ builder.addColumn("test", "date", "DATE");
+
+ builder.addColumn("test", "nullbool", "BOOL");
+ builder.addColumn("test", "nullint64", "INT64");
+ builder.addColumn("test", "nullfloat64", "FLOAT64");
+ builder.addColumn("test", "nullstring", "STRING");
+ builder.addColumn("test", "nullbytes", "BYTES");
+ builder.addColumn("test", "nulltimestamp", "TIMESTAMP");
+ builder.addColumn("test", "nulldate", "DATE");
+
+ builder.addColumn("test", "arrbool", "ARRAY<BOOL>");
+ builder.addColumn("test", "arrint64", "ARRAY<INT64>");
+ builder.addColumn("test", "arrfloat64", "ARRAY<FLOAT64>");
+ builder.addColumn("test", "arrstring", "ARRAY<STRING>");
+ builder.addColumn("test", "arrbytes", "ARRAY<BYTES>");
+ builder.addColumn("test", "arrtimestamp", "ARRAY<TIMESTAMP>");
+ builder.addColumn("test", "arrdate", "ARRAY<DATE>");
+
+ builder.addColumn("test", "nullarrbool", "ARRAY<BOOL>");
+ builder.addColumn("test", "nullarrint64", "ARRAY<INT64>");
+ builder.addColumn("test", "nullarrfloat64", "ARRAY<FLOAT64>");
+ builder.addColumn("test", "nullarrstring", "ARRAY<STRING>");
+ builder.addColumn("test", "nullarrbytes", "ARRAY<BYTES>");
+ builder.addColumn("test", "nullarrtimestamp", "ARRAY<TIMESTAMP>");
+ builder.addColumn("test", "nullarrdate", "ARRAY<DATE>");
+
+ allTypesSchema = builder.build();
+ }
+
+ @Test
+ public void testAllTypesSingleMutation() throws Exception {
+ encodeAndVerify(g(appendAllTypes(Mutation.newInsertOrUpdateBuilder("test")).build()));
+ encodeAndVerify(g(appendAllTypes(Mutation.newInsertBuilder("test")).build()));
+ encodeAndVerify(g(appendAllTypes(Mutation.newUpdateBuilder("test")).build()));
+ encodeAndVerify(g(appendAllTypes(Mutation.newReplaceBuilder("test")).build()));
+ }
+
+ @Test
+ public void testAllTypesMultipleMutations() throws Exception {
+ encodeAndVerify(g(
+ appendAllTypes(Mutation.newInsertOrUpdateBuilder("test")).build(),
+ appendAllTypes(Mutation.newInsertBuilder("test")).build(),
+ appendAllTypes(Mutation.newUpdateBuilder("test")).build(),
+ appendAllTypes(Mutation.newReplaceBuilder("test")).build(),
+ Mutation
+ .delete("test", KeySet.range(KeyRange.closedClosed(Key.of(1L), Key.of(2L))))));
+ }
+
+ @Test
+ public void testUnknownColumn() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+ builder.addKeyPart("test", "bool_field", false);
+ builder.addColumn("test", "bool_field", "BOOL");
+ SpannerSchema schema = builder.build();
+
+ Mutation mutation = Mutation.newInsertBuilder("test").set("unknown")
+ .to(true).build();
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("Columns [unknown] were not defined in table test");
+ encodeAndVerify(g(mutation), schema);
+ }
+
+ @Test
+ public void testUnknownTable() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+ builder.addKeyPart("test", "bool_field", false);
+ builder.addColumn("test", "bool_field", "BOOL");
+ SpannerSchema schema = builder.build();
+
+ Mutation mutation = Mutation.newInsertBuilder("unknown").set("bool_field")
+ .to(true).build();
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("Unknown table 'unknown'");
+ encodeAndVerify(g(mutation), schema);
+ }
+
+ @Test
+ public void testMutationCaseInsensitive() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+ builder.addKeyPart("test", "bool_field", false);
+ builder.addColumn("test", "bool_field", "BOOL");
+ SpannerSchema schema = builder.build();
+
+ Mutation mutation = Mutation.newInsertBuilder("TEsT").set("BoOL_FiELd").to(true).build();
+ encodeAndVerify(g(mutation), schema);
+ }
+
+ @Test
+ public void testDeleteCaseInsensitive() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+ builder.addKeyPart("test", "bool_field", false);
+ builder.addColumn("test", "int_field", "INT64");
+ SpannerSchema schema = builder.build();
+
+ Mutation mutation = Mutation.delete("TeSt", Key.of(1L));
+ encodeAndVerify(g(mutation), schema);
+ }
+
+ @Test
+ public void testDeletes() throws Exception {
+ encodeAndVerify(g(Mutation.delete("test", Key.of(1L))));
+ encodeAndVerify(g(Mutation.delete("test", Key.of((Long) null))));
+
+ KeySet allTypes = KeySet.newBuilder()
+ .addKey(Key.of(1L))
+ .addKey(Key.of((Long) null))
+ .addKey(Key.of(1.2))
+ .addKey(Key.of((Double) null))
+ .addKey(Key.of("one"))
+ .addKey(Key.of((String) null))
+ .addKey(Key.of(ByteArray.fromBase64("abcd")))
+ .addKey(Key.of((ByteArray) null))
+ .addKey(Key.of(Timestamp.now()))
+ .addKey(Key.of((Timestamp) null))
+ .addKey(Key.of(Date.fromYearMonthDay(2012, 1, 1)))
+ .addKey(Key.of((Date) null))
+ .build();
+
+ encodeAndVerify(g(Mutation.delete("test", allTypes)));
+
+ encodeAndVerify(
+ g(Mutation
+ .delete("test", KeySet.range(KeyRange.closedClosed(Key.of(1L), Key.of(2L))))));
+ }
+
+ private Mutation.WriteBuilder appendAllTypes(Mutation.WriteBuilder builder) {
+ Timestamp ts = Timestamp.now();
+ Date date = Date.fromYearMonthDay(2017, 1, 1);
+ return builder
+ .set("bool").to(true)
+ .set("int64").to(1L)
+ .set("float64").to(1.0)
+ .set("string").to("my string")
+ .set("bytes").to(ByteArray.fromBase64("abcdedf"))
+ .set("timestamp").to(ts)
+ .set("date").to(date)
+
+ .set("arrbool").toBoolArray(Arrays.asList(true, false, null, true, null, false))
+ .set("arrint64").toInt64Array(Arrays.asList(10L, -12L, null, null, 100000L))
+ .set("arrfloat64").toFloat64Array(Arrays.asList(10., -12.23, null, null, 100000.33231))
+ .set("arrstring").toStringArray(Arrays.asList("one", "two", null, null, "three"))
+ .set("arrbytes").toBytesArray(Arrays.asList(ByteArray.fromBase64("abcs"), null))
+ .set("arrtimestamp").toTimestampArray(Arrays.asList(Timestamp.MIN_VALUE, null, ts))
+ .set("arrdate").toDateArray(Arrays.asList(null, date))
+
+ .set("nullbool").to((Boolean) null)
+ .set("nullint64").to((Long) null)
+ .set("nullfloat64").to((Double) null)
+ .set("nullstring").to((String) null)
+ .set("nullbytes").to((ByteArray) null)
+ .set("nulltimestamp").to((Timestamp) null)
+ .set("nulldate").to((Date) null)
+
+ .set("nullarrbool").toBoolArray((Iterable<Boolean>) null)
+ .set("nullarrint64").toInt64Array((Iterable<Long>) null)
+ .set("nullarrfloat64").toFloat64Array((Iterable<Double>) null)
+ .set("nullarrstring").toStringArray(null)
+ .set("nullarrbytes").toBytesArray(null)
+ .set("nullarrtimestamp").toTimestampArray(null)
+ .set("nullarrdate").toDateArray(null);
+ }
+
+ @Test
+ public void int64Keys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "INT64");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "INT64");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(1L)
+ .set("keydesc").to(0L)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2L)
+ .set("keydesc").to((Long) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2L)
+ .set("keydesc").to(10L)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2L)
+ .set("keydesc").to(9L)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to((Long) null)
+ .set("keydesc").to(0L)
+ .build());
+
+ List<Key> keys = Arrays.asList(
+ Key.of(1L, 0L),
+ Key.of(2L, null),
+ Key.of(2L, 10L),
+ Key.of(2L, 9L),
+ Key.of(2L, 0L)
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void float64Keys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "FLOAT64");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "FLOAT64");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(1.0)
+ .set("keydesc").to(0.)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2.)
+ .set("keydesc").to((Long) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2.)
+ .set("keydesc").to(10.)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2.)
+ .set("keydesc").to(9.)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(2.)
+ .set("keydesc").to(0.)
+ .build());
+ List<Key> keys = Arrays.asList(
+ Key.of(1., 0.),
+ Key.of(2., null),
+ Key.of(2., 10.),
+ Key.of(2., 9.),
+ Key.of(2., 0.)
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void stringKeys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "STRING");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "STRING");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to("a")
+ .set("keydesc").to("bc")
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to("b")
+ .set("keydesc").to((String) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to("b")
+ .set("keydesc").to("z")
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to("b")
+ .set("keydesc").to("y")
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to("b")
+ .set("keydesc").to("a")
+ .build());
+
+ List<Key> keys = Arrays.asList(
+ Key.of("a", "bc"),
+ Key.of("b", null),
+ Key.of("b", "z"),
+ Key.of("b", "y"),
+ Key.of("b", "a")
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void bytesKeys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "BYTES");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "BYTES");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(ByteArray.fromBase64("abc"))
+ .set("keydesc").to(ByteArray.fromBase64("zzz"))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(ByteArray.fromBase64("xxx"))
+ .set("keydesc").to((ByteArray) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(ByteArray.fromBase64("xxx"))
+ .set("keydesc").to(ByteArray.fromBase64("zzzz"))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(ByteArray.fromBase64("xxx"))
+ .set("keydesc").to(ByteArray.fromBase64("ssss"))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(ByteArray.fromBase64("xxx"))
+ .set("keydesc").to(ByteArray.fromBase64("aaa"))
+ .build());
+
+ List<Key> keys = Arrays.asList(
+ Key.of(ByteArray.fromBase64("abc"), ByteArray.fromBase64("zzz")),
+ Key.of(ByteArray.fromBase64("xxx"), null),
+ Key.of(ByteArray.fromBase64("xxx"), ByteArray.fromBase64("zzz")),
+ Key.of(ByteArray.fromBase64("xxx"), ByteArray.fromBase64("sss")),
+ Key.of(ByteArray.fromBase64("xxx"), ByteArray.fromBase64("aaa"))
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void dateKeys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "DATE");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "DATE");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Date.fromYearMonthDay(2012, 10, 10))
+ .set("keydesc").to(Date.fromYearMonthDay(2000, 10, 10))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Date.fromYearMonthDay(2020, 10, 10))
+ .set("keydesc").to((Date) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Date.fromYearMonthDay(2020, 10, 10))
+ .set("keydesc").to(Date.fromYearMonthDay(2050, 10, 10))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Date.fromYearMonthDay(2020, 10, 10))
+ .set("keydesc").to(Date.fromYearMonthDay(2000, 10, 10))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Date.fromYearMonthDay(2020, 10, 10))
+ .set("keydesc").to(Date.fromYearMonthDay(1900, 10, 10))
+ .build());
+
+ List<Key> keys = Arrays.asList(
+ Key.of(Date.fromYearMonthDay(2012, 10, 10), ByteArray.fromBase64("zzz")),
+ Key.of(Date.fromYearMonthDay(2015, 10, 10), null),
+ Key.of(Date.fromYearMonthDay(2015, 10, 10), Date.fromYearMonthDay(2050, 10, 10)),
+ Key.of(Date.fromYearMonthDay(2015, 10, 10), Date.fromYearMonthDay(2000, 10, 10)),
+ Key.of(Date.fromYearMonthDay(2015, 10, 10), Date.fromYearMonthDay(1900, 10, 10))
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void timestampKeys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "key", "TIMESTAMP");
+ builder.addKeyPart("test", "key", false);
+
+ builder.addColumn("test", "keydesc", "TIMESTAMP");
+ builder.addKeyPart("test", "keydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Timestamp.ofTimeMicroseconds(10000))
+ .set("keydesc").to(Timestamp.ofTimeMicroseconds(50000))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Timestamp.ofTimeMicroseconds(20000))
+ .set("keydesc").to((Timestamp) null)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Timestamp.ofTimeMicroseconds(20000))
+ .set("keydesc").to(Timestamp.ofTimeMicroseconds(90000))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Timestamp.ofTimeMicroseconds(20000))
+ .set("keydesc").to(Timestamp.ofTimeMicroseconds(50000))
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("key").to(Timestamp.ofTimeMicroseconds(20000))
+ .set("keydesc").to(Timestamp.ofTimeMicroseconds(10000))
+ .build());
+
+
+ List<Key> keys = Arrays.asList(
+ Key.of(Timestamp.ofTimeMicroseconds(10000), ByteArray.fromBase64("zzz")),
+ Key.of(Timestamp.ofTimeMicroseconds(20000), null),
+ Key.of(Timestamp.ofTimeMicroseconds(20000), Timestamp.ofTimeMicroseconds(90000)),
+ Key.of(Timestamp.ofTimeMicroseconds(20000), Timestamp.ofTimeMicroseconds(50000)),
+ Key.of(Timestamp.ofTimeMicroseconds(20000), Timestamp.ofTimeMicroseconds(10000))
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ @Test
+ public void boolKeys() throws Exception {
+ SpannerSchema.Builder builder = SpannerSchema.builder();
+
+ builder.addColumn("test", "boolkey", "BOOL");
+ builder.addKeyPart("test", "boolkey", false);
+
+ builder.addColumn("test", "boolkeydesc", "BOOL");
+ builder.addKeyPart("test", "boolkeydesc", true);
+
+ SpannerSchema schema = builder.build();
+
+ List<Mutation> mutations = Arrays.asList(
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("boolkey").to(true)
+ .set("boolkeydesc").to(false)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("boolkey").to(false)
+ .set("boolkeydesc").to(false)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("boolkey").to(false)
+ .set("boolkeydesc").to(true)
+ .build(),
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("boolkey").to((Boolean) null)
+ .set("boolkeydesc").to(false)
+ .build()
+ );
+
+ List<Key> keys = Arrays.asList(
+ Key.of(true, ByteArray.fromBase64("zzz")),
+ Key.of(false, null),
+ Key.of(false, false),
+ Key.of(false, true),
+ Key.of(null, false)
+ );
+
+ verifyEncodedOrdering(schema, mutations);
+ verifyEncodedOrdering(schema, "test", keys);
+ }
+
+ private void verifyEncodedOrdering(SpannerSchema schema, List<Mutation> mutations) {
+ MutationGroupEncoder encoder = new MutationGroupEncoder(schema);
+ List<byte[]> mutationEncodings = new ArrayList<>(mutations.size());
+ for (Mutation m : mutations) {
+ mutationEncodings.add(encoder.encodeKey(m));
+ }
+ List<byte[]> copy = new ArrayList<>(mutationEncodings);
+ Collections.sort(copy, UnsignedBytes.lexicographicalComparator());
+
+ Assert.assertEquals(mutationEncodings, copy);
+ }
+
+ private void verifyEncodedOrdering(SpannerSchema schema, String table, List<Key> keys) {
+ MutationGroupEncoder encoder = new MutationGroupEncoder(schema);
+ List<byte[]> keyEncodings = new ArrayList<>(keys.size());
+ for (Key k : keys) {
+ keyEncodings.add(encoder.encodeKey(table, k));
+ }
+ List<byte[]> copy = new ArrayList<>(keyEncodings);
+ Collections.sort(copy, UnsignedBytes.lexicographicalComparator());
+
+ Assert.assertEquals(keyEncodings, copy);
+ }
+
+ private MutationGroup g(Mutation mutation, Mutation... other) {
+ return MutationGroup.create(mutation, other);
+ }
+
+ private void encodeAndVerify(MutationGroup expected) {
+ SpannerSchema schema = this.allTypesSchema;
+ encodeAndVerify(expected, schema);
+ }
+
+ private static void encodeAndVerify(MutationGroup expected, SpannerSchema schema) {
+ MutationGroupEncoder coder = new MutationGroupEncoder(schema);
+ byte[] encode = coder.encode(expected);
+ MutationGroup actual = coder.decode(encode);
+
+ Assert.assertTrue(mutationGroupsEqual(expected, actual));
+ }
+
+ private static boolean mutationGroupsEqual(MutationGroup a, MutationGroup b) {
+ ImmutableList<Mutation> alist = ImmutableList.copyOf(a);
+ ImmutableList<Mutation> blist = ImmutableList.copyOf(b);
+
+ if (alist.size() != blist.size()) {
+ return false;
+ }
+
+ for (int i = 0; i < alist.size(); i++) {
+ if (!mutationsEqual(alist.get(i), blist.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // Is different from Mutation#equals. Case insensitive for table/column names, the order of
+ // the columns doesn't matter.
+ private static boolean mutationsEqual(Mutation a, Mutation b) {
+ if (a == b) {
+ return true;
+ }
+ if (a == null || b == null) {
+ return false;
+ }
+ if (a.getOperation() != b.getOperation()) {
+ return false;
+ }
+ if (!a.getTable().equalsIgnoreCase(b.getTable())) {
+ return false;
+ }
+ if (a.getOperation() == Mutation.Op.DELETE) {
+ return a.getKeySet().equals(b.getKeySet());
+ }
+
+ // Compare pairs instead? This seems to be good enough...
+ return ImmutableSet.copyOf(getNormalizedColumns(a))
+ .equals(ImmutableSet.copyOf(getNormalizedColumns(b))) && ImmutableSet.copyOf(a.getValues())
+ .equals(ImmutableSet.copyOf(b.getValues()));
+ }
+
+ // Pray for Java 8 support.
+ private static Iterable<String> getNormalizedColumns(Mutation a) {
+ return Iterables.transform(a.getColumns(), new Function<String, String>() {
+
+ @Override
+ public String apply(String input) {
+ return input.toLowerCase();
+ }
+ });
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
new file mode 100644
index 0000000..013b83d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.Date;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.Mutation;
+import java.util.Arrays;
+import org.junit.Test;
+
+/** A set of unit tests for {@link MutationSizeEstimator}. */
+public class MutationSizeEstimatorTest {
+
+ @Test
+ public void primitives() throws Exception {
+ Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+ Mutation float64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2.9).build();
+ Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").to(false).build();
+
+ assertThat(MutationSizeEstimator.sizeOf(int64), is(8L));
+ assertThat(MutationSizeEstimator.sizeOf(float64), is(8L));
+ assertThat(MutationSizeEstimator.sizeOf(bool), is(1L));
+ }
+
+ @Test
+ public void primitiveArrays() throws Exception {
+ Mutation int64 =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toInt64Array(new long[] {1L, 2L, 3L})
+ .build();
+ Mutation float64 =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toFloat64Array(new double[] {1., 2.})
+ .build();
+ Mutation bool =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toBoolArray(new boolean[] {true, true, false, true})
+ .build();
+
+ assertThat(MutationSizeEstimator.sizeOf(int64), is(24L));
+ assertThat(MutationSizeEstimator.sizeOf(float64), is(16L));
+ assertThat(MutationSizeEstimator.sizeOf(bool), is(4L));
+ }
+
+ @Test
+ public void strings() throws Exception {
+ Mutation emptyString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("").build();
+ Mutation nullString =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to((String) null).build();
+ Mutation sampleString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("abc").build();
+ Mutation sampleArray =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toStringArray(Arrays.asList("one", "two", null))
+ .build();
+
+ assertThat(MutationSizeEstimator.sizeOf(emptyString), is(0L));
+ assertThat(MutationSizeEstimator.sizeOf(nullString), is(0L));
+ assertThat(MutationSizeEstimator.sizeOf(sampleString), is(3L));
+ assertThat(MutationSizeEstimator.sizeOf(sampleArray), is(6L));
+ }
+
+ @Test
+ public void bytes() throws Exception {
+ Mutation empty =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to(ByteArray.fromBase64("")).build();
+ Mutation nullValue =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to((ByteArray) null).build();
+ Mutation sample =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .to(ByteArray.fromBase64("abcdabcd"))
+ .build();
+
+ assertThat(MutationSizeEstimator.sizeOf(empty), is(0L));
+ assertThat(MutationSizeEstimator.sizeOf(nullValue), is(0L));
+ assertThat(MutationSizeEstimator.sizeOf(sample), is(6L));
+ }
+
+ @Test
+ public void dates() throws Exception {
+ Mutation timestamp =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to(Timestamp.now()).build();
+ Mutation nullTimestamp =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to((Timestamp) null).build();
+ Mutation date =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .to(Date.fromYearMonthDay(2017, 10, 10))
+ .build();
+ Mutation nullDate =
+ Mutation.newInsertOrUpdateBuilder("test").set("one").to((Date) null).build();
+ Mutation timestampArray =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toTimestampArray(Arrays.asList(Timestamp.now(), null))
+ .build();
+ Mutation dateArray =
+ Mutation.newInsertOrUpdateBuilder("test")
+ .set("one")
+ .toDateArray(
+ Arrays.asList(
+ null,
+ Date.fromYearMonthDay(2017, 1, 1),
+ null,
+ Date.fromYearMonthDay(2017, 1, 2)))
+ .build();
+
+ assertThat(MutationSizeEstimator.sizeOf(timestamp), is(12L));
+ assertThat(MutationSizeEstimator.sizeOf(date), is(12L));
+ assertThat(MutationSizeEstimator.sizeOf(nullTimestamp), is(12L));
+ assertThat(MutationSizeEstimator.sizeOf(nullDate), is(12L));
+ assertThat(MutationSizeEstimator.sizeOf(timestampArray), is(24L));
+ assertThat(MutationSizeEstimator.sizeOf(dateArray), is(48L));
+ }
+
+ @Test
+ public void group() throws Exception {
+ Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+ Mutation float64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2.9).build();
+ Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").to(false).build();
+
+ MutationGroup group = MutationGroup.create(int64, float64, bool);
+
+ assertThat(MutationSizeEstimator.sizeOf(group), is(17L));
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java
new file mode 100644
index 0000000..5be4826
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java
@@ -0,0 +1,890 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.BaseEncoding;
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.UnsignedBytes;
+import com.google.common.primitives.UnsignedInteger;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A set of unit tests to verify {@link OrderedCode}.
+ */
+@RunWith(JUnit4.class)
+public class OrderedCodeTest {
+ /** Data for a generic coding test case with known encoded outputs. */
+ abstract static class CodingTestCase<T> {
+ /** The test value. */
+ abstract T value();
+
+ /**
+ * Test value's encoding in increasing order (obtained from the C++
+ * implementation).
+ */
+ abstract String increasingBytes();
+
+ /**
+ * Test value's encoding in dencreasing order (obtained from the C++
+ * implementation).
+ */
+ abstract String decreasingBytes();
+
+ // Helper methods to implement in concrete classes.
+
+ abstract byte[] encodeIncreasing();
+ abstract byte[] encodeDecreasing();
+
+ T decodeIncreasing() {
+ return decodeIncreasing(
+ new OrderedCode(bytesFromHexString(increasingBytes())));
+ }
+
+ T decodeDecreasing() {
+ return decodeDecreasing(
+ new OrderedCode(bytesFromHexString(decreasingBytes())));
+ }
+
+ abstract T decodeIncreasing(OrderedCode orderedCode);
+ abstract T decodeDecreasing(OrderedCode orderedCode);
+ }
+
+ @AutoValue
+ abstract static class UnsignedNumber extends CodingTestCase<Long> {
+ @Override
+ byte[] encodeIncreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumIncreasing(value());
+ return orderedCode.getEncodedBytes();
+ }
+
+ @Override
+ byte[] encodeDecreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumDecreasing(value());
+ return orderedCode.getEncodedBytes();
+ }
+
+ @Override
+ Long decodeIncreasing(OrderedCode orderedCode) {
+ return orderedCode.readNumIncreasing();
+ }
+
+ @Override
+ Long decodeDecreasing(OrderedCode orderedCode) {
+ return orderedCode.readNumDecreasing();
+ }
+
+ private static UnsignedNumber testCase(
+ long value, String increasingBytes, String decreasingBytes) {
+ return new AutoValue_OrderedCodeTest_UnsignedNumber(
+ value, increasingBytes, decreasingBytes);
+ }
+
+ /** Test cases for unsigned numbers, in increasing (unsigned) order by value. */
+ private static final ImmutableList<UnsignedNumber> TEST_CASES =
+ ImmutableList.of(
+ testCase(0, "00", "ff"),
+ testCase(1, "0101", "fefe"),
+ testCase(33, "0121", "fede"),
+ testCase(55000, "02d6d8", "fd2927"),
+ testCase(Integer.MAX_VALUE, "047fffffff", "fb80000000"),
+ testCase(Long.MAX_VALUE, "087fffffffffffffff", "f78000000000000000"),
+ testCase(Long.MIN_VALUE, "088000000000000000", "f77fffffffffffffff"),
+ testCase(-100, "08ffffffffffffff9c", "f70000000000000063"),
+ testCase(-1, "08ffffffffffffffff", "f70000000000000000"));
+ }
+
+ @AutoValue
+ abstract static class BytesTest extends CodingTestCase<String> {
+ @Override
+ byte[] encodeIncreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytes(bytesFromHexString(value()));
+ return orderedCode.getEncodedBytes();
+ }
+
+ @Override
+ byte[] encodeDecreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytesDecreasing(bytesFromHexString(value()));
+ return orderedCode.getEncodedBytes();
+ }
+
+ @Override
+ String decodeIncreasing(OrderedCode orderedCode) {
+ return bytesToHexString(orderedCode.readBytes());
+ }
+
+ @Override
+ String decodeDecreasing(OrderedCode orderedCode) {
+ return bytesToHexString(orderedCode.readBytesDecreasing());
+ }
+
+ private static BytesTest testCase(
+ String value, String increasingBytes, String decreasingBytes) {
+ return new AutoValue_OrderedCodeTest_BytesTest(
+ value, increasingBytes, decreasingBytes);
+ }
+
+ /** Test cases for byte arrays, in increasing order by value. */
+ private static final ImmutableList<BytesTest> TEST_CASES =
+ ImmutableList.of(
+ testCase("", "0001", "fffe"),
+ testCase("00", "00ff0001", "ff00fffe"),
+ testCase("0000", "00ff00ff0001", "ff00ff00fffe"),
+ testCase("0001", "00ff010001", "ff00fefffe"),
+ testCase("0041", "00ff410001", "ff00befffe"),
+ testCase("00ff", "00ffff000001", "ff0000fffffe"),
+ testCase("01", "010001", "fefffe"),
+ testCase("0100", "0100ff0001", "feff00fffe"),
+ testCase("6f776c", "6f776c0001", "908893fffe"),
+ testCase("ff", "ff000001", "00fffffe"),
+ testCase("ff00", "ff0000ff0001", "00ffff00fffe"),
+ testCase("ff01", "ff00010001", "00fffefffe"),
+ testCase("ffff", "ff00ff000001", "00ff00fffffe"),
+ testCase("ffffff", "ff00ff00ff000001", "00ff00ff00fffffe"));
+ }
+
+ @Test
+ public void testUnsignedEncoding() {
+ testEncoding(UnsignedNumber.TEST_CASES);
+ }
+
+ @Test
+ public void testUnsignedDecoding() {
+ testDecoding(UnsignedNumber.TEST_CASES);
+ }
+
+ @Test
+ public void testUnsignedOrdering() {
+ testOrdering(UnsignedNumber.TEST_CASES);
+ }
+
+ @Test
+ public void testBytesEncoding() {
+ testEncoding(BytesTest.TEST_CASES);
+ }
+
+ @Test
+ public void testBytesDecoding() {
+ testDecoding(BytesTest.TEST_CASES);
+ }
+
+ @Test
+ public void testBytesOrdering() {
+ testOrdering(BytesTest.TEST_CASES);
+ }
+
+ private void testEncoding(List<? extends CodingTestCase<?>> testCases) {
+ for (CodingTestCase<?> testCase : testCases) {
+ byte[] actualIncreasing = testCase.encodeIncreasing();
+ byte[] expectedIncreasing =
+ bytesFromHexString(testCase.increasingBytes());
+ assertEquals(0, compare(actualIncreasing, expectedIncreasing));
+
+ byte[] actualDecreasing = testCase.encodeDecreasing();
+ byte[] expectedDecreasing =
+ bytesFromHexString(testCase.decreasingBytes());
+ assertEquals(0, compare(actualDecreasing, expectedDecreasing));
+ }
+ }
+
+ private void testDecoding(List<? extends CodingTestCase<?>> testCases) {
+ for (CodingTestCase<?> testCase : testCases) {
+ assertEquals(testCase.value(), testCase.decodeIncreasing());
+ assertEquals(testCase.value(), testCase.decodeDecreasing());
+ }
+ }
+
+ private void testOrdering(List<? extends CodingTestCase<?>> testCases) {
+ // This is verifiable by inspection of the C++ encodings, but it seems
+ // worth checking explicitly
+ for (int caseIndex = 0; caseIndex < testCases.size() - 1; caseIndex++) {
+ byte[] encodedValue = testCases.get(caseIndex).encodeIncreasing();
+ byte[] nextEncodedValue = testCases.get(caseIndex + 1).encodeIncreasing();
+ assertTrue(compare(encodedValue, nextEncodedValue) < 0);
+
+ encodedValue = testCases.get(caseIndex).encodeDecreasing();
+ nextEncodedValue = testCases.get(caseIndex + 1).encodeDecreasing();
+ assertTrue(compare(encodedValue, nextEncodedValue) > 0);
+ }
+ }
+
+ @Test
+ public void testWriteInfinity() {
+ OrderedCode orderedCode = new OrderedCode();
+ try {
+ orderedCode.readInfinity();
+ fail("Expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ orderedCode.writeInfinity();
+ assertTrue(orderedCode.readInfinity());
+ try {
+ orderedCode.readInfinity();
+ fail("Expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testWriteInfinityDecreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ try {
+ orderedCode.readInfinityDecreasing();
+ fail("Expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ orderedCode.writeInfinityDecreasing();
+ assertTrue(orderedCode.readInfinityDecreasing());
+ try {
+ orderedCode.readInfinityDecreasing();
+ fail("Expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testWriteBytes() {
+ byte[] first = { 'a', 'b', 'c'};
+ byte[] second = { 'd', 'e', 'f'};
+ byte[] last = { 'x', 'y', 'z'};
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytes(first);
+ byte[] firstEncoded = orderedCode.getEncodedBytes();
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+
+ orderedCode.writeBytes(first);
+ orderedCode.writeBytes(second);
+ orderedCode.writeBytes(last);
+ byte[] allEncoded = orderedCode.getEncodedBytes();
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), last));
+
+ orderedCode = new OrderedCode(firstEncoded);
+ orderedCode.writeBytes(second);
+ orderedCode.writeBytes(last);
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), allEncoded));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), last));
+
+ orderedCode = new OrderedCode(allEncoded);
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), last));
+ }
+
+ @Test
+ public void testWriteBytesDecreasing() {
+ byte[] first = { 'a', 'b', 'c'};
+ byte[] second = { 'd', 'e', 'f'};
+ byte[] last = { 'x', 'y', 'z'};
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytesDecreasing(first);
+ byte[] firstEncoded = orderedCode.getEncodedBytes();
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), first));
+
+ orderedCode.writeBytesDecreasing(first);
+ orderedCode.writeBytesDecreasing(second);
+ orderedCode.writeBytesDecreasing(last);
+ byte[] allEncoded = orderedCode.getEncodedBytes();
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), last));
+
+ orderedCode = new OrderedCode(firstEncoded);
+ orderedCode.writeBytesDecreasing(second);
+ orderedCode.writeBytesDecreasing(last);
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), allEncoded));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), last));
+
+ orderedCode = new OrderedCode(allEncoded);
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), second));
+ assertTrue(Arrays.equals(orderedCode.readBytesDecreasing(), last));
+ }
+
+ @Test
+ public void testWriteNumIncreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumIncreasing(0);
+ orderedCode.writeNumIncreasing(1);
+ orderedCode.writeNumIncreasing(Long.MIN_VALUE);
+ orderedCode.writeNumIncreasing(Long.MAX_VALUE);
+ assertEquals(0, orderedCode.readNumIncreasing());
+ assertEquals(1, orderedCode.readNumIncreasing());
+ assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
+ }
+
+ @Test
+ public void testWriteNumIncreasing_unsignedInt() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumIncreasing(UnsignedInteger.fromIntBits(0));
+ orderedCode.writeNumIncreasing(UnsignedInteger.fromIntBits(1));
+ orderedCode.writeNumIncreasing(UnsignedInteger.fromIntBits(Integer.MIN_VALUE));
+ orderedCode.writeNumIncreasing(UnsignedInteger.fromIntBits(Integer.MAX_VALUE));
+ assertEquals(0, orderedCode.readNumIncreasing());
+ assertEquals(1, orderedCode.readNumIncreasing());
+ assertEquals(Long.valueOf(Integer.MAX_VALUE) + 1L, orderedCode.readNumIncreasing());
+ assertEquals(Integer.MAX_VALUE, orderedCode.readNumIncreasing());
+ }
+
+ @Test
+ public void testWriteNumDecreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumDecreasing(0);
+ orderedCode.writeNumDecreasing(1);
+ orderedCode.writeNumDecreasing(Long.MIN_VALUE);
+ orderedCode.writeNumDecreasing(Long.MAX_VALUE);
+ assertEquals(0, orderedCode.readNumDecreasing());
+ assertEquals(1, orderedCode.readNumDecreasing());
+ assertEquals(Long.MIN_VALUE, orderedCode.readNumDecreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readNumDecreasing());
+ }
+
+ @Test
+ public void testWriteNumDecreasing_unsignedInt() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeNumDecreasing(UnsignedInteger.fromIntBits(0));
+ orderedCode.writeNumDecreasing(UnsignedInteger.fromIntBits(1));
+ orderedCode.writeNumDecreasing(UnsignedInteger.fromIntBits(Integer.MIN_VALUE));
+ orderedCode.writeNumDecreasing(UnsignedInteger.fromIntBits(Integer.MAX_VALUE));
+ assertEquals(0, orderedCode.readNumDecreasing());
+ assertEquals(1, orderedCode.readNumDecreasing());
+ assertEquals(Long.valueOf(Integer.MAX_VALUE) + 1L, orderedCode.readNumDecreasing());
+ assertEquals(Integer.MAX_VALUE, orderedCode.readNumDecreasing());
+ }
+
+ /**
+ * Assert that encoding the specified long via
+ * {@link OrderedCode#writeSignedNumIncreasing(long)} results in the bytes
+ * represented by the specified string of hex digits.
+ * E.g. assertSignedNumIncreasingEncodingEquals("3fbf", -65) asserts that
+ * -65 is encoded as { (byte) 0x3f, (byte) 0xbf }.
+ */
+ private static void assertSignedNumIncreasingEncodingEquals(
+ String expectedHexEncoding, long num) {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeSignedNumIncreasing(num);
+ assertEquals(
+ "Unexpected encoding for " + num,
+ expectedHexEncoding,
+ bytesToHexString(orderedCode.getEncodedBytes()));
+ }
+
+ /**
+ * Assert that encoding various long values via
+ * {@link OrderedCode#writeSignedNumIncreasing(long)} produces the expected
+ * bytes. Expected byte sequences were generated via the c++ (authoritative)
+ * implementation of OrderedCode::WriteSignedNumIncreasing.
+ */
+ @Test
+ public void testSignedNumIncreasing_write() {
+ assertSignedNumIncreasingEncodingEquals(
+ "003f8000000000000000", Long.MIN_VALUE);
+ assertSignedNumIncreasingEncodingEquals(
+ "003f8000000000000001", Long.MIN_VALUE + 1);
+ assertSignedNumIncreasingEncodingEquals(
+ "077fffffff", Integer.MIN_VALUE - 1L);
+ assertSignedNumIncreasingEncodingEquals("0780000000", Integer.MIN_VALUE);
+ assertSignedNumIncreasingEncodingEquals(
+ "0780000001", Integer.MIN_VALUE + 1);
+ assertSignedNumIncreasingEncodingEquals("3fbf", -65);
+ assertSignedNumIncreasingEncodingEquals("40", -64);
+ assertSignedNumIncreasingEncodingEquals("41", -63);
+ assertSignedNumIncreasingEncodingEquals("7d", -3);
+ assertSignedNumIncreasingEncodingEquals("7e", -2);
+ assertSignedNumIncreasingEncodingEquals("7f", -1);
+ assertSignedNumIncreasingEncodingEquals("80", 0);
+ assertSignedNumIncreasingEncodingEquals("81", 1);
+ assertSignedNumIncreasingEncodingEquals("82", 2);
+ assertSignedNumIncreasingEncodingEquals("83", 3);
+ assertSignedNumIncreasingEncodingEquals("bf", 63);
+ assertSignedNumIncreasingEncodingEquals("c040", 64);
+ assertSignedNumIncreasingEncodingEquals("c041", 65);
+ assertSignedNumIncreasingEncodingEquals(
+ "f87ffffffe", Integer.MAX_VALUE - 1);
+ assertSignedNumIncreasingEncodingEquals("f87fffffff", Integer.MAX_VALUE);
+ assertSignedNumIncreasingEncodingEquals(
+ "f880000000", Integer.MAX_VALUE + 1L);
+ assertSignedNumIncreasingEncodingEquals(
+ "ffc07ffffffffffffffe", Long.MAX_VALUE - 1);
+ assertSignedNumIncreasingEncodingEquals(
+ "ffc07fffffffffffffff", Long.MAX_VALUE);
+ }
+
+ /**
+ * Convert a string of hex digits (e.g. "3fbf") to a byte[]
+ * (e.g. { (byte) 0x3f, (byte) 0xbf }).
+ */
+ private static byte[] bytesFromHexString(String hexDigits) {
+ return BaseEncoding.base16().lowerCase().decode(hexDigits);
+ }
+
+ /**
+ * Convert a byte[] (e.g. { (byte) 0x3f, (byte) 0xbf }) to a string of hex
+ * digits (e.g. "3fbf").
+ */
+ private static String bytesToHexString(byte[] bytes) {
+ return BaseEncoding.base16().lowerCase().encode(bytes);
+ }
+
+ /**
+ * Assert that decoding (via {@link OrderedCode#readSignedNumIncreasing()})
+ * the bytes represented by the specified string of hex digits results in the
+ * expected long value.
+ * E.g. assertDecodedSignedNumIncreasingEquals(-65, "3fbf") asserts that the
+ * byte array { (byte) 0x3f, (byte) 0xbf } is decoded as -65.
+ */
+ private static void assertDecodedSignedNumIncreasingEquals(
+ long expectedNum, String encodedHexString) {
+ OrderedCode orderedCode =
+ new OrderedCode(bytesFromHexString(encodedHexString));
+ assertEquals(
+ "Unexpected value when decoding 0x" + encodedHexString,
+ expectedNum,
+ orderedCode.readSignedNumIncreasing());
+ assertFalse(
+ "Unexpected encoded bytes remain after decoding 0x" + encodedHexString,
+ orderedCode.hasRemainingEncodedBytes());
+ }
+
+ /**
+ * Assert that decoding various sequences of bytes via
+ * {@link OrderedCode#readSignedNumIncreasing()} produces the expected long
+ * value.
+ * Input byte sequences were generated via the c++ (authoritative)
+ * implementation of OrderedCode::WriteSignedNumIncreasing.
+ */
+ @Test
+ public void testSignedNumIncreasing_read() {
+ assertDecodedSignedNumIncreasingEquals(
+ Long.MIN_VALUE, "003f8000000000000000");
+ assertDecodedSignedNumIncreasingEquals(
+ Long.MIN_VALUE + 1, "003f8000000000000001");
+ assertDecodedSignedNumIncreasingEquals(
+ Integer.MIN_VALUE - 1L, "077fffffff");
+ assertDecodedSignedNumIncreasingEquals(Integer.MIN_VALUE, "0780000000");
+ assertDecodedSignedNumIncreasingEquals(Integer.MIN_VALUE + 1, "0780000001");
+ assertDecodedSignedNumIncreasingEquals(-65, "3fbf");
+ assertDecodedSignedNumIncreasingEquals(-64, "40");
+ assertDecodedSignedNumIncreasingEquals(-63, "41");
+ assertDecodedSignedNumIncreasingEquals(-3, "7d");
+ assertDecodedSignedNumIncreasingEquals(-2, "7e");
+ assertDecodedSignedNumIncreasingEquals(-1, "7f");
+ assertDecodedSignedNumIncreasingEquals(0, "80");
+ assertDecodedSignedNumIncreasingEquals(1, "81");
+ assertDecodedSignedNumIncreasingEquals(2, "82");
+ assertDecodedSignedNumIncreasingEquals(3, "83");
+ assertDecodedSignedNumIncreasingEquals(63, "bf");
+ assertDecodedSignedNumIncreasingEquals(64, "c040");
+ assertDecodedSignedNumIncreasingEquals(65, "c041");
+ assertDecodedSignedNumIncreasingEquals(Integer.MAX_VALUE - 1, "f87ffffffe");
+ assertDecodedSignedNumIncreasingEquals(Integer.MAX_VALUE, "f87fffffff");
+ assertDecodedSignedNumIncreasingEquals(
+ Integer.MAX_VALUE + 1L, "f880000000");
+ assertDecodedSignedNumIncreasingEquals(
+ Long.MAX_VALUE - 1, "ffc07ffffffffffffffe");
+ assertDecodedSignedNumIncreasingEquals(
+ Long.MAX_VALUE, "ffc07fffffffffffffff");
+ }
+
+ /**
+ * Assert that encoding (via
+ * {@link OrderedCode#writeSignedNumIncreasing(long)}) the specified long
+ * value and then decoding (via {@link OrderedCode#readSignedNumIncreasing()})
+ * results in the original value.
+ */
+ private static void assertSignedNumIncreasingWriteAndReadIsLossless(
+ long num) {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeSignedNumIncreasing(num);
+ assertEquals(
+ "Unexpected result when decoding writeSignedNumIncreasing(" + num + ")",
+ num,
+ orderedCode.readSignedNumIncreasing());
+ assertFalse("Unexpected remaining encoded bytes after decoding " + num,
+ orderedCode.hasRemainingEncodedBytes());
+ }
+
+ /**
+ * Assert that for various long values, encoding (via
+ * {@link OrderedCode#writeSignedNumIncreasing(long)}) and then decoding (via
+ * {@link OrderedCode#readSignedNumIncreasing()}) results in the original
+ * value.
+ */
+ @Test
+ public void testSignedNumIncreasing_writeAndRead() {
+ assertSignedNumIncreasingWriteAndReadIsLossless(Long.MIN_VALUE);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Long.MIN_VALUE + 1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE - 1L);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE + 1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-65);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-64);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-63);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-3);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-2);
+ assertSignedNumIncreasingWriteAndReadIsLossless(-1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(0);
+ assertSignedNumIncreasingWriteAndReadIsLossless(1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(2);
+ assertSignedNumIncreasingWriteAndReadIsLossless(3);
+ assertSignedNumIncreasingWriteAndReadIsLossless(63);
+ assertSignedNumIncreasingWriteAndReadIsLossless(64);
+ assertSignedNumIncreasingWriteAndReadIsLossless(65);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE - 1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE + 1L);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Long.MAX_VALUE - 1);
+ assertSignedNumIncreasingWriteAndReadIsLossless(Long.MAX_VALUE);
+ }
+
+ /**
+ * Assert that encoding (via
+ * {@link OrderedCode#writeSignedNumDecreasing(long)}) the specified long
+ * value and then decoding (via {@link OrderedCode#readSignedNumDecreasing()})
+ * results in the original value.
+ */
+ private static void assertSignedNumDecreasingWriteAndReadIsLossless(
+ long num) {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeSignedNumDecreasing(num);
+ assertEquals(
+ "Unexpected result when decoding writeSignedNumDecreasing(" + num + ")",
+ num,
+ orderedCode.readSignedNumDecreasing());
+ assertFalse("Unexpected remaining encoded bytes after decoding " + num,
+ orderedCode.hasRemainingEncodedBytes());
+ }
+
+ /**
+ * Assert that for various long values, encoding (via
+ * {@link OrderedCode#writeSignedNumDecreasing(long)}) and then decoding (via
+ * {@link OrderedCode#readSignedNumDecreasing()}) results in the original
+ * value.
+ */
+ @Test
+ public void testSignedNumDecreasing_writeAndRead() {
+ assertSignedNumDecreasingWriteAndReadIsLossless(Long.MIN_VALUE);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Long.MIN_VALUE + 1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MIN_VALUE - 1L);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MIN_VALUE);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MIN_VALUE + 1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-65);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-64);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-63);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-3);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-2);
+ assertSignedNumDecreasingWriteAndReadIsLossless(-1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(0);
+ assertSignedNumDecreasingWriteAndReadIsLossless(1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(2);
+ assertSignedNumDecreasingWriteAndReadIsLossless(3);
+ assertSignedNumDecreasingWriteAndReadIsLossless(63);
+ assertSignedNumDecreasingWriteAndReadIsLossless(64);
+ assertSignedNumDecreasingWriteAndReadIsLossless(65);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MAX_VALUE - 1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MAX_VALUE);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Integer.MAX_VALUE + 1L);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Long.MAX_VALUE - 1);
+ assertSignedNumDecreasingWriteAndReadIsLossless(Long.MAX_VALUE);
+ }
+
+ /** Ensures that numbers encoded as "decreasing" do indeed sort in reverse order. */
+ @Test
+ public void testDecreasing() {
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeSignedNumDecreasing(10L);
+ byte[] ten = orderedCode.getEncodedBytes();
+ orderedCode = new OrderedCode();
+ orderedCode.writeSignedNumDecreasing(20L);
+ byte[] twenty = orderedCode.getEncodedBytes();
+ // In decreasing order, twenty preceeds ten.
+ assertTrue(compare(twenty, ten) < 0);
+ }
+
+ @Test
+ public void testLog2Floor_Positive() {
+ OrderedCode orderedCode = new OrderedCode();
+ assertEquals(0, orderedCode.log2Floor(1));
+ assertEquals(1, orderedCode.log2Floor(2));
+ assertEquals(1, orderedCode.log2Floor(3));
+ assertEquals(2, orderedCode.log2Floor(4));
+ assertEquals(5, orderedCode.log2Floor(63));
+ assertEquals(6, orderedCode.log2Floor(64));
+ assertEquals(62, orderedCode.log2Floor(Long.MAX_VALUE));
+ }
+
+ /**
+ * OrderedCode.log2Floor(long) is defined to return -1 given an input of zero
+ * (because that's what Bits::Log2Floor64(uint64) does).
+ */
+ @Test
+ public void testLog2Floor_zero() {
+ OrderedCode orderedCode = new OrderedCode();
+ assertEquals(-1, orderedCode.log2Floor(0));
+ }
+
+ @Test
+ public void testLog2Floor_negative() {
+ OrderedCode orderedCode = new OrderedCode();
+ try {
+ orderedCode.log2Floor(-1);
+ fail("Expected an IllegalArgumentException.");
+ } catch (IllegalArgumentException expected) {
+ // Expected!
+ }
+ }
+
+ @Test
+ public void testGetSignedEncodingLength() {
+ OrderedCode orderedCode = new OrderedCode();
+ assertEquals(10, orderedCode.getSignedEncodingLength(Long.MIN_VALUE));
+ assertEquals(10, orderedCode.getSignedEncodingLength(~(1L << 62)));
+ assertEquals(9, orderedCode.getSignedEncodingLength(~(1L << 62) + 1));
+ assertEquals(3, orderedCode.getSignedEncodingLength(-8193));
+ assertEquals(2, orderedCode.getSignedEncodingLength(-8192));
+ assertEquals(2, orderedCode.getSignedEncodingLength(-65));
+ assertEquals(1, orderedCode.getSignedEncodingLength(-64));
+ assertEquals(1, orderedCode.getSignedEncodingLength(-2));
+ assertEquals(1, orderedCode.getSignedEncodingLength(-1));
+ assertEquals(1, orderedCode.getSignedEncodingLength(0));
+ assertEquals(1, orderedCode.getSignedEncodingLength(1));
+ assertEquals(1, orderedCode.getSignedEncodingLength(63));
+ assertEquals(2, orderedCode.getSignedEncodingLength(64));
+ assertEquals(2, orderedCode.getSignedEncodingLength(8191));
+ assertEquals(3, orderedCode.getSignedEncodingLength(8192));
+ assertEquals(9, orderedCode.getSignedEncodingLength((1L << 62)) - 1);
+ assertEquals(10, orderedCode.getSignedEncodingLength(1L << 62));
+ assertEquals(10, orderedCode.getSignedEncodingLength(Long.MAX_VALUE));
+ }
+
+ @Test
+ public void testWriteTrailingBytes() {
+ byte[] escapeChars = new byte[] { OrderedCode.ESCAPE1,
+ OrderedCode.NULL_CHARACTER, OrderedCode.SEPARATOR, OrderedCode.ESCAPE2,
+ OrderedCode.INFINITY, OrderedCode.FF_CHARACTER};
+ byte[] anotherArray = new byte[] { 'a', 'b', 'c', 'd', 'e' };
+
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeTrailingBytes(escapeChars);
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), escapeChars));
+ assertTrue(Arrays.equals(orderedCode.readTrailingBytes(), escapeChars));
+ try {
+ orderedCode.readInfinity();
+ fail("Expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ orderedCode = new OrderedCode();
+ orderedCode.writeTrailingBytes(anotherArray);
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), anotherArray));
+ assertTrue(Arrays.equals(orderedCode.readTrailingBytes(), anotherArray));
+ }
+
+ @Test
+ public void testMixedWrite() {
+ byte[] first = { 'a', 'b', 'c'};
+ byte[] second = { 'd', 'e', 'f'};
+ byte[] last = { 'x', 'y', 'z'};
+ byte[] escapeChars = new byte[] { OrderedCode.ESCAPE1,
+ OrderedCode.NULL_CHARACTER, OrderedCode.SEPARATOR, OrderedCode.ESCAPE2,
+ OrderedCode.INFINITY, OrderedCode.FF_CHARACTER};
+
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytes(first);
+ orderedCode.writeBytes(second);
+ orderedCode.writeBytes(last);
+ orderedCode.writeInfinity();
+ orderedCode.writeNumIncreasing(0);
+ orderedCode.writeNumIncreasing(1);
+ orderedCode.writeNumIncreasing(Long.MIN_VALUE);
+ orderedCode.writeNumIncreasing(Long.MAX_VALUE);
+ orderedCode.writeSignedNumIncreasing(0);
+ orderedCode.writeSignedNumIncreasing(1);
+ orderedCode.writeSignedNumIncreasing(Long.MIN_VALUE);
+ orderedCode.writeSignedNumIncreasing(Long.MAX_VALUE);
+ orderedCode.writeTrailingBytes(escapeChars);
+ byte[] allEncoded = orderedCode.getEncodedBytes();
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), second));
+ assertFalse(orderedCode.readInfinity());
+ assertTrue(Arrays.equals(orderedCode.readBytes(), last));
+ assertTrue(orderedCode.readInfinity());
+ assertEquals(0, orderedCode.readNumIncreasing());
+ assertEquals(1, orderedCode.readNumIncreasing());
+ assertFalse(orderedCode.readInfinity());
+ assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
+ assertEquals(0, orderedCode.readSignedNumIncreasing());
+ assertEquals(1, orderedCode.readSignedNumIncreasing());
+ assertFalse(orderedCode.readInfinity());
+ assertEquals(Long.MIN_VALUE, orderedCode.readSignedNumIncreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readSignedNumIncreasing());
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), escapeChars));
+ assertTrue(Arrays.equals(orderedCode.readTrailingBytes(), escapeChars));
+
+ orderedCode = new OrderedCode(allEncoded);
+ assertTrue(Arrays.equals(orderedCode.readBytes(), first));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), second));
+ assertFalse(orderedCode.readInfinity());
+ assertTrue(Arrays.equals(orderedCode.readBytes(), last));
+ assertTrue(orderedCode.readInfinity());
+ assertEquals(0, orderedCode.readNumIncreasing());
+ assertEquals(1, orderedCode.readNumIncreasing());
+ assertFalse(orderedCode.readInfinity());
+ assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
+ assertEquals(0, orderedCode.readSignedNumIncreasing());
+ assertEquals(1, orderedCode.readSignedNumIncreasing());
+ assertFalse(orderedCode.readInfinity());
+ assertEquals(Long.MIN_VALUE, orderedCode.readSignedNumIncreasing());
+ assertEquals(Long.MAX_VALUE, orderedCode.readSignedNumIncreasing());
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(), escapeChars));
+ assertTrue(Arrays.equals(orderedCode.readTrailingBytes(), escapeChars));
+ }
+
+ @Test
+ public void testEdgeCases() {
+ byte[] ffChar = {OrderedCode.FF_CHARACTER};
+ byte[] nullChar = {OrderedCode.NULL_CHARACTER};
+
+ byte[] separatorEncoded = {OrderedCode.ESCAPE1, OrderedCode.SEPARATOR};
+ byte[] ffCharEncoded = {OrderedCode.ESCAPE1, OrderedCode.NULL_CHARACTER};
+ byte[] nullCharEncoded = {OrderedCode.ESCAPE2, OrderedCode.FF_CHARACTER};
+ byte[] infinityEncoded = {OrderedCode.ESCAPE2, OrderedCode.INFINITY};
+
+ OrderedCode orderedCode = new OrderedCode();
+ orderedCode.writeBytes(ffChar);
+ orderedCode.writeBytes(nullChar);
+ orderedCode.writeInfinity();
+ assertTrue(Arrays.equals(orderedCode.getEncodedBytes(),
+ Bytes.concat(ffCharEncoded, separatorEncoded,
+ nullCharEncoded, separatorEncoded,
+ infinityEncoded)));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), ffChar));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), nullChar));
+ assertTrue(orderedCode.readInfinity());
+
+ orderedCode = new OrderedCode(
+ Bytes.concat(ffCharEncoded, separatorEncoded));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), ffChar));
+
+ orderedCode = new OrderedCode(
+ Bytes.concat(nullCharEncoded, separatorEncoded));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), nullChar));
+
+ byte[] invalidEncodingForRead = {OrderedCode.ESCAPE2, OrderedCode.ESCAPE2,
+ OrderedCode.ESCAPE1, OrderedCode.SEPARATOR};
+ orderedCode = new OrderedCode(invalidEncodingForRead);
+ try {
+ orderedCode.readBytes();
+ fail("Should have failed.");
+ } catch (Exception e) {
+ // Expected
+ }
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ }
+
+ @Test
+ public void testHasRemainingEncodedBytes() {
+ byte[] bytes = { 'a', 'b', 'c'};
+ long number = 12345;
+
+ // Empty
+ OrderedCode orderedCode = new OrderedCode();
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ // First and only field of each type.
+ orderedCode.writeBytes(bytes);
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertTrue(Arrays.equals(orderedCode.readBytes(), bytes));
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ orderedCode.writeNumIncreasing(number);
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertEquals(orderedCode.readNumIncreasing(), number);
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ orderedCode.writeSignedNumIncreasing(number);
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertEquals(orderedCode.readSignedNumIncreasing(), number);
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ orderedCode.writeInfinity();
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertTrue(orderedCode.readInfinity());
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ orderedCode.writeTrailingBytes(bytes);
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertTrue(Arrays.equals(orderedCode.readTrailingBytes(), bytes));
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+
+ // Two fields of same type.
+ orderedCode.writeBytes(bytes);
+ orderedCode.writeBytes(bytes);
+ assertTrue(orderedCode.hasRemainingEncodedBytes());
+ assertTrue(Arrays.equals(orderedCode.readBytes(), bytes));
+ assertTrue(Arrays.equals(orderedCode.readBytes(), bytes));
+ assertFalse(orderedCode.hasRemainingEncodedBytes());
+ }
+
+ @Test
+ public void testOrderingInfinity() {
+ OrderedCode inf = new OrderedCode();
+ inf.writeInfinity();
+
+ OrderedCode negInf = new OrderedCode();
+ negInf.writeInfinityDecreasing();
+
+ OrderedCode longValue = new OrderedCode();
+ longValue.writeSignedNumIncreasing(1);
+
+ assertTrue(compare(inf.getEncodedBytes(), negInf.getEncodedBytes()) > 0);
+ assertTrue(compare(longValue.getEncodedBytes(), negInf.getEncodedBytes()) > 0);
+ assertTrue(compare(inf.getEncodedBytes(), longValue.getEncodedBytes()) > 0);
+ }
+
+ private int compare(byte[] bytes1, byte[] bytes2) {
+ return UnsignedBytes.lexicographicalComparator().compare(bytes1, bytes2);
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/RandomUtils.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/RandomUtils.java
new file mode 100644
index 0000000..f479b4a
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/RandomUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import java.util.Random;
+
+/**
+ * Useful randomness related utilities.
+ */
+public class RandomUtils {
+
+ private static final char[] ALPHANUMERIC = "1234567890abcdefghijklmnopqrstuvwxyz".toCharArray();
+
+ private RandomUtils() {
+ }
+
+ public static String randomAlphaNumeric(int length) {
+ Random random = new Random();
+ char[] result = new char[length];
+ for (int i = 0; i < length; i++) {
+ result[i] = ALPHANUMERIC[random.nextInt(ALPHANUMERIC.length)];
+ }
+ return new String(result);
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchemaTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchemaTest.java
new file mode 100644
index 0000000..25dc6dc
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchemaTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.hamcrest.Matchers.contains;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSets;
+import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.Type;
+import com.google.cloud.spanner.Value;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * A test of {@link ReadSpannerSchemaTest}.
+ */
+public class ReadSpannerSchemaTest {
+
+ @Rule
+ public final transient ExpectedException thrown = ExpectedException.none();
+
+ private FakeServiceFactory serviceFactory;
+ private ReadOnlyTransaction mockTx;
+
+
+ private static Struct columnMetadata(String tableName, String columnName, String type) {
+ return Struct.newBuilder().add("table_name", Value.string(tableName))
+ .add("column_name", Value.string(columnName)).add("spanner_type", Value.string(type))
+ .build();
+ }
+
+ private static Struct pkMetadata(String tableName, String columnName, String ordering) {
+ return Struct.newBuilder().add("table_name", Value.string(tableName))
+ .add("column_name", Value.string(columnName)).add("column_ordering", Value.string(ordering))
+ .build();
+ }
+
+ private void prepareColumnMetadata(ReadOnlyTransaction tx, List<Struct> rows) {
+ Type type = Type.struct(Type.StructField.of("table_name", Type.string()),
+ Type.StructField.of("column_name", Type.string()),
+ Type.StructField.of("spanner_type", Type.string()));
+ when(tx.executeQuery(argThat(new ArgumentMatcher<Statement>() {
+
+ @Override public boolean matches(Object argument) {
+ if (!(argument instanceof Statement)) {
+ return false;
+ }
+ Statement st = (Statement) argument;
+ return st.getSql().contains("information_schema.columns");
+ }
+ }))).thenReturn(ResultSets.forRows(type, rows));
+ }
+
+ private void preparePkMetadata(ReadOnlyTransaction tx, List<Struct> rows) {
+ Type type = Type.struct(Type.StructField.of("table_name", Type.string()),
+ Type.StructField.of("column_name", Type.string()),
+ Type.StructField.of("column_ordering", Type.string()));
+ when(tx.executeQuery(argThat(new ArgumentMatcher<Statement>() {
+
+ @Override public boolean matches(Object argument) {
+ if (!(argument instanceof Statement)) {
+ return false;
+ }
+ Statement st = (Statement) argument;
+ return st.getSql().contains("information_schema.index_columns");
+ }
+ }))).thenReturn(ResultSets.forRows(type, rows));
+ }
+
+ @Before
+ @SuppressWarnings("unchecked")
+ public void setUp() throws Exception {
+ serviceFactory = new FakeServiceFactory();
+ mockTx = mock(ReadOnlyTransaction.class);
+ }
+
+ @Test
+ public void simple() throws Exception {
+ // Simplest schema: a table with int64 key
+ ReadOnlyTransaction tx = mock(ReadOnlyTransaction.class);
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction()).thenReturn(tx);
+
+ preparePkMetadata(tx, Arrays.asList(pkMetadata("test", "key", "ASC")));
+ prepareColumnMetadata(tx, Arrays.asList(columnMetadata("test", "key", "INT64")));
+
+ SpannerConfig config = SpannerConfig.create().withProjectId("test-project")
+ .withInstanceId("test-instance").withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory);
+
+ DoFnTester<Void, SpannerSchema> tester = DoFnTester.of(new ReadSpannerSchema(config));
+ List<SpannerSchema> schemas = tester.processBundle(Arrays.asList((Void) null));
+
+ assertEquals(1, schemas.size());
+
+ SpannerSchema schema = schemas.get(0);
+
+ assertEquals(1, schema.getTables().size());
+
+ SpannerSchema.Column column = SpannerSchema.Column.create("key", Type.int64());
+ SpannerSchema.KeyPart keyPart = SpannerSchema.KeyPart.create("key", false);
+
+ assertThat(schema.getColumns("test"), contains(column));
+ assertThat(schema.getKeyParts("test"), contains(keyPart));
+ }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOReadTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOReadTest.java
new file mode 100644
index 0000000..ad4e47b
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOReadTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSets;
+import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.TimestampBound;
+import com.google.cloud.spanner.Type;
+import com.google.cloud.spanner.Value;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+/** Unit tests for {@link SpannerIO}. */
+@RunWith(JUnit4.class)
+public class SpannerIOReadTest implements Serializable {
+
+ @Rule
+ public final transient TestPipeline pipeline = TestPipeline.create();
+ @Rule
+ public final transient ExpectedException thrown = ExpectedException.none();
+
+ private FakeServiceFactory serviceFactory;
+ private ReadOnlyTransaction mockTx;
+
+ private static final Type FAKE_TYPE =
+ Type.struct(
+ Type.StructField.of("id", Type.int64()), Type.StructField.of("name", Type.string()));
+
+ private static final List<Struct> FAKE_ROWS =
+ Arrays.asList(
+ Struct.newBuilder().add("id", Value.int64(1)).add("name", Value.string("Alice")).build(),
+ Struct.newBuilder().add("id", Value.int64(2)).add("name", Value.string("Bob")).build(),
+ Struct.newBuilder().add("id", Value.int64(3)).add("name", Value.string("Carl")).build(),
+ Struct.newBuilder().add("id", Value.int64(4)).add("name", Value.string("Dan")).build());
+
+ @Before
+ @SuppressWarnings("unchecked")
+ public void setUp() throws Exception {
+ serviceFactory = new FakeServiceFactory();
+ mockTx = Mockito.mock(ReadOnlyTransaction.class);
+ }
+
+ @Test
+ public void runQuery() throws Exception {
+ SpannerIO.Read read =
+ SpannerIO.read()
+ .withProjectId("test")
+ .withInstanceId("123")
+ .withDatabaseId("aaa")
+ .withQuery("SELECT * FROM users")
+ .withServiceFactory(serviceFactory);
+
+ NaiveSpannerReadFn readFn = new NaiveSpannerReadFn(read.getSpannerConfig());
+ DoFnTester<ReadOperation, Struct> fnTester = DoFnTester.of(readFn);
+
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction(any(TimestampBound.class)))
+ .thenReturn(mockTx);
+ when(mockTx.executeQuery(any(Statement.class)))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS));
+
+ List<Struct> result = fnTester.processBundle(read.getReadOperation());
+ assertThat(result, Matchers.containsInAnyOrder(FAKE_ROWS.toArray()));
+
+ verify(serviceFactory.mockDatabaseClient()).readOnlyTransaction(TimestampBound
+ .strong());
+ verify(mockTx).executeQuery(Statement.of("SELECT * FROM users"));
+ }
+
+ @Test
+ public void runRead() throws Exception {
+ SpannerIO.Read read =
+ SpannerIO.read()
+ .withProjectId("test")
+ .withInstanceId("123")
+ .withDatabaseId("aaa")
+ .withTable("users")
+ .withColumns("id", "name")
+ .withServiceFactory(serviceFactory);
+
+ NaiveSpannerReadFn readFn = new NaiveSpannerReadFn(read.getSpannerConfig());
+ DoFnTester<ReadOperation, Struct> fnTester = DoFnTester.of(readFn);
+
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction(any(TimestampBound.class)))
+ .thenReturn(mockTx);
+ when(mockTx.read("users", KeySet.all(), Arrays.asList("id", "name")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS));
+
+ List<Struct> result = fnTester.processBundle(read.getReadOperation());
+ assertThat(result, Matchers.containsInAnyOrder(FAKE_ROWS.toArray()));
+
+ verify(serviceFactory.mockDatabaseClient()).readOnlyTransaction(TimestampBound.strong());
+ verify(mockTx).read("users", KeySet.all(), Arrays.asList("id", "name"));
+ }
+
+ @Test
+ public void runReadUsingIndex() throws Exception {
+ SpannerIO.Read read =
+ SpannerIO.read()
+ .withProjectId("test")
+ .withInstanceId("123")
+ .withDatabaseId("aaa")
+ .withTimestamp(Timestamp.now())
+ .withTable("users")
+ .withColumns("id", "name")
+ .withIndex("theindex")
+ .withServiceFactory(serviceFactory);
+
+ NaiveSpannerReadFn readFn = new NaiveSpannerReadFn(read.getSpannerConfig());
+ DoFnTester<ReadOperation, Struct> fnTester = DoFnTester.of(readFn);
+
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction(any(TimestampBound.class)))
+ .thenReturn(mockTx);
+ when(mockTx.readUsingIndex("users", "theindex", KeySet.all(), Arrays.asList("id", "name")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS));
+
+ List<Struct> result = fnTester.processBundle(read.getReadOperation());
+ assertThat(result, Matchers.containsInAnyOrder(FAKE_ROWS.toArray()));
+
+ verify(serviceFactory.mockDatabaseClient()).readOnlyTransaction(TimestampBound.strong());
+ verify(mockTx).readUsingIndex("users", "theindex", KeySet.all(), Arrays.asList("id", "name"));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void readPipeline() throws Exception {
+ Timestamp timestamp = Timestamp.ofTimeMicroseconds(12345);
+
+ SpannerConfig spannerConfig =
+ SpannerConfig.create()
+ .withProjectId("test")
+ .withInstanceId("123")
+ .withDatabaseId("aaa")
+ .withServiceFactory(serviceFactory);
+
+ PCollectionView<Transaction> tx =
+ pipeline.apply("tx", SpannerIO.createTransaction().withSpannerConfig(spannerConfig));
+
+ PCollection<Struct> one =
+ pipeline.apply(
+ "read q",
+ SpannerIO.read()
+ .withSpannerConfig(spannerConfig)
+ .withQuery("SELECT * FROM users")
+ .withTransaction(tx));
+ PCollection<Struct> two =
+ pipeline.apply(
+ "read r",
+ SpannerIO.read()
+ .withSpannerConfig(spannerConfig)
+ .withTimestamp(Timestamp.now())
+ .withTable("users")
+ .withColumns("id", "name")
+ .withTransaction(tx));
+
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction(any(TimestampBound.class)))
+ .thenReturn(mockTx);
+
+ when(mockTx.executeQuery(Statement.of("SELECT 1"))).thenReturn(ResultSets.forRows(Type.struct(),
+ Collections.<Struct>emptyList()));
+
+ when(mockTx.executeQuery(Statement.of("SELECT * FROM users")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS));
+ when(mockTx.read("users", KeySet.all(), Arrays.asList("id", "name")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS));
+ when(mockTx.getReadTimestamp()).thenReturn(timestamp);
+
+ PAssert.that(one).containsInAnyOrder(FAKE_ROWS);
+ PAssert.that(two).containsInAnyOrder(FAKE_ROWS);
+
+ pipeline.run();
+
+ verify(serviceFactory.mockDatabaseClient(), times(2))
+ .readOnlyTransaction(TimestampBound.ofReadTimestamp(timestamp));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void readAllPipeline() throws Exception {
+ Timestamp timestamp = Timestamp.ofTimeMicroseconds(12345);
+
+ SpannerConfig spannerConfig =
+ SpannerConfig.create()
+ .withProjectId("test")
+ .withInstanceId("123")
+ .withDatabaseId("aaa")
+ .withServiceFactory(serviceFactory);
+
+ PCollectionView<Transaction> tx =
+ pipeline.apply("tx", SpannerIO.createTransaction().withSpannerConfig(spannerConfig));
+
+ PCollection<ReadOperation> reads =
+ pipeline.apply(
+ Create.of(
+ ReadOperation.create().withQuery("SELECT * FROM users"),
+ ReadOperation.create().withTable("users").withColumns("id", "name")));
+
+ PCollection<Struct> one =
+ reads.apply(
+ "read all", SpannerIO.readAll().withSpannerConfig(spannerConfig).withTransaction(tx));
+
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction(any(TimestampBound.class)))
+ .thenReturn(mockTx);
+
+ when(mockTx.executeQuery(Statement.of("SELECT 1")))
+ .thenReturn(ResultSets.forRows(Type.struct(), Collections.<Struct>emptyList()));
+
+ when(mockTx.executeQuery(Statement.of("SELECT * FROM users")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS.subList(0, 2)));
+ when(mockTx.read("users", KeySet.all(), Arrays.asList("id", "name")))
+ .thenReturn(ResultSets.forRows(FAKE_TYPE, FAKE_ROWS.subList(2, 4)));
+ when(mockTx.getReadTimestamp()).thenReturn(timestamp);
+
+ PAssert.that(one).containsInAnyOrder(FAKE_ROWS);
+
+ pipeline.run();
+
+ verify(serviceFactory.mockDatabaseClient(), times(2))
+ .readOnlyTransaction(TimestampBound.ofReadTimestamp(timestamp));
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java
new file mode 100644
index 0000000..de1d403
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeyRange;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.ReadOnlyTransaction;
+import com.google.cloud.spanner.ResultSets;
+import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.Type;
+import com.google.cloud.spanner.Value;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.hamcrest.Description;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Unit tests for {@link SpannerIO}.
+ */
+@RunWith(JUnit4.class)
+public class SpannerIOWriteTest implements Serializable {
+
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+ private FakeServiceFactory serviceFactory;
+
+ @Before @SuppressWarnings("unchecked") public void setUp() throws Exception {
+ serviceFactory = new FakeServiceFactory();
+
+ ReadOnlyTransaction tx = mock(ReadOnlyTransaction.class);
+ when(serviceFactory.mockDatabaseClient().readOnlyTransaction()).thenReturn(tx);
+
+ // Simplest schema: a table with int64 key
+ preparePkMetadata(tx, Arrays.asList(pkMetadata("test", "key", "ASC")));
+ prepareColumnMetadata(tx, Arrays.asList(columnMetadata("test", "key", "INT64")));
+ }
+
+ private static Struct columnMetadata(String tableName, String columnName, String type) {
+ return Struct.newBuilder().add("table_name", Value.string(tableName))
+ .add("column_name", Value.string(columnName)).add("spanner_type", Value.string(type))
+ .build();
+ }
+
+ private static Struct pkMetadata(String tableName, String columnName, String ordering) {
+ return Struct.newBuilder().add("table_name", Value.string(tableName))
+ .add("column_name", Value.string(columnName)).add("column_ordering", Value.string(ordering))
+ .build();
+ }
+
+ private void prepareColumnMetadata(ReadOnlyTransaction tx, List<Struct> rows) {
+ Type type = Type.struct(Type.StructField.of("table_name", Type.string()),
+ Type.StructField.of("column_name", Type.string()),
+ Type.StructField.of("spanner_type", Type.string()));
+ when(tx.executeQuery(argThat(new ArgumentMatcher<Statement>() {
+
+ @Override public boolean matches(Object argument) {
+ if (!(argument instanceof Statement)) {
+ return false;
+ }
+ Statement st = (Statement) argument;
+ return st.getSql().contains("information_schema.columns");
+ }
+ }))).thenReturn(ResultSets.forRows(type, rows));
+ }
+
+ private void preparePkMetadata(ReadOnlyTransaction tx, List<Struct> rows) {
+ Type type = Type.struct(Type.StructField.of("table_name", Type.string()),
+ Type.StructField.of("column_name", Type.string()),
+ Type.StructField.of("column_ordering", Type.string()));
+ when(tx.executeQuery(argThat(new ArgumentMatcher<Statement>() {
+
+ @Override public boolean matches(Object argument) {
+ if (!(argument instanceof Statement)) {
+ return false;
+ }
+ Statement st = (Statement) argument;
+ return st.getSql().contains("information_schema.index_columns");
+ }
+ }))).thenReturn(ResultSets.forRows(type, rows));
+ }
+
+
+ @Test
+ public void emptyTransform() throws Exception {
+ SpannerIO.Write write = SpannerIO.write();
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage("requires instance id to be set with");
+ write.expand(null);
+ }
+
+ @Test
+ public void emptyInstanceId() throws Exception {
+ SpannerIO.Write write = SpannerIO.write().withDatabaseId("123");
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage("requires instance id to be set with");
+ write.expand(null);
+ }
+
+ @Test
+ public void emptyDatabaseId() throws Exception {
+ SpannerIO.Write write = SpannerIO.write().withInstanceId("123");
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage("requires database id to be set with");
+ write.expand(null);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void singleMutationPipeline() throws Exception {
+ Mutation mutation = m(2L);
+ PCollection<Mutation> mutations = pipeline.apply(Create.of(mutation));
+
+ mutations.apply(
+ SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory));
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(2L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void singleMutationGroupPipeline() throws Exception {
+ PCollection<MutationGroup> mutations = pipeline
+ .apply(Create.<MutationGroup>of(g(m(1L), m(2L), m(3L))));
+ mutations.apply(
+ SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L), m(2L), m(3L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void batching() throws Exception {
+ MutationGroup one = g(m(1L));
+ MutationGroup two = g(m(2L));
+ PCollection<MutationGroup> mutations = pipeline.apply(Create.of(one, two));
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1000000000)
+ .withSampler(fakeSampler(m(1000L)))
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L), m(2L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void batchingWithDeletes() throws Exception {
+ PCollection<MutationGroup> mutations = pipeline
+ .apply(Create.of(g(m(1L)), g(m(2L)), g(del(3L)), g(del(4L))));
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1000000000)
+ .withSampler(fakeSampler(m(1000L)))
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L), m(2L), del(3L), del(4L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void noBatchingRangeDelete() throws Exception {
+ Mutation all = Mutation.delete("test", KeySet.all());
+ Mutation prefix = Mutation.delete("test", KeySet.prefixRange(Key.of(1L)));
+ Mutation range = Mutation.delete("test", KeySet.range(KeyRange.openOpen(Key.of(1L), Key
+ .newBuilder().build())));
+
+ PCollection<MutationGroup> mutations = pipeline.apply(Create
+ .of(
+ g(m(1L)),
+ g(m(2L)),
+ g(del(5L, 6L)),
+ g(delRange(50L, 55L)),
+ g(delRange(11L, 20L)),
+ g(all),
+ g(prefix), g(range)
+ )
+ );
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1000000000)
+ .withSampler(fakeSampler(m(1000L)))
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L), m(2L)),
+ batch(del(5L, 6L)),
+ batch(delRange(11L, 20L)),
+ batch(delRange(50L, 55L)),
+ batch(all),
+ batch(prefix),
+ batch(range)
+ );
+ }
+
+ private void verifyBatches(Iterable<Mutation>... batches) {
+ for (Iterable<Mutation> b : batches) {
+ verify(serviceFactory.mockDatabaseClient(), times(1)).writeAtLeastOnce(mutationsInNoOrder(b));
+ }
+
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void batchingGroups() throws Exception {
+
+ // Have a room to accumulate one more item.
+ long batchSize = MutationSizeEstimator.sizeOf(g(m(1L))) + 1;
+
+ PCollection<MutationGroup> mutations = pipeline.apply(Create.of(g(m(1L)), g(m(2L)), g(m(3L))));
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(batchSize)
+ .withSampler(fakeSampler(m(1000L)))
+ .grouped());
+
+ pipeline.run();
+
+ // The content of batches is not deterministic. Just verify that the size is correct.
+ verify(serviceFactory.mockDatabaseClient(), times(1))
+ .writeAtLeastOnce(iterableOfSize(2));
+ verify(serviceFactory.mockDatabaseClient(), times(1))
+ .writeAtLeastOnce(iterableOfSize(1));
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void noBatching() throws Exception {
+ PCollection<MutationGroup> mutations = pipeline.apply(Create.of(g(m(1L)), g(m(2L))));
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1)
+ .withSampler(fakeSampler(m(1000L)))
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L)),
+ batch(m(2L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void batchingPlusSampling() throws Exception {
+ PCollection<MutationGroup> mutations = pipeline
+ .apply(Create.of(
+ g(m(1L)), g(m(2L)), g(m(3L)), g(m(4L)), g(m(5L)),
+ g(m(6L)), g(m(7L)), g(m(8L)), g(m(9L)), g(m(10L)))
+ );
+
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1000000000)
+ .withSampler(fakeSampler(m(2L), m(5L), m(10L)))
+ .grouped());
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L), m(2L)),
+ batch(m(3L), m(4L), m(5L)),
+ batch(m(6L), m(7L), m(8L), m(9L), m(10L))
+ );
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void noBatchingPlusSampling() throws Exception {
+ PCollection<MutationGroup> mutations = pipeline
+ .apply(Create.of(g(m(1L)), g(m(2L)), g(m(3L)), g(m(4L)), g(m(5L))));
+ mutations.apply(SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withServiceFactory(serviceFactory)
+ .withBatchSizeBytes(1)
+ .withSampler(fakeSampler(m(2L)))
+ .grouped());
+
+ pipeline.run();
+
+ verifyBatches(
+ batch(m(1L)),
+ batch(m(2L)),
+ batch(m(3L)),
+ batch(m(4L)),
+ batch(m(5L))
+ );
+ }
+
+ @Test
+ public void displayData() throws Exception {
+ SpannerIO.Write write =
+ SpannerIO.write()
+ .withProjectId("test-project")
+ .withInstanceId("test-instance")
+ .withDatabaseId("test-database")
+ .withBatchSizeBytes(123);
+
+ DisplayData data = DisplayData.from(write);
+ assertThat(data.items(), hasSize(4));
+ assertThat(data, hasDisplayItem("projectId", "test-project"));
+ assertThat(data, hasDisplayItem("instanceId", "test-instance"));
+ assertThat(data, hasDisplayItem("databaseId", "test-database"));
+ assertThat(data, hasDisplayItem("batchSizeBytes", 123));
+ }
+
+ private static MutationGroup g(Mutation m, Mutation... other) {
+ return MutationGroup.create(m, other);
+ }
+
+ private static Mutation m(Long key) {
+ return Mutation.newInsertOrUpdateBuilder("test").set("key").to(key).build();
+ }
+
+ private static Iterable<Mutation> batch(Mutation... m) {
+ return Arrays.asList(m);
+ }
+
+ private static Mutation del(Long... keys) {
+
+ KeySet.Builder builder = KeySet.newBuilder();
+ for (Long key : keys) {
+ builder.addKey(Key.of(key));
+ }
+ return Mutation.delete("test", builder.build());
+ }
+
+ private static Mutation delRange(Long start, Long end) {
+ return Mutation.delete("test", KeySet.range(KeyRange.closedClosed(Key.of(start), Key.of(end))));
+ }
+
+ private static Iterable<Mutation> mutationsInNoOrder(Iterable<Mutation> expected) {
+ final ImmutableSet<Mutation> mutations = ImmutableSet.copyOf(expected);
+ return argThat(new ArgumentMatcher<Iterable<Mutation>>() {
+
+ @Override
+ public boolean matches(Object argument) {
+ if (!(argument instanceof Iterable)) {
+ return false;
+ }
+ ImmutableSet<Mutation> actual = ImmutableSet.copyOf((Iterable) argument);
+ return actual.equals(mutations);
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("Iterable must match ").appendValue(mutations);
+ }
+
+ });
+ }
+
+ private Iterable<Mutation> iterableOfSize(final int size) {
+ return argThat(new ArgumentMatcher<Iterable<Mutation>>() {
+
+ @Override
+ public boolean matches(Object argument) {
+ return argument instanceof Iterable && Iterables.size((Iterable<?>) argument) == size;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("The size of the iterable must equal ").appendValue(size);
+ }
+ });
+ }
+
+ private static FakeSampler fakeSampler(Mutation... mutations) {
+ SpannerSchema.Builder schema = SpannerSchema.builder();
+ schema.addColumn("test", "key", "INT64");
+ schema.addKeyPart("test", "key", false);
+ return new FakeSampler(schema.build(), Arrays.asList(mutations));
+ }
+
+ private static class FakeSampler
+ extends PTransform<PCollection<KV<String, byte[]>>, PCollection<KV<String, List<byte[]>>>> {
+
+ private final SpannerSchema schema;
+ private final List<Mutation> mutations;
+
+ private FakeSampler(SpannerSchema schema, List<Mutation> mutations) {
+ this.schema = schema;
+ this.mutations = mutations;
+ }
+
+ @Override
+ public PCollection<KV<String, List<byte[]>>> expand(
+ PCollection<KV<String, byte[]>> input) {
+ MutationGroupEncoder coder = new MutationGroupEncoder(schema);
+ Map<String, List<byte[]>> map = new HashMap<>();
+ for (Mutation m : mutations) {
+ String table = m.getTable();
+ List<byte[]> list = map.get(table);
+ if (list == null) {
+ list = new ArrayList<>();
+ map.put(table, list);
+ }
+ list.add(coder.encodeKey(m));
+ }
+ List<KV<String, List<byte[]>>> result = new ArrayList<>();
+ for (Map.Entry<String, List<byte[]>> entry : map.entrySet()) {
+ Collections.sort(entry.getValue(), SpannerIO.SerializableBytesComparator.INSTANCE);
+ result.add(KV.of(entry.getKey(), entry.getValue()));
+ }
+ return input.getPipeline().apply(Create.of(result));
+ }
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java
new file mode 100644
index 0000000..d866975
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Operation;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.TimestampBound;
+import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** End-to-end test of Cloud Spanner Source. */
+@RunWith(JUnit4.class)
+public class SpannerReadIT {
+
+ private static final int MAX_DB_NAME_LENGTH = 30;
+
+ @Rule public final transient TestPipeline p = TestPipeline.create();
+
+ /** Pipeline options for this test. */
+ public interface SpannerTestPipelineOptions extends TestPipelineOptions {
+ @Description("Instance ID to write to in Spanner")
+ @Default.String("beam-test")
+ String getInstanceId();
+ void setInstanceId(String value);
+
+ @Description("Database ID prefix to write to in Spanner")
+ @Default.String("beam-testdb")
+ String getDatabaseIdPrefix();
+ void setDatabaseIdPrefix(String value);
+
+ @Description("Table name")
+ @Default.String("users")
+ String getTable();
+ void setTable(String value);
+ }
+
+ private Spanner spanner;
+ private DatabaseAdminClient databaseAdminClient;
+ private SpannerTestPipelineOptions options;
+ private String databaseName;
+ private String project;
+
+ @Before
+ public void setUp() throws Exception {
+ PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
+ options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);
+
+ project = options.as(GcpOptions.class).getProject();
+
+ spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();
+
+ databaseName = generateDatabaseName();
+
+ databaseAdminClient = spanner.getDatabaseAdminClient();
+
+ // Delete database if exists.
+ databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);
+
+ Operation<Database, CreateDatabaseMetadata> op =
+ databaseAdminClient.createDatabase(
+ options.getInstanceId(),
+ databaseName,
+ Collections.singleton(
+ "CREATE TABLE "
+ + options.getTable()
+ + " ("
+ + " Key INT64,"
+ + " Value STRING(MAX),"
+ + ") PRIMARY KEY (Key)"));
+ op.waitFor();
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ DatabaseClient databaseClient =
+ spanner.getDatabaseClient(
+ DatabaseId.of(
+ project, options.getInstanceId(), databaseName));
+
+ List<Mutation> mutations = new ArrayList<>();
+ for (int i = 0; i < 5L; i++) {
+ mutations.add(
+ Mutation.newInsertOrUpdateBuilder(options.getTable())
+ .set("key")
+ .to((long) i)
+ .set("value")
+ .to(RandomUtils.randomAlphaNumeric(100))
+ .build());
+ }
+
+ databaseClient.writeAtLeastOnce(mutations);
+
+ SpannerConfig spannerConfig = SpannerConfig.create()
+ .withProjectId(project)
+ .withInstanceId(options.getInstanceId())
+ .withDatabaseId(databaseName);
+
+ PCollectionView<Transaction> tx =
+ p.apply(
+ SpannerIO.createTransaction()
+ .withSpannerConfig(spannerConfig)
+ .withTimestampBound(TimestampBound.strong()));
+
+ PCollection<Struct> output =
+ p.apply(
+ SpannerIO.read()
+ .withSpannerConfig(spannerConfig)
+ .withQuery("SELECT * FROM " + options.getTable())
+ .withTransaction(tx));
+ PAssert.thatSingleton(output.apply("Count rows", Count.<Struct>globally())).isEqualTo(5L);
+ p.run();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);
+ spanner.close();
+ }
+
+ private String generateDatabaseName() {
+ String random = RandomUtils
+ .randomAlphaNumeric(MAX_DB_NAME_LENGTH - 1 - options.getDatabaseIdPrefix().length());
+ return options.getDatabaseIdPrefix() + "-" + random;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java
new file mode 100644
index 0000000..fcb23dc
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+/**
+ * A test of {@link SpannerSchema}.
+ */
+public class SpannerSchemaTest {
+
+ @Test
+ public void testSingleTable() throws Exception {
+ SpannerSchema schema = SpannerSchema.builder()
+ .addColumn("test", "pk", "STRING(48)")
+ .addKeyPart("test", "pk", false)
+ .addColumn("test", "maxKey", "STRING(MAX)").build();
+
+ assertEquals(1, schema.getTables().size());
+ assertEquals(2, schema.getColumns("test").size());
+ assertEquals(1, schema.getKeyParts("test").size());
+ }
+
+ @Test
+ public void testTwoTables() throws Exception {
+ SpannerSchema schema = SpannerSchema.builder()
+ .addColumn("test", "pk", "STRING(48)")
+ .addKeyPart("test", "pk", false)
+ .addColumn("test", "maxKey", "STRING(MAX)")
+
+ .addColumn("other", "pk", "INT64")
+ .addKeyPart("other", "pk", true)
+ .addColumn("other", "maxKey", "STRING(MAX)")
+
+ .build();
+
+ assertEquals(2, schema.getTables().size());
+ assertEquals(2, schema.getColumns("test").size());
+ assertEquals(1, schema.getKeyParts("test").size());
+
+ assertEquals(2, schema.getColumns("other").size());
+ assertEquals(1, schema.getKeyParts("other").size());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java
new file mode 100644
index 0000000..89be159
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Operation;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.cloud.spanner.Statement;
+import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
+import java.util.Collections;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** End-to-end test of Cloud Spanner Sink. */
+@RunWith(JUnit4.class)
+public class SpannerWriteIT {
+
+ private static final int MAX_DB_NAME_LENGTH = 30;
+
+ @Rule public final transient TestPipeline p = TestPipeline.create();
+
+ /** Pipeline options for this test. */
+ public interface SpannerTestPipelineOptions extends TestPipelineOptions {
+ @Description("Instance ID to write to in Spanner")
+ @Default.String("beam-test")
+ String getInstanceId();
+ void setInstanceId(String value);
+
+ @Description("Database ID prefix to write to in Spanner")
+ @Default.String("beam-testdb")
+ String getDatabaseIdPrefix();
+ void setDatabaseIdPrefix(String value);
+
+ @Description("Table name")
+ @Default.String("users")
+ String getTable();
+ void setTable(String value);
+ }
+
+ private Spanner spanner;
+ private DatabaseAdminClient databaseAdminClient;
+ private SpannerTestPipelineOptions options;
+ private String databaseName;
+ private String project;
+
+ @Before
+ public void setUp() throws Exception {
+ PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
+ options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);
+
+ project = options.as(GcpOptions.class).getProject();
+
+ spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();
+
+ databaseName = generateDatabaseName();
+
+ databaseAdminClient = spanner.getDatabaseAdminClient();
+
+ // Delete database if exists.
+ databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);
+
+ Operation<Database, CreateDatabaseMetadata> op =
+ databaseAdminClient.createDatabase(
+ options.getInstanceId(),
+ databaseName,
+ Collections.singleton(
+ "CREATE TABLE "
+ + options.getTable()
+ + " ("
+ + " Key INT64,"
+ + " Value STRING(MAX),"
+ + ") PRIMARY KEY (Key)"));
+ op.waitFor();
+ }
+
+ private String generateDatabaseName() {
+ String random = RandomUtils
+ .randomAlphaNumeric(MAX_DB_NAME_LENGTH - 1 - options.getDatabaseIdPrefix().length());
+ return options.getDatabaseIdPrefix() + "-" + random;
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ int numRecords = 100;
+ p.apply(GenerateSequence.from(0).to(numRecords))
+ .apply(ParDo.of(new GenerateMutations(options.getTable())))
+ .apply(
+ SpannerIO.write()
+ .withProjectId(project)
+ .withInstanceId(options.getInstanceId())
+ .withDatabaseId(databaseName));
+
+ p.run();
+ DatabaseClient databaseClient =
+ spanner.getDatabaseClient(
+ DatabaseId.of(
+ project, options.getInstanceId(), databaseName));
+
+ ResultSet resultSet =
+ databaseClient
+ .singleUse()
+ .executeQuery(Statement.of("SELECT COUNT(*) FROM " + options.getTable()));
+ assertThat(resultSet.next(), is(true));
+ assertThat(resultSet.getLong(0), equalTo((long) numRecords));
+ assertThat(resultSet.next(), is(false));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);
+ spanner.close();
+ }
+
+ private static class GenerateMutations extends DoFn<Long, Mutation> {
+ private final String table;
+ private final int valueSize = 100;
+
+ public GenerateMutations(String table) {
+ this.table = table;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(table);
+ Long key = c.element();
+ builder.set("Key").to(key);
+ builder.set("Value").to(RandomUtils.randomAlphaNumeric(valueSize));
+ Mutation mutation = builder.build();
+ c.output(mutation);
+ }
+ }
+}
diff --git a/sdks/java/io/hadoop-common/pom.xml b/sdks/java/io/hadoop-common/pom.xml
index 8749243..b722bc4 100644
--- a/sdks/java/io/hadoop-common/pom.xml
+++ b/sdks/java/io/hadoop-common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java
index 8101f4b..33c660a 100644
--- a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java
+++ b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java
@@ -49,21 +49,21 @@
return conf;
}
+
@Override
public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(conf.size());
- for (Map.Entry<String, String> entry : conf) {
- out.writeUTF(entry.getKey());
- out.writeUTF(entry.getValue());
- }
+ out.writeUTF(conf.getClass().getCanonicalName());
+ conf.write(out);
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- conf = new Configuration(false);
- int size = in.readInt();
- for (int i = 0; i < size; i++) {
- conf.set(in.readUTF(), in.readUTF());
+ String className = in.readUTF();
+ try {
+ conf = (Configuration) Class.forName(className).newInstance();
+ conf.readFields(in);
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new IOException("Unable to create configuration: " + e);
}
}
diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml
index db5a1db..775efb5 100644
--- a/sdks/java/io/hadoop-file-system/pom.xml
+++ b/sdks/java/io/hadoop-file-system/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -30,51 +30,6 @@
<name>Apache Beam :: SDKs :: Java :: IO :: Hadoop File System</name>
<description>Library to read and write Hadoop/HDFS file formats from Beam.</description>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamUseDummyRunner>false</beamUseDummyRunner>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <properties>
- <!--
- This is the version of Hadoop used to compile the hadoop-common module.
- This dependency is defined with a provided scope.
- Users must supply their own Hadoop version at runtime.
- -->
- <hadoop.version>2.7.3</hadoop.version>
- </properties>
-
- <dependencyManagement>
- <!--
- We define dependencies here instead of sdks/java/io because
- of a version mimatch between this Hadoop version and other
- Hadoop versions declared in other io submodules.
- -->
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <classifier>tests</classifier>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
index d519a8c..f7a1a49 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
@@ -28,6 +28,7 @@
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.beam.sdk.io.FileSystem;
import org.apache.beam.sdk.io.fs.CreateOptions;
@@ -78,6 +79,12 @@
for (String spec : specs) {
try {
FileStatus[] fileStatuses = fileSystem.globStatus(new Path(spec));
+ if (fileStatuses == null) {
+ resultsBuilder.add(MatchResult.create(Status.NOT_FOUND,
+ Collections.<Metadata>emptyList()));
+ continue;
+ }
+
List<Metadata> metadata = new ArrayList<>();
for (FileStatus fileStatus : fileStatuses) {
if (fileStatus.isFile()) {
@@ -182,7 +189,25 @@
if (closed) {
throw new IOException("Channel is closed");
}
- return inputStream.read(dst);
+ // O length read must be supported
+ int read = 0;
+ // We avoid using the ByteBuffer based read for Hadoop because some FSDataInputStream
+ // implementations are not ByteBufferReadable,
+ // See https://issues.apache.org/jira/browse/HADOOP-14603
+ if (dst.hasArray()) {
+ // does the same as inputStream.read(dst):
+ // stores up to dst.remaining() bytes into dst.array() starting at dst.position().
+ // But dst can have an offset with its backing array hence the + dst.arrayOffset()
+ read = inputStream.read(dst.array(), dst.position() + dst.arrayOffset(), dst.remaining());
+ } else {
+ // TODO: Add support for off heap ByteBuffers in case the underlying FSDataInputStream
+ // does not support reading from a ByteBuffer.
+ read = inputStream.read(dst);
+ }
+ if (read > 0) {
+ dst.position(dst.position() + read);
+ }
+ return read;
}
@Override
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
index 88275f4..538141f 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
@@ -19,6 +19,8 @@
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -26,11 +28,13 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
+import java.io.InputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
+import java.util.Arrays;
import java.util.List;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.TextIO;
@@ -84,8 +88,28 @@
@Test
public void testCreateAndReadFile() throws Exception {
- create("testFile", "testData".getBytes());
- assertArrayEquals("testData".getBytes(), read("testFile"));
+ byte[] bytes = "testData".getBytes();
+ create("testFile", bytes);
+ assertArrayEquals(bytes, read("testFile", 0));
+ }
+
+ @Test
+ public void testCreateAndReadFileWithShift() throws Exception {
+ byte[] bytes = "testData".getBytes();
+ create("testFile", bytes);
+ int bytesToSkip = 3;
+ byte[] expected = Arrays.copyOfRange(bytes, bytesToSkip, bytes.length);
+ byte[] actual = read("testFile", bytesToSkip);
+ assertArrayEquals(expected, actual);
+ }
+
+ @Test
+ public void testCreateAndReadFileWithShiftToEnd() throws Exception {
+ byte[] bytes = "testData".getBytes();
+ create("testFile", bytes);
+ int bytesToSkip = bytes.length;
+ byte[] expected = Arrays.copyOfRange(bytes, bytesToSkip, bytes.length);
+ assertArrayEquals(expected, read("testFile", bytesToSkip));
}
@Test
@@ -99,10 +123,10 @@
ImmutableList.of(
testPath("copyTestFileA"),
testPath("copyTestFileB")));
- assertArrayEquals("testDataA".getBytes(), read("testFileA"));
- assertArrayEquals("testDataB".getBytes(), read("testFileB"));
- assertArrayEquals("testDataA".getBytes(), read("copyTestFileA"));
- assertArrayEquals("testDataB".getBytes(), read("copyTestFileB"));
+ assertArrayEquals("testDataA".getBytes(), read("testFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("testFileB", 0));
+ assertArrayEquals("testDataA".getBytes(), read("copyTestFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("copyTestFileB", 0));
}
@Test
@@ -112,9 +136,9 @@
create("testFileC", "testDataC".getBytes());
// ensure files exist
- assertArrayEquals("testDataA".getBytes(), read("testFileA"));
- assertArrayEquals("testDataB".getBytes(), read("testFileB"));
- assertArrayEquals("testDataC".getBytes(), read("testFileC"));
+ assertArrayEquals("testDataA".getBytes(), read("testFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("testFileB", 0));
+ assertArrayEquals("testDataC".getBytes(), read("testFileC", 0));
fileSystem.delete(ImmutableList.of(
testPath("testFileA"),
@@ -137,9 +161,9 @@
create("testFileB", "testDataB".getBytes());
// ensure files exist
- assertArrayEquals("testDataAA".getBytes(), read("testFileAA"));
- assertArrayEquals("testDataA".getBytes(), read("testFileA"));
- assertArrayEquals("testDataB".getBytes(), read("testFileB"));
+ assertArrayEquals("testDataAA".getBytes(), read("testFileAA", 0));
+ assertArrayEquals("testDataA".getBytes(), read("testFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("testFileB", 0));
List<MatchResult> results =
fileSystem.match(ImmutableList.of(testPath("testFileA*").toString()));
@@ -158,13 +182,44 @@
}
@Test
+ public void testMatchForNonExistentFile() throws Exception {
+ create("testFileAA", "testDataAA".getBytes());
+ create("testFileBB", "testDataBB".getBytes());
+
+ // ensure files exist
+ assertArrayEquals("testDataAA".getBytes(), read("testFileAA", 0));
+ assertArrayEquals("testDataBB".getBytes(), read("testFileBB", 0));
+
+ List<MatchResult> matchResults = fileSystem.match(ImmutableList.of(
+ testPath("testFileAA").toString(),
+ testPath("testFileA").toString(),
+ testPath("testFileBB").toString()));
+
+ assertThat(matchResults, hasSize(3));
+
+ final List<MatchResult> expected = ImmutableList.of(
+ MatchResult.create(Status.OK, ImmutableList.of(Metadata.builder()
+ .setResourceId(testPath("testFileAA"))
+ .setIsReadSeekEfficient(true)
+ .setSizeBytes("testDataAA".getBytes().length)
+ .build())),
+ MatchResult.create(Status.NOT_FOUND, ImmutableList.<Metadata>of()),
+ MatchResult.create(Status.OK, ImmutableList.of(Metadata.builder()
+ .setResourceId(testPath("testFileBB"))
+ .setIsReadSeekEfficient(true)
+ .setSizeBytes("testDataBB".getBytes().length)
+ .build())));
+ assertThat(matchResults, equalTo(expected));
+ }
+
+ @Test
public void testRename() throws Exception {
create("testFileA", "testDataA".getBytes());
create("testFileB", "testDataB".getBytes());
// ensure files exist
- assertArrayEquals("testDataA".getBytes(), read("testFileA"));
- assertArrayEquals("testDataB".getBytes(), read("testFileB"));
+ assertArrayEquals("testDataA".getBytes(), read("testFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("testFileB", 0));
fileSystem.rename(
ImmutableList.of(
@@ -188,8 +243,8 @@
.build()));
// ensure files exist
- assertArrayEquals("testDataA".getBytes(), read("renameFileA"));
- assertArrayEquals("testDataB".getBytes(), read("renameFileB"));
+ assertArrayEquals("testDataA".getBytes(), read("renameFileA", 0));
+ assertArrayEquals("testDataB".getBytes(), read("renameFileB", 0));
}
@Test
@@ -234,9 +289,13 @@
}
}
- private byte[] read(String relativePath) throws Exception {
+ private byte[] read(String relativePath, long bytesToSkip) throws Exception {
try (ReadableByteChannel channel = fileSystem.open(testPath(relativePath))) {
- return ByteStreams.toByteArray(Channels.newInputStream(channel));
+ InputStream inputStream = Channels.newInputStream(channel);
+ if (bytesToSkip > 0) {
+ ByteStreams.skipFully(inputStream, bytesToSkip);
+ }
+ return ByteStreams.toByteArray(inputStream);
}
}
diff --git a/sdks/java/io/hadoop/input-format/pom.xml b/sdks/java/io/hadoop/input-format/pom.xml
index 06f9f11..c698b40 100644
--- a/sdks/java/io/hadoop/input-format/pom.xml
+++ b/sdks/java/io/hadoop/input-format/pom.xml
@@ -20,22 +20,17 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-hadoop-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
<name>Apache Beam :: SDKs :: Java :: IO :: Hadoop :: input-format</name>
<description>IO to read data from data sources which implement Hadoop Input Format.</description>
- <properties>
- <guava.version>19.0</guava.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/sdks/java/io/hadoop/input-format/src/main/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIO.java b/sdks/java/io/hadoop/input-format/src/main/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIO.java
index 336740c..89df555 100644
--- a/sdks/java/io/hadoop/input-format/src/main/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIO.java
+++ b/sdks/java/io/hadoop/input-format/src/main/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIO.java
@@ -15,7 +15,6 @@
package org.apache.beam.sdk.io.hadoop.inputformat;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
@@ -23,11 +22,8 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.AtomicDouble;
-import java.io.Externalizable;
import java.io.IOException;
-import java.io.ObjectInput;
import java.io.ObjectInputStream;
-import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.math.BigDecimal;
@@ -46,6 +42,7 @@
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.hadoop.SerializableConfiguration;
import org.apache.beam.sdk.io.hadoop.WritableCoder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.PTransform;
@@ -168,7 +165,7 @@
* }
* </pre>
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class HadoopInputFormatIO {
private static final Logger LOG = LoggerFactory.getLogger(HadoopInputFormatIO.class);
@@ -221,12 +218,7 @@
abstract Read<K, V> build();
}
- /**
- * Returns a new {@link HadoopInputFormatIO.Read} that will read from the source using the
- * options provided by the given configuration.
- *
- * <p>Does not modify this object.
- */
+ /** Reads from the source using the options provided by the given configuration. */
public Read<K, V> withConfiguration(Configuration configuration) {
validateConfiguration(configuration);
TypeDescriptor<?> inputFormatClass =
@@ -257,27 +249,17 @@
return builder.build();
}
- /**
- * Returns a new {@link HadoopInputFormatIO.Read} that will transform the keys read from the
- * source using the given key translation function.
- *
- * <p>Does not modify this object.
- */
+ /** Transforms the keys read from the source using the given key translation function. */
public Read<K, V> withKeyTranslation(SimpleFunction<?, K> function) {
- checkNotNull(function, "function");
+ checkArgument(function != null, "function can not be null");
// Sets key class to key translation function's output class type.
return toBuilder().setKeyTranslationFunction(function)
.setKeyTypeDescriptor((TypeDescriptor<K>) function.getOutputTypeDescriptor()).build();
}
- /**
- * Returns a new {@link HadoopInputFormatIO.Read} that will transform the values read from the
- * source using the given value translation function.
- *
- * <p>Does not modify this object.
- */
+ /** Transforms the values read from the source using the given value translation function. */
public Read<K, V> withValueTranslation(SimpleFunction<?, V> function) {
- checkNotNull(function, "function");
+ checkArgument(function != null, "function can not be null");
// Sets value class to value translation function's output class type.
return toBuilder().setValueTranslationFunction(function)
.setValueTypeDescriptor((TypeDescriptor<V>) function.getOutputTypeDescriptor()).build();
@@ -304,12 +286,14 @@
* key and value classes are provided in the Hadoop configuration.
*/
private void validateConfiguration(Configuration configuration) {
- checkNotNull(configuration, "configuration");
- checkNotNull(configuration.get("mapreduce.job.inputformat.class"),
- "configuration.get(\"mapreduce.job.inputformat.class\")");
- checkNotNull(configuration.get("key.class"), "configuration.get(\"key.class\")");
- checkNotNull(configuration.get("value.class"),
- "configuration.get(\"value.class\")");
+ checkArgument(configuration != null, "configuration can not be null");
+ checkArgument(
+ configuration.get("mapreduce.job.inputformat.class") != null,
+ "Configuration must contain \"mapreduce.job.inputformat.class\"");
+ checkArgument(
+ configuration.get("key.class") != null, "configuration must contain \"key.class\"");
+ checkArgument(
+ configuration.get("value.class") != null, "configuration must contain \"value.class\"");
}
/**
@@ -317,7 +301,7 @@
*/
@VisibleForTesting
void validateTransform() {
- checkNotNull(getConfiguration(), "getConfiguration()");
+ checkArgument(getConfiguration() != null, "withConfiguration() is required");
// Validate that the key translation input type must be same as key class of InputFormat.
validateTranslationFunction(getinputFormatKeyClass(), getKeyTranslationFunction(),
"Key translation's input type is not same as hadoop InputFormat : %s key class : %s");
@@ -424,15 +408,15 @@
@Override
public void validate() {
- checkNotNull(conf, "conf");
- checkNotNull(keyCoder, "keyCoder");
- checkNotNull(valueCoder, "valueCoder");
+ checkArgument(conf != null, "conf can not be null");
+ checkArgument(keyCoder != null, "keyCoder can not be null");
+ checkArgument(valueCoder != null, "valueCoder can not be null");
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- Configuration hadoopConfig = getConfiguration().getHadoopConfiguration();
+ Configuration hadoopConfig = getConfiguration().get();
if (hadoopConfig != null) {
builder.addIfNotNull(DisplayData.item("mapreduce.job.inputformat.class",
hadoopConfig.get("mapreduce.job.inputformat.class"))
@@ -493,7 +477,7 @@
}
createInputFormatInstance();
List<InputSplit> splits =
- inputFormatObj.getSplits(Job.getInstance(conf.getHadoopConfiguration()));
+ inputFormatObj.getSplits(Job.getInstance(conf.get()));
if (splits == null) {
throw new IOException("Error in computing splits, getSplits() returns null.");
}
@@ -520,12 +504,12 @@
if (inputFormatObj == null) {
try {
taskAttemptContext =
- new TaskAttemptContextImpl(conf.getHadoopConfiguration(), new TaskAttemptID());
+ new TaskAttemptContextImpl(conf.get(), new TaskAttemptID());
inputFormatObj =
(InputFormat<?, ?>) conf
- .getHadoopConfiguration()
+ .get()
.getClassByName(
- conf.getHadoopConfiguration().get("mapreduce.job.inputformat.class"))
+ conf.get().get("mapreduce.job.inputformat.class"))
.newInstance();
/*
* If InputFormat explicitly implements interface {@link Configurable}, then setConf()
@@ -535,7 +519,7 @@
* org.apache.hadoop.hbase.mapreduce.TableInputFormat TableInputFormat}, etc.
*/
if (Configurable.class.isAssignableFrom(inputFormatObj.getClass())) {
- ((Configurable) inputFormatObj).setConf(conf.getHadoopConfiguration());
+ ((Configurable) inputFormatObj).setConf(conf.get());
}
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new IOException("Unable to create InputFormat object: ", e);
@@ -554,7 +538,7 @@
}
@Override
- public Coder<KV<K, V>> getDefaultOutputCoder() {
+ public Coder<KV<K, V>> getOutputCoder() {
return KvCoder.of(keyCoder, valueCoder);
}
@@ -802,41 +786,4 @@
new ObjectWritable(inputSplit).write(out);
}
}
-
- /**
- * A wrapper to allow Hadoop {@link org.apache.hadoop.conf.Configuration} to be serialized using
- * Java's standard serialization mechanisms. Note that the org.apache.hadoop.conf.Configuration
- * is Writable.
- */
- public static class SerializableConfiguration implements Externalizable {
-
- private Configuration conf;
-
- public SerializableConfiguration() {}
-
- public SerializableConfiguration(Configuration conf) {
- this.conf = conf;
- }
-
- public Configuration getHadoopConfiguration() {
- return conf;
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeUTF(conf.getClass().getCanonicalName());
- ((Writable) conf).write(out);
- }
-
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- String className = in.readUTF();
- try {
- conf = (Configuration) Class.forName(className).newInstance();
- conf.readFields(in);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new IOException("Unable to create configuration: " + e);
- }
- }
- }
}
diff --git a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIOTest.java b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIOTest.java
index aeeeb17..a474744 100644
--- a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIOTest.java
+++ b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HadoopInputFormatIOTest.java
@@ -26,11 +26,11 @@
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
+import org.apache.beam.sdk.io.hadoop.SerializableConfiguration;
import org.apache.beam.sdk.io.hadoop.WritableCoder;
import org.apache.beam.sdk.io.hadoop.inputformat.EmployeeInputFormat.EmployeeRecordReader;
import org.apache.beam.sdk.io.hadoop.inputformat.EmployeeInputFormat.NewObjectsEmployeeInputSplit;
import org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIO.HadoopInputFormatBoundedSource;
-import org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIO.SerializableConfiguration;
import org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIO.SerializableSplit;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.testing.PAssert;
@@ -94,11 +94,11 @@
@Test
public void testReadBuildsCorrectly() {
HadoopInputFormatIO.Read<String, String> read = HadoopInputFormatIO.<String, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslate)
.withValueTranslation(myValueTranslate);
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(myKeyTranslate, read.getKeyTranslationFunction());
assertEquals(myValueTranslate, read.getValueTranslationFunction());
assertEquals(myValueTranslate.getOutputTypeDescriptor(), read.getValueTypeDescriptor());
@@ -116,10 +116,10 @@
HadoopInputFormatIO.Read<String, String> read =
HadoopInputFormatIO.<String, String>read()
.withValueTranslation(myValueTranslate)
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslate);
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(myKeyTranslate, read.getKeyTranslationFunction());
assertEquals(myValueTranslate, read.getValueTranslationFunction());
assertEquals(myKeyTranslate.getOutputTypeDescriptor(), read.getKeyTypeDescriptor());
@@ -142,15 +142,15 @@
Employee.class,
Text.class);
HadoopInputFormatIO.Read<String, String> read = HadoopInputFormatIO.<String, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslate)
- .withConfiguration(diffConf.getHadoopConfiguration());
- assertEquals(diffConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ .withConfiguration(diffConf.get());
+ assertEquals(diffConf.get(),
+ read.getConfiguration().get());
assertEquals(myKeyTranslate, read.getKeyTranslationFunction());
assertEquals(null, read.getValueTranslationFunction());
assertEquals(myKeyTranslate.getOutputTypeDescriptor(), read.getKeyTypeDescriptor());
- assertEquals(diffConf.getHadoopConfiguration().getClass("value.class", Object.class), read
+ assertEquals(diffConf.get().getClass("value.class", Object.class), read
.getValueTypeDescriptor().getRawType());
}
@@ -161,7 +161,7 @@
*/
@Test
public void testReadObjectCreationFailsIfConfigurationIsNull() {
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<Text, Employee>read()
.withConfiguration(null);
}
@@ -173,14 +173,14 @@
@Test
public void testReadObjectCreationWithConfiguration() {
HadoopInputFormatIO.Read<Text, Employee> read = HadoopInputFormatIO.<Text, Employee>read()
- .withConfiguration(serConf.getHadoopConfiguration());
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ .withConfiguration(serConf.get());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(null, read.getKeyTranslationFunction());
assertEquals(null, read.getValueTranslationFunction());
- assertEquals(serConf.getHadoopConfiguration().getClass("key.class", Object.class), read
+ assertEquals(serConf.get().getClass("key.class", Object.class), read
.getKeyTypeDescriptor().getRawType());
- assertEquals(serConf.getHadoopConfiguration().getClass("value.class", Object.class), read
+ assertEquals(serConf.get().getClass("value.class", Object.class), read
.getValueTypeDescriptor().getRawType());
}
@@ -192,9 +192,9 @@
*/
@Test
public void testReadObjectCreationFailsIfKeyTranslationFunctionIsNull() {
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<String, Employee>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(null);
}
@@ -205,15 +205,15 @@
@Test
public void testReadObjectCreationWithConfigurationKeyTranslation() {
HadoopInputFormatIO.Read<String, Employee> read = HadoopInputFormatIO.<String, Employee>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslate);
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(myKeyTranslate, read.getKeyTranslationFunction());
assertEquals(null, read.getValueTranslationFunction());
assertEquals(myKeyTranslate.getOutputTypeDescriptor().getRawType(),
read.getKeyTypeDescriptor().getRawType());
- assertEquals(serConf.getHadoopConfiguration().getClass("value.class", Object.class),
+ assertEquals(serConf.get().getClass("value.class", Object.class),
read.getValueTypeDescriptor().getRawType());
}
@@ -225,9 +225,9 @@
*/
@Test
public void testReadObjectCreationFailsIfValueTranslationFunctionIsNull() {
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<Text, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withValueTranslation(null);
}
@@ -238,13 +238,13 @@
@Test
public void testReadObjectCreationWithConfigurationValueTranslation() {
HadoopInputFormatIO.Read<Text, String> read = HadoopInputFormatIO.<Text, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withValueTranslation(myValueTranslate);
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(null, read.getKeyTranslationFunction());
assertEquals(myValueTranslate, read.getValueTranslationFunction());
- assertEquals(serConf.getHadoopConfiguration().getClass("key.class", Object.class),
+ assertEquals(serConf.get().getClass("key.class", Object.class),
read.getKeyTypeDescriptor().getRawType());
assertEquals(myValueTranslate.getOutputTypeDescriptor().getRawType(),
read.getValueTypeDescriptor().getRawType());
@@ -257,11 +257,11 @@
@Test
public void testReadObjectCreationWithConfigurationKeyTranslationValueTranslation() {
HadoopInputFormatIO.Read<String, String> read = HadoopInputFormatIO.<String, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslate)
.withValueTranslation(myValueTranslate);
- assertEquals(serConf.getHadoopConfiguration(),
- read.getConfiguration().getHadoopConfiguration());
+ assertEquals(serConf.get(),
+ read.getConfiguration().get());
assertEquals(myKeyTranslate, read.getKeyTranslationFunction());
assertEquals(myValueTranslate, read.getValueTranslationFunction());
assertEquals(myKeyTranslate.getOutputTypeDescriptor().getRawType(),
@@ -278,7 +278,7 @@
@Test
public void testReadValidationFailsMissingConfiguration() {
HadoopInputFormatIO.Read<String, String> read = HadoopInputFormatIO.<String, String>read();
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
read.validateTransform();
}
@@ -292,7 +292,7 @@
Configuration configuration = new Configuration();
configuration.setClass("key.class", Text.class, Object.class);
configuration.setClass("value.class", Employee.class, Object.class);
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<Text, Employee>read()
.withConfiguration(configuration);
}
@@ -307,7 +307,7 @@
configuration.setClass("mapreduce.job.inputformat.class", EmployeeInputFormat.class,
InputFormat.class);
configuration.setClass("value.class", Employee.class, Object.class);
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<Text, Employee>read()
.withConfiguration(configuration);
}
@@ -322,7 +322,7 @@
configuration.setClass("mapreduce.job.inputformat.class", EmployeeInputFormat.class,
InputFormat.class);
configuration.setClass("key.class", Text.class, Object.class);
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
HadoopInputFormatIO.<Text, Employee>read().withConfiguration(configuration);
}
@@ -342,13 +342,13 @@
}
};
HadoopInputFormatIO.Read<String, Employee> read = HadoopInputFormatIO.<String, Employee>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withKeyTranslation(myKeyTranslateWithWrongInputType);
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(String.format(
"Key translation's input type is not same as hadoop InputFormat : %s key " + "class : %s",
- serConf.getHadoopConfiguration().getClass("mapreduce.job.inputformat.class",
- InputFormat.class), serConf.getHadoopConfiguration()
+ serConf.get().getClass("mapreduce.job.inputformat.class",
+ InputFormat.class), serConf.get()
.getClass("key.class", Object.class)));
read.validateTransform();
}
@@ -370,15 +370,15 @@
};
HadoopInputFormatIO.Read<Text, String> read =
HadoopInputFormatIO.<Text, String>read()
- .withConfiguration(serConf.getHadoopConfiguration())
+ .withConfiguration(serConf.get())
.withValueTranslation(myValueTranslateWithWrongInputType);
String expectedMessage =
String.format(
"Value translation's input type is not same as hadoop InputFormat : "
+ "%s value class : %s",
- serConf.getHadoopConfiguration().getClass("mapreduce.job.inputformat.class",
+ serConf.get().getClass("mapreduce.job.inputformat.class",
InputFormat.class),
- serConf.getHadoopConfiguration().getClass("value.class", Object.class));
+ serConf.get().getClass("value.class", Object.class));
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(expectedMessage);
read.validateTransform();
@@ -387,7 +387,7 @@
@Test
public void testReadingData() throws Exception {
HadoopInputFormatIO.Read<Text, Employee> read = HadoopInputFormatIO.<Text, Employee>read()
- .withConfiguration(serConf.getHadoopConfiguration());
+ .withConfiguration(serConf.get());
List<KV<Text, Employee>> expected = TestEmployeeDataSet.getEmployeeData();
PCollection<KV<Text, Employee>> actual = p.apply("ReadTest", read);
PAssert.that(actual).containsInAnyOrder(expected);
@@ -413,11 +413,11 @@
assertThat(
displayData,
hasDisplayItem("mapreduce.job.inputformat.class",
- serConf.getHadoopConfiguration().get("mapreduce.job.inputformat.class")));
+ serConf.get().get("mapreduce.job.inputformat.class")));
assertThat(displayData,
- hasDisplayItem("key.class", serConf.getHadoopConfiguration().get("key.class")));
+ hasDisplayItem("key.class", serConf.get().get("key.class")));
assertThat(displayData,
- hasDisplayItem("value.class", serConf.getHadoopConfiguration().get("value.class")));
+ hasDisplayItem("value.class", serConf.get().get("value.class")));
}
/**
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
index 9f84e88..550d31d 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
+++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-hadoop-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>beam-sdks-java-io-hadoop-jdk1.8-tests</artifactId>
@@ -39,7 +39,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
- <version>1.4.1</version>
<executions>
<execution>
<id>enforce</id>
@@ -108,13 +107,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
index 99d371d..3f866a4 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
+++ b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
@@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
+import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -74,9 +75,9 @@
public class HIFIOWithElasticTest implements Serializable {
private static final long serialVersionUID = 1L;
- private static final Logger LOGGER = LoggerFactory.getLogger(HIFIOWithElasticTest.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HIFIOWithElasticTest.class);
private static final String ELASTIC_IN_MEM_HOSTNAME = "127.0.0.1";
- private static final String ELASTIC_IN_MEM_PORT = "9200";
+ private static String elasticInMemPort = "9200";
private static final String ELASTIC_INTERNAL_VERSION = "5.x";
private static final String TRUE = "true";
private static final String ELASTIC_INDEX_NAME = "beamdb";
@@ -94,6 +95,10 @@
@BeforeClass
public static void startServer()
throws NodeValidationException, InterruptedException, IOException {
+ ServerSocket serverSocket = new ServerSocket(0);
+ int port = serverSocket.getLocalPort();
+ serverSocket.close();
+ elasticInMemPort = String.valueOf(port);
ElasticEmbeddedServer.startElasticEmbeddedServer();
}
@@ -173,7 +178,7 @@
public Configuration getConfiguration() {
Configuration conf = new Configuration();
conf.set(ConfigurationOptions.ES_NODES, ELASTIC_IN_MEM_HOSTNAME);
- conf.set(ConfigurationOptions.ES_PORT, String.format("%s", ELASTIC_IN_MEM_PORT));
+ conf.set(ConfigurationOptions.ES_PORT, String.format("%s", elasticInMemPort));
conf.set(ConfigurationOptions.ES_RESOURCE, ELASTIC_RESOURCE);
conf.set("es.internal.es.version", ELASTIC_INTERNAL_VERSION);
conf.set(ConfigurationOptions.ES_NODES_DISCOVERY, TRUE);
@@ -209,7 +214,7 @@
Settings settings = Settings.builder()
.put("node.data", TRUE)
.put("network.host", ELASTIC_IN_MEM_HOSTNAME)
- .put("http.port", ELASTIC_IN_MEM_PORT)
+ .put("http.port", elasticInMemPort)
.put("path.data", elasticTempFolder.getRoot().getPath())
.put("path.home", elasticTempFolder.getRoot().getPath())
.put("transport.type", "local")
@@ -217,9 +222,9 @@
.put("node.ingest", TRUE).build();
node = new PluginNode(settings);
node.start();
- LOGGER.info("Elastic in memory server started.");
+ LOG.info("Elastic in memory server started.");
prepareElasticIndex();
- LOGGER.info("Prepared index " + ELASTIC_INDEX_NAME
+ LOG.info("Prepared index " + ELASTIC_INDEX_NAME
+ "and populated data on elastic in memory server.");
}
@@ -243,9 +248,9 @@
public static void shutdown() throws IOException {
DeleteIndexRequest indexRequest = new DeleteIndexRequest(ELASTIC_INDEX_NAME);
node.client().admin().indices().delete(indexRequest).actionGet();
- LOGGER.info("Deleted index " + ELASTIC_INDEX_NAME + " from elastic in memory server");
+ LOG.info("Deleted index " + ELASTIC_INDEX_NAME + " from elastic in memory server");
node.close();
- LOGGER.info("Closed elastic in memory server node.");
+ LOG.info("Closed elastic in memory server node.");
deleteElasticDataDirectory();
}
diff --git a/sdks/java/io/hadoop/pom.xml b/sdks/java/io/hadoop/pom.xml
index a1c7a2e..0d63423 100644
--- a/sdks/java/io/hadoop/pom.xml
+++ b/sdks/java/io/hadoop/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<packaging>pom</packaging>
diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml
index 746b993..221e988 100644
--- a/sdks/java/io/hbase/pom.xml
+++ b/sdks/java/io/hbase/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -31,8 +31,7 @@
<description>Library to read and write from/to HBase</description>
<properties>
- <hbase.version>1.2.5</hbase.version>
- <hbase.hadoop.version>2.5.1</hbase.hadoop.version>
+ <hbase.version>1.2.6</hbase.version>
</properties>
<build>
@@ -64,6 +63,12 @@
</dependency>
<dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase.version}</version>
@@ -103,15 +108,26 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>${hbase.hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hbase.hadoop.version}</version>
<scope>test</scope>
+ <exclusions>
+ <!-- Fix build on JDK-9 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java
new file mode 100644
index 0000000..f836ebe
--- /dev/null
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hbase;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.hadoop.hbase.client.Result;
+
+/** A {@link CoderProviderRegistrar} for standard types used with {@link HBaseIO}. */
+@AutoService(CoderProviderRegistrar.class)
+public class HBaseCoderProviderRegistrar implements CoderProviderRegistrar {
+ @Override
+ public List<CoderProvider> getCoderProviders() {
+ return ImmutableList.of(
+ HBaseMutationCoder.getCoderProvider(),
+ CoderProviders.forCoder(TypeDescriptor.of(Result.class), HBaseResultCoder.of()));
+ }
+}
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
index 3c42da9..bcdaefa 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
@@ -31,20 +31,17 @@
import java.util.TreeSet;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.hadoop.SerializableConfiguration;
import org.apache.beam.sdk.io.range.ByteKey;
import org.apache.beam.sdk.io.range.ByteKeyRange;
+import org.apache.beam.sdk.io.range.ByteKeyRangeTracker;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
@@ -74,19 +71,19 @@
/**
* A bounded source and sink for HBase.
*
- * <p>For more information, see the online documentation at
- * <a href="https://hbase.apache.org/">HBase</a>.
+ * <p>For more information, see the online documentation at <a
+ * href="https://hbase.apache.org/">HBase</a>.
*
* <h3>Reading from HBase</h3>
*
- * <p>The HBase source returns a set of rows from a single table, returning a
- * {@code PCollection<Result>}.
+ * <p>The HBase source returns a set of rows from a single table, returning a {@code
+ * PCollection<Result>}.
*
- * <p>To configure a HBase source, you must supply a table id and a {@link Configuration}
- * to identify the HBase instance. By default, {@link HBaseIO.Read} will read all rows in the
- * table. The row range to be read can optionally be restricted using with a {@link Scan} object
- * or using the {@link HBaseIO.Read#withKeyRange}, and a {@link Filter} using
- * {@link HBaseIO.Read#withFilter}, for example:
+ * <p>To configure a HBase source, you must supply a table id and a {@link Configuration} to
+ * identify the HBase instance. By default, {@link HBaseIO.Read} will read all rows in the table.
+ * The row range to be read can optionally be restricted using with a {@link Scan} object or using
+ * the {@link HBaseIO.Read#withKeyRange}, and a {@link Filter} using {@link
+ * HBaseIO.Read#withFilter}, for example:
*
* <pre>{@code
* // Scan the entire table.
@@ -121,18 +118,16 @@
*
* <h3>Writing to HBase</h3>
*
- * <p>The HBase sink executes a set of row mutations on a single table. It takes as input a
- * {@link PCollection PCollection<KV<byte[], Iterable<Mutation>>>}, where the
- * {@code byte[]} is the key of the row being mutated, and each {@link Mutation} represents an
- * idempotent transformation to that row.
+ * <p>The HBase sink executes a set of row mutations on a single table. It takes as input a {@link
+ * PCollection PCollection<Mutation>}, where each {@link Mutation} represents an idempotent
+ * transformation on a row.
*
- * <p>To configure a HBase sink, you must supply a table id and a {@link Configuration}
- * to identify the HBase instance, for example:
+ * <p>To configure a HBase sink, you must supply a table id and a {@link Configuration} to identify
+ * the HBase instance, for example:
*
* <pre>{@code
* Configuration configuration = ...;
- * PCollection<KV<byte[], Iterable<Mutation>>> data = ...;
- * data.setCoder(HBaseIO.WRITE_CODER);
+ * PCollection<Mutation> data = ...;
*
* data.apply("write",
* HBaseIO.write()
@@ -142,542 +137,563 @@
*
* <h3>Experimental</h3>
*
- * <p>The design of the API for HBaseIO is currently related to the BigtableIO one,
- * it can evolve or be different in some aspects, but the idea is that users can easily migrate
- * from one to the other</p>.
+ * <p>The design of the API for HBaseIO is currently related to the BigtableIO one, it can evolve or
+ * be different in some aspects, but the idea is that users can easily migrate from one to the other
+ * .
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class HBaseIO {
- private static final Logger LOG = LoggerFactory.getLogger(HBaseIO.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseIO.class);
- /** Disallow construction of utility class. */
- private HBaseIO() {
+ /** Disallow construction of utility class. */
+ private HBaseIO() {}
+
+ /**
+ * Creates an uninitialized {@link HBaseIO.Read}. Before use, the {@code Read} must be initialized
+ * with a {@link HBaseIO.Read#withConfiguration(Configuration)} that specifies the HBase instance,
+ * and a {@link HBaseIO.Read#withTableId tableId} that specifies which table to read. A {@link
+ * Filter} may also optionally be specified using {@link HBaseIO.Read#withFilter}.
+ */
+ @Experimental
+ public static Read read() {
+ return new Read(null, "", new SerializableScan(new Scan()));
+ }
+
+ /**
+ * A {@link PTransform} that reads from HBase. See the class-level Javadoc on {@link HBaseIO} for*
+ * more information.
+ *
+ * @see HBaseIO
+ */
+ public static class Read extends PTransform<PBegin, PCollection<Result>> {
+ /** Reads from the HBase instance indicated by the* given configuration. */
+ public Read withConfiguration(Configuration configuration) {
+ checkArgument(configuration != null, "configuration can not be null");
+ return new Read(new SerializableConfiguration(configuration), tableId, serializableScan);
+ }
+
+ /** Reads from the specified table. */
+ public Read withTableId(String tableId) {
+ checkArgument(tableId != null, "tableIdcan not be null");
+ return new Read(serializableConfiguration, tableId, serializableScan);
+ }
+
+ /** Filters the rows read from HBase using the given* scan. */
+ public Read withScan(Scan scan) {
+ checkArgument(scan != null, "scancan not be null");
+ return new Read(serializableConfiguration, tableId, new SerializableScan(scan));
+ }
+
+ /** Filters the rows read from HBase using the given* row filter. */
+ public Read withFilter(Filter filter) {
+ checkArgument(filter != null, "filtercan not be null");
+ return withScan(serializableScan.get().setFilter(filter));
+ }
+
+ /** Reads only rows in the specified range. */
+ public Read withKeyRange(ByteKeyRange keyRange) {
+ checkArgument(keyRange != null, "keyRangecan not be null");
+ byte[] startRow = keyRange.getStartKey().getBytes();
+ byte[] stopRow = keyRange.getEndKey().getBytes();
+ return withScan(serializableScan.get().setStartRow(startRow).setStopRow(stopRow));
+ }
+
+ /** Reads only rows in the specified range. */
+ public Read withKeyRange(byte[] startRow, byte[] stopRow) {
+ checkArgument(startRow != null, "startRowcan not be null");
+ checkArgument(stopRow != null, "stopRowcan not be null");
+ ByteKeyRange keyRange =
+ ByteKeyRange.of(ByteKey.copyFrom(startRow), ByteKey.copyFrom(stopRow));
+ return withKeyRange(keyRange);
+ }
+
+ private Read(
+ SerializableConfiguration serializableConfiguration,
+ String tableId,
+ SerializableScan serializableScan) {
+ this.serializableConfiguration = serializableConfiguration;
+ this.tableId = tableId;
+ this.serializableScan = serializableScan;
+ }
+
+ @Override
+ public PCollection<Result> expand(PBegin input) {
+ checkArgument(serializableConfiguration != null, "withConfiguration() is required");
+ checkArgument(!tableId.isEmpty(), "withTableId() is required");
+ try (Connection connection =
+ ConnectionFactory.createConnection(serializableConfiguration.get())) {
+ Admin admin = connection.getAdmin();
+ checkArgument(
+ admin.tableExists(TableName.valueOf(tableId)), "Table %s does not exist", tableId);
+ } catch (IOException e) {
+ LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
+ }
+ HBaseSource source = new HBaseSource(this, null /* estimatedSizeBytes */);
+ return input.getPipeline().apply(org.apache.beam.sdk.io.Read.from(source));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("configuration", serializableConfiguration.get().toString()));
+ builder.add(DisplayData.item("tableId", tableId));
+ builder.addIfNotNull(DisplayData.item("scan", serializableScan.get().toString()));
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ public Configuration getConfiguration() {
+ return serializableConfiguration.get();
+ }
+
+ /** Returns the range of keys that will be read from the table. */
+ public ByteKeyRange getKeyRange() {
+ byte[] startRow = serializableScan.get().getStartRow();
+ byte[] stopRow = serializableScan.get().getStopRow();
+ return ByteKeyRange.of(ByteKey.copyFrom(startRow), ByteKey.copyFrom(stopRow));
+ }
+
+ private final SerializableConfiguration serializableConfiguration;
+ private final String tableId;
+ private final SerializableScan serializableScan;
+ }
+
+ static class HBaseSource extends BoundedSource<Result> {
+ private final Read read;
+ @Nullable private Long estimatedSizeBytes;
+
+ HBaseSource(Read read, @Nullable Long estimatedSizeBytes) {
+ this.read = read;
+ this.estimatedSizeBytes = estimatedSizeBytes;
+ }
+
+ HBaseSource withStartKey(ByteKey startKey) throws IOException {
+ checkNotNull(startKey, "startKey");
+ Read newRead =
+ new Read(
+ read.serializableConfiguration,
+ read.tableId,
+ new SerializableScan(
+ new Scan(read.serializableScan.get()).setStartRow(startKey.getBytes())));
+ return new HBaseSource(newRead, estimatedSizeBytes);
+ }
+
+ HBaseSource withEndKey(ByteKey endKey) throws IOException {
+ checkNotNull(endKey, "endKey");
+ Read newRead =
+ new Read(
+ read.serializableConfiguration,
+ read.tableId,
+ new SerializableScan(
+ new Scan(read.serializableScan.get()).setStopRow(endKey.getBytes())));
+ return new HBaseSource(newRead, estimatedSizeBytes);
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
+ if (estimatedSizeBytes == null) {
+ estimatedSizeBytes = estimateSizeBytes();
+ LOG.debug(
+ "Estimated size {} bytes for table {} and scan {}",
+ estimatedSizeBytes,
+ read.tableId,
+ read.serializableScan.get());
+ }
+ return estimatedSizeBytes;
}
/**
- * Creates an uninitialized {@link HBaseIO.Read}. Before use, the {@code Read} must be
- * initialized with a
- * {@link HBaseIO.Read#withConfiguration(Configuration)} that specifies
- * the HBase instance, and a {@link HBaseIO.Read#withTableId tableId} that
- * specifies which table to read. A {@link Filter} may also optionally be specified using
- * {@link HBaseIO.Read#withFilter}.
+ * This estimates the real size, it can be the compressed size depending on the HBase
+ * configuration.
*/
- @Experimental
- public static Read read() {
- return new Read(null, "", new SerializableScan(new Scan()));
+ private long estimateSizeBytes() throws Exception {
+ // This code is based on RegionSizeCalculator in hbase-server
+ long estimatedSizeBytes = 0L;
+ Configuration configuration = this.read.serializableConfiguration.get();
+ try (Connection connection = ConnectionFactory.createConnection(configuration)) {
+ // filter regions for the given table/scan
+ List<HRegionLocation> regionLocations = getRegionLocations(connection);
+
+ // builds set of regions who are part of the table scan
+ Set<byte[]> tableRegions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
+ for (HRegionLocation regionLocation : regionLocations) {
+ tableRegions.add(regionLocation.getRegionInfo().getRegionName());
+ }
+
+ // calculate estimated size for the regions
+ Admin admin = connection.getAdmin();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
+ Collection<ServerName> servers = clusterStatus.getServers();
+ for (ServerName serverName : servers) {
+ ServerLoad serverLoad = clusterStatus.getLoad(serverName);
+ for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
+ byte[] regionId = regionLoad.getName();
+ if (tableRegions.contains(regionId)) {
+ long regionSizeBytes = regionLoad.getStorefileSizeMB() * 1_048_576L;
+ estimatedSizeBytes += regionSizeBytes;
+ }
+ }
+ }
+ }
+ return estimatedSizeBytes;
}
- /**
- * A {@link PTransform} that reads from HBase. See the class-level Javadoc on
- * {@link HBaseIO} for more information.
- *
- * @see HBaseIO
- */
- public static class Read extends PTransform<PBegin, PCollection<Result>> {
- /**
- * Returns a new {@link HBaseIO.Read} that will read from the HBase instance
- * indicated by the given configuration.
- */
- public Read withConfiguration(Configuration configuration) {
- checkNotNull(configuration, "conf");
- return new Read(new SerializableConfiguration(configuration),
- tableId, serializableScan);
- }
+ private List<HRegionLocation> getRegionLocations(Connection connection) throws Exception {
+ final Scan scan = read.serializableScan.get();
+ byte[] startRow = scan.getStartRow();
+ byte[] stopRow = scan.getStopRow();
- /**
- * Returns a new {@link HBaseIO.Read} that will read from the specified table.
- *
- * <p>Does not modify this object.
- */
- public Read withTableId(String tableId) {
- checkNotNull(tableId, "tableId");
- return new Read(serializableConfiguration, tableId, serializableScan);
- }
+ final List<HRegionLocation> regionLocations = new ArrayList<>();
- /**
- * Returns a new {@link HBaseIO.Read} that will filter the rows read from HBase
- * using the given scan.
- *
- * <p>Does not modify this object.
- */
- public Read withScan(Scan scan) {
- checkNotNull(scan, "scan");
- return new Read(serializableConfiguration, tableId, new SerializableScan(scan));
- }
+ final boolean scanWithNoLowerBound = startRow.length == 0;
+ final boolean scanWithNoUpperBound = stopRow.length == 0;
- /**
- * Returns a new {@link HBaseIO.Read} that will filter the rows read from HBase
- * using the given row filter.
- *
- * <p>Does not modify this object.
- */
- public Read withFilter(Filter filter) {
- checkNotNull(filter, "filter");
- return withScan(serializableScan.get().setFilter(filter));
+ TableName tableName = TableName.valueOf(read.tableId);
+ RegionLocator regionLocator = connection.getRegionLocator(tableName);
+ List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
+ for (HRegionLocation regionLocation : tableRegionInfos) {
+ final byte[] startKey = regionLocation.getRegionInfo().getStartKey();
+ final byte[] endKey = regionLocation.getRegionInfo().getEndKey();
+ boolean isLastRegion = endKey.length == 0;
+ // filters regions who are part of the scan
+ if ((scanWithNoLowerBound || isLastRegion || Bytes.compareTo(startRow, endKey) < 0)
+ && (scanWithNoUpperBound || Bytes.compareTo(stopRow, startKey) > 0)) {
+ regionLocations.add(regionLocation);
}
+ }
- /**
- * Returns a new {@link HBaseIO.Read} that will read only rows in the specified range.
- *
- * <p>Does not modify this object.
- */
- public Read withKeyRange(ByteKeyRange keyRange) {
- checkNotNull(keyRange, "keyRange");
- byte[] startRow = keyRange.getStartKey().getBytes();
- byte[] stopRow = keyRange.getEndKey().getBytes();
- return withScan(serializableScan.get().setStartRow(startRow).setStopRow(stopRow));
- }
-
- /**
- * Returns a new {@link HBaseIO.Read} that will read only rows in the specified range.
- *
- * <p>Does not modify this object.
- */
- public Read withKeyRange(byte[] startRow, byte[] stopRow) {
- checkNotNull(startRow, "startRow");
- checkNotNull(stopRow, "stopRow");
- ByteKeyRange keyRange =
- ByteKeyRange.of(ByteKey.copyFrom(startRow), ByteKey.copyFrom(stopRow));
- return withKeyRange(keyRange);
- }
-
- private Read(SerializableConfiguration serializableConfiguration, String tableId,
- SerializableScan serializableScan) {
- this.serializableConfiguration = serializableConfiguration;
- this.tableId = tableId;
- this.serializableScan = serializableScan;
- }
-
- @Override
- public PCollection<Result> expand(PBegin input) {
- HBaseSource source = new HBaseSource(this, null /* estimatedSizeBytes */);
- return input.getPipeline().apply(org.apache.beam.sdk.io.Read.from(source));
- }
-
- @Override
- public void validate(PipelineOptions options) {
- checkArgument(serializableConfiguration != null,
- "Configuration not provided");
- checkArgument(!tableId.isEmpty(), "Table ID not specified");
- try (Connection connection = ConnectionFactory.createConnection(
- serializableConfiguration.get())) {
- Admin admin = connection.getAdmin();
- checkArgument(admin.tableExists(TableName.valueOf(tableId)),
- "Table %s does not exist", tableId);
- } catch (IOException e) {
- LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
- }
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
- builder.add(DisplayData.item("configuration",
- serializableConfiguration.get().toString()));
- builder.add(DisplayData.item("tableId", tableId));
- builder.addIfNotNull(DisplayData.item("scan", serializableScan.get().toString()));
- }
-
- public String getTableId() {
- return tableId;
- }
-
- public Configuration getConfiguration() {
- return serializableConfiguration.get();
- }
-
- /**
- * Returns the range of keys that will be read from the table.
- */
- public ByteKeyRange getKeyRange() {
- byte[] startRow = serializableScan.get().getStartRow();
- byte[] stopRow = serializableScan.get().getStopRow();
- return ByteKeyRange.of(ByteKey.copyFrom(startRow), ByteKey.copyFrom(stopRow));
- }
-
- private final SerializableConfiguration serializableConfiguration;
- private final String tableId;
- private final SerializableScan serializableScan;
+ return regionLocations;
}
- static class HBaseSource extends BoundedSource<Result> {
- private final Read read;
- @Nullable private Long estimatedSizeBytes;
+ private List<HBaseSource> splitBasedOnRegions(
+ List<HRegionLocation> regionLocations, int numSplits) throws Exception {
+ final Scan scan = read.serializableScan.get();
+ byte[] startRow = scan.getStartRow();
+ byte[] stopRow = scan.getStopRow();
- HBaseSource(Read read, @Nullable Long estimatedSizeBytes) {
- this.read = read;
- this.estimatedSizeBytes = estimatedSizeBytes;
- }
+ final List<HBaseSource> sources = new ArrayList<>(numSplits);
+ final boolean scanWithNoLowerBound = startRow.length == 0;
+ final boolean scanWithNoUpperBound = stopRow.length == 0;
- @Override
- public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
- if (estimatedSizeBytes == null) {
- estimatedSizeBytes = estimateSizeBytes();
- LOG.debug("Estimated size {} bytes for table {} and scan {}", estimatedSizeBytes,
- read.tableId, read.serializableScan.get());
- }
- return estimatedSizeBytes;
- }
+ for (HRegionLocation regionLocation : regionLocations) {
+ final byte[] startKey = regionLocation.getRegionInfo().getStartKey();
+ final byte[] endKey = regionLocation.getRegionInfo().getEndKey();
+ boolean isLastRegion = endKey.length == 0;
+ String host = regionLocation.getHostnamePort();
- /**
- * This estimates the real size, it can be the compressed size depending on the HBase
- * configuration.
- */
- private long estimateSizeBytes() throws Exception {
- // This code is based on RegionSizeCalculator in hbase-server
- long estimatedSizeBytes = 0L;
- Configuration configuration = this.read.serializableConfiguration.get();
- try (Connection connection = ConnectionFactory.createConnection(configuration)) {
- // filter regions for the given table/scan
- List<HRegionLocation> regionLocations = getRegionLocations(connection);
+ final byte[] splitStart =
+ (scanWithNoLowerBound || Bytes.compareTo(startKey, startRow) >= 0)
+ ? startKey
+ : startRow;
+ final byte[] splitStop =
+ (scanWithNoUpperBound || Bytes.compareTo(endKey, stopRow) <= 0) && !isLastRegion
+ ? endKey
+ : stopRow;
- // builds set of regions who are part of the table scan
- Set<byte[]> tableRegions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
- for (HRegionLocation regionLocation : regionLocations) {
- tableRegions.add(regionLocation.getRegionInfo().getRegionName());
- }
+ LOG.debug(
+ "{} {} {} {} {}",
+ sources.size(),
+ host,
+ read.tableId,
+ Bytes.toString(splitStart),
+ Bytes.toString(splitStop));
- // calculate estimated size for the regions
- Admin admin = connection.getAdmin();
- ClusterStatus clusterStatus = admin.getClusterStatus();
- Collection<ServerName> servers = clusterStatus.getServers();
- for (ServerName serverName : servers) {
- ServerLoad serverLoad = clusterStatus.getLoad(serverName);
- for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
- byte[] regionId = regionLoad.getName();
- if (tableRegions.contains(regionId)) {
- long regionSizeBytes = regionLoad.getStorefileSizeMB() * 1_048_576L;
- estimatedSizeBytes += regionSizeBytes;
- }
- }
- }
- }
- return estimatedSizeBytes;
- }
-
- private List<HRegionLocation> getRegionLocations(Connection connection) throws Exception {
- final Scan scan = read.serializableScan.get();
- byte[] startRow = scan.getStartRow();
- byte[] stopRow = scan.getStopRow();
-
- final List<HRegionLocation> regionLocations = new ArrayList<>();
-
- final boolean scanWithNoLowerBound = startRow.length == 0;
- final boolean scanWithNoUpperBound = stopRow.length == 0;
-
- TableName tableName = TableName.valueOf(read.tableId);
- RegionLocator regionLocator = connection.getRegionLocator(tableName);
- List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
- for (HRegionLocation regionLocation : tableRegionInfos) {
- final byte[] startKey = regionLocation.getRegionInfo().getStartKey();
- final byte[] endKey = regionLocation.getRegionInfo().getEndKey();
- boolean isLastRegion = endKey.length == 0;
- // filters regions who are part of the scan
- if ((scanWithNoLowerBound
- || isLastRegion || Bytes.compareTo(startRow, endKey) < 0)
- && (scanWithNoUpperBound || Bytes.compareTo(stopRow, startKey) > 0)) {
- regionLocations.add(regionLocation);
- }
- }
-
- return regionLocations;
- }
-
- private List<HBaseSource>
- splitBasedOnRegions(List<HRegionLocation> regionLocations, int numSplits)
- throws Exception {
- final Scan scan = read.serializableScan.get();
- byte[] startRow = scan.getStartRow();
- byte[] stopRow = scan.getStopRow();
-
- final List<HBaseSource> sources = new ArrayList<>(numSplits);
- final boolean scanWithNoLowerBound = startRow.length == 0;
- final boolean scanWithNoUpperBound = stopRow.length == 0;
-
- for (HRegionLocation regionLocation : regionLocations) {
- final byte[] startKey = regionLocation.getRegionInfo().getStartKey();
- final byte[] endKey = regionLocation.getRegionInfo().getEndKey();
- boolean isLastRegion = endKey.length == 0;
- String host = regionLocation.getHostnamePort();
-
- final byte[] splitStart = (scanWithNoLowerBound
- || Bytes.compareTo(startKey, startRow) >= 0) ? startKey : startRow;
- final byte[] splitStop =
- (scanWithNoUpperBound || Bytes.compareTo(endKey, stopRow) <= 0)
- && !isLastRegion ? endKey : stopRow;
-
- LOG.debug("{} {} {} {} {}", sources.size(), host, read.tableId,
- Bytes.toString(splitStart), Bytes.toString(splitStop));
-
- // We need to create a new copy of the scan and read to add the new ranges
- Scan newScan = new Scan(scan).setStartRow(splitStart).setStopRow(splitStop);
- Read newRead = new Read(read.serializableConfiguration, read.tableId,
- new SerializableScan(newScan));
- sources.add(new HBaseSource(newRead, estimatedSizeBytes));
- }
- return sources;
- }
+ // We need to create a new copy of the scan and read to add the new ranges
+ Scan newScan = new Scan(scan).setStartRow(splitStart).setStopRow(splitStop);
+ Read newRead =
+ new Read(read.serializableConfiguration, read.tableId, new SerializableScan(newScan));
+ sources.add(new HBaseSource(newRead, estimatedSizeBytes));
+ }
+ return sources;
+ }
@Override
public List<? extends BoundedSource<Result>> split(
long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
- LOG.debug("desiredBundleSize {} bytes", desiredBundleSizeBytes);
- long estimatedSizeBytes = getEstimatedSizeBytes(options);
- int numSplits = 1;
- if (estimatedSizeBytes > 0 && desiredBundleSizeBytes > 0) {
- numSplits = (int) Math.ceil((double) estimatedSizeBytes / desiredBundleSizeBytes);
- }
+ LOG.debug("desiredBundleSize {} bytes", desiredBundleSizeBytes);
+ long estimatedSizeBytes = getEstimatedSizeBytes(options);
+ int numSplits = 1;
+ if (estimatedSizeBytes > 0 && desiredBundleSizeBytes > 0) {
+ numSplits = (int) Math.ceil((double) estimatedSizeBytes / desiredBundleSizeBytes);
+ }
- try (Connection connection = ConnectionFactory.createConnection(
- read.getConfiguration())) {
- List<HRegionLocation> regionLocations = getRegionLocations(connection);
- int realNumSplits =
- numSplits < regionLocations.size() ? regionLocations.size() : numSplits;
- LOG.debug("Suggested {} bundle(s) based on size", numSplits);
- LOG.debug("Suggested {} bundle(s) based on number of regions",
- regionLocations.size());
- final List<HBaseSource> sources = splitBasedOnRegions(regionLocations,
- realNumSplits);
- LOG.debug("Split into {} bundle(s)", sources.size());
- if (numSplits >= 1) {
- return sources;
- }
- return Collections.singletonList(this);
- }
+ try (Connection connection = ConnectionFactory.createConnection(read.getConfiguration())) {
+ List<HRegionLocation> regionLocations = getRegionLocations(connection);
+ int realNumSplits = numSplits < regionLocations.size() ? regionLocations.size() : numSplits;
+ LOG.debug("Suggested {} bundle(s) based on size", numSplits);
+ LOG.debug("Suggested {} bundle(s) based on number of regions", regionLocations.size());
+ final List<HBaseSource> sources = splitBasedOnRegions(regionLocations, realNumSplits);
+ LOG.debug("Split into {} bundle(s)", sources.size());
+ if (numSplits >= 1) {
+ return sources;
}
-
- @Override
- public BoundedReader<Result> createReader(PipelineOptions pipelineOptions)
- throws IOException {
- return new HBaseReader(this);
- }
-
- @Override
- public void validate() {
- read.validate(null /* input */);
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- read.populateDisplayData(builder);
- }
-
- @Override
- public Coder<Result> getDefaultOutputCoder() {
- return HBaseResultCoder.of();
- }
+ return Collections.singletonList(this);
+ }
}
- private static class HBaseReader extends BoundedSource.BoundedReader<Result> {
- private final HBaseSource source;
- private Connection connection;
- private ResultScanner scanner;
- private Iterator<Result> iter;
- private Result current;
- private long recordsReturned;
-
- HBaseReader(HBaseSource source) {
- this.source = source;
- }
-
- @Override
- public boolean start() throws IOException {
- Configuration configuration = source.read.serializableConfiguration.get();
- String tableId = source.read.tableId;
- connection = ConnectionFactory.createConnection(configuration);
- TableName tableName = TableName.valueOf(tableId);
- Table table = connection.getTable(tableName);
- Scan scan = source.read.serializableScan.get();
- scanner = table.getScanner(scan);
- iter = scanner.iterator();
- return advance();
- }
-
- @Override
- public Result getCurrent() throws NoSuchElementException {
- return current;
- }
-
- @Override
- public boolean advance() throws IOException {
- boolean hasRecord = iter.hasNext();
- if (hasRecord) {
- current = iter.next();
- ++recordsReturned;
- }
- return hasRecord;
- }
-
- @Override
- public void close() throws IOException {
- LOG.debug("Closing reader after reading {} records.", recordsReturned);
- if (scanner != null) {
- scanner.close();
- scanner = null;
- }
- if (connection != null) {
- connection.close();
- connection = null;
- }
- }
-
- @Override
- public BoundedSource<Result> getCurrentSource() {
- return source;
- }
+ @Override
+ public BoundedReader<Result> createReader(PipelineOptions pipelineOptions) throws IOException {
+ return new HBaseReader(this);
}
- /**
- * Creates an uninitialized {@link HBaseIO.Write}. Before use, the {@code Write} must be
- * initialized with a
- * {@link HBaseIO.Write#withConfiguration(Configuration)} that specifies
- * the destination HBase instance, and a {@link HBaseIO.Write#withTableId tableId}
- * that specifies which table to write.
- */
- public static Write write() {
- return new Write(null /* SerializableConfiguration */, "");
+ @Override
+ public void validate() {
+ read.validate(null /* input */);
}
- /**
- * A {@link PTransform} that writes to HBase. See the class-level Javadoc on
- * {@link HBaseIO} for more information.
- *
- * @see HBaseIO
- */
- public static class Write
- extends PTransform<PCollection<KV<byte[], Iterable<Mutation>>>, PDone> {
-
- /**
- * Returns a new {@link HBaseIO.Write} that will write to the HBase instance
- * indicated by the given Configuration, and using any other specified customizations.
- *
- * <p>Does not modify this object.
- */
- public Write withConfiguration(Configuration configuration) {
- checkNotNull(configuration, "conf");
- return new Write(new SerializableConfiguration(configuration), tableId);
- }
-
- /**
- * Returns a new {@link HBaseIO.Write} that will write to the specified table.
- *
- * <p>Does not modify this object.
- */
- public Write withTableId(String tableId) {
- checkNotNull(tableId, "tableId");
- return new Write(serializableConfiguration, tableId);
- }
-
- private Write(SerializableConfiguration serializableConfiguration, String tableId) {
- this.serializableConfiguration = serializableConfiguration;
- this.tableId = tableId;
- }
-
- @Override
- public PDone expand(PCollection<KV<byte[], Iterable<Mutation>>> input) {
- input.apply(ParDo.of(new HBaseWriterFn(tableId, serializableConfiguration)));
- return PDone.in(input.getPipeline());
- }
-
- @Override
- public void validate(PipelineOptions options) {
- checkArgument(serializableConfiguration != null, "Configuration not specified");
- checkArgument(!tableId.isEmpty(), "Table ID not specified");
- try (Connection connection = ConnectionFactory.createConnection(
- serializableConfiguration.get())) {
- Admin admin = connection.getAdmin();
- checkArgument(admin.tableExists(TableName.valueOf(tableId)),
- "Table %s does not exist", tableId);
- } catch (IOException e) {
- LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
- }
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
- builder.add(DisplayData.item("configuration",
- serializableConfiguration.get().toString()));
- builder.add(DisplayData.item("tableId", tableId));
- }
-
- public String getTableId() {
- return tableId;
- }
-
- public Configuration getConfiguration() {
- return serializableConfiguration.get();
- }
-
- private final String tableId;
- private final SerializableConfiguration serializableConfiguration;
-
- private class HBaseWriterFn extends DoFn<KV<byte[], Iterable<Mutation>>, Void> {
-
- public HBaseWriterFn(String tableId,
- SerializableConfiguration serializableConfiguration) {
- this.tableId = checkNotNull(tableId, "tableId");
- this.serializableConfiguration = checkNotNull(serializableConfiguration,
- "serializableConfiguration");
- }
-
- @Setup
- public void setup() throws Exception {
- Configuration configuration = this.serializableConfiguration.get();
- connection = ConnectionFactory.createConnection(configuration);
-
- TableName tableName = TableName.valueOf(tableId);
- BufferedMutatorParams params =
- new BufferedMutatorParams(tableName);
- mutator = connection.getBufferedMutator(params);
-
- recordsWritten = 0;
- }
-
- @ProcessElement
- public void processElement(ProcessContext ctx) throws Exception {
- KV<byte[], Iterable<Mutation>> record = ctx.element();
- List<Mutation> mutations = new ArrayList<>();
- for (Mutation mutation : record.getValue()) {
- mutations.add(mutation);
- ++recordsWritten;
- }
- mutator.mutate(mutations);
- }
-
- @FinishBundle
- public void finishBundle() throws Exception {
- mutator.flush();
- }
-
- @Teardown
- public void tearDown() throws Exception {
- if (mutator != null) {
- mutator.close();
- mutator = null;
- }
- if (connection != null) {
- connection.close();
- connection = null;
- }
- LOG.debug("Wrote {} records", recordsWritten);
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- builder.delegate(Write.this);
- }
-
- private final String tableId;
- private final SerializableConfiguration serializableConfiguration;
-
- private Connection connection;
- private BufferedMutator mutator;
-
- private long recordsWritten;
- }
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ read.populateDisplayData(builder);
}
- public static final Coder<KV<byte[], Iterable<Mutation>>> WRITE_CODER =
- KvCoder.of(ByteArrayCoder.of(), IterableCoder.of(HBaseMutationCoder.of()));
+ @Override
+ public Coder<Result> getOutputCoder() {
+ return HBaseResultCoder.of();
+ }
+ }
+
+ private static class HBaseReader extends BoundedSource.BoundedReader<Result> {
+ private HBaseSource source;
+ private Connection connection;
+ private ResultScanner scanner;
+ private Iterator<Result> iter;
+ private Result current;
+ private final ByteKeyRangeTracker rangeTracker;
+ private long recordsReturned;
+
+ HBaseReader(HBaseSource source) {
+ this.source = source;
+ Scan scan = source.read.serializableScan.get();
+ ByteKeyRange range =
+ ByteKeyRange.of(
+ ByteKey.copyFrom(scan.getStartRow()), ByteKey.copyFrom(scan.getStopRow()));
+ rangeTracker = ByteKeyRangeTracker.of(range);
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ HBaseSource source = getCurrentSource();
+ Configuration configuration = source.read.serializableConfiguration.get();
+ String tableId = source.read.tableId;
+ connection = ConnectionFactory.createConnection(configuration);
+ TableName tableName = TableName.valueOf(tableId);
+ Table table = connection.getTable(tableName);
+ // [BEAM-2319] We have to clone the Scan because the underlying scanner may mutate it.
+ Scan scanClone = new Scan(source.read.serializableScan.get());
+ scanner = table.getScanner(scanClone);
+ iter = scanner.iterator();
+ return advance();
+ }
+
+ @Override
+ public Result getCurrent() throws NoSuchElementException {
+ return current;
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ if (!iter.hasNext()) {
+ return rangeTracker.markDone();
+ }
+ final Result next = iter.next();
+ boolean hasRecord =
+ rangeTracker.tryReturnRecordAt(true, ByteKey.copyFrom(next.getRow()))
+ || rangeTracker.markDone();
+ if (hasRecord) {
+ current = next;
+ ++recordsReturned;
+ }
+ return hasRecord;
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOG.debug("Closing reader after reading {} records.", recordsReturned);
+ if (scanner != null) {
+ scanner.close();
+ scanner = null;
+ }
+ if (connection != null) {
+ connection.close();
+ connection = null;
+ }
+ }
+
+ @Override
+ public synchronized HBaseSource getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ public final Double getFractionConsumed() {
+ return rangeTracker.getFractionConsumed();
+ }
+
+ @Override
+ public final long getSplitPointsConsumed() {
+ return rangeTracker.getSplitPointsConsumed();
+ }
+
+ @Override
+ @Nullable
+ public final synchronized HBaseSource splitAtFraction(double fraction) {
+ ByteKey splitKey;
+ try {
+ splitKey = rangeTracker.getRange().interpolateKey(fraction);
+ } catch (RuntimeException e) {
+ LOG.info(
+ "{}: Failed to interpolate key for fraction {}.", rangeTracker.getRange(), fraction, e);
+ return null;
+ }
+ LOG.info("Proposing to split {} at fraction {} (key {})", rangeTracker, fraction, splitKey);
+ HBaseSource primary;
+ HBaseSource residual;
+ try {
+ primary = source.withEndKey(splitKey);
+ residual = source.withStartKey(splitKey);
+ } catch (Exception e) {
+ LOG.info(
+ "{}: Interpolating for fraction {} yielded invalid split key {}.",
+ rangeTracker.getRange(),
+ fraction,
+ splitKey,
+ e);
+ return null;
+ }
+ if (!rangeTracker.trySplitAtPosition(splitKey)) {
+ return null;
+ }
+ this.source = primary;
+ return residual;
+ }
+ }
+
+ /**
+ * Creates an uninitialized {@link HBaseIO.Write}. Before use, the {@code Write} must be
+ * initialized with a {@link HBaseIO.Write#withConfiguration(Configuration)} that specifies the
+ * destination HBase instance, and a {@link HBaseIO.Write#withTableId tableId} that specifies
+ * which table to write.
+ */
+ public static Write write() {
+ return new Write(null /* SerializableConfiguration */, "");
+ }
+
+ /**
+ * A {@link PTransform} that writes to HBase. See the class-level Javadoc on {@link HBaseIO} for*
+ * more information.
+ *
+ * @see HBaseIO
+ */
+ public static class Write extends PTransform<PCollection<Mutation>, PDone> {
+ /** Writes to the HBase instance indicated by the* given Configuration. */
+ public Write withConfiguration(Configuration configuration) {
+ checkArgument(configuration != null, "configuration can not be null");
+ return new Write(new SerializableConfiguration(configuration), tableId);
+ }
+
+ /** Writes to the specified table. */
+ public Write withTableId(String tableId) {
+ checkArgument(tableId != null, "tableIdcan not be null");
+ return new Write(serializableConfiguration, tableId);
+ }
+
+ private Write(SerializableConfiguration serializableConfiguration, String tableId) {
+ this.serializableConfiguration = serializableConfiguration;
+ this.tableId = tableId;
+ }
+
+ @Override
+ public PDone expand(PCollection<Mutation> input) {
+ checkArgument(serializableConfiguration != null, "withConfiguration() is required");
+ checkArgument(tableId != null && !tableId.isEmpty(), "withTableId() is required");
+ try (Connection connection =
+ ConnectionFactory.createConnection(serializableConfiguration.get())) {
+ Admin admin = connection.getAdmin();
+ checkArgument(
+ admin.tableExists(TableName.valueOf(tableId)), "Table %s does not exist", tableId);
+ } catch (IOException e) {
+ LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
+ }
+ input.apply(ParDo.of(new HBaseWriterFn(tableId, serializableConfiguration)));
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("configuration", serializableConfiguration.get().toString()));
+ builder.add(DisplayData.item("tableId", tableId));
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ public Configuration getConfiguration() {
+ return serializableConfiguration.get();
+ }
+
+ private final String tableId;
+ private final SerializableConfiguration serializableConfiguration;
+
+ private class HBaseWriterFn extends DoFn<Mutation, Void> {
+
+ public HBaseWriterFn(String tableId, SerializableConfiguration serializableConfiguration) {
+ this.tableId = checkNotNull(tableId, "tableId");
+ this.serializableConfiguration =
+ checkNotNull(serializableConfiguration, "serializableConfiguration");
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ connection = ConnectionFactory.createConnection(serializableConfiguration.get());
+ }
+
+ @StartBundle
+ public void startBundle(StartBundleContext c) throws IOException {
+ BufferedMutatorParams params = new BufferedMutatorParams(TableName.valueOf(tableId));
+ mutator = connection.getBufferedMutator(params);
+ recordsWritten = 0;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ mutator.mutate(c.element());
+ ++recordsWritten;
+ }
+
+ @FinishBundle
+ public void finishBundle() throws Exception {
+ mutator.flush();
+ LOG.debug("Wrote {} records", recordsWritten);
+ }
+
+ @Teardown
+ public void tearDown() throws Exception {
+ if (mutator != null) {
+ mutator.close();
+ mutator = null;
+ }
+ if (connection != null) {
+ connection.close();
+ connection = null;
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.delegate(Write.this);
+ }
+
+ private final String tableId;
+ private final SerializableConfiguration serializableConfiguration;
+
+ private Connection connection;
+ private BufferedMutator mutator;
+
+ private long recordsWritten;
+ }
+ }
}
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
index 501fe09..e7a36d5 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
@@ -21,8 +21,12 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
+import java.util.List;
import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -65,4 +69,41 @@
throw new IllegalArgumentException("Only Put and Delete are supported");
}
}
+
+ /**
+ * Returns a {@link CoderProvider} which uses the {@link HBaseMutationCoder} for {@link Mutation
+ * mutations}.
+ */
+ static CoderProvider getCoderProvider() {
+ return HBASE_MUTATION_CODER_PROVIDER;
+ }
+
+ private static final CoderProvider HBASE_MUTATION_CODER_PROVIDER =
+ new HBaseMutationCoderProvider();
+
+ /** A {@link CoderProvider} for {@link Mutation mutations}. */
+ private static class HBaseMutationCoderProvider extends CoderProvider {
+ @Override
+ public <T> Coder<T> coderFor(
+ TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
+ throws CannotProvideCoderException {
+ if (!typeDescriptor.isSubtypeOf(HBASE_MUTATION_TYPE_DESCRIPTOR)) {
+ throw new CannotProvideCoderException(
+ String.format(
+ "Cannot provide %s because %s is not a subclass of %s",
+ HBaseMutationCoder.class.getSimpleName(),
+ typeDescriptor,
+ Mutation.class.getName()));
+ }
+
+ try {
+ return (Coder<T>) HBaseMutationCoder.of();
+ } catch (IllegalArgumentException e) {
+ throw new CannotProvideCoderException(e);
+ }
+ }
+ }
+
+ private static final TypeDescriptor<Mutation> HBASE_MUTATION_TYPE_DESCRIPTOR =
+ new TypeDescriptor<Mutation>() {};
}
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
index 1d06635..bce1567 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
@@ -41,14 +41,12 @@
}
@Override
- public void encode(Result value, OutputStream outputStream)
- throws IOException {
+ public void encode(Result value, OutputStream outputStream) throws IOException {
ProtobufUtil.toResult(value).writeDelimitedTo(outputStream);
}
@Override
- public Result decode(InputStream inputStream)
- throws IOException {
+ public Result decode(InputStream inputStream) throws IOException {
return ProtobufUtil.toResult(ClientProtos.Result.parseDelimitedFrom(inputStream));
}
}
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/SerializableScan.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/SerializableScan.java
index f3bc7ac..6ed3c51 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/SerializableScan.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/SerializableScan.java
@@ -25,31 +25,28 @@
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
-/**
- * This is just a wrapper class to serialize HBase {@link Scan} using Protobuf.
- */
+/** This is just a wrapper class to serialize HBase {@link Scan} using Protobuf. */
class SerializableScan implements Serializable {
- private transient Scan scan;
+ private transient Scan scan;
- public SerializableScan() {
- }
+ public SerializableScan() {}
- public SerializableScan(Scan scan) {
- if (scan == null) {
- throw new NullPointerException("Scan must not be null.");
- }
- this.scan = scan;
+ public SerializableScan(Scan scan) {
+ if (scan == null) {
+ throw new NullPointerException("Scan must not be null.");
}
+ this.scan = scan;
+ }
- private void writeObject(ObjectOutputStream out) throws IOException {
- ProtobufUtil.toScan(scan).writeDelimitedTo(out);
- }
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ ProtobufUtil.toScan(scan).writeDelimitedTo(out);
+ }
- private void readObject(ObjectInputStream in) throws IOException {
- scan = ProtobufUtil.toScan(ClientProtos.Scan.parseDelimitedFrom(in));
- }
+ private void readObject(ObjectInputStream in) throws IOException {
+ scan = ProtobufUtil.toScan(ClientProtos.Scan.parseDelimitedFrom(in));
+ }
- public Scan get() {
- return scan;
- }
+ public Scan get() {
+ return scan;
+ }
}
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrarTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrarTest.java
new file mode 100644
index 0000000..25369fc
--- /dev/null
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrarTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hbase;
+
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link HBaseCoderProviderRegistrar}. */
+@RunWith(JUnit4.class)
+public class HBaseCoderProviderRegistrarTest {
+ @Test
+ public void testResultCoderIsRegistered() throws Exception {
+ CoderRegistry.createDefault().getCoder(Result.class);
+ }
+
+ @Test
+ public void testMutationCoderIsRegistered() throws Exception {
+ CoderRegistry.createDefault().getCoder(Mutation.class);
+ CoderRegistry.createDefault().getCoder(Put.class);
+ CoderRegistry.createDefault().getCoder(Delete.class);
+ }
+}
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
index dbeab04..fd42024 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
@@ -18,6 +18,9 @@
package org.apache.beam.sdk.io.hbase;
import static org.apache.beam.sdk.testing.SourceTestUtils.assertSourcesEqualReferenceSource;
+import static org.apache.beam.sdk.testing.SourceTestUtils.assertSplitAtFractionExhaustive;
+import static org.apache.beam.sdk.testing.SourceTestUtils.assertSplitAtFractionFails;
+import static org.apache.beam.sdk.testing.SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
@@ -33,11 +36,11 @@
import org.apache.beam.sdk.io.range.ByteKey;
import org.apache.beam.sdk.io.range.ByteKeyRange;
import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -46,6 +49,7 @@
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.Connection;
@@ -71,350 +75,406 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Test HBaseIO.
- */
+/** Test HBaseIO. */
@RunWith(JUnit4.class)
public class HBaseIOTest {
- @Rule public final transient TestPipeline p = TestPipeline.create();
- @Rule public ExpectedException thrown = ExpectedException.none();
+ @Rule public final transient TestPipeline p = TestPipeline.create();
+ @Rule public ExpectedException thrown = ExpectedException.none();
- private static HBaseTestingUtility htu;
- private static HBaseAdmin admin;
+ private static HBaseTestingUtility htu;
+ private static HBaseAdmin admin;
- private static Configuration conf = HBaseConfiguration.create();
- private static final byte[] COLUMN_FAMILY = Bytes.toBytes("info");
- private static final byte[] COLUMN_NAME = Bytes.toBytes("name");
- private static final byte[] COLUMN_EMAIL = Bytes.toBytes("email");
+ private static final Configuration conf = HBaseConfiguration.create();
+ private static final byte[] COLUMN_FAMILY = Bytes.toBytes("info");
+ private static final byte[] COLUMN_NAME = Bytes.toBytes("name");
+ private static final byte[] COLUMN_EMAIL = Bytes.toBytes("email");
- @BeforeClass
- public static void beforeClass() throws Exception {
- conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
- // Try to bind the hostname to localhost to solve an issue when it is not configured or
- // no DNS resolution available.
- conf.setStrings("hbase.master.hostname", "localhost");
- conf.setStrings("hbase.regionserver.hostname", "localhost");
- htu = new HBaseTestingUtility(conf);
- htu.startMiniCluster(1, 4);
- admin = htu.getHBaseAdmin();
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+ // Try to bind the hostname to localhost to solve an issue when it is not configured or
+ // no DNS resolution available.
+ conf.setStrings("hbase.master.hostname", "localhost");
+ conf.setStrings("hbase.regionserver.hostname", "localhost");
+ htu = new HBaseTestingUtility(conf);
+
+ // We don't use the full htu.startMiniCluster() to avoid starting unneeded HDFS/MR daemons
+ htu.startMiniZKCluster();
+ MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 4);
+ hbm.waitForActiveAndReadyMaster();
+
+ admin = htu.getHBaseAdmin();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ if (admin != null) {
+ admin.close();
+ admin = null;
}
-
- @AfterClass
- public static void afterClass() throws Exception {
- if (admin != null) {
- admin.close();
- admin = null;
- }
- if (htu != null) {
- htu.shutdownMiniCluster();
- htu = null;
- }
+ if (htu != null) {
+ htu.shutdownMiniHBaseCluster();
+ htu.shutdownMiniZKCluster();
+ htu = null;
}
+ }
- @Test
- public void testReadBuildsCorrectly() {
- HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId("table");
- assertEquals("table", read.getTableId());
- assertNotNull("configuration", read.getConfiguration());
+ @Test
+ public void testReadBuildsCorrectly() {
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId("table");
+ assertEquals("table", read.getTableId());
+ assertNotNull("configuration", read.getConfiguration());
+ }
+
+ @Test
+ public void testReadBuildsCorrectlyInDifferentOrder() {
+ HBaseIO.Read read = HBaseIO.read().withTableId("table").withConfiguration(conf);
+ assertEquals("table", read.getTableId());
+ assertNotNull("configuration", read.getConfiguration());
+ }
+
+ @Test
+ public void testWriteBuildsCorrectly() {
+ HBaseIO.Write write = HBaseIO.write().withConfiguration(conf).withTableId("table");
+ assertEquals("table", write.getTableId());
+ assertNotNull("configuration", write.getConfiguration());
+ }
+
+ @Test
+ public void testWriteBuildsCorrectlyInDifferentOrder() {
+ HBaseIO.Write write = HBaseIO.write().withTableId("table").withConfiguration(conf);
+ assertEquals("table", write.getTableId());
+ assertNotNull("configuration", write.getConfiguration());
+ }
+
+ @Test
+ public void testWriteValidationFailsMissingTable() {
+ HBaseIO.Write write = HBaseIO.write().withConfiguration(conf);
+ thrown.expect(IllegalArgumentException.class);
+ write.expand(null /* input */);
+ }
+
+ @Test
+ public void testWriteValidationFailsMissingConfiguration() {
+ HBaseIO.Write write = HBaseIO.write().withTableId("table");
+ thrown.expect(IllegalArgumentException.class);
+ write.expand(null /* input */);
+ }
+
+ /** Tests that when reading from a non-existent table, the read fails. */
+ @Test
+ public void testReadingFailsTableDoesNotExist() throws Exception {
+ final String table = "TEST-TABLE-INVALID";
+ // Exception will be thrown by read.expand() when read is applied.
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage(String.format("Table %s does not exist", table));
+ runReadTest(HBaseIO.read().withConfiguration(conf).withTableId(table), new ArrayList<Result>());
+ }
+
+ /** Tests that when reading from an empty table, the read succeeds. */
+ @Test
+ public void testReadingEmptyTable() throws Exception {
+ final String table = "TEST-EMPTY-TABLE";
+ createTable(table);
+ runReadTest(HBaseIO.read().withConfiguration(conf).withTableId(table), new ArrayList<Result>());
+ }
+
+ @Test
+ public void testReading() throws Exception {
+ final String table = "TEST-MANY-ROWS-TABLE";
+ final int numRows = 1001;
+ createTable(table);
+ writeData(table, numRows);
+ runReadTestLength(HBaseIO.read().withConfiguration(conf).withTableId(table), 1001);
+ }
+
+ /** Tests reading all rows from a split table. */
+ @Test
+ public void testReadingWithSplits() throws Exception {
+ final String table = "TEST-MANY-ROWS-SPLITS-TABLE";
+ final int numRows = 1500;
+ final int numRegions = 4;
+ final long bytesPerRow = 100L;
+
+ // Set up test table data and sample row keys for size estimation and splitting.
+ createTable(table);
+ writeData(table, numRows);
+
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
+ HBaseSource source = new HBaseSource(read, null /* estimatedSizeBytes */);
+ List<? extends BoundedSource<Result>> splits =
+ source.split(numRows * bytesPerRow / numRegions, null /* options */);
+
+ // Test num splits and split equality.
+ assertThat(splits, hasSize(4));
+ assertSourcesEqualReferenceSource(source, splits, null /* options */);
+ }
+
+ /** Tests that a {@link HBaseSource} can be read twice, verifying its immutability. */
+ @Test
+ public void testReadingSourceTwice() throws Exception {
+ final String table = "TEST-READING-TWICE";
+ final int numRows = 10;
+
+ // Set up test table data and sample row keys for size estimation and splitting.
+ createTable(table);
+ writeData(table, numRows);
+
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
+ HBaseSource source = new HBaseSource(read, null /* estimatedSizeBytes */);
+ assertThat(SourceTestUtils.readFromSource(source, null), hasSize(numRows));
+ // second read.
+ assertThat(SourceTestUtils.readFromSource(source, null), hasSize(numRows));
+ }
+
+ /** Tests reading all rows using a filter. */
+ @Test
+ public void testReadingWithFilter() throws Exception {
+ final String table = "TEST-FILTER-TABLE";
+ final int numRows = 1001;
+
+ createTable(table);
+ writeData(table, numRows);
+
+ String regex = ".*17.*";
+ Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
+ HBaseIO.Read read =
+ HBaseIO.read().withConfiguration(conf).withTableId(table).withFilter(filter);
+ runReadTestLength(read, 20);
+ }
+
+ /**
+ * Tests reading all rows using key ranges. Tests a prefix [), a suffix (], and a restricted range
+ * [] and that some properties hold across them.
+ */
+ @Test
+ public void testReadingWithKeyRange() throws Exception {
+ final String table = "TEST-KEY-RANGE-TABLE";
+ final int numRows = 1001;
+ final byte[] startRow = "2".getBytes();
+ final byte[] stopRow = "9".getBytes();
+ final ByteKey startKey = ByteKey.copyFrom(startRow);
+
+ createTable(table);
+ writeData(table, numRows);
+
+ // Test prefix: [beginning, startKey).
+ final ByteKeyRange prefixRange = ByteKeyRange.ALL_KEYS.withEndKey(startKey);
+ runReadTestLength(
+ HBaseIO.read().withConfiguration(conf).withTableId(table).withKeyRange(prefixRange), 126);
+
+ // Test suffix: [startKey, end).
+ final ByteKeyRange suffixRange = ByteKeyRange.ALL_KEYS.withStartKey(startKey);
+ runReadTestLength(
+ HBaseIO.read().withConfiguration(conf).withTableId(table).withKeyRange(suffixRange), 875);
+
+ // Test restricted range: [startKey, endKey).
+ // This one tests the second signature of .withKeyRange
+ runReadTestLength(
+ HBaseIO.read().withConfiguration(conf).withTableId(table).withKeyRange(startRow, stopRow),
+ 441);
+ }
+
+ /** Tests dynamic work rebalancing exhaustively. */
+ @Test
+ public void testReadingSplitAtFractionExhaustive() throws Exception {
+ final String table = "TEST-FEW-ROWS-SPLIT-EXHAUSTIVE-TABLE";
+ final int numRows = 7;
+
+ createTable(table);
+ writeData(table, numRows);
+
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
+ HBaseSource source =
+ new HBaseSource(read, null /* estimatedSizeBytes */)
+ .withStartKey(ByteKey.of(48))
+ .withEndKey(ByteKey.of(58));
+
+ assertSplitAtFractionExhaustive(source, null);
+ }
+
+ /** Unit tests of splitAtFraction. */
+ @Test
+ public void testReadingSplitAtFraction() throws Exception {
+ final String table = "TEST-SPLIT-AT-FRACTION";
+ final int numRows = 10;
+
+ createTable(table);
+ writeData(table, numRows);
+
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
+ HBaseSource source = new HBaseSource(read, null /* estimatedSizeBytes */);
+
+ // The value k is based on the partitioning schema for the data, in this test case,
+ // the partitioning is HEX-based, so we start from 1/16m and the value k will be
+ // around 1/256, so the tests are done in approximately k ~= 0.003922 steps
+ double k = 0.003922;
+
+ assertSplitAtFractionFails(source, 0, k, null /* options */);
+ assertSplitAtFractionFails(source, 0, 1.0, null /* options */);
+ // With 1 items read, all split requests past k will succeed.
+ assertSplitAtFractionSucceedsAndConsistent(source, 1, k, null /* options */);
+ assertSplitAtFractionSucceedsAndConsistent(source, 1, 0.666, null /* options */);
+ // With 3 items read, all split requests past 3k will succeed.
+ assertSplitAtFractionFails(source, 3, 2 * k, null /* options */);
+ assertSplitAtFractionSucceedsAndConsistent(source, 3, 3 * k, null /* options */);
+ assertSplitAtFractionSucceedsAndConsistent(source, 3, 4 * k, null /* options */);
+ // With 6 items read, all split requests past 6k will succeed.
+ assertSplitAtFractionFails(source, 6, 5 * k, null /* options */);
+ assertSplitAtFractionSucceedsAndConsistent(source, 6, 0.7, null /* options */);
+ }
+
+ @Test
+ public void testReadingDisplayData() {
+ HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId("fooTable");
+ DisplayData displayData = DisplayData.from(read);
+ assertThat(displayData, hasDisplayItem("tableId", "fooTable"));
+ assertThat(displayData, hasDisplayItem("configuration"));
+ }
+
+ /** Tests that a record gets written to the service and messages are logged. */
+ @Test
+ public void testWriting() throws Exception {
+ final String table = "table";
+ final String key = "key";
+ final String value = "value";
+ final int numMutations = 100;
+
+ createTable(table);
+
+ p.apply("multiple rows", Create.of(makeMutations(key, value, numMutations)))
+ .apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
+ p.run().waitUntilFinish();
+
+ List<Result> results = readTable(table, new Scan());
+ assertEquals(numMutations, results.size());
+ }
+
+ /** Tests that when writing to a non-existent table, the write fails. */
+ @Test
+ public void testWritingFailsTableDoesNotExist() throws Exception {
+ final String table = "TEST-TABLE-DOES-NOT-EXIST";
+
+ // Exception will be thrown by write.expand() when write is applied.
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage(String.format("Table %s does not exist", table));
+ p.apply(Create.empty(HBaseMutationCoder.of()))
+ .apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
+ }
+
+ /** Tests that when writing an element fails, the write fails. */
+ @Test
+ public void testWritingFailsBadElement() throws Exception {
+ final String table = "TEST-TABLE-BAD-ELEMENT";
+ final String key = "KEY";
+ createTable(table);
+
+ p.apply(Create.of(makeBadMutation(key)))
+ .apply(HBaseIO.write().withConfiguration(conf).withTableId(table));
+
+ thrown.expect(Pipeline.PipelineExecutionException.class);
+ thrown.expectCause(Matchers.<Throwable>instanceOf(IllegalArgumentException.class));
+ thrown.expectMessage("No columns to insert");
+ p.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testWritingDisplayData() {
+ HBaseIO.Write write = HBaseIO.write().withTableId("fooTable").withConfiguration(conf);
+ DisplayData displayData = DisplayData.from(write);
+ assertThat(displayData, hasDisplayItem("tableId", "fooTable"));
+ }
+
+ // HBase helper methods
+ private static void createTable(String tableId) throws Exception {
+ byte[][] splitKeys = {"4".getBytes(), "8".getBytes(), "C".getBytes()};
+ createTable(tableId, COLUMN_FAMILY, splitKeys);
+ }
+
+ private static void createTable(String tableId, byte[] columnFamily, byte[][] splitKeys)
+ throws Exception {
+ TableName tableName = TableName.valueOf(tableId);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ HColumnDescriptor colDef = new HColumnDescriptor(columnFamily);
+ desc.addFamily(colDef);
+ admin.createTable(desc, splitKeys);
+ }
+
+ /** Helper function to create a table and return the rows that it created. */
+ private static void writeData(String tableId, int numRows) throws Exception {
+ Connection connection = admin.getConnection();
+ TableName tableName = TableName.valueOf(tableId);
+ BufferedMutator mutator = connection.getBufferedMutator(tableName);
+ List<Mutation> mutations = makeTableData(numRows);
+ mutator.mutate(mutations);
+ mutator.flush();
+ mutator.close();
+ }
+
+ private static List<Mutation> makeTableData(int numRows) {
+ List<Mutation> mutations = new ArrayList<>(numRows);
+ for (int i = 0; i < numRows; ++i) {
+ // We pad values in hex order 0,1, ... ,F,0, ...
+ String prefix = String.format("%X", i % 16);
+ // This 21 is to have a key longer than an input
+ byte[] rowKey = Bytes.toBytes(StringUtils.leftPad("_" + String.valueOf(i), 21, prefix));
+ byte[] value = Bytes.toBytes(String.valueOf(i));
+ byte[] valueEmail = Bytes.toBytes(String.valueOf(i) + "@email.com");
+ mutations.add(new Put(rowKey).addColumn(COLUMN_FAMILY, COLUMN_NAME, value));
+ mutations.add(new Put(rowKey).addColumn(COLUMN_FAMILY, COLUMN_EMAIL, valueEmail));
}
+ return mutations;
+ }
- @Test
- public void testReadBuildsCorrectlyInDifferentOrder() {
- HBaseIO.Read read = HBaseIO.read().withTableId("table").withConfiguration(conf);
- assertEquals("table", read.getTableId());
- assertNotNull("configuration", read.getConfiguration());
+ private static ResultScanner scanTable(String tableId, Scan scan) throws Exception {
+ Connection connection = ConnectionFactory.createConnection(conf);
+ TableName tableName = TableName.valueOf(tableId);
+ Table table = connection.getTable(tableName);
+ return table.getScanner(scan);
+ }
+
+ private static List<Result> readTable(String tableId, Scan scan) throws Exception {
+ ResultScanner scanner = scanTable(tableId, scan);
+ List<Result> results = new ArrayList<>();
+ for (Result result : scanner) {
+ results.add(result);
}
+ scanner.close();
+ return results;
+ }
- @Test
- public void testWriteBuildsCorrectly() {
- HBaseIO.Write write = HBaseIO.write().withConfiguration(conf).withTableId("table");
- assertEquals("table", write.getTableId());
- assertNotNull("configuration", write.getConfiguration());
+ // Beam helper methods
+ /** Helper function to make a single row mutation to be written. */
+ private static Iterable<Mutation> makeMutations(String key, String value, int numMutations) {
+ List<Mutation> mutations = new ArrayList<>();
+ for (int i = 0; i < numMutations; i++) {
+ mutations.add(makeMutation(key + i, value));
}
+ return mutations;
+ }
- @Test
- public void testWriteBuildsCorrectlyInDifferentOrder() {
- HBaseIO.Write write = HBaseIO.write().withTableId("table").withConfiguration(conf);
- assertEquals("table", write.getTableId());
- assertNotNull("configuration", write.getConfiguration());
- }
+ private static Mutation makeMutation(String key, String value) {
+ return new Put(key.getBytes(StandardCharsets.UTF_8))
+ .addColumn(COLUMN_FAMILY, COLUMN_NAME, Bytes.toBytes(value))
+ .addColumn(COLUMN_FAMILY, COLUMN_EMAIL, Bytes.toBytes(value + "@email.com"));
+ }
- @Test
- public void testWriteValidationFailsMissingTable() {
- HBaseIO.Write write = HBaseIO.write().withConfiguration(conf);
- thrown.expect(IllegalArgumentException.class);
- write.validate(null /* input */);
- }
+ private static Mutation makeBadMutation(String key) {
+ return new Put(key.getBytes());
+ }
- @Test
- public void testWriteValidationFailsMissingConfiguration() {
- HBaseIO.Write write = HBaseIO.write().withTableId("table");
- thrown.expect(IllegalArgumentException.class);
- write.validate(null /* input */);
- }
+ private void runReadTest(HBaseIO.Read read, List<Result> expected) {
+ final String transformId = read.getTableId() + "_" + read.getKeyRange();
+ PCollection<Result> rows = p.apply("Read" + transformId, read);
+ PAssert.that(rows).containsInAnyOrder(expected);
+ p.run().waitUntilFinish();
+ }
- /** Tests that when reading from a non-existent table, the read fails. */
- @Test
- public void testReadingFailsTableDoesNotExist() throws Exception {
- final String table = "TEST-TABLE-INVALID";
- // Exception will be thrown by read.validate() when read is applied.
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage(String.format("Table %s does not exist", table));
- runReadTest(HBaseIO.read().withConfiguration(conf).withTableId(table),
- new ArrayList<Result>());
- }
-
- /** Tests that when reading from an empty table, the read succeeds. */
- @Test
- public void testReadingEmptyTable() throws Exception {
- final String table = "TEST-EMPTY-TABLE";
- createTable(table);
- runReadTest(HBaseIO.read().withConfiguration(conf).withTableId(table),
- new ArrayList<Result>());
- }
-
- @Test
- public void testReading() throws Exception {
- final String table = "TEST-MANY-ROWS-TABLE";
- final int numRows = 1001;
- createTable(table);
- writeData(table, numRows);
- runReadTestLength(HBaseIO.read().withConfiguration(conf).withTableId(table), 1001);
- }
-
- /** Tests reading all rows from a split table. */
- @Test
- public void testReadingWithSplits() throws Exception {
- final String table = "TEST-MANY-ROWS-SPLITS-TABLE";
- final int numRows = 1500;
- final int numRegions = 4;
- final long bytesPerRow = 100L;
-
- // Set up test table data and sample row keys for size estimation and splitting.
- createTable(table);
- writeData(table, numRows);
-
- HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
- HBaseSource source = new HBaseSource(read, null /* estimatedSizeBytes */);
- List<? extends BoundedSource<Result>> splits =
- source.split(numRows * bytesPerRow / numRegions,
- null /* options */);
-
- // Test num splits and split equality.
- assertThat(splits, hasSize(4));
- assertSourcesEqualReferenceSource(source, splits, null /* options */);
- }
-
-
- /** Tests reading all rows using a filter. */
- @Test
- public void testReadingWithFilter() throws Exception {
- final String table = "TEST-FILTER-TABLE";
- final int numRows = 1001;
-
- createTable(table);
- writeData(table, numRows);
-
- String regex = ".*17.*";
- Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL,
- new RegexStringComparator(regex));
- HBaseIO.Read read =
- HBaseIO.read().withConfiguration(conf).withTableId(table).withFilter(filter);
- runReadTestLength(read, 20);
- }
-
- /**
- * Tests reading all rows using key ranges. Tests a prefix [), a suffix (], and a restricted
- * range [] and that some properties hold across them.
- */
- @Test
- public void testReadingWithKeyRange() throws Exception {
- final String table = "TEST-KEY-RANGE-TABLE";
- final int numRows = 1001;
- final byte[] startRow = "2".getBytes();
- final byte[] stopRow = "9".getBytes();
- final ByteKey startKey = ByteKey.copyFrom(startRow);
-
- createTable(table);
- writeData(table, numRows);
-
- // Test prefix: [beginning, startKey).
- final ByteKeyRange prefixRange = ByteKeyRange.ALL_KEYS.withEndKey(startKey);
- runReadTestLength(HBaseIO.read().withConfiguration(conf).withTableId(table)
- .withKeyRange(prefixRange), 126);
-
- // Test suffix: [startKey, end).
- final ByteKeyRange suffixRange = ByteKeyRange.ALL_KEYS.withStartKey(startKey);
- runReadTestLength(HBaseIO.read().withConfiguration(conf).withTableId(table)
- .withKeyRange(suffixRange), 875);
-
- // Test restricted range: [startKey, endKey).
- // This one tests the second signature of .withKeyRange
- runReadTestLength(HBaseIO.read().withConfiguration(conf).withTableId(table)
- .withKeyRange(startRow, stopRow), 441);
- }
-
- @Test
- public void testReadingDisplayData() {
- HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId("fooTable");
- DisplayData displayData = DisplayData.from(read);
- assertThat(displayData, hasDisplayItem("tableId", "fooTable"));
- assertThat(displayData, hasDisplayItem("configuration"));
- }
-
- /** Tests that a record gets written to the service and messages are logged. */
- @Test
- public void testWriting() throws Exception {
- final String table = "table";
- final String key = "key";
- final String value = "value";
-
- createTable(table);
-
- p.apply("single row", Create.of(makeWrite(key, value)).withCoder(HBaseIO.WRITE_CODER))
- .apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
- p.run().waitUntilFinish();
-
- List<Result> results = readTable(table, new Scan());
- assertEquals(1, results.size());
- }
-
- /** Tests that when writing to a non-existent table, the write fails. */
- @Test
- public void testWritingFailsTableDoesNotExist() throws Exception {
- final String table = "TEST-TABLE";
-
- PCollection<KV<byte[], Iterable<Mutation>>> emptyInput =
- p.apply(Create.empty(HBaseIO.WRITE_CODER));
-
- emptyInput.apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
-
- // Exception will be thrown by write.validate() when write is applied.
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage(String.format("Table %s does not exist", table));
- p.run();
- }
-
- /** Tests that when writing an element fails, the write fails. */
- @Test
- public void testWritingFailsBadElement() throws Exception {
- final String table = "TEST-TABLE";
- final String key = "KEY";
- createTable(table);
-
- p.apply(Create.of(makeBadWrite(key)).withCoder(HBaseIO.WRITE_CODER))
- .apply(HBaseIO.write().withConfiguration(conf).withTableId(table));
-
- thrown.expect(Pipeline.PipelineExecutionException.class);
- thrown.expectCause(Matchers.<Throwable>instanceOf(IllegalArgumentException.class));
- thrown.expectMessage("No columns to insert");
- p.run().waitUntilFinish();
- }
-
- @Test
- public void testWritingDisplayData() {
- HBaseIO.Write write = HBaseIO.write().withTableId("fooTable").withConfiguration(conf);
- DisplayData displayData = DisplayData.from(write);
- assertThat(displayData, hasDisplayItem("tableId", "fooTable"));
- }
-
- // HBase helper methods
- private static void createTable(String tableId) throws Exception {
- byte[][] splitKeys = {"4".getBytes(), "8".getBytes(), "C".getBytes()};
- createTable(tableId, COLUMN_FAMILY, splitKeys);
- }
-
- private static void createTable(String tableId, byte[] columnFamily, byte[][] splitKeys)
- throws Exception {
- TableName tableName = TableName.valueOf(tableId);
- HTableDescriptor desc = new HTableDescriptor(tableName);
- HColumnDescriptor colDef = new HColumnDescriptor(columnFamily);
- desc.addFamily(colDef);
- admin.createTable(desc, splitKeys);
- }
-
- /**
- * Helper function to create a table and return the rows that it created.
- */
- private static void writeData(String tableId, int numRows) throws Exception {
- Connection connection = admin.getConnection();
- TableName tableName = TableName.valueOf(tableId);
- BufferedMutator mutator = connection.getBufferedMutator(tableName);
- List<Mutation> mutations = makeTableData(numRows);
- mutator.mutate(mutations);
- mutator.flush();
- mutator.close();
- }
-
- private static List<Mutation> makeTableData(int numRows) {
- List<Mutation> mutations = new ArrayList<>(numRows);
- for (int i = 0; i < numRows; ++i) {
- // We pad values in hex order 0,1, ... ,F,0, ...
- String prefix = String.format("%X", i % 16);
- // This 21 is to have a key longer than an input
- byte[] rowKey = Bytes.toBytes(
- StringUtils.leftPad("_" + String.valueOf(i), 21, prefix));
- byte[] value = Bytes.toBytes(String.valueOf(i));
- byte[] valueEmail = Bytes.toBytes(String.valueOf(i) + "@email.com");
- mutations.add(new Put(rowKey).addColumn(COLUMN_FAMILY, COLUMN_NAME, value));
- mutations.add(new Put(rowKey).addColumn(COLUMN_FAMILY, COLUMN_EMAIL, valueEmail));
- }
- return mutations;
- }
-
- private static ResultScanner scanTable(String tableId, Scan scan) throws Exception {
- Connection connection = ConnectionFactory.createConnection(conf);
- TableName tableName = TableName.valueOf(tableId);
- Table table = connection.getTable(tableName);
- return table.getScanner(scan);
- }
-
- private static List<Result> readTable(String tableId, Scan scan) throws Exception {
- ResultScanner scanner = scanTable(tableId, scan);
- List<Result> results = new ArrayList<>();
- for (Result result : scanner) {
- results.add(result);
- }
- scanner.close();
- return results;
- }
-
- // Beam helper methods
- /** Helper function to make a single row mutation to be written. */
- private static KV<byte[], Iterable<Mutation>> makeWrite(String key, String value) {
- byte[] rowKey = key.getBytes(StandardCharsets.UTF_8);
- List<Mutation> mutations = new ArrayList<>();
- mutations.add(makeMutation(key, value));
- return KV.of(rowKey, (Iterable<Mutation>) mutations);
- }
-
-
- private static Mutation makeMutation(String key, String value) {
- byte[] rowKey = key.getBytes(StandardCharsets.UTF_8);
- return new Put(rowKey)
- .addColumn(COLUMN_FAMILY, COLUMN_NAME, Bytes.toBytes(value))
- .addColumn(COLUMN_FAMILY, COLUMN_EMAIL, Bytes.toBytes(value + "@email.com"));
- }
-
- private static KV<byte[], Iterable<Mutation>> makeBadWrite(String key) {
- Put put = new Put(key.getBytes());
- List<Mutation> mutations = new ArrayList<>();
- mutations.add(put);
- return KV.of(key.getBytes(StandardCharsets.UTF_8), (Iterable<Mutation>) mutations);
- }
-
- private void runReadTest(HBaseIO.Read read, List<Result> expected) {
- final String transformId = read.getTableId() + "_" + read.getKeyRange();
- PCollection<Result> rows = p.apply("Read" + transformId, read);
- PAssert.that(rows).containsInAnyOrder(expected);
- p.run().waitUntilFinish();
- }
-
- private void runReadTestLength(HBaseIO.Read read, long numElements) {
- final String transformId = read.getTableId() + "_" + read.getKeyRange();
- PCollection<Result> rows = p.apply("Read" + transformId, read);
- PAssert.thatSingleton(rows.apply("Count" + transformId,
- Count.<Result>globally())).isEqualTo(numElements);
- p.run().waitUntilFinish();
- }
+ private void runReadTestLength(HBaseIO.Read read, long numElements) {
+ final String transformId = read.getTableId() + "_" + read.getKeyRange();
+ PCollection<Result> rows = p.apply("Read" + transformId, read);
+ PAssert.thatSingleton(rows.apply("Count" + transformId, Count.<Result>globally()))
+ .isEqualTo(numElements);
+ p.run().waitUntilFinish();
+ }
}
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoderTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoderTest.java
index 5bf2d80..41525dc 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoderTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoderTest.java
@@ -28,9 +28,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for HBaseMutationCoder.
- */
+/** Tests for HBaseMutationCoder. */
@RunWith(JUnit4.class)
public class HBaseMutationCoderTest {
@Rule public final ExpectedException thrown = ExpectedException.none();
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseResultCoderTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseResultCoderTest.java
index c6b27d6..5af5e16 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseResultCoderTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseResultCoderTest.java
@@ -25,9 +25,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for HBaseResultCoder.
- */
+/** Tests for HBaseResultCoder. */
@RunWith(JUnit4.class)
public class HBaseResultCoderTest {
@Rule public final ExpectedException thrown = ExpectedException.none();
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
index 49eb4e3..7d2fd28 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
@@ -28,14 +28,12 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for SerializableScan.
- */
+/** Tests for SerializableScan. */
@RunWith(JUnit4.class)
public class SerializableScanTest {
@Rule public final ExpectedException thrown = ExpectedException.none();
private static final SerializableScan DEFAULT_SERIALIZABLE_SCAN =
- new SerializableScan(new Scan());
+ new SerializableScan(new Scan());
@Test
public void testSerializationDeserialization() throws Exception {
diff --git a/sdks/java/io/hcatalog/pom.xml b/sdks/java/io/hcatalog/pom.xml
new file mode 100644
index 0000000..307f595
--- /dev/null
+++ b/sdks/java/io/hcatalog/pom.xml
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-hcatalog</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: HCatalog</name>
+ <description>IO to read and write for HCatalog source.</description>
+
+ <properties>
+ <hive.version>2.1.0</hive.version>
+ <apache.commons.version>2.5</apache.commons.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <!-- Fix build on JDK-9 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <!-- Fix build on JDK-9 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <version>${hive.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${apache.commons.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-cli</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
new file mode 100644
index 0000000..8ff9a28
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.hadoop.WritableCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.hive.hcatalog.common.HCatException;
+import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.data.DefaultHCatRecord;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
+import org.apache.hive.hcatalog.data.transfer.HCatReader;
+import org.apache.hive.hcatalog.data.transfer.HCatWriter;
+import org.apache.hive.hcatalog.data.transfer.ReadEntity;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.apache.hive.hcatalog.data.transfer.WriteEntity;
+import org.apache.hive.hcatalog.data.transfer.WriterContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IO to read and write data using HCatalog.
+ *
+ * <h3>Reading using HCatalog</h3>
+ *
+ * <p>HCatalog source supports reading of HCatRecord from a HCatalog managed source, for eg. Hive.
+ *
+ * <p>To configure a HCatalog source, you must specify a metastore URI and a table name. Other
+ * optional parameters are database & filter For instance:
+ *
+ * <pre>{@code
+ * Map<String, String> configProperties = new HashMap<String, String>();
+ * configProperties.put("hive.metastore.uris","thrift://metastore-host:port");
+ *
+ * pipeline
+ * .apply(HCatalogIO.read()
+ * .withConfigProperties(configProperties)
+ * .withDatabase("default") //optional, assumes default if none specified
+ * .withTable("employee")
+ * .withFilter(filterString) //optional, may be specified if the table is partitioned
+ * }</pre>
+ *
+ * <h3>Writing using HCatalog</h3>
+ *
+ * <p>HCatalog sink supports writing of HCatRecord to a HCatalog managed source, for eg. Hive.
+ *
+ * <p>To configure a HCatalog sink, you must specify a metastore URI and a table name. Other
+ * optional parameters are database, partition & batchsize The destination table should exist
+ * beforehand, the transform does not create a new table if it does not exist For instance:
+ *
+ * <pre>{@code
+ * Map<String, String> configProperties = new HashMap<String, String>();
+ * configProperties.put("hive.metastore.uris","thrift://metastore-host:port");
+ *
+ * pipeline
+ * .apply(...)
+ * .apply(HiveIO.write()
+ * .withConfigProperties(configProperties)
+ * .withDatabase("default") //optional, assumes default if none specified
+ * .withTable("employee")
+ * .withPartition(partitionValues) //optional, may be specified if the table is partitioned
+ * .withBatchSize(1024L)) //optional, assumes a default batch size of 1024 if none specified
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class HCatalogIO {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HCatalogIO.class);
+
+ private static final long BATCH_SIZE = 1024L;
+ private static final String DEFAULT_DATABASE = "default";
+
+ /** Write data to Hive. */
+ public static Write write() {
+ return new AutoValue_HCatalogIO_Write.Builder().setBatchSize(BATCH_SIZE).build();
+ }
+
+ /** Read data from Hive. */
+ public static Read read() {
+ return new AutoValue_HCatalogIO_Read.Builder().setDatabase(DEFAULT_DATABASE).build();
+ }
+
+ private HCatalogIO() {}
+
+ /** A {@link PTransform} to read data using HCatalog. */
+ @VisibleForTesting
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<HCatRecord>> {
+ @Nullable abstract Map<String, String> getConfigProperties();
+ @Nullable abstract String getDatabase();
+ @Nullable abstract String getTable();
+ @Nullable abstract String getFilter();
+ @Nullable abstract ReaderContext getContext();
+ @Nullable abstract Integer getSplitId();
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConfigProperties(Map<String, String> configProperties);
+ abstract Builder setDatabase(String database);
+ abstract Builder setTable(String table);
+ abstract Builder setFilter(String filter);
+ abstract Builder setSplitId(Integer splitId);
+ abstract Builder setContext(ReaderContext context);
+ abstract Read build();
+ }
+
+ /** Sets the configuration properties like metastore URI. */
+ public Read withConfigProperties(Map<String, String> configProperties) {
+ return toBuilder().setConfigProperties(new HashMap<>(configProperties)).build();
+ }
+
+ /** Sets the database name. This is optional, assumes 'default' database if none specified */
+ public Read withDatabase(String database) {
+ return toBuilder().setDatabase(database).build();
+ }
+
+ /** Sets the table name to read from. */
+ public Read withTable(String table) {
+ return toBuilder().setTable(table).build();
+ }
+
+ /** Sets the filter details. This is optional, assumes none if not specified */
+ public Read withFilter(String filter) {
+ return toBuilder().setFilter(filter).build();
+ }
+
+ Read withSplitId(int splitId) {
+ checkArgument(splitId >= 0, "Invalid split id-" + splitId);
+ return toBuilder().setSplitId(splitId).build();
+ }
+
+ Read withContext(ReaderContext context) {
+ return toBuilder().setContext(context).build();
+ }
+
+ @Override
+ public PCollection<HCatRecord> expand(PBegin input) {
+ checkArgument(getTable() != null, "withTable() is required");
+ checkArgument(getConfigProperties() != null, "withConfigProperties() is required");
+
+ return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedHCatalogSource(this)));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("configProperties", getConfigProperties().toString()));
+ builder.add(DisplayData.item("table", getTable()));
+ builder.addIfNotNull(DisplayData.item("database", getDatabase()));
+ builder.addIfNotNull(DisplayData.item("filter", getFilter()));
+ }
+ }
+
+ /** A HCatalog {@link BoundedSource} reading {@link HCatRecord} from a given instance. */
+ @VisibleForTesting
+ static class BoundedHCatalogSource extends BoundedSource<HCatRecord> {
+ private final Read spec;
+
+ BoundedHCatalogSource(Read spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Coder<HCatRecord> getOutputCoder() {
+ return (Coder) WritableCoder.of(DefaultHCatRecord.class);
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ spec.populateDisplayData(builder);
+ }
+
+ @Override
+ public BoundedReader<HCatRecord> createReader(PipelineOptions options) {
+ return new BoundedHCatalogReader(this);
+ }
+
+ /**
+ * Returns the size of the table in bytes, does not take into consideration filter/partition
+ * details passed, if any.
+ */
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
+ Configuration conf = new Configuration();
+ for (Entry<String, String> entry : spec.getConfigProperties().entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ IMetaStoreClient client = null;
+ try {
+ HiveConf hiveConf = HCatUtil.getHiveConf(conf);
+ client = HCatUtil.getHiveMetastoreClient(hiveConf);
+ Table table = HCatUtil.getTable(client, spec.getDatabase(), spec.getTable());
+ return StatsUtils.getFileSizeForTable(hiveConf, table);
+ } finally {
+ // IMetaStoreClient is not AutoCloseable, closing it manually
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+ /**
+ * Calculates the 'desired' number of splits based on desiredBundleSizeBytes which is passed as
+ * a hint to native API. Retrieves the actual splits generated by native API, which could be
+ * different from the 'desired' split count calculated using desiredBundleSizeBytes
+ */
+ @Override
+ public List<BoundedSource<HCatRecord>> split(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ int desiredSplitCount = 1;
+ long estimatedSizeBytes = getEstimatedSizeBytes(options);
+ if (desiredBundleSizeBytes > 0 && estimatedSizeBytes > 0) {
+ desiredSplitCount = (int) Math.ceil((double) estimatedSizeBytes / desiredBundleSizeBytes);
+ }
+ ReaderContext readerContext = getReaderContext(desiredSplitCount);
+ //process the splits returned by native API
+ //this could be different from 'desiredSplitCount' calculated above
+ LOG.info(
+ "Splitting into bundles of {} bytes: "
+ + "estimated size {}, desired split count {}, actual split count {}",
+ desiredBundleSizeBytes,
+ estimatedSizeBytes,
+ desiredSplitCount,
+ readerContext.numSplits());
+
+ List<BoundedSource<HCatRecord>> res = new ArrayList<>();
+ for (int split = 0; split < readerContext.numSplits(); split++) {
+ res.add(new BoundedHCatalogSource(spec.withContext(readerContext).withSplitId(split)));
+ }
+ return res;
+ }
+
+ private ReaderContext getReaderContext(long desiredSplitCount) throws HCatException {
+ ReadEntity entity =
+ new ReadEntity.Builder()
+ .withDatabase(spec.getDatabase())
+ .withTable(spec.getTable())
+ .withFilter(spec.getFilter())
+ .build();
+ // pass the 'desired' split count as an hint to the API
+ Map<String, String> configProps = new HashMap<>(spec.getConfigProperties());
+ configProps.put(
+ HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
+ return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
+ }
+
+ static class BoundedHCatalogReader extends BoundedSource.BoundedReader<HCatRecord> {
+ private final BoundedHCatalogSource source;
+ private HCatRecord current;
+ private Iterator<HCatRecord> hcatIterator;
+
+ public BoundedHCatalogReader(BoundedHCatalogSource source) {
+ this.source = source;
+ }
+
+ @Override
+ public boolean start() throws HCatException {
+ HCatReader reader =
+ DataTransferFactory.getHCatReader(source.spec.getContext(), source.spec.getSplitId());
+ hcatIterator = reader.read();
+ return advance();
+ }
+
+ @Override
+ public boolean advance() {
+ if (hcatIterator.hasNext()) {
+ current = hcatIterator.next();
+ return true;
+ } else {
+ current = null;
+ return false;
+ }
+ }
+
+ @Override
+ public BoundedHCatalogSource getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ public HCatRecord getCurrent() {
+ if (current == null) {
+ throw new NoSuchElementException("Current element is null");
+ }
+ return current;
+ }
+
+ @Override
+ public void close() {
+ // nothing to close/release
+ }
+ }
+ }
+
+ /** A {@link PTransform} to write to a HCatalog managed source. */
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<HCatRecord>, PDone> {
+ @Nullable abstract Map<String, String> getConfigProperties();
+ @Nullable abstract String getDatabase();
+ @Nullable abstract String getTable();
+ @Nullable abstract Map<String, String> getPartition();
+ abstract long getBatchSize();
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConfigProperties(Map<String, String> configProperties);
+ abstract Builder setDatabase(String database);
+ abstract Builder setTable(String table);
+ abstract Builder setPartition(Map<String, String> partition);
+ abstract Builder setBatchSize(long batchSize);
+ abstract Write build();
+ }
+
+ /** Sets the configuration properties like metastore URI. */
+ public Write withConfigProperties(Map<String, String> configProperties) {
+ return toBuilder().setConfigProperties(new HashMap<>(configProperties)).build();
+ }
+
+ /** Sets the database name. This is optional, assumes 'default' database if none specified */
+ public Write withDatabase(String database) {
+ return toBuilder().setDatabase(database).build();
+ }
+
+ /** Sets the table name to write to, the table should exist beforehand. */
+ public Write withTable(String table) {
+ return toBuilder().setTable(table).build();
+ }
+
+ /** Sets the partition details. */
+ public Write withPartition(Map<String, String> partition) {
+ return toBuilder().setPartition(partition).build();
+ }
+
+ /**
+ * Sets batch size for the write operation. This is optional, assumes a default batch size of
+ * 1024 if not set
+ */
+ public Write withBatchSize(long batchSize) {
+ return toBuilder().setBatchSize(batchSize).build();
+ }
+
+ @Override
+ public PDone expand(PCollection<HCatRecord> input) {
+ checkArgument(getConfigProperties() != null, "withConfigProperties() is required");
+ checkArgument(getTable() != null, "withTable() is required");
+ input.apply(ParDo.of(new WriteFn(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ private static class WriteFn extends DoFn<HCatRecord, Void> {
+ private final Write spec;
+ private WriterContext writerContext;
+ private HCatWriter slaveWriter;
+ private HCatWriter masterWriter;
+ private List<HCatRecord> hCatRecordsBatch;
+
+ public WriteFn(Write spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.addIfNotNull(DisplayData.item("database", spec.getDatabase()));
+ builder.add(DisplayData.item("table", spec.getTable()));
+ builder.addIfNotNull(DisplayData.item("partition", String.valueOf(spec.getPartition())));
+ builder.add(DisplayData.item("configProperties", spec.getConfigProperties().toString()));
+ builder.add(DisplayData.item("batchSize", spec.getBatchSize()));
+ }
+
+ @Setup
+ public void initiateWrite() throws HCatException {
+ WriteEntity entity =
+ new WriteEntity.Builder()
+ .withDatabase(spec.getDatabase())
+ .withTable(spec.getTable())
+ .withPartition(spec.getPartition())
+ .build();
+ masterWriter = DataTransferFactory.getHCatWriter(entity, spec.getConfigProperties());
+ writerContext = masterWriter.prepareWrite();
+ slaveWriter = DataTransferFactory.getHCatWriter(writerContext);
+ }
+
+ @StartBundle
+ public void startBundle() {
+ hCatRecordsBatch = new ArrayList<>();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext ctx) throws HCatException {
+ hCatRecordsBatch.add(ctx.element());
+ if (hCatRecordsBatch.size() >= spec.getBatchSize()) {
+ flush();
+ }
+ }
+
+ @FinishBundle
+ public void finishBundle() throws HCatException {
+ flush();
+ }
+
+ private void flush() throws HCatException {
+ if (hCatRecordsBatch.isEmpty()) {
+ return;
+ }
+ try {
+ slaveWriter.write(hCatRecordsBatch.iterator());
+ masterWriter.commit(writerContext);
+ } catch (HCatException e) {
+ LOG.error("Exception in flush - write/commit data to Hive", e);
+ //abort on exception
+ masterWriter.abort(writerContext);
+ throw e;
+ } finally {
+ hCatRecordsBatch.clear();
+ }
+ }
+
+ @Teardown
+ public void tearDown() throws Exception {
+ if (slaveWriter != null) {
+ slaveWriter = null;
+ }
+ if (masterWriter != null) {
+ masterWriter = null;
+ }
+ if (writerContext != null) {
+ writerContext = null;
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
new file mode 100644
index 0000000..dff5bd1
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transforms for reading and writing using HCatalog.
+ */
+package org.apache.beam.sdk.io.hcatalog;
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
new file mode 100644
index 0000000..31e5b1c
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static org.apache.hive.hcatalog.common.HCatUtil.makePathASafeFileName;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a light-weight embedded metastore. This class is a trimmed-down version of <a
+ * href="https://github.com/apache/hive/blob/master
+ * /hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java">
+ * https://github.com/apache/hive/blob/master/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce
+ * /HCatBaseTest.java </a>
+ */
+final class EmbeddedMetastoreService implements AutoCloseable {
+ private final Driver driver;
+ private final HiveConf hiveConf;
+ private final SessionState sessionState;
+
+ EmbeddedMetastoreService(String baseDirPath) throws IOException {
+ FileUtils.forceDeleteOnExit(new File(baseDirPath));
+
+ String hiveDirPath = makePathASafeFileName(baseDirPath + "/hive");
+ String testDataDirPath =
+ makePathASafeFileName(
+ hiveDirPath
+ + "/data/"
+ + EmbeddedMetastoreService.class.getCanonicalName()
+ + System.currentTimeMillis());
+ String testWarehouseDirPath = makePathASafeFileName(testDataDirPath + "/warehouse");
+
+ hiveConf = new HiveConf(getClass());
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, "");
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, testWarehouseDirPath);
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
+ hiveConf.setVar(
+ HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd."
+ + "SQLStdHiveAuthorizerFactory");
+ hiveConf.set("test.tmp.dir", hiveDirPath);
+
+ System.setProperty("derby.stream.error.file", "/dev/null");
+ driver = new Driver(hiveConf);
+ sessionState = SessionState.start(new CliSessionState(hiveConf));
+ }
+
+ /** Executes the passed query on the embedded metastore service. */
+ void executeQuery(String query) throws CommandNeedRetryException {
+ driver.run(query);
+ }
+
+ /** Returns the HiveConf object for the embedded metastore. */
+ HiveConf getHiveConf() {
+ return hiveConf;
+ }
+
+ @Override
+ public void close() throws Exception {
+ driver.close();
+ sessionState.close();
+ }
+}
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
new file mode 100644
index 0000000..dc53c84
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_DATABASE;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_FILTER;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_RECORDS_COUNT;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_TABLE;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getConfigPropertiesAsMap;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getExpectedRecords;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getHCatRecords;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getReaderContext;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.insertTestData;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.isA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/** Test for HCatalogIO. */
+public class HCatalogIOTest implements Serializable {
+ private static final PipelineOptions OPTIONS = PipelineOptionsFactory.create();
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+
+ @Rule public final transient TestPipeline defaultPipeline = TestPipeline.create();
+
+ @Rule public final transient TestPipeline readAfterWritePipeline = TestPipeline.create();
+
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+ @Rule
+ public final transient TestRule testDataSetupRule =
+ new TestWatcher() {
+ public Statement apply(final Statement base, final Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ if (description.getAnnotation(NeedsTestData.class) != null) {
+ prepareTestData();
+ } else if (description.getAnnotation(NeedsEmptyTestTables.class) != null) {
+ reCreateTestTable();
+ }
+ base.evaluate();
+ }
+ };
+ }
+ };
+
+ private static EmbeddedMetastoreService service;
+
+ /** Use this annotation to setup complete test data(table populated with records). */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ElementType.METHOD})
+ private @interface NeedsTestData {}
+
+ /** Use this annotation to setup test tables alone(empty tables, no records are populated). */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ElementType.METHOD})
+ private @interface NeedsEmptyTestTables {}
+
+ @BeforeClass
+ public static void setupEmbeddedMetastoreService () throws IOException {
+ service = new EmbeddedMetastoreService(TMP_FOLDER.getRoot().getAbsolutePath());
+ }
+
+ @AfterClass
+ public static void shutdownEmbeddedMetastoreService () throws Exception {
+ service.executeQuery("drop table " + TEST_TABLE);
+ service.close();
+ }
+
+ /** Perform end-to-end test of Write-then-Read operation. */
+ @Test
+ @NeedsEmptyTestTables
+ public void testWriteThenReadSuccess() throws Exception {
+ defaultPipeline
+ .apply(Create.of(getHCatRecords(TEST_RECORDS_COUNT)))
+ .apply(
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withDatabase(TEST_DATABASE)
+ .withTable(TEST_TABLE)
+ .withPartition(new java.util.HashMap<String, String>())
+ .withBatchSize(512L));
+ defaultPipeline.run();
+
+ PCollection<String> output = readAfterWritePipeline
+ .apply(
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withDatabase(TEST_DATABASE)
+ .withTable(TEST_TABLE)
+ .withFilter(TEST_FILTER))
+ .apply(
+ ParDo.of(
+ new DoFn<HCatRecord, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().get(0).toString());
+ }
+ }));
+ PAssert.that(output).containsInAnyOrder(getExpectedRecords(TEST_RECORDS_COUNT));
+ readAfterWritePipeline.run();
+ }
+
+ /** Test of Write to a non-existent table. */
+ @Test
+ public void testWriteFailureTableDoesNotExist() throws Exception {
+ thrown.expectCause(isA(UserCodeException.class));
+ thrown.expectMessage(containsString("org.apache.hive.hcatalog.common.HCatException"));
+ thrown.expectMessage(containsString("NoSuchObjectException"));
+ defaultPipeline
+ .apply(Create.of(getHCatRecords(TEST_RECORDS_COUNT)))
+ .apply(
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable("myowntable"));
+ defaultPipeline.run();
+ }
+
+ /** Test of Write without specifying a table. */
+ @Test
+ public void testWriteFailureValidationTable() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("withTable() is required");
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .expand(null);
+ }
+
+ /** Test of Write without specifying configuration properties. */
+ @Test
+ public void testWriteFailureValidationConfigProp() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("withConfigProperties() is required");
+ HCatalogIO.write().withTable("myowntable").expand(null);
+ }
+
+ /** Test of Read from a non-existent table. */
+ @Test
+ public void testReadFailureTableDoesNotExist() throws Exception {
+ defaultPipeline.apply(
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable("myowntable"));
+ thrown.expectCause(isA(NoSuchObjectException.class));
+ defaultPipeline.run();
+ }
+
+ /** Test of Read without specifying configuration properties. */
+ @Test
+ public void testReadFailureValidationConfig() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("withConfigProperties() is required");
+ HCatalogIO.read().withTable("myowntable").expand(null);
+ }
+
+ /** Test of Read without specifying a table. */
+ @Test
+ public void testReadFailureValidationTable() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("withTable() is required");
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .expand(null);
+ }
+
+ /** Test of Read using SourceTestUtils.readFromSource(..). */
+ @Test
+ @NeedsTestData
+ public void testReadFromSource() throws Exception {
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE);
+
+ List<String> records = new ArrayList<>();
+ for (int i = 0; i < context.numSplits(); i++) {
+ BoundedHCatalogSource source = new BoundedHCatalogSource(spec.withSplitId(i));
+ for (HCatRecord record : SourceTestUtils.readFromSource(source, OPTIONS)) {
+ records.add(record.get(0).toString());
+ }
+ }
+ assertThat(records, containsInAnyOrder(getExpectedRecords(TEST_RECORDS_COUNT).toArray()));
+ }
+
+ /** Test of Read using SourceTestUtils.assertSourcesEqualReferenceSource(..). */
+ @Test
+ @NeedsTestData
+ public void testSourceEqualsSplits() throws Exception {
+ final int numRows = 1500;
+ final int numSamples = 10;
+ final long bytesPerRow = 15;
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE);
+
+ BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
+ List<BoundedSource<HCatRecord>> unSplitSource = source.split(-1, OPTIONS);
+ assertEquals(1, unSplitSource.size());
+
+ List<BoundedSource<HCatRecord>> splits =
+ source.split(numRows * bytesPerRow / numSamples, OPTIONS);
+ assertTrue(splits.size() >= 1);
+
+ SourceTestUtils.assertSourcesEqualReferenceSource(unSplitSource.get(0), splits, OPTIONS);
+ }
+
+ private void reCreateTestTable() throws CommandNeedRetryException {
+ service.executeQuery("drop table " + TEST_TABLE);
+ service.executeQuery("create table " + TEST_TABLE + "(mycol1 string, mycol2 int)");
+ }
+
+ private void prepareTestData() throws Exception {
+ reCreateTestTable();
+ insertTestData(getConfigPropertiesAsMap(service.getHiveConf()));
+ }
+}
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
new file mode 100644
index 0000000..ae1eb50
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.hcatalog.common.HCatException;
+import org.apache.hive.hcatalog.data.DefaultHCatRecord;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
+import org.apache.hive.hcatalog.data.transfer.ReadEntity;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.apache.hive.hcatalog.data.transfer.WriteEntity;
+import org.apache.hive.hcatalog.data.transfer.WriterContext;
+
+/** Utility class for HCatalogIOTest. */
+class HCatalogIOTestUtils {
+ static final String TEST_DATABASE = "default";
+ static final String TEST_TABLE = "mytable";
+ static final String TEST_FILTER = "myfilter";
+ static final int TEST_RECORDS_COUNT = 1000;
+
+ private static final ReadEntity READ_ENTITY =
+ new ReadEntity.Builder().withTable(TEST_TABLE).build();
+ private static final WriteEntity WRITE_ENTITY =
+ new WriteEntity.Builder().withTable(TEST_TABLE).build();
+
+ /** Returns a ReaderContext instance for the passed datastore config params. */
+ static ReaderContext getReaderContext(Map<String, String> config) throws HCatException {
+ return DataTransferFactory.getHCatReader(READ_ENTITY, config).prepareRead();
+ }
+
+ /** Returns a WriterContext instance for the passed datastore config params. */
+ private static WriterContext getWriterContext(Map<String, String> config) throws HCatException {
+ return DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).prepareWrite();
+ }
+
+ /** Writes records to the table using the passed WriterContext. */
+ private static void writeRecords(WriterContext context) throws HCatException {
+ DataTransferFactory.getHCatWriter(context).write(getHCatRecords(TEST_RECORDS_COUNT).iterator());
+ }
+
+ /** Commits the pending writes to the database. */
+ private static void commitRecords(Map<String, String> config, WriterContext context)
+ throws IOException {
+ DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).commit(context);
+ }
+
+ /** Returns a list of strings containing 'expected' test data for verification. */
+ static List<String> getExpectedRecords(int count) {
+ List<String> expected = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ expected.add("record " + i);
+ }
+ return expected;
+ }
+
+ /** Returns a list of HCatRecords of passed size. */
+ static List<HCatRecord> getHCatRecords(int size) {
+ List<HCatRecord> expected = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ expected.add(toHCatRecord(i));
+ }
+ return expected;
+ }
+
+ /** Inserts data into test datastore. */
+ static void insertTestData(Map<String, String> configMap) throws Exception {
+ WriterContext cntxt = getWriterContext(configMap);
+ writeRecords(cntxt);
+ commitRecords(configMap, cntxt);
+ }
+
+ /** Returns config params for the test datastore as a Map. */
+ static Map<String, String> getConfigPropertiesAsMap(HiveConf hiveConf) {
+ Map<String, String> map = new HashMap<>();
+ for (Entry<String, String> kv : hiveConf) {
+ map.put(kv.getKey(), kv.getValue());
+ }
+ return map;
+ }
+
+ /** returns a DefaultHCatRecord instance for passed value. */
+ private static DefaultHCatRecord toHCatRecord(int value) {
+ return new DefaultHCatRecord(Arrays.<Object>asList("record " + value, value));
+ }
+}
diff --git a/sdks/java/io/hcatalog/src/test/resources/hive-site.xml b/sdks/java/io/hcatalog/src/test/resources/hive-site.xml
new file mode 100644
index 0000000..5bb1496
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/resources/hive-site.xml
@@ -0,0 +1,301 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This file is a copy of https://github.com/apache/hive/blob/master/data/conf/hive-site.xml used to support embedded Hive metastore-->
+<configuration>
+
+<property>
+ <name>hive.in.test</name>
+ <value>true</value>
+ <description>Internal marker for test. Used for masking env-dependent values</description>
+</property>
+
+<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
+<!-- that are implied by Hadoop setup variables. -->
+<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
+<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
+<!-- resource). -->
+
+<!-- Hive Execution Parameters -->
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>${test.tmp.dir}/hadoop-tmp</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+<!--
+<property>
+ <name>hive.exec.reducers.max</name>
+ <value>1</value>
+ <description>maximum number of reducers</description>
+</property>
+-->
+
+<property>
+ <name>hive.exec.scratchdir</name>
+ <value>${test.tmp.dir}/scratchdir</value>
+ <description>Scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>hive.exec.local.scratchdir</name>
+ <value>${test.tmp.dir}/localscratchdir/</value>
+ <description>Local scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>APP</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>mine</value>
+</property>
+
+<property>
+ <!-- this should eventually be deprecated since the metastore should supply this -->
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${test.warehouse.dir}</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.metadb.dir</name>
+ <value>file://${test.tmp.dir}/metadb/</value>
+ <description>
+ Required by metastore server or if the uris argument below is not supplied
+ </description>
+</property>
+
+<property>
+ <name>test.log.dir</name>
+ <value>${test.tmp.dir}/log/</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.files</name>
+ <value>${hive.root}/data/files</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.jar.path</name>
+ <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.rawstore.impl</name>
+ <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
+ <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description>
+</property>
+
+<property>
+ <name>hive.querylog.location</name>
+ <value>${test.tmp.dir}/tmp</value>
+ <description>Location of the structured hive logs</description>
+</property>
+
+<property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
+ <description>Pre Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <description>Post Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description>
+</property>
+
+<property>
+ <key>hive.unlock.numretries</key>
+ <value>2</value>
+ <description>The number of times you want to retry to do one unlock</description>
+</property>
+
+<property>
+ <key>hive.lock.sleep.between.retries</key>
+ <value>2</value>
+ <description>The sleep time (in seconds) between various retries</description>
+</property>
+
+
+<property>
+ <name>fs.pfile.impl</name>
+ <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+ <description>A proxy for local file system used for cross file system testing</description>
+</property>
+
+<property>
+ <name>hive.exec.mode.local.auto</name>
+ <value>false</value>
+ <description>
+ Let hive determine whether to run in local mode automatically
+ Disabling this for tests so that minimr is not affected
+ </description>
+</property>
+
+<property>
+ <name>hive.auto.convert.join</name>
+ <value>false</value>
+ <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description>
+</property>
+
+<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>false</value>
+ <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
+ <name>hive.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
+ <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
+</property>
+
+<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile format</description>
+</property>
+
+<property>
+ <name>hive.stats.key.prefix.reserve.length</name>
+ <value>0</value>
+</property>
+
+<property>
+ <name>hive.conf.restricted.list</name>
+ <value>dummy.config.value</value>
+ <description>Using dummy config value above because you cannot override config with empty value</description>
+</property>
+
+<property>
+ <name>hive.exec.submit.local.task.via.child</name>
+ <value>false</value>
+</property>
+
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.override</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.hivesite</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+ <name>test.var.hiveconf.property</name>
+ <value>${hive.exec.default.partition.name}</value>
+ <description>Test hiveconf property substitution</description>
+</property>
+
+<property>
+ <name>test.property1</name>
+ <value>value1</value>
+ <description>Test property defined in hive-site.xml only</description>
+</property>
+
+<property>
+ <name>hive.test.dummystats.aggregator</name>
+ <value>value2</value>
+</property>
+
+<property>
+ <name>hive.fetch.task.conversion</name>
+ <value>minimal</value>
+</property>
+
+<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hive_admin_user</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.size</name>
+ <value>8388608</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.arena.size</name>
+ <value>8388608</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.alloc.max</name>
+ <value>2097152</value>
+</property>
+
+
+<property>
+ <name>hive.llap.io.cache.orc.alloc.min</name>
+ <value>32768</value>
+</property>
+
+<property>
+ <name>hive.llap.cache.allow.synthetic.fileid</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.llap.io.use.lrfu</name>
+ <value>true</value>
+</property>
+
+
+<property>
+ <name>hive.llap.io.allocator.direct</name>
+ <value>false</value>
+</property>
+
+
+</configuration>
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index 17c26a0..3a5f53b 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -49,13 +49,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
@@ -67,6 +65,178 @@
</dependencies>
</profile>
+ <!--
+ This profile invokes PerfKitBenchmarker, which does benchmarking of
+ the IO ITs. The arguments passed to it allow it to invoke mvn again
+ with the desired benchmark.
+
+ To invoke this, run:
+
+ mvn verify -Dio-it-suite -pl sdks/java/io/jdbc
+ -DpkbLocation="path-to-pkb.py" \
+ -DintegrationTestPipelineOptions='["-tempRoot=gs://bucket/staging"]'
+ -->
+ <profile>
+ <id>io-it-suite</id>
+ <activation>
+ <property><name>io-it-suite</name></property>
+ </activation>
+ <properties>
+ <!-- This is based on the location of the current pom relative to the root
+ See discussion in BEAM-2460 -->
+ <beamRootProjectDir>${project.parent.parent.parent.parent.basedir}</beamRootProjectDir>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.gmaven</groupId>
+ <artifactId>groovy-maven-plugin</artifactId>
+ <version>${groovy-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>find-supported-python-for-compile</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>execute</goal>
+ </goals>
+ <configuration>
+ <source>${beamRootProjectDir}/sdks/python/findSupportedPython.groovy</source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven-exec-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>${python.interpreter.bin}</executable>
+ <arguments>
+ <argument>${pkbLocation}</argument>
+ <argument>-benchmarks=beam_integration_benchmark</argument>
+ <argument>-beam_it_profile=io-it</argument>
+ <argument>-beam_location=${beamRootProjectDir}</argument>
+ <argument>-beam_prebuilt=true</argument>
+ <argument>-beam_sdk=java</argument>
+ <argument>-kubeconfig=${kubeconfig}</argument>
+ <argument>-kubectl=${kubectl}</argument>
+ <!-- runner overrides, controlled via forceDirectRunner -->
+ <argument>${pkbBeamRunnerProfile}</argument>
+ <argument>${pkbBeamRunnerOption}</argument>
+ <!-- specific to this IO -->
+ <argument>-beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config.yml</argument>
+ <argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml</argument>
+ <argument>-beam_it_module=sdks/java/io/jdbc</argument>
+ <argument>-beam_it_class=org.apache.beam.sdk.io.jdbc.JdbcIOIT</argument>
+ <!-- arguments typically defined by user -->
+ <argument>-beam_it_options=${integrationTestPipelineOptions}</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire-plugin.version}</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!--
+ io-it-suite overrides part of io-it-suite, allowing users to run tests
+ when they are on a separate network from the kubernetes cluster by
+ creating a LoadBalancer service.
+ -->
+ <profile>
+ <id>io-it-suite-local</id>
+ <activation><property><name>io-it-suite-local</name></property></activation>
+ <properties>
+ <!-- This is based on the location of the current pom relative to the root
+ See discussion in BEAM-2460 -->
+ <beamRootProjectDir>${project.parent.parent.parent.parent.basedir}</beamRootProjectDir>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.gmaven</groupId>
+ <artifactId>groovy-maven-plugin</artifactId>
+ <version>${groovy-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>find-supported-python-for-compile</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>execute</goal>
+ </goals>
+ <configuration>
+ <source>${beamRootProjectDir}/sdks/python/findSupportedPython.groovy</source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven-exec-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>${python.interpreter.bin}</executable>
+ <arguments>
+ <argument>${pkbLocation}</argument>
+ <argument>-benchmarks=beam_integration_benchmark</argument>
+ <argument>-beam_it_profile=io-it</argument>
+ <argument>-beam_location=${beamRootProjectDir}</argument>
+ <argument>-beam_prebuilt=true</argument>
+ <argument>-beam_sdk=java</argument>
+ <argument>-kubeconfig=${kubeconfig}</argument>
+ <argument>-kubectl=${kubectl}</argument>
+ <!-- runner overrides, controlled via forceDirectRunner -->
+ <argument>${pkbBeamRunnerProfile}</argument>
+ <argument>${pkbBeamRunnerOption}</argument>
+ <!-- specific to this IO -->
+ <argument>-beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml</argument>
+ <argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml</argument>
+ <argument>-beam_it_module=sdks/java/io/jdbc</argument>
+ <argument>-beam_it_class=org.apache.beam.sdk.io.jdbc.JdbcIOIT</argument>
+ <!-- arguments typically defined by user -->
+ <argument>-beam_it_options=${integrationTestPipelineOptions}</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire-plugin.version}</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- Include the Google Cloud Dataflow runner -P dataflow-runner -->
<profile>
<id>dataflow-runner</id>
@@ -87,11 +257,6 @@
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -107,11 +272,6 @@
<version>2.1.1</version>
</dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
-
<!-- compile dependencies -->
<dependency>
<groupId>com.google.auto.value</groupId>
@@ -155,6 +315,11 @@
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<scope>test</scope>
</dependency>
@@ -170,5 +335,10 @@
<scope>test</scope>
<classifier>tests</classifier>
</dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-common</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
index 8092da6..b134ec0 100644
--- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
+++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
@@ -18,7 +18,6 @@
package org.apache.beam.sdk.io.jdbc;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
@@ -26,23 +25,23 @@
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Random;
import javax.annotation.Nullable;
import javax.sql.DataSource;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PDone;
import org.apache.commons.dbcp2.BasicDataSource;
@@ -133,7 +132,7 @@
* Consider using <a href="https://en.wikipedia.org/wiki/Merge_(SQL)">MERGE ("upsert")
* statements</a> supported by your database instead.
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class JdbcIO {
/**
* Read data from a JDBC datasource.
@@ -145,6 +144,17 @@
}
/**
+ * Like {@link #read}, but executes multiple instances of the query substituting each element
+ * of a {@link PCollection} as query parameters.
+ *
+ * @param <ParameterT> Type of the data representing query parameters.
+ * @param <OutputT> Type of the data to be read.
+ */
+ public static <ParameterT, OutputT> ReadAll<ParameterT, OutputT> readAll() {
+ return new AutoValue_JdbcIO_ReadAll.Builder<ParameterT, OutputT>().build();
+ }
+
+ /**
* Write data to a JDBC datasource.
*
* @param <T> Type of the data to be written.
@@ -159,6 +169,7 @@
* An interface used by {@link JdbcIO.Read} for converting each row of the {@link ResultSet} into
* an element of the resulting {@link PCollection}.
*/
+ @FunctionalInterface
public interface RowMapper<T> extends Serializable {
T mapRow(ResultSet resultSet) throws Exception;
}
@@ -190,20 +201,16 @@
}
public static DataSourceConfiguration create(DataSource dataSource) {
- checkArgument(dataSource != null, "DataSourceConfiguration.create(dataSource) called with "
- + "null data source");
- checkArgument(dataSource instanceof Serializable,
- "DataSourceConfiguration.create(dataSource) called with a dataSource not Serializable");
+ checkArgument(dataSource != null, "dataSource can not be null");
+ checkArgument(dataSource instanceof Serializable, "dataSource must be Serializable");
return new AutoValue_JdbcIO_DataSourceConfiguration.Builder()
.setDataSource(dataSource)
.build();
}
public static DataSourceConfiguration create(String driverClassName, String url) {
- checkArgument(driverClassName != null,
- "DataSourceConfiguration.create(driverClassName, url) called with null driverClassName");
- checkArgument(url != null,
- "DataSourceConfiguration.create(driverClassName, url) called with null url");
+ checkArgument(driverClassName != null, "driverClassName can not be null");
+ checkArgument(url != null, "url can not be null");
return new AutoValue_JdbcIO_DataSourceConfiguration.Builder()
.setDriverClassName(driverClassName)
.setUrl(url)
@@ -226,9 +233,7 @@
* {@link #withPassword(String)}, so they do not need to be included here.
*/
public DataSourceConfiguration withConnectionProperties(String connectionProperties) {
- checkArgument(connectionProperties != null, "DataSourceConfiguration.create(driver, url)"
- + ".withConnectionProperties(connectionProperties) "
- + "called with null connectionProperties");
+ checkArgument(connectionProperties != null, "connectionProperties can not be null");
return builder().setConnectionProperties(connectionProperties).build();
}
@@ -264,15 +269,16 @@
* An interface used by the JdbcIO Write to set the parameters of the {@link PreparedStatement}
* used to setParameters into the database.
*/
+ @FunctionalInterface
public interface StatementPreparator extends Serializable {
void setParameters(PreparedStatement preparedStatement) throws Exception;
}
- /** A {@link PTransform} to read data from a JDBC datasource. */
+ /** Implementation of {@link #read}. */
@AutoValue
public abstract static class Read<T> extends PTransform<PBegin, PCollection<T>> {
@Nullable abstract DataSourceConfiguration getDataSourceConfiguration();
- @Nullable abstract String getQuery();
+ @Nullable abstract ValueProvider<String> getQuery();
@Nullable abstract StatementPreparator getStatementPreparator();
@Nullable abstract RowMapper<T> getRowMapper();
@Nullable abstract Coder<T> getCoder();
@@ -282,7 +288,7 @@
@AutoValue.Builder
abstract static class Builder<T> {
abstract Builder<T> setDataSourceConfiguration(DataSourceConfiguration config);
- abstract Builder<T> setQuery(String query);
+ abstract Builder<T> setQuery(ValueProvider<String> query);
abstract Builder<T> setStatementPreparator(StatementPreparator statementPreparator);
abstract Builder<T> setRowMapper(RowMapper<T> rowMapper);
abstract Builder<T> setCoder(Coder<T> coder);
@@ -290,66 +296,61 @@
}
public Read<T> withDataSourceConfiguration(DataSourceConfiguration configuration) {
- checkArgument(configuration != null, "JdbcIO.read().withDataSourceConfiguration"
- + "(configuration) called with null configuration");
+ checkArgument(configuration != null, "configuration can not be null");
return toBuilder().setDataSourceConfiguration(configuration).build();
}
public Read<T> withQuery(String query) {
- checkArgument(query != null, "JdbcIO.read().withQuery(query) called with null query");
+ checkArgument(query != null, "query can not be null");
+ return withQuery(ValueProvider.StaticValueProvider.of(query));
+ }
+
+ public Read<T> withQuery(ValueProvider<String> query) {
+ checkArgument(query != null, "query can not be null");
return toBuilder().setQuery(query).build();
}
public Read<T> withStatementPreparator(StatementPreparator statementPreparator) {
- checkArgument(statementPreparator != null,
- "JdbcIO.read().withStatementPreparator(statementPreparator) called "
- + "with null statementPreparator");
+ checkArgument(statementPreparator != null, "statementPreparator can not be null");
return toBuilder().setStatementPreparator(statementPreparator).build();
}
public Read<T> withRowMapper(RowMapper<T> rowMapper) {
- checkArgument(rowMapper != null,
- "JdbcIO.read().withRowMapper(rowMapper) called with null rowMapper");
+ checkArgument(rowMapper != null, "rowMapper can not be null");
return toBuilder().setRowMapper(rowMapper).build();
}
public Read<T> withCoder(Coder<T> coder) {
- checkArgument(coder != null, "JdbcIO.read().withCoder(coder) called with null coder");
+ checkArgument(coder != null, "coder can not be null");
return toBuilder().setCoder(coder).build();
}
@Override
public PCollection<T> expand(PBegin input) {
- return input
- .apply(Create.of(getQuery()))
- .apply(ParDo.of(new ReadFn<>(this))).setCoder(getCoder())
- .apply(ParDo.of(new DoFn<T, KV<Integer, T>>() {
- private Random random;
- @Setup
- public void setup() {
- random = new Random();
- }
- @ProcessElement
- public void processElement(ProcessContext context) {
- context.output(KV.of(random.nextInt(), context.element()));
- }
- }))
- .apply(GroupByKey.<Integer, T>create())
- .apply(Values.<Iterable<T>>create())
- .apply(Flatten.<T>iterables());
- }
+ checkArgument(getQuery() != null, "withQuery() is required");
+ checkArgument(getRowMapper() != null, "withRowMapper() is required");
+ checkArgument(getCoder() != null, "withCoder() is required");
+ checkArgument(
+ getDataSourceConfiguration() != null, "withDataSourceConfiguration() is required");
- @Override
- public void validate(PipelineOptions options) {
- checkState(getQuery() != null,
- "JdbcIO.read() requires a query to be set via withQuery(query)");
- checkState(getRowMapper() != null,
- "JdbcIO.read() requires a rowMapper to be set via withRowMapper(rowMapper)");
- checkState(getCoder() != null,
- "JdbcIO.read() requires a coder to be set via withCoder(coder)");
- checkState(getDataSourceConfiguration() != null,
- "JdbcIO.read() requires a DataSource configuration to be set via "
- + "withDataSourceConfiguration(dataSourceConfiguration)");
+ return input
+ .apply(Create.of((Void) null))
+ .apply(
+ JdbcIO.<Void, T>readAll()
+ .withDataSourceConfiguration(getDataSourceConfiguration())
+ .withQuery(getQuery())
+ .withCoder(getCoder())
+ .withRowMapper(getRowMapper())
+ .withParameterSetter(
+ new PreparedStatementSetter<Void>() {
+ @Override
+ public void setParameters(Void element, PreparedStatement preparedStatement)
+ throws Exception {
+ if (getStatementPreparator() != null) {
+ getStatementPreparator().setParameters(preparedStatement);
+ }
+ }
+ }));
}
@Override
@@ -360,44 +361,138 @@
builder.add(DisplayData.item("coder", getCoder().getClass().getName()));
getDataSourceConfiguration().populateDisplayData(builder);
}
+ }
- /** A {@link DoFn} executing the SQL query to read from the database. */
- static class ReadFn<T> extends DoFn<String, T> {
- private JdbcIO.Read<T> spec;
- private DataSource dataSource;
- private Connection connection;
+ /** Implementation of {@link #readAll}. */
- private ReadFn(Read<T> spec) {
- this.spec = spec;
- }
+ /** Implementation of {@link #read}. */
+ @AutoValue
+ public abstract static class ReadAll<ParameterT, OutputT>
+ extends PTransform<PCollection<ParameterT>, PCollection<OutputT>> {
+ @Nullable abstract DataSourceConfiguration getDataSourceConfiguration();
+ @Nullable abstract ValueProvider<String> getQuery();
+ @Nullable abstract PreparedStatementSetter<ParameterT> getParameterSetter();
+ @Nullable abstract RowMapper<OutputT> getRowMapper();
+ @Nullable abstract Coder<OutputT> getCoder();
- @Setup
- public void setup() throws Exception {
- dataSource = spec.getDataSourceConfiguration().buildDatasource();
- connection = dataSource.getConnection();
- }
+ abstract Builder<ParameterT, OutputT> toBuilder();
- @ProcessElement
- public void processElement(ProcessContext context) throws Exception {
- String query = context.element();
- try (PreparedStatement statement = connection.prepareStatement(query)) {
- if (this.spec.getStatementPreparator() != null) {
- this.spec.getStatementPreparator().setParameters(statement);
- }
- try (ResultSet resultSet = statement.executeQuery()) {
- while (resultSet.next()) {
- context.output(spec.getRowMapper().mapRow(resultSet));
- }
+ @AutoValue.Builder
+ abstract static class Builder<ParameterT, OutputT> {
+ abstract Builder<ParameterT, OutputT> setDataSourceConfiguration(
+ DataSourceConfiguration config);
+ abstract Builder<ParameterT, OutputT> setQuery(ValueProvider<String> query);
+ abstract Builder<ParameterT, OutputT> setParameterSetter(
+ PreparedStatementSetter<ParameterT> parameterSetter);
+ abstract Builder<ParameterT, OutputT> setRowMapper(RowMapper<OutputT> rowMapper);
+ abstract Builder<ParameterT, OutputT> setCoder(Coder<OutputT> coder);
+ abstract ReadAll<ParameterT, OutputT> build();
+ }
+
+ public ReadAll<ParameterT, OutputT> withDataSourceConfiguration(
+ DataSourceConfiguration configuration) {
+ checkArgument(configuration != null, "JdbcIO.readAll().withDataSourceConfiguration"
+ + "(configuration) called with null configuration");
+ return toBuilder().setDataSourceConfiguration(configuration).build();
+ }
+
+ public ReadAll<ParameterT, OutputT> withQuery(String query) {
+ checkArgument(query != null, "JdbcIO.readAll().withQuery(query) called with null query");
+ return withQuery(ValueProvider.StaticValueProvider.of(query));
+ }
+
+ public ReadAll<ParameterT, OutputT> withQuery(ValueProvider<String> query) {
+ checkArgument(query != null, "JdbcIO.readAll().withQuery(query) called with null query");
+ return toBuilder().setQuery(query).build();
+ }
+
+ public ReadAll<ParameterT, OutputT> withParameterSetter(
+ PreparedStatementSetter<ParameterT> parameterSetter) {
+ checkArgument(parameterSetter != null,
+ "JdbcIO.readAll().withParameterSetter(parameterSetter) called "
+ + "with null statementPreparator");
+ return toBuilder().setParameterSetter(parameterSetter).build();
+ }
+
+ public ReadAll<ParameterT, OutputT> withRowMapper(RowMapper<OutputT> rowMapper) {
+ checkArgument(rowMapper != null,
+ "JdbcIO.readAll().withRowMapper(rowMapper) called with null rowMapper");
+ return toBuilder().setRowMapper(rowMapper).build();
+ }
+
+ public ReadAll<ParameterT, OutputT> withCoder(Coder<OutputT> coder) {
+ checkArgument(coder != null, "JdbcIO.readAll().withCoder(coder) called with null coder");
+ return toBuilder().setCoder(coder).build();
+ }
+
+ @Override
+ public PCollection<OutputT> expand(PCollection<ParameterT> input) {
+ return input
+ .apply(
+ ParDo.of(
+ new ReadFn<>(
+ getDataSourceConfiguration(),
+ getQuery(),
+ getParameterSetter(),
+ getRowMapper())))
+ .setCoder(getCoder())
+ .apply(new Reparallelize<OutputT>());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("query", getQuery()));
+ builder.add(DisplayData.item("rowMapper", getRowMapper().getClass().getName()));
+ builder.add(DisplayData.item("coder", getCoder().getClass().getName()));
+ getDataSourceConfiguration().populateDisplayData(builder);
+ }
+ }
+
+ /** A {@link DoFn} executing the SQL query to read from the database. */
+ private static class ReadFn<ParameterT, OutputT> extends DoFn<ParameterT, OutputT> {
+ private final DataSourceConfiguration dataSourceConfiguration;
+ private final ValueProvider<String> query;
+ private final PreparedStatementSetter<ParameterT> parameterSetter;
+ private final RowMapper<OutputT> rowMapper;
+
+ private DataSource dataSource;
+ private Connection connection;
+
+ private ReadFn(
+ DataSourceConfiguration dataSourceConfiguration,
+ ValueProvider<String> query,
+ PreparedStatementSetter<ParameterT> parameterSetter,
+ RowMapper<OutputT> rowMapper) {
+ this.dataSourceConfiguration = dataSourceConfiguration;
+ this.query = query;
+ this.parameterSetter = parameterSetter;
+ this.rowMapper = rowMapper;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ dataSource = dataSourceConfiguration.buildDatasource();
+ connection = dataSource.getConnection();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) throws Exception {
+ try (PreparedStatement statement = connection.prepareStatement(query.get())) {
+ parameterSetter.setParameters(context.element(), statement);
+ try (ResultSet resultSet = statement.executeQuery()) {
+ while (resultSet.next()) {
+ context.output(rowMapper.mapRow(resultSet));
}
}
}
+ }
- @Teardown
- public void teardown() throws Exception {
- connection.close();
- if (dataSource instanceof AutoCloseable) {
- ((AutoCloseable) dataSource).close();
- }
+ @Teardown
+ public void teardown() throws Exception {
+ connection.close();
+ if (dataSource instanceof AutoCloseable) {
+ ((AutoCloseable) dataSource).close();
}
}
}
@@ -406,6 +501,7 @@
* An interface used by the JdbcIO Write to set the parameters of the {@link PreparedStatement}
* used to setParameters into the database.
*/
+ @FunctionalInterface
public interface PreparedStatementSetter<T> extends Serializable {
void setParameters(T element, PreparedStatement preparedStatement) throws Exception;
}
@@ -440,22 +536,16 @@
@Override
public PDone expand(PCollection<T> input) {
+ checkArgument(
+ getDataSourceConfiguration() != null, "withDataSourceConfiguration() is required");
+ checkArgument(getStatement() != null, "withStatement() is required");
+ checkArgument(
+ getPreparedStatementSetter() != null, "withPreparedStatementSetter() is required");
+
input.apply(ParDo.of(new WriteFn<T>(this)));
return PDone.in(input.getPipeline());
}
- @Override
- public void validate(PipelineOptions options) {
- checkArgument(getDataSourceConfiguration() != null,
- "JdbcIO.write() requires a configuration to be set via "
- + ".withDataSourceConfiguration(configuration)");
- checkArgument(getStatement() != null,
- "JdbcIO.write() requires a statement to be set via .withStatement(statement)");
- checkArgument(getPreparedStatementSetter() != null,
- "JdbcIO.write() requires a preparedStatementSetter to be set via "
- + ".withPreparedStatementSetter(preparedStatementSetter)");
- }
-
private static class WriteFn<T> extends DoFn<T, Void> {
private static final int DEFAULT_BATCH_SIZE = 1000;
@@ -528,4 +618,36 @@
}
}
}
+
+ private static class Reparallelize<T> extends PTransform<PCollection<T>, PCollection<T>> {
+ @Override
+ public PCollection<T> expand(PCollection<T> input) {
+ // See https://issues.apache.org/jira/browse/BEAM-2803
+ // We use a combined approach to "break fusion" here:
+ // (see https://cloud.google.com/dataflow/service/dataflow-service-desc#preventing-fusion)
+ // 1) force the data to be materialized by passing it as a side input to an identity fn,
+ // then 2) reshuffle it with a random key. Initial materialization provides some parallelism
+ // and ensures that data to be shuffled can be generated in parallel, while reshuffling
+ // provides perfect parallelism.
+ // In most cases where a "fusion break" is needed, a simple reshuffle would be sufficient.
+ // The current approach is necessary only to support the particular case of JdbcIO where
+ // a single query may produce many gigabytes of query results.
+ PCollectionView<Iterable<T>> empty =
+ input
+ .apply("Consume", Filter.by(SerializableFunctions.<T, Boolean>constant(false)))
+ .apply(View.<T>asIterable());
+ PCollection<T> materialized =
+ input.apply(
+ "Identity",
+ ParDo.of(
+ new DoFn<T, T>() {
+ @ProcessElement
+ public void process(ProcessContext c) {
+ c.output(c.element());
+ }
+ })
+ .withSideInputs(empty));
+ return materialized.apply(Reshuffle.<T>viaRandomKey());
+ }
+ }
}
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
index e8ffad6..32d6d9e 100644
--- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
+++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
@@ -17,41 +17,39 @@
*/
package org.apache.beam.sdk.io.jdbc;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
+import java.text.ParseException;
import java.util.List;
-import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
+
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.io.common.HashingFn;
import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.io.common.TestRow;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Top;
import org.apache.beam.sdk.values.PCollection;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.postgresql.ds.PGSimpleDataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A test of {@link org.apache.beam.sdk.io.jdbc.JdbcIO} on an independent Postgres instance.
*
- * <p>This test requires a running instance of Postgres, and the test dataset must exist in the
- * database. `JdbcTestDataSet` will create the read table.
- *
- * <p>You can run this test by doing the following:
+ * <p>This test requires a running instance of Postgres. Pass in connection information using
+ * PipelineOptions:
* <pre>
* mvn -e -Pio-it verify -pl sdks/java/io/jdbc -DintegrationTestPipelineOptions='[
* "--postgresServerName=1.2.3.4",
@@ -67,112 +65,123 @@
*/
@RunWith(JUnit4.class)
public class JdbcIOIT {
+ private static final Logger LOG = LoggerFactory.getLogger(JdbcIOIT.class);
+ public static final int EXPECTED_ROW_COUNT = 1000;
private static PGSimpleDataSource dataSource;
- private static String writeTableName;
+ private static String tableName;
+
+ @Rule
+ public TestPipeline pipelineWrite = TestPipeline.create();
+ @Rule
+ public TestPipeline pipelineRead = TestPipeline.create();
@BeforeClass
- public static void setup() throws SQLException {
+ public static void setup() throws SQLException, ParseException {
PipelineOptionsFactory.register(IOTestPipelineOptions.class);
IOTestPipelineOptions options = TestPipeline.testingPipelineOptions()
.as(IOTestPipelineOptions.class);
- // We do dataSource set up in BeforeClass rather than Before since we don't need to create a new
- // dataSource for each test.
- dataSource = JdbcTestDataSet.getDataSource(options);
+ dataSource = getDataSource(options);
+
+ tableName = JdbcTestHelper.getTableName("IT");
+ JdbcTestHelper.createDataTable(dataSource, tableName);
+ }
+
+ private static PGSimpleDataSource getDataSource(IOTestPipelineOptions options)
+ throws SQLException {
+ PGSimpleDataSource dataSource = new PGSimpleDataSource();
+
+ dataSource.setDatabaseName(options.getPostgresDatabaseName());
+ dataSource.setServerName(options.getPostgresServerName());
+ dataSource.setPortNumber(options.getPostgresPort());
+ dataSource.setUser(options.getPostgresUsername());
+ dataSource.setPassword(options.getPostgresPassword());
+ dataSource.setSsl(options.getPostgresSsl());
+
+ return dataSource;
}
@AfterClass
public static void tearDown() throws SQLException {
- // Only do write table clean up once for the class since we don't want to clean up after both
- // read and write tests, only want to do it once after all the tests are done.
- JdbcTestDataSet.cleanUpDataTable(dataSource, writeTableName);
+ JdbcTestHelper.cleanUpDataTable(dataSource, tableName);
}
- private static class CreateKVOfNameAndId implements JdbcIO.RowMapper<KV<String, Integer>> {
- @Override
- public KV<String, Integer> mapRow(ResultSet resultSet) throws Exception {
- KV<String, Integer> kv =
- KV.of(resultSet.getString("name"), resultSet.getInt("id"));
- return kv;
- }
- }
-
- private static class PutKeyInColumnOnePutValueInColumnTwo
- implements JdbcIO.PreparedStatementSetter<KV<Integer, String>> {
- @Override
- public void setParameters(KV<Integer, String> element, PreparedStatement statement)
- throws SQLException {
- statement.setInt(1, element.getKey());
- statement.setString(2, element.getValue());
- }
- }
-
- @Rule
- public TestPipeline pipeline = TestPipeline.create();
-
/**
- * Does a test read of a few rows from a postgres database.
- *
- * <p>Note that IT read tests must not do any data table manipulation (setup/clean up.)
- * @throws SQLException
+ * Tests writing then reading data for a postgres database.
*/
@Test
- public void testRead() throws SQLException {
- String writeTableName = JdbcTestDataSet.READ_TABLE_NAME;
+ public void testWriteThenRead() {
+ runWrite();
+ runRead();
+ }
- PCollection<KV<String, Integer>> output = pipeline.apply(JdbcIO.<KV<String, Integer>>read()
+ /**
+ * Writes the test dataset to postgres.
+ *
+ * <p>This method does not attempt to validate the data - we do so in the read test. This does
+ * make it harder to tell whether a test failed in the write or read phase, but the tests are much
+ * easier to maintain (don't need any separate code to write test data for read tests to
+ * the database.)
+ */
+ private void runWrite() {
+ pipelineWrite.apply(GenerateSequence.from(0).to((long) EXPECTED_ROW_COUNT))
+ .apply(ParDo.of(new TestRow.DeterministicallyConstructTestRowFn()))
+ .apply(JdbcIO.<TestRow>write()
.withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
- .withQuery("select name,id from " + writeTableName)
- .withRowMapper(new CreateKVOfNameAndId())
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
+ .withStatement(String.format("insert into %s values(?, ?)", tableName))
+ .withPreparedStatementSetter(new JdbcTestHelper.PrepareStatementFromTestRow()));
- // TODO: validate actual contents of rows, not just count.
+ pipelineWrite.run().waitUntilFinish();
+ }
+
+ /**
+ * Read the test dataset from postgres and validate its contents.
+ *
+ * <p>When doing the validation, we wish to ensure that we:
+ * 1. Ensure *all* the rows are correct
+ * 2. Provide enough information in assertions such that it is easy to spot obvious errors (e.g.
+ * all elements have a similar mistake, or "only 5 elements were generated" and the user wants
+ * to see what the problem was.
+ *
+ * <p>We do not wish to generate and compare all of the expected values, so this method uses
+ * hashing to ensure that all expected data is present. However, hashing does not provide easy
+ * debugging information (failures like "every element was empty string" are hard to see),
+ * so we also:
+ * 1. Generate expected values for the first and last 500 rows
+ * 2. Use containsInAnyOrder to verify that their values are correct.
+ * Where first/last 500 rows is determined by the fact that we know all rows have a unique id - we
+ * can use the natural ordering of that key.
+ */
+ private void runRead() {
+ PCollection<TestRow> namesAndIds =
+ pipelineRead.apply(JdbcIO.<TestRow>read()
+ .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
+ .withQuery(String.format("select name,id from %s;", tableName))
+ .withRowMapper(new JdbcTestHelper.CreateTestRowOfNameAndId())
+ .withCoder(SerializableCoder.of(TestRow.class)));
+
PAssert.thatSingleton(
- output.apply("Count All", Count.<KV<String, Integer>>globally()))
- .isEqualTo(1000L);
+ namesAndIds.apply("Count All", Count.<TestRow>globally()))
+ .isEqualTo((long) EXPECTED_ROW_COUNT);
- List<KV<String, Long>> expectedCounts = new ArrayList<>();
- for (String scientist : JdbcTestDataSet.SCIENTISTS) {
- expectedCounts.add(KV.of(scientist, 100L));
- }
- PAssert.that(output.apply("Count Scientist", Count.<String, Integer>perKey()))
- .containsInAnyOrder(expectedCounts);
+ PCollection<String> consolidatedHashcode = namesAndIds
+ .apply(ParDo.of(new TestRow.SelectNameFn()))
+ .apply("Hash row contents", Combine.globally(new HashingFn()).withoutDefaults());
+ PAssert.that(consolidatedHashcode)
+ .containsInAnyOrder(TestRow.getExpectedHashForRowCount(EXPECTED_ROW_COUNT));
- pipeline.run().waitUntilFinish();
- }
+ PCollection<List<TestRow>> frontOfList =
+ namesAndIds.apply(Top.<TestRow>smallest(500));
+ Iterable<TestRow> expectedFrontOfList = TestRow.getExpectedValues(0, 500);
+ PAssert.thatSingletonIterable(frontOfList).containsInAnyOrder(expectedFrontOfList);
- /**
- * Tests writes to a postgres database.
- *
- * <p>Write Tests must clean up their data - in this case, it uses a new table every test run so
- * that it won't interfere with read tests/other write tests. It uses finally to attempt to
- * clean up data at the end of the test run.
- * @throws SQLException
- */
- @Test
- public void testWrite() throws SQLException {
- writeTableName = JdbcTestDataSet.createWriteDataTable(dataSource);
+ PCollection<List<TestRow>> backOfList =
+ namesAndIds.apply(Top.<TestRow>largest(500));
+ Iterable<TestRow> expectedBackOfList =
+ TestRow.getExpectedValues(EXPECTED_ROW_COUNT - 500,
+ EXPECTED_ROW_COUNT);
+ PAssert.thatSingletonIterable(backOfList).containsInAnyOrder(expectedBackOfList);
- ArrayList<KV<Integer, String>> data = new ArrayList<>();
- for (int i = 0; i < 1000; i++) {
- KV<Integer, String> kv = KV.of(i, "Test");
- data.add(kv);
- }
- pipeline.apply(Create.of(data))
- .apply(JdbcIO.<KV<Integer, String>>write()
- .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
- .withStatement(String.format("insert into %s values(?, ?)", writeTableName))
- .withPreparedStatementSetter(new PutKeyInColumnOnePutValueInColumnTwo()));
-
- pipeline.run().waitUntilFinish();
-
- try (Connection connection = dataSource.getConnection();
- Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery("select count(*) from " + writeTableName)) {
- resultSet.next();
- int count = resultSet.getInt(1);
- Assert.assertEquals(2000, count);
- }
- // TODO: Actually verify contents of the rows.
+ pipelineRead.run().waitUntilFinish();
}
}
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java
index 984ce1a..f35c8b1 100644
--- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java
+++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.io.jdbc;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.PrintWriter;
@@ -28,18 +27,21 @@
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
-import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import java.util.Collections;
+import javax.sql.DataSource;
+
import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.io.common.TestRow;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.derby.drda.NetworkServerControl;
@@ -49,7 +51,6 @@
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,11 +59,13 @@
*/
public class JdbcIOTest implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(JdbcIOTest.class);
+ public static final int EXPECTED_ROW_COUNT = 1000;
private static NetworkServerControl derbyServer;
private static ClientDataSource dataSource;
private static int port;
+ private static String readTableName;
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
@@ -108,14 +111,16 @@
dataSource.setServerName("localhost");
dataSource.setPortNumber(port);
+ readTableName = JdbcTestHelper.getTableName("UT_READ");
- JdbcTestDataSet.createReadDataTable(dataSource);
+ JdbcTestHelper.createDataTable(dataSource, readTableName);
+ addInitialData(dataSource, readTableName);
}
@AfterClass
public static void shutDownDatabase() throws Exception {
try {
- JdbcTestDataSet.cleanUpDataTable(dataSource, JdbcTestDataSet.READ_TABLE_NAME);
+ JdbcTestHelper.cleanUpDataTable(dataSource, readTableName);
} finally {
if (derbyServer != null) {
derbyServer.shutdown();
@@ -177,84 +182,82 @@
}
}
- @Test
- @Category(NeedsRunner.class)
- public void testRead() throws Exception {
+ /**
+ * Create test data that is consistent with that generated by TestRow.
+ */
+ private static void addInitialData(DataSource dataSource, String tableName)
+ throws SQLException {
+ try (Connection connection = dataSource.getConnection()) {
+ connection.setAutoCommit(false);
+ try (PreparedStatement preparedStatement =
+ connection.prepareStatement(
+ String.format("insert into %s values (?,?)", tableName))) {
+ for (int i = 0; i < EXPECTED_ROW_COUNT; i++) {
+ preparedStatement.clearParameters();
+ preparedStatement.setInt(1, i);
+ preparedStatement.setString(2, TestRow.getNameForSeed(i));
+ preparedStatement.executeUpdate();
+ }
+ }
+ connection.commit();
+ }
+ }
- PCollection<KV<String, Integer>> output = pipeline.apply(
- JdbcIO.<KV<String, Integer>>read()
+ @Test
+ public void testRead() throws Exception {
+ PCollection<TestRow> rows = pipeline.apply(
+ JdbcIO.<TestRow>read()
.withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
- .withQuery("select name,id from " + JdbcTestDataSet.READ_TABLE_NAME)
- .withRowMapper(new JdbcIO.RowMapper<KV<String, Integer>>() {
- @Override
- public KV<String, Integer> mapRow(ResultSet resultSet) throws Exception {
- KV<String, Integer> kv =
- KV.of(resultSet.getString("name"), resultSet.getInt("id"));
- return kv;
- }
- })
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
+ .withQuery("select name,id from " + readTableName)
+ .withRowMapper(new JdbcTestHelper.CreateTestRowOfNameAndId())
+ .withCoder(SerializableCoder.of(TestRow.class)));
PAssert.thatSingleton(
- output.apply("Count All", Count.<KV<String, Integer>>globally()))
- .isEqualTo(1000L);
+ rows.apply("Count All", Count.<TestRow>globally()))
+ .isEqualTo((long) EXPECTED_ROW_COUNT);
- PAssert.that(output
- .apply("Count Scientist", Count.<String, Integer>perKey())
- ).satisfies(new SerializableFunction<Iterable<KV<String, Long>>, Void>() {
- @Override
- public Void apply(Iterable<KV<String, Long>> input) {
- for (KV<String, Long> element : input) {
- assertEquals(element.getKey(), 100L, element.getValue().longValue());
- }
- return null;
- }
- });
+ Iterable<TestRow> expectedValues = TestRow.getExpectedValues(0, EXPECTED_ROW_COUNT);
+ PAssert.that(rows).containsInAnyOrder(expectedValues);
pipeline.run();
}
@Test
- @Category(NeedsRunner.class)
public void testReadWithSingleStringParameter() throws Exception {
-
- PCollection<KV<String, Integer>> output = pipeline.apply(
- JdbcIO.<KV<String, Integer>>read()
+ PCollection<TestRow> rows = pipeline.apply(
+ JdbcIO.<TestRow>read()
.withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
.withQuery(String.format("select name,id from %s where name = ?",
- JdbcTestDataSet.READ_TABLE_NAME))
+ readTableName))
.withStatementPreparator(new JdbcIO.StatementPreparator() {
@Override
public void setParameters(PreparedStatement preparedStatement)
- throws Exception {
- preparedStatement.setString(1, "Darwin");
+ throws Exception {
+ preparedStatement.setString(1, TestRow.getNameForSeed(1));
}
})
- .withRowMapper(new JdbcIO.RowMapper<KV<String, Integer>>() {
- @Override
- public KV<String, Integer> mapRow(ResultSet resultSet) throws Exception {
- KV<String, Integer> kv =
- KV.of(resultSet.getString("name"), resultSet.getInt("id"));
- return kv;
- }
- })
- .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));
+ .withRowMapper(new JdbcTestHelper.CreateTestRowOfNameAndId())
+ .withCoder(SerializableCoder.of(TestRow.class)));
PAssert.thatSingleton(
- output.apply("Count One Scientist", Count.<KV<String, Integer>>globally()))
- .isEqualTo(100L);
+ rows.apply("Count All", Count.<TestRow>globally()))
+ .isEqualTo(1L);
+
+ Iterable<TestRow> expectedValues = Collections.singletonList(TestRow.fromSeed(1));
+ PAssert.that(rows).containsInAnyOrder(expectedValues);
pipeline.run();
}
@Test
- @Category(NeedsRunner.class)
public void testWrite() throws Exception {
+ final long rowsToAdd = 1000L;
- String tableName = JdbcTestDataSet.createWriteDataTable(dataSource);
+ String tableName = JdbcTestHelper.getTableName("UT_WRITE");
+ JdbcTestHelper.createDataTable(dataSource, tableName);
try {
ArrayList<KV<Integer, String>> data = new ArrayList<>();
- for (int i = 0; i < 1000; i++) {
+ for (int i = 0; i < rowsToAdd; i++) {
KV<Integer, String> kv = KV.of(i, "Test");
data.add(kv);
}
@@ -282,19 +285,17 @@
resultSet.next();
int count = resultSet.getInt(1);
- Assert.assertEquals(2000, count);
+ Assert.assertEquals(EXPECTED_ROW_COUNT, count);
}
}
}
} finally {
- JdbcTestDataSet.cleanUpDataTable(dataSource, tableName);
+ JdbcTestHelper.cleanUpDataTable(dataSource, tableName);
}
}
@Test
- @Category(NeedsRunner.class)
public void testWriteWithEmptyPCollection() throws Exception {
-
pipeline
.apply(Create.empty(KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of())))
.apply(JdbcIO.<KV<Integer, String>>write()
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestDataSet.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestDataSet.java
deleted file mode 100644
index 0b88be2..0000000
--- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestDataSet.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.jdbc;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import javax.sql.DataSource;
-import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.postgresql.ds.PGSimpleDataSource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Manipulates test data used by the {@link org.apache.beam.sdk.io.jdbc.JdbcIO} tests.
- *
- * <p>This is independent from the tests so that for read tests it can be run separately after data
- * store creation rather than every time (which can be more fragile.)
- */
-public class JdbcTestDataSet {
- private static final Logger LOG = LoggerFactory.getLogger(JdbcTestDataSet.class);
- public static final String[] SCIENTISTS = {"Einstein", "Darwin", "Copernicus", "Pasteur", "Curie",
- "Faraday", "McClintock", "Herschel", "Hopper", "Lovelace"};
- /**
- * Use this to create the read tables before IT read tests.
- *
- * <p>To invoke this class, you can use this command line:
- * (run from the jdbc root directory)
- * mvn test-compile exec:java -Dexec.mainClass=org.apache.beam.sdk.io.jdbc.JdbcTestDataSet \
- * -Dexec.args="--postgresServerName=127.0.0.1 --postgresUsername=postgres \
- * --postgresDatabaseName=myfancydb \
- * --postgresPassword=yourpassword --postgresSsl=false" \
- * -Dexec.classpathScope=test
- * @param args Please pass options from IOTestPipelineOptions used for connection to postgres as
- * shown above.
- */
- public static void main(String[] args) throws SQLException {
- PipelineOptionsFactory.register(IOTestPipelineOptions.class);
- IOTestPipelineOptions options =
- PipelineOptionsFactory.fromArgs(args).as(IOTestPipelineOptions.class);
-
- createReadDataTable(getDataSource(options));
- }
-
- public static PGSimpleDataSource getDataSource(IOTestPipelineOptions options)
- throws SQLException {
- PGSimpleDataSource dataSource = new PGSimpleDataSource();
-
- // Tests must receive parameters for connections from PipelineOptions
- // Parameters should be generic to all tests that use a particular datasource, not
- // the particular test.
- dataSource.setDatabaseName(options.getPostgresDatabaseName());
- dataSource.setServerName(options.getPostgresServerName());
- dataSource.setPortNumber(options.getPostgresPort());
- dataSource.setUser(options.getPostgresUsername());
- dataSource.setPassword(options.getPostgresPassword());
- dataSource.setSsl(options.getPostgresSsl());
-
- return dataSource;
- }
-
- public static final String READ_TABLE_NAME = "BEAM_TEST_READ";
-
- public static void createReadDataTable(DataSource dataSource) throws SQLException {
- createDataTable(dataSource, READ_TABLE_NAME);
- }
-
- public static String createWriteDataTable(DataSource dataSource) throws SQLException {
- String tableName = "BEAMTEST" + org.joda.time.Instant.now().getMillis();
- createDataTable(dataSource, tableName);
- return tableName;
- }
-
- private static void createDataTable(DataSource dataSource, String tableName) throws SQLException {
- try (Connection connection = dataSource.getConnection()) {
- // something like this will need to happen in tests on a newly created postgres server,
- // but likely it will happen in perfkit, not here
- // alternatively, we may have a pipelineoption indicating whether we want to
- // re-use the database or create a new one
- try (Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("create table %s (id INT, name VARCHAR(500))", tableName));
- }
-
- connection.setAutoCommit(false);
- try (PreparedStatement preparedStatement =
- connection.prepareStatement(
- String.format("insert into %s values (?,?)", tableName))) {
- for (int i = 0; i < 1000; i++) {
- int index = i % SCIENTISTS.length;
- preparedStatement.clearParameters();
- preparedStatement.setInt(1, i);
- preparedStatement.setString(2, SCIENTISTS[index]);
- preparedStatement.executeUpdate();
- }
- }
- connection.commit();
- }
-
- LOG.info("Created table {}", tableName);
- }
-
- public static void cleanUpDataTable(DataSource dataSource, String tableName)
- throws SQLException {
- if (tableName != null) {
- try (Connection connection = dataSource.getConnection();
- Statement statement = connection.createStatement()) {
- statement.executeUpdate(String.format("drop table %s", tableName));
- }
- }
- }
-
-}
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestHelper.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestHelper.java
new file mode 100644
index 0000000..fedae51
--- /dev/null
+++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestHelper.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.jdbc;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import javax.sql.DataSource;
+import org.apache.beam.sdk.io.common.TestRow;
+
+/**
+ * Contains Test helper methods used by both Integration and Unit Tests in
+ * {@link org.apache.beam.sdk.io.jdbc.JdbcIO}.
+ */
+class JdbcTestHelper {
+ static String getTableName(String testIdentifier) throws ParseException {
+ SimpleDateFormat formatter = new SimpleDateFormat();
+ formatter.applyPattern("yyyy_MM_dd_HH_mm_ss_S");
+ return String.format("BEAMTEST_%s_%s", testIdentifier, formatter.format(new Date()));
+ }
+
+ static void createDataTable(
+ DataSource dataSource, String tableName)
+ throws SQLException {
+ try (Connection connection = dataSource.getConnection()) {
+ try (Statement statement = connection.createStatement()) {
+ statement.execute(
+ String.format("create table %s (id INT, name VARCHAR(500))", tableName));
+ }
+ }
+ }
+
+ static void cleanUpDataTable(DataSource dataSource, String tableName)
+ throws SQLException {
+ if (tableName != null) {
+ try (Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(String.format("drop table %s", tableName));
+ }
+ }
+ }
+
+ static class CreateTestRowOfNameAndId implements JdbcIO.RowMapper<TestRow> {
+ @Override
+ public TestRow mapRow(ResultSet resultSet) throws Exception {
+ return TestRow.create(
+ resultSet.getInt("id"), resultSet.getString("name"));
+ }
+ }
+
+ static class PrepareStatementFromTestRow
+ implements JdbcIO.PreparedStatementSetter<TestRow> {
+ @Override
+ public void setParameters(TestRow element, PreparedStatement statement)
+ throws SQLException {
+ statement.setLong(1, element.id());
+ statement.setString(2, element.name());
+ }
+ }
+
+}
diff --git a/sdks/java/io/jms/pom.xml b/sdks/java/io/jms/pom.xml
index 58009a1..6f030ee 100644
--- a/sdks/java/io/jms/pom.xml
+++ b/sdks/java/io/jms/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
index b8355ad..b3a9c8b 100644
--- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
+++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
@@ -18,7 +18,6 @@
package org.apache.beam.sdk.io.jms;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
@@ -98,7 +97,7 @@
*
* }</pre>
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class JmsIO {
private static final Logger LOG = LoggerFactory.getLogger(JmsIO.class);
@@ -165,8 +164,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Read withConnectionFactory(ConnectionFactory connectionFactory) {
- checkArgument(connectionFactory != null, "withConnectionFactory(connectionFactory) called"
- + " with null connectionFactory");
+ checkArgument(connectionFactory != null, "connectionFactory can not be null");
return builder().setConnectionFactory(connectionFactory).build();
}
@@ -189,7 +187,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Read withQueue(String queue) {
- checkArgument(queue != null, "withQueue(queue) called with null queue");
+ checkArgument(queue != null, "queue can not be null");
return builder().setQueue(queue).build();
}
@@ -212,7 +210,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Read withTopic(String topic) {
- checkArgument(topic != null, "withTopic(topic) called with null topic");
+ checkArgument(topic != null, "topic can not be null");
return builder().setTopic(topic).build();
}
@@ -220,8 +218,7 @@
* Define the username to connect to the JMS broker (authenticated).
*/
public Read withUsername(String username) {
- checkArgument(username != null, "JmsIO.read().withUsername(username) called with null "
- + "username");
+ checkArgument(username != null, "username can not be null");
return builder().setUsername(username).build();
}
@@ -229,8 +226,7 @@
* Define the password to connect to the JMS broker (authenticated).
*/
public Read withPassword(String password) {
- checkArgument(password != null, "JmsIO.read().withPassword(password) called with null "
- + "password");
+ checkArgument(password != null, "password can not be null");
return builder().setPassword(password).build();
}
@@ -251,8 +247,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Read withMaxNumRecords(long maxNumRecords) {
- checkArgument(maxNumRecords >= 0, "withMaxNumRecords(maxNumRecords) called with invalid "
- + "maxNumRecords");
+ checkArgument(maxNumRecords >= 0, "maxNumRecords must be > 0, but was: %d", maxNumRecords);
return builder().setMaxNumRecords(maxNumRecords).build();
}
@@ -273,13 +268,20 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Read withMaxReadTime(Duration maxReadTime) {
- checkArgument(maxReadTime != null, "withMaxReadTime(maxReadTime) called with null "
- + "maxReadTime");
+ checkArgument(maxReadTime != null, "maxReadTime can not be null");
return builder().setMaxReadTime(maxReadTime).build();
}
@Override
public PCollection<JmsRecord> expand(PBegin input) {
+ checkArgument(getConnectionFactory() != null, "withConnectionFactory() is required");
+ checkArgument(
+ getQueue() != null || getTopic() != null,
+ "Either withQueue() or withTopic() is required");
+ checkArgument(
+ getQueue() == null || getTopic() == null,
+ "withQueue() and withTopic() are exclusive");
+
// handles unbounded source to bounded conversion if maxNumRecords is set.
Unbounded<JmsRecord> unbounded = org.apache.beam.sdk.io.Read.from(createSource());
@@ -295,15 +297,6 @@
}
@Override
- public void validate(PipelineOptions options) {
- checkState(getConnectionFactory() != null, "JmsIO.read() requires a JMS connection "
- + "factory to be set via withConnectionFactory(connectionFactory)");
- checkState((getQueue() != null || getTopic() != null), "JmsIO.read() requires a JMS "
- + "destination (queue or topic) to be set via withQueue(queueName) or withTopic"
- + "(topicName)");
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder.addIfNotNull(DisplayData.item("queue", getQueue()));
@@ -363,23 +356,19 @@
}
@Override
- public void validate() {
- spec.validate(null);
- }
-
- @Override
public Coder<JmsCheckpointMark> getCheckpointMarkCoder() {
return AvroCoder.of(JmsCheckpointMark.class);
}
@Override
- public Coder<JmsRecord> getDefaultOutputCoder() {
+ public Coder<JmsRecord> getOutputCoder() {
return SerializableCoder.of(JmsRecord.class);
}
}
- private static class UnboundedJmsReader extends UnboundedReader<JmsRecord> {
+ @VisibleForTesting
+ static class UnboundedJmsReader extends UnboundedReader<JmsRecord> {
private UnboundedJmsSource source;
private JmsCheckpointMark checkpointMark;
@@ -421,7 +410,7 @@
}
try {
- this.session = this.connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
+ this.session = this.connection.createSession(false, Session.CLIENT_ACKNOWLEDGE);
} catch (Exception e) {
throw new IOException("Error creating JMS session", e);
}
@@ -578,8 +567,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Write withConnectionFactory(ConnectionFactory connectionFactory) {
- checkArgument(connectionFactory != null, "withConnectionFactory(connectionFactory) called"
- + " with null connectionFactory");
+ checkArgument(connectionFactory != null, "connectionFactory can not be null");
return builder().setConnectionFactory(connectionFactory).build();
}
@@ -602,7 +590,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Write withQueue(String queue) {
- checkArgument(queue != null, "withQueue(queue) called with null queue");
+ checkArgument(queue != null, "queue can not be null");
return builder().setQueue(queue).build();
}
@@ -625,7 +613,7 @@
* @return The corresponding {@link JmsIO.Read}.
*/
public Write withTopic(String topic) {
- checkArgument(topic != null, "withTopic(topic) called with null topic");
+ checkArgument(topic != null, "topic can not be null");
return builder().setTopic(topic).build();
}
@@ -633,8 +621,7 @@
* Define the username to connect to the JMS broker (authenticated).
*/
public Write withUsername(String username) {
- checkArgument(username != null, "JmsIO.write().withUsername(username) called with null "
- + "username");
+ checkArgument(username != null, "username can not be null");
return builder().setUsername(username).build();
}
@@ -642,25 +629,24 @@
* Define the password to connect to the JMS broker (authenticated).
*/
public Write withPassword(String password) {
- checkArgument(password != null, "JmsIO.write().withPassword(password) called with null "
- + "password");
+ checkArgument(password != null, "password can not be null");
return builder().setPassword(password).build();
}
@Override
public PDone expand(PCollection<String> input) {
+ checkArgument(getConnectionFactory() != null, "withConnectionFactory() is required");
+ checkArgument(
+ getQueue() != null || getTopic() != null,
+ "Either withQueue(queue) or withTopic(topic) is required");
+ checkArgument(
+ getQueue() == null || getTopic() == null,
+ "withQueue(queue) and withTopic(topic) are exclusive");
+
input.apply(ParDo.of(new WriterFn(this)));
return PDone.in(input.getPipeline());
}
- @Override
- public void validate(PipelineOptions options) {
- checkState(getConnectionFactory() != null, "JmsIO.write() requires a JMS connection "
- + "factory to be set via withConnectionFactory(connectionFactory)");
- checkState((getQueue() != null || getTopic() != null), "JmsIO.write() requires a JMS "
- + "destination (queue or topic) to be set via withQueue(queue) or withTopic(topic)");
- }
-
private static class WriterFn extends DoFn<String, Void> {
private Write spec;
diff --git a/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java b/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java
index 7edda1a..43c050e 100644
--- a/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java
+++ b/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java
@@ -23,10 +23,12 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Enumeration;
import java.util.List;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
@@ -34,6 +36,7 @@
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
+import javax.jms.QueueBrowser;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.activemq.ActiveMQConnectionFactory;
@@ -71,6 +74,7 @@
private BrokerService broker;
private ConnectionFactory connectionFactory;
+ private ConnectionFactory connectionFactoryWithoutPrefetch;
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
@@ -98,6 +102,8 @@
// create JMS connection factory
connectionFactory = new ActiveMQConnectionFactory(BROKER_URL);
+ connectionFactoryWithoutPrefetch =
+ new ActiveMQConnectionFactory(BROKER_URL + "?jms.prefetchPolicy.all=0");
}
@After
@@ -236,4 +242,76 @@
assertEquals(1, splits.size());
}
+ @Test
+ public void testCheckpointMark() throws Exception {
+ // we are using no prefetch here
+ // prefetch is an ActiveMQ feature: to make efficient use of network resources the broker
+ // utilizes a 'push' model to dispatch messages to consumers. However, in the case of our
+ // test, it means that we can have some latency between the receiveNoWait() method used by
+ // the consumer and the prefetch buffer populated by the broker. Using a prefetch to 0 means
+ // that the consumer will poll for message, which is exactly what we want for the test.
+ Connection connection = connectionFactoryWithoutPrefetch.createConnection(USERNAME, PASSWORD);
+ connection.start();
+ Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
+ MessageProducer producer = session.createProducer(session.createQueue(QUEUE));
+ for (int i = 0; i < 10; i++) {
+ producer.send(session.createTextMessage("test " + i));
+ }
+ producer.close();
+ session.close();
+ connection.close();
+
+ JmsIO.Read spec = JmsIO.read()
+ .withConnectionFactory(connectionFactoryWithoutPrefetch)
+ .withUsername(USERNAME)
+ .withPassword(PASSWORD)
+ .withQueue(QUEUE);
+ JmsIO.UnboundedJmsSource source = new JmsIO.UnboundedJmsSource(spec);
+ JmsIO.UnboundedJmsReader reader = source.createReader(null, null);
+
+ // start the reader and move to the first record
+ assertTrue(reader.start());
+
+ // consume 3 messages (NB: start already consumed the first message)
+ for (int i = 0; i < 3; i++) {
+ assertTrue(reader.advance());
+ }
+
+ // the messages are still pending in the queue (no ACK yet)
+ assertEquals(10, count(QUEUE));
+
+ // we finalize the checkpoint
+ reader.getCheckpointMark().finalizeCheckpoint();
+
+ // the checkpoint finalize ack the messages, and so they are not pending in the queue anymore
+ assertEquals(6, count(QUEUE));
+
+ // we read the 6 pending messages
+ for (int i = 0; i < 6; i++) {
+ assertTrue(reader.advance());
+ }
+
+ // still 6 pending messages as we didn't finalize the checkpoint
+ assertEquals(6, count(QUEUE));
+
+ // we finalize the checkpoint: no more message in the queue
+ reader.getCheckpointMark().finalizeCheckpoint();
+
+ assertEquals(0, count(QUEUE));
+ }
+
+ private int count(String queue) throws Exception {
+ Connection connection = connectionFactory.createConnection(USERNAME, PASSWORD);
+ connection.start();
+ Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
+ QueueBrowser browser = session.createBrowser(session.createQueue(queue));
+ Enumeration<Message> messages = browser.getEnumeration();
+ int count = 0;
+ while (messages.hasMoreElements()) {
+ messages.nextElement();
+ count++;
+ }
+ return count;
+ }
+
}
diff --git a/sdks/java/io/kafka/README.md b/sdks/java/io/kafka/README.md
new file mode 100644
index 0000000..07d00a1
--- /dev/null
+++ b/sdks/java/io/kafka/README.md
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+KafkaIO contains I/O transforms which allow you to read/write messages from/to [Apache Kafka](http://kafka.apache.org/).
+
+## Dependencies
+
+To use KafkaIO you must first add a dependency on `beam-sdks-java-io-kafka`
+
+```maven
+<dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-kafka</artifactId>
+ <version>...</version>
+</dependency>
+```
+
+## Documentation
+
+- [KafkaIO.java](https://github.com/apache/beam/blob/master/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java)
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index f6f0385..f1ddb51 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -29,10 +29,6 @@
<name>Apache Beam :: SDKs :: Java :: IO :: Kafka</name>
<description>Library to read Kafka topics.</description>
- <properties>
- <kafka.clients.version>0.9.0.1</kafka.clients.version>
- </properties>
-
<build>
<pluginManagement>
<plugins>
@@ -46,18 +42,6 @@
</plugin>
</plugins>
</pluginManagement>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamUseDummyRunner>false</beamUseDummyRunner>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- </plugins>
</build>
<dependencies>
@@ -69,7 +53,6 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
- <version>${kafka.clients.version}</version>
</dependency>
<dependency>
@@ -97,7 +80,17 @@
<artifactId>auto-value</artifactId>
<scope>provided</scope>
</dependency>
-
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-expression</artifactId>
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java
index 8fe17c1..8cdad22 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java
@@ -17,12 +17,18 @@
*/
package org.apache.beam.sdk.io.kafka;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Collection;
+import java.util.Map;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
+import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.expression.Expression;
@@ -51,13 +57,28 @@
private Method timestampMethod;
private boolean hasRecordTimestamp = false;
+ private Method offsetGetterMethod;
+ private Method offsetsForTimesMethod;
+ private boolean hasOffsetsForTimes = false;
+
public ConsumerSpEL() {
try {
+ // It is supported by Kafka Client 0.10.0.0 onwards.
timestampMethod = ConsumerRecord.class.getMethod("timestamp", (Class<?>[]) null);
hasRecordTimestamp = timestampMethod.getReturnType().equals(Long.TYPE);
} catch (NoSuchMethodException | SecurityException e) {
LOG.debug("Timestamp for Kafka message is not available.");
}
+
+ try {
+ // It is supported by Kafka Client 0.10.1.0 onwards.
+ offsetGetterMethod = Class.forName("org.apache.kafka.clients.consumer.OffsetAndTimestamp")
+ .getMethod("offset", (Class<?>[]) null);
+ offsetsForTimesMethod = Consumer.class.getMethod("offsetsForTimes", Map.class);
+ hasOffsetsForTimes = offsetsForTimesMethod.getReturnType().equals(Map.class);
+ } catch (NoSuchMethodException | SecurityException | ClassNotFoundException e) {
+ LOG.debug("OffsetsForTimes is not available.");
+ }
}
public void evaluateSeek2End(Consumer consumer, TopicPartition topicPartitions) {
@@ -88,4 +109,39 @@
}
return timestamp;
}
+
+ public boolean hasOffsetsForTimes() {
+ return hasOffsetsForTimes;
+ }
+
+ /**
+ * Look up the offset for the given partition by timestamp.
+ * Throws RuntimeException if there are no messages later than timestamp or if this partition
+ * does not support timestamp based offset.
+ */
+ @SuppressWarnings("unchecked")
+ public long offsetForTime(Consumer<?, ?> consumer, TopicPartition topicPartition, Instant time) {
+
+ checkArgument(hasOffsetsForTimes,
+ "This Kafka Client must support Consumer.OffsetsForTimes().");
+
+ Map<TopicPartition, Long> timestampsToSearch =
+ ImmutableMap.of(topicPartition, time.getMillis());
+ try {
+ Map offsetsByTimes = (Map) offsetsForTimesMethod.invoke(consumer, timestampsToSearch);
+ Object offsetAndTimestamp = Iterables.getOnlyElement(offsetsByTimes.values());
+
+ if (offsetAndTimestamp == null) {
+ throw new RuntimeException("There are no messages has a timestamp that is greater than or "
+ + "equals to the target time or the message format version in this partition is "
+ + "before 0.10.0, topicPartition is: " + topicPartition);
+ } else {
+ return (long) offsetGetterMethod.invoke(offsetAndTimestamp);
+ }
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
}
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index a1130fc..33fc289 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -21,10 +21,20 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
+import com.google.common.base.MoreObjects;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -49,14 +59,18 @@
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
@@ -69,17 +83,28 @@
import org.apache.beam.sdk.io.kafka.KafkaCheckpointMark.PartitionMark;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.metrics.Gauge;
+import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.metrics.SinkMetrics;
import org.apache.beam.sdk.metrics.SourceMetrics;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
@@ -89,20 +114,27 @@
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.AppInfoParser;
+import org.joda.time.DateTimeUtils;
+import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.joda.time.format.DateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,11 +175,14 @@
* .updateConsumerProperties(ImmutableMap.of("receive.buffer.bytes", 1024 * 1024))
*
* // custom function for calculating record timestamp (default is processing time)
- * .withTimestampFn(new MyTypestampFunction())
+ * .withTimestampFn(new MyTimestampFunction())
*
* // custom function for watermark (default is record timestamp)
* .withWatermarkFn(new MyWatermarkFunction())
*
+ * // restrict reader to committed messages on Kafka (see method documentation).
+ * .withReadCommitted()
+ *
* // finally, if you don't need Kafka metadata, you can drop it
* .withoutMetadata() // PCollection<KV<Long, String>>
* )
@@ -204,6 +239,9 @@
* // you can further customize KafkaProducer used to write the records by adding more
* // settings for ProducerConfig. e.g, to enable compression :
* .updateProducerProperties(ImmutableMap.of("compression.type", "gzip"))
+ *
+ * // Optionally enable exactly-once sink (on supported runners). See JavaDoc for withEOS().
+ * .withEOS(20, "eos-sink-group-id");
* );
* }</pre>
*
@@ -215,7 +253,7 @@
* strings.apply(KafkaIO.<Void, String>write()
* .withBootstrapServers("broker_1:9092,broker_2:9092")
* .withTopic("results")
- * .withValueSerializer(new StringSerializer()) // just need serializer for value
+ * .withValueSerializer(StringSerializer.class) // just need serializer for value
* .values()
* );
* }</pre>
@@ -234,7 +272,7 @@
* Note that {@link KafkaRecord#getTimestamp()} reflects timestamp provided by Kafka if any,
* otherwise it is set to processing time.
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class KafkaIO {
/**
@@ -279,6 +317,9 @@
public static <K, V> Write<K, V> write() {
return new AutoValue_KafkaIO_Write.Builder<K, V>()
.setProducerConfig(Write.DEFAULT_PRODUCER_PROPERTIES)
+ .setEOS(false)
+ .setNumShards(0)
+ .setConsumerFactoryFn(Read.KAFKA_CONSUMER_FACTORY_FN)
.build();
}
@@ -306,6 +347,8 @@
abstract long getMaxNumRecords();
@Nullable abstract Duration getMaxReadTime();
+ @Nullable abstract Instant getStartReadTime();
+
abstract Builder<K, V> toBuilder();
@AutoValue.Builder
@@ -324,6 +367,7 @@
abstract Builder<K, V> setWatermarkFn(SerializableFunction<KafkaRecord<K, V>, Instant> fn);
abstract Builder<K, V> setMaxNumRecords(long maxNumRecords);
abstract Builder<K, V> setMaxReadTime(Duration maxReadTime);
+ abstract Builder<K, V> setStartReadTime(Instant startReadTime);
abstract Read<K, V> build();
}
@@ -448,6 +492,24 @@
}
/**
+ * Use timestamp to set up start offset.
+ * It is only supported by Kafka Client 0.10.1.0 onwards and the message format version
+ * after 0.10.0.
+ *
+ * <p>Note that this take priority over start offset configuration
+ * {@code ConsumerConfig.AUTO_OFFSET_RESET_CONFIG} and any auto committed offsets.
+ *
+ * <p>This results in hard failures in either of the following two cases :
+ * 1. If one of more partitions do not contain any messages with timestamp larger than or
+ * equal to desired timestamp.
+ * 2. If the message format version in a partition is before 0.10.0, i.e. the messages do
+ * not have timestamps.
+ */
+ public Read<K, V> withStartReadTime(Instant startReadTime) {
+ return toBuilder().setStartReadTime(startReadTime).build();
+ }
+
+ /**
* Similar to
* {@link org.apache.beam.sdk.io.Read.Unbounded#withMaxReadTime(Duration)}.
* Mainly used for tests and demo
@@ -462,7 +524,7 @@
*/
public Read<K, V> withTimestampFn2(
SerializableFunction<KafkaRecord<K, V>, Instant> timestampFn) {
- checkNotNull(timestampFn);
+ checkArgument(timestampFn != null, "timestampFn can not be null");
return toBuilder().setTimestampFn(timestampFn).build();
}
@@ -472,7 +534,7 @@
*/
public Read<K, V> withWatermarkFn2(
SerializableFunction<KafkaRecord<K, V>, Instant> watermarkFn) {
- checkNotNull(watermarkFn);
+ checkArgument(watermarkFn != null, "watermarkFn can not be null");
return toBuilder().setWatermarkFn(watermarkFn).build();
}
@@ -480,7 +542,7 @@
* A function to assign a timestamp to a record. Default is processing timestamp.
*/
public Read<K, V> withTimestampFn(SerializableFunction<KV<K, V>, Instant> timestampFn) {
- checkNotNull(timestampFn);
+ checkArgument(timestampFn != null, "timestampFn can not be null");
return withTimestampFn2(unwrapKafkaAndThen(timestampFn));
}
@@ -489,11 +551,23 @@
* @see #withTimestampFn(SerializableFunction)
*/
public Read<K, V> withWatermarkFn(SerializableFunction<KV<K, V>, Instant> watermarkFn) {
- checkNotNull(watermarkFn);
+ checkArgument(watermarkFn != null, "watermarkFn can not be null");
return withWatermarkFn2(unwrapKafkaAndThen(watermarkFn));
}
/**
+ * Sets "isolation_level" to "read_committed" in Kafka consumer configuration. This is
+ * ensures that the consumer does not read uncommitted messages. Kafka version 0.11
+ * introduced transactional writes. Applications requiring end-to-end exactly-once
+ * semantics should only read committed messages. See JavaDoc for {@link KafkaConsumer}
+ * for more description.
+ */
+ public Read<K, V> withReadCommitted() {
+ return updateConsumerProperties(
+ ImmutableMap.<String, Object>of("isolation.level", "read_committed"));
+ }
+
+ /**
* Returns a {@link PTransform} for PCollection of {@link KV}, dropping Kafka metatdata.
*/
public PTransform<PBegin, PCollection<KV<K, V>>> withoutMetadata() {
@@ -501,32 +575,38 @@
}
@Override
- public void validate(PipelineOptions options) {
- checkNotNull(getConsumerConfig().get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG),
- "Kafka bootstrap servers should be set");
- checkArgument(getTopics().size() > 0 || getTopicPartitions().size() > 0,
- "Kafka topics or topic_partitions are required");
- checkNotNull(getKeyDeserializer(), "Key deserializer must be set");
- checkNotNull(getValueDeserializer(), "Value deserializer must be set");
- }
-
- @Override
public PCollection<KafkaRecord<K, V>> expand(PBegin input) {
+ checkArgument(
+ getConsumerConfig().get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG) != null,
+ "withBootstrapServers() is required");
+ checkArgument(getTopics().size() > 0 || getTopicPartitions().size() > 0,
+ "Either withTopic(), withTopics() or withTopicPartitions() is required");
+ checkArgument(getKeyDeserializer() != null, "withKeyDeserializer() is required");
+ checkArgument(getValueDeserializer() != null, "withValueDeserializer() is required");
+ if (getStartReadTime() != null) {
+ checkArgument(new ConsumerSpEL().hasOffsetsForTimes(),
+ "Consumer.offsetsForTimes is only supported by Kafka Client 0.10.1.0 onwards, "
+ + "current version of Kafka Client is " + AppInfoParser.getVersion()
+ + ". If you are building with maven, set \"kafka.clients.version\" "
+ + "maven property to 0.10.1.0 or newer.");
+ }
+
// Infer key/value coders if not specified explicitly
CoderRegistry registry = input.getPipeline().getCoderRegistry();
Coder<K> keyCoder =
- checkNotNull(
- getKeyCoder() != null ? getKeyCoder() : inferCoder(registry, getKeyDeserializer()),
- "Key coder could not be inferred from key deserializer. Please provide"
- + "key coder explicitly using withKeyDeserializerAndCoder()");
+ getKeyCoder() != null ? getKeyCoder() : inferCoder(registry, getKeyDeserializer());
+ checkArgument(
+ keyCoder != null,
+ "Key coder could not be inferred from key deserializer. Please provide"
+ + "key coder explicitly using withKeyDeserializerAndCoder()");
Coder<V> valueCoder =
- checkNotNull(
- getValueCoder() != null ? getValueCoder()
- : inferCoder(registry, getValueDeserializer()),
- "Value coder could not be inferred from value deserializer. Please provide"
- + "value coder explicitly using withValueDeserializerAndCoder()");
+ getValueCoder() != null ? getValueCoder() : inferCoder(registry, getValueDeserializer());
+ checkArgument(
+ valueCoder != null,
+ "Value coder could not be inferred from value deserializer. Please provide"
+ + "value coder explicitly using withValueDeserializerAndCoder()");
// Handles unbounded source to bounded conversion if maxNumRecords or maxReadTime is set.
Unbounded<KafkaRecord<K, V>> unbounded =
@@ -808,12 +888,7 @@
}
@Override
- public void validate() {
- spec.validate(null);
- }
-
- @Override
- public Coder<KafkaRecord<K, V>> getDefaultOutputCoder() {
+ public Coder<KafkaRecord<K, V>> getOutputCoder() {
return KafkaRecordCoder.of(spec.getKeyCoder(), spec.getValueCoder());
}
}
@@ -853,10 +928,8 @@
// Backlog support :
// Kafka consumer does not have an API to fetch latest offset for topic. We need to seekToEnd()
// then look at position(). Use another consumer to do this so that the primary consumer does
- // not need to be interrupted. The latest offsets are fetched periodically on another thread.
- // This is still a hack. There could be unintended side effects, e.g. if user enabled offset
- // auto commit in consumer config, this could interfere with the primary consumer (we will
- // handle this particular problem). We might have to make this optional.
+ // not need to be interrupted. The latest offsets are fetched periodically on a thread. This is
+ // still a bit of a hack, but so far there haven't been any issues reported by the users.
private Consumer<byte[], byte[]> offsetConsumer;
private final ScheduledExecutorService offsetFetcherThread =
Executors.newSingleThreadScheduledExecutor();
@@ -868,13 +941,29 @@
private transient ConsumerSpEL consumerSpEL;
/** watermark before any records have been read. */
- private static Instant initialWatermark = new Instant(Long.MIN_VALUE);
+ private static Instant initialWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
@Override
public String toString() {
return name;
}
+ // Maintains approximate average over last 1000 elements
+ private static class MovingAvg {
+ private static final int MOVING_AVG_WINDOW = 1000;
+ private double avg = 0;
+ private long numUpdates = 0;
+
+ void update(double quantity) {
+ numUpdates++;
+ avg += (quantity - avg) / Math.min(MOVING_AVG_WINDOW, numUpdates);
+ }
+
+ double get() {
+ return avg;
+ }
+ }
+
// maintains state of each assigned partition (buffered records, consumed offset, etc)
private static class PartitionState {
private final TopicPartition topicPartition;
@@ -882,9 +971,8 @@
private long latestOffset;
private Iterator<ConsumerRecord<byte[], byte[]>> recordIter = Collections.emptyIterator();
- // simple moving average for size of each record in bytes
- private double avgRecordSize = 0;
- private static final int movingAvgWindow = 1000; // very roughly avg of last 1000 elements
+ private MovingAvg avgRecordSize = new MovingAvg();
+ private MovingAvg avgOffsetGap = new MovingAvg(); // > 0 only when log compaction is enabled.
PartitionState(TopicPartition partition, long nextOffset) {
this.topicPartition = partition;
@@ -892,17 +980,13 @@
this.latestOffset = UNINITIALIZED_OFFSET;
}
- // update consumedOffset and avgRecordSize
- void recordConsumed(long offset, int size) {
+ // Update consumedOffset, avgRecordSize, and avgOffsetGap
+ void recordConsumed(long offset, int size, long offsetGap) {
nextOffset = offset + 1;
- // this is always updated from single thread. probably not worth making it an AtomicDouble
- if (avgRecordSize <= 0) {
- avgRecordSize = size;
- } else {
- // initially, first record heavily contributes to average.
- avgRecordSize += ((size - avgRecordSize) / movingAvgWindow);
- }
+ // This is always updated from single thread. Probably not worth making atomic.
+ avgRecordSize.update(size);
+ avgOffsetGap.update(offsetGap);
}
synchronized void setLatestOffset(long latestOffset) {
@@ -915,14 +999,15 @@
if (backlogMessageCount == UnboundedReader.BACKLOG_UNKNOWN) {
return UnboundedReader.BACKLOG_UNKNOWN;
}
- return (long) (backlogMessageCount * avgRecordSize);
+ return (long) (backlogMessageCount * avgRecordSize.get());
}
synchronized long backlogMessageCount() {
if (latestOffset < 0 || nextOffset < 0) {
return UnboundedReader.BACKLOG_UNKNOWN;
}
- return Math.max(0, (latestOffset - nextOffset));
+ double remaining = (latestOffset - nextOffset) / (1 + avgOffsetGap.get());
+ return Math.max(0, (long) Math.ceil(remaining));
}
}
@@ -1020,8 +1105,32 @@
curBatch = Iterators.cycle(nonEmpty);
}
+ private void setupInitialOffset(PartitionState pState) {
+ Read<K, V> spec = source.spec;
+
+ if (pState.nextOffset != UNINITIALIZED_OFFSET) {
+ consumer.seek(pState.topicPartition, pState.nextOffset);
+ } else {
+ // nextOffset is unininitialized here, meaning start reading from latest record as of now
+ // ('latest' is the default, and is configurable) or 'look up offset by startReadTime.
+ // Remember the current position without waiting until the first record is read. This
+ // ensures checkpoint is accurate even if the reader is closed before reading any records.
+ Instant startReadTime = spec.getStartReadTime();
+ if (startReadTime != null) {
+ pState.nextOffset =
+ consumerSpEL.offsetForTime(consumer, pState.topicPartition, spec.getStartReadTime());
+ consumer.seek(pState.topicPartition, pState.nextOffset);
+ } else {
+ pState.nextOffset = consumer.position(pState.topicPartition);
+ }
+ }
+ }
+
@Override
public boolean start() throws IOException {
+ final int defaultPartitionInitTimeout = 60 * 1000;
+ final int kafkaRequestTimeoutMultiple = 2;
+
Read<K, V> spec = source.spec;
consumer = spec.getConsumerFactoryFn().apply(spec.getConsumerConfig());
consumerSpEL.evaluateAssign(consumer, spec.getTopicPartitions());
@@ -1036,18 +1145,38 @@
keyDeserializerInstance.configure(spec.getConsumerConfig(), true);
valueDeserializerInstance.configure(spec.getConsumerConfig(), false);
- for (PartitionState p : partitionStates) {
- if (p.nextOffset != UNINITIALIZED_OFFSET) {
- consumer.seek(p.topicPartition, p.nextOffset);
- } else {
- // nextOffset is unininitialized here, meaning start reading from latest record as of now
- // ('latest' is the default, and is configurable). Remember the current position without
- // waiting until the first record read. This ensures checkpoint is accurate even if the
- // reader is closed before reading any records.
- p.nextOffset = consumer.position(p.topicPartition);
- }
+ // Seek to start offset for each partition. This is the first interaction with the server.
+ // Unfortunately it can block forever in case of network issues like incorrect ACLs.
+ // Initialize partition in a separate thread and cancel it if takes longer than a minute.
+ for (final PartitionState pState : partitionStates) {
+ Future<?> future = consumerPollThread.submit(new Runnable() {
+ public void run() {
+ setupInitialOffset(pState);
+ }
+ });
- LOG.info("{}: reading from {} starting at offset {}", name, p.topicPartition, p.nextOffset);
+ try {
+ // Timeout : 1 minute OR 2 * Kafka consumer request timeout if it is set.
+ Integer reqTimeout = (Integer) source.spec.getConsumerConfig().get(
+ ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG);
+ future.get(reqTimeout != null ? kafkaRequestTimeoutMultiple * reqTimeout
+ : defaultPartitionInitTimeout,
+ TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ consumer.wakeup(); // This unblocks consumer stuck on network I/O.
+ // Likely reason : Kafka servers are configured to advertise internal ips, but
+ // those ips are not accessible from workers outside.
+ String msg = String.format(
+ "%s: Timeout while initializing partition '%s'. "
+ + "Kafka client may not be able to connect to servers.",
+ this, pState.topicPartition);
+ LOG.error("{}", msg);
+ throw new IOException(msg);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ LOG.info("{}: reading from {} starting at offset {}",
+ name, pState.topicPartition, pState.nextOffset);
}
// Start consumer read loop.
@@ -1118,21 +1247,17 @@
continue;
}
- // sanity check
- if (offset != expected) {
- LOG.warn("{}: gap in offsets for {} at {}. {} records missing.",
- this, pState.topicPartition, expected, offset - expected);
- }
+ long offsetGap = offset - expected; // could be > 0 when Kafka log compaction is enabled.
if (curRecord == null) {
LOG.info("{}: first record offset {}", name, offset);
+ offsetGap = 0;
}
- curRecord = null; // user coders below might throw.
-
- // apply user deserializers.
+ // Apply user deserializers. User deserializers might throw, which will be propagated up
+ // and 'curRecord' remains unchanged. The runner should close this reader.
// TODO: write records that can't be deserialized to a "dead-letter" additional output.
- KafkaRecord<K, V> record = new KafkaRecord<K, V>(
+ KafkaRecord<K, V> record = new KafkaRecord<>(
rawRecord.topic(),
rawRecord.partition(),
rawRecord.offset(),
@@ -1146,7 +1271,7 @@
int recordSize = (rawRecord.key() == null ? 0 : rawRecord.key().length)
+ (rawRecord.value() == null ? 0 : rawRecord.value().length);
- pState.recordConsumed(offset, recordSize);
+ pState.recordConsumed(offset, recordSize, offsetGap);
bytesRead.inc(recordSize);
bytesReadBySplit.inc(recordSize);
return true;
@@ -1166,6 +1291,9 @@
private void updateLatestOffsets() {
for (PartitionState p : partitionStates) {
try {
+ // If "read_committed" is enabled in the config, this seeks to 'Last Stable Offset'.
+ // As a result uncommitted messages are not counted in backlog. It is correct since
+ // the reader can not read them anyway.
consumerSpEL.evaluateSeek2End(offsetConsumer, p.topicPartition);
long offset = offsetConsumer.position(p.topicPartition);
p.setLatestOffset(offset);
@@ -1242,7 +1370,6 @@
return curTimestamp;
}
-
@Override
public long getSplitBacklogBytes() {
long backlogBytes = 0;
@@ -1284,8 +1411,12 @@
// might block to enqueue right after availableRecordsQueue.poll() below.
while (!isShutdown) {
- consumer.wakeup();
- offsetConsumer.wakeup();
+ if (consumer != null) {
+ consumer.wakeup();
+ }
+ if (offsetConsumer != null) {
+ offsetConsumer.wakeup();
+ }
availableRecordsQueue.poll(); // drain unread batch, this unblocks consumer thread.
try {
isShutdown = consumerPollThread.awaitTermination(10, TimeUnit.SECONDS)
@@ -1324,6 +1455,13 @@
@Nullable abstract Class<? extends Serializer<K>> getKeySerializer();
@Nullable abstract Class<? extends Serializer<V>> getValueSerializer();
+ // Configuration for EOS sink
+ abstract boolean isEOS();
+ @Nullable abstract String getSinkGroupId();
+ abstract int getNumShards();
+ @Nullable abstract
+ SerializableFunction<Map<String, Object>, ? extends Consumer<?, ?>> getConsumerFactoryFn();
+
abstract Builder<K, V> toBuilder();
@AutoValue.Builder
@@ -1334,6 +1472,11 @@
SerializableFunction<Map<String, Object>, Producer<K, V>> fn);
abstract Builder<K, V> setKeySerializer(Class<? extends Serializer<K>> serializer);
abstract Builder<K, V> setValueSerializer(Class<? extends Serializer<V>> serializer);
+ abstract Builder<K, V> setEOS(boolean eosEnabled);
+ abstract Builder<K, V> setSinkGroupId(String sinkGroupId);
+ abstract Builder<K, V> setNumShards(int numShards);
+ abstract Builder<K, V> setConsumerFactoryFn(
+ SerializableFunction<Map<String, Object>, ? extends Consumer<?, ?>> fn);
abstract Write<K, V> build();
}
@@ -1390,24 +1533,118 @@
}
/**
+ * Provides exactly-once semantics while writing to Kafka, which enables applications with
+ * end-to-end exactly-once guarantees on top of exactly-once semantics <i>within</i> Beam
+ * pipelines. It ensures that records written to sink are committed on Kafka exactly once,
+ * even in the case of retries during pipeline execution even when some processing is retried.
+ * Retries typically occur when workers restart (as in failure recovery), or when the work is
+ * redistributed (as in an autoscaling event).
+ *
+ * <p>Beam runners typically provide exactly-once semantics for results of a pipeline, but not
+ * for side effects from user code in transform. If a transform such as Kafka sink writes
+ * to an external system, those writes might occur more than once. When EOS is enabled here,
+ * the sink transform ties checkpointing semantics in compatible Beam runners and transactions
+ * in Kafka (version 0.11+) to ensure a record is written only once. As the implementation
+ * relies on runners checkpoint semantics, not all the runners are compatible. The sink throws
+ * an exception during initialization if the runner is not whitelisted. Flink runner is
+ * one of the runners whose checkpoint semantics are not compatible with current
+ * implementation (hope to provide a solution in near future). Dataflow runner and Spark
+ * runners are whitelisted as compatible.
+ *
+ * <p>Note on performance: Exactly-once sink involves two shuffles of the records. In addition
+ * to cost of shuffling the records among workers, the records go through 2
+ * serialization-deserialization cycles. Depending on volume and cost of serialization,
+ * the CPU cost might be noticeable. The CPU cost can be reduced by writing byte arrays
+ * (i.e. serializing them to byte before writing to Kafka sink).
+ *
+ * @param numShards Sets sink parallelism. The state metadata stored on Kafka is stored across
+ * this many virtual partitions using {@code sinkGroupId}. A good rule of thumb is to set
+ * this to be around number of partitions in Kafka topic.
+ *
+ * @param sinkGroupId The <i>group id</i> used to store small amount of state as metadata on
+ * Kafka. It is similar to <i>consumer group id</i> used with a {@link KafkaConsumer}. Each
+ * job should use a unique group id so that restarts/updates of job preserve the state to
+ * ensure exactly-once semantics. The state is committed atomically with sink transactions
+ * on Kafka. See {@link KafkaProducer#sendOffsetsToTransaction(Map, String)} for more
+ * information. The sink performs multiple sanity checks during initialization to catch
+ * common mistakes so that it does not end up using state that does not <i>seem</i> to
+ * be written by the same job.
+ */
+ public Write<K, V> withEOS(int numShards, String sinkGroupId) {
+ EOSWrite.ensureEOSSupport();
+ checkArgument(numShards >= 1, "numShards should be >= 1");
+ checkArgument(sinkGroupId != null, "sinkGroupId is required for exactly-once sink");
+ return toBuilder()
+ .setEOS(true)
+ .setNumShards(numShards)
+ .setSinkGroupId(sinkGroupId)
+ .build();
+ }
+
+ /**
+ * When exactly-once semantics are enabled (see {@link #withEOS(int, String)}), the sink needs
+ * to fetch previously stored state with Kafka topic. Fetching the metadata requires a
+ * consumer. Similar to {@link Read#withConsumerFactoryFn(SerializableFunction)}, a factory
+ * function can be supplied if required in a specific case.
+ * The default is {@link KafkaConsumer}.
+ */
+ public Write<K, V> withConsumerFactoryFn(
+ SerializableFunction<Map<String, Object>, ? extends Consumer<?, ?>> consumerFactoryFn) {
+ return toBuilder().setConsumerFactoryFn(consumerFactoryFn).build();
+ }
+
+ /**
* Writes just the values to Kafka. This is useful for writing collections of values rather
* thank {@link KV}s.
*/
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public PTransform<PCollection<V>, PDone> values() {
- return new KafkaValueWrite<>(toBuilder().build());
+ return new KafkaValueWrite<>(
+ toBuilder()
+ .setKeySerializer((Class) StringSerializer.class)
+ .build()
+ );
}
@Override
public PDone expand(PCollection<KV<K, V>> input) {
- input.apply(ParDo.of(new KafkaWriter<>(this)));
+ checkArgument(
+ getProducerConfig().get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) != null,
+ "withBootstrapServers() is required");
+ checkArgument(getTopic() != null, "withTopic() is required");
+ checkArgument(getKeySerializer() != null, "withKeySerializer() is required");
+ checkArgument(getValueSerializer() != null, "withValueSerializer() is required");
+
+ if (isEOS()) {
+ EOSWrite.ensureEOSSupport();
+
+ // TODO: Verify that the group_id does not have existing state stored on Kafka unless
+ // this is an upgrade. This avoids issues with simple mistake of reusing group_id
+ // across multiple runs or across multiple jobs. This is checked when the sink
+ // transform initializes while processing the output. It might be better to
+ // check here to catch common mistake.
+
+ input.apply(new EOSWrite<>(this));
+ } else {
+ input.apply(ParDo.of(new KafkaWriter<>(this)));
+ }
return PDone.in(input.getPipeline());
}
- @Override
public void validate(PipelineOptions options) {
- checkNotNull(getProducerConfig().get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG),
- "Kafka bootstrap servers should be set");
- checkNotNull(getTopic(), "Kafka topic should be set");
+ if (isEOS()) {
+ String runner = options.getRunner().getName();
+ if (runner.equals("org.apache.beam.runners.direct.DirectRunner")
+ || runner.startsWith("org.apache.beam.runners.dataflow.")
+ || runner.startsWith("org.apache.beam.runners.spark.")) {
+ return;
+ }
+ throw new UnsupportedOperationException(
+ runner + " is not whitelisted among runners compatible with Kafka exactly-once sink. "
+ + "This implementation of exactly-once sink relies on specific checkpoint guarantees. "
+ + "Only the runners with known to have compatible checkpoint semantics are whitelisted."
+ );
+ }
}
// set config defaults
@@ -1471,6 +1708,7 @@
}
}
+
private static class NullOnlyCoder<T> extends AtomicCoder<T> {
@Override
public void encode(T value, OutputStream outStream) {
@@ -1615,4 +1853,562 @@
throw new RuntimeException(String.format(
"Could not extract the Kafka Deserializer type from %s", deserializer));
}
+
+ ////////////////////////////////// Exactly-Once Sink \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
+
+ /**
+ * Exactly-once sink transform.
+ */
+ private static class EOSWrite<K, V> extends PTransform<PCollection<KV<K, V>>, PCollection<Void>> {
+ //
+ // Dataflow ensures at-least once processing for side effects like sinks. In order to provide
+ // exactly-once semantics, a sink needs to be idempotent or it should avoid writing records
+ // that have already been written. This snk does the latter. All the the records are ordered
+ // across a fixed number of shards and records in each shard are written in order. It drops
+ // any records that are already written and buffers those arriving out of order.
+ //
+ // Exactly once sink involves two shuffles of the records:
+ // A : Assign a shard ---> B : Assign sequential ID ---> C : Write to Kafka in order
+ //
+ // Processing guarantees also require deterministic processing within user transforms.
+ // Here, that requires order of the records committed to Kafka by C should not be affected by
+ // restarts in C and its upstream stages.
+ //
+ // A : Assigns a random shard for message. Note that there are no ordering guarantees for
+ // writing user records to Kafka. User can still control partitioning among topic
+ // partitions as with regular sink (of course, there are no ordering guarantees in
+ // regular Kafka sink either).
+ // B : Assigns an id sequentially for each messages within a shard.
+ // C : Writes each shard to Kafka in sequential id order. In Dataflow, when C sees a record
+ // and id, it implies that record and the associated id are checkpointed to persistent
+ // storage and this record will always have same id, even in retries.
+ // Exactly-once semantics are achieved by writing records in the strict order of
+ // these check-pointed sequence ids.
+ //
+ // Parallelism for B and C is fixed to 'numShards', which defaults to number of partitions
+ // for the topic. A few reasons for that:
+ // - B & C implement their functionality using per-key state. Shard id makes it independent
+ // of cardinality of user key.
+ // - We create one producer per shard, and its 'transactional id' is based on shard id. This
+ // requires that number of shards to be finite. This also helps with batching. and avoids
+ // initializing producers and transactions.
+ // - Most importantly, each of sharded writers stores 'next message id' in partition
+ // metadata, which is committed atomically with Kafka transactions. This is critical
+ // to handle retries of C correctly. Initial testing showed number of shards could be
+ // larger than number of partitions for the topic.
+ //
+ // Number of shards can change across multiple runs of a pipeline (job upgrade in Dataflow).
+ //
+
+ private final Write<K, V> spec;
+
+ static void ensureEOSSupport() {
+ checkArgument(
+ ProducerSpEL.supportsTransactions(), "%s %s",
+ "This version of Kafka client does not support transactions required to support",
+ "exactly-once semantics. Please use Kafka client version 0.11 or newer.");
+ }
+
+ EOSWrite(Write<K, V> spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ public PCollection<Void> expand(PCollection<KV<K, V>> input) {
+
+ int numShards = spec.getNumShards();
+ if (numShards <= 0) {
+ try (Consumer<?, ?> consumer = openConsumer(spec)) {
+ numShards = consumer.partitionsFor(spec.getTopic()).size();
+ LOG.info("Using {} shards for exactly-once writer, matching number of partitions "
+ + "for topic '{}'", numShards, spec.getTopic());
+ }
+ }
+ checkState(numShards > 0, "Could not set number of shards");
+
+ return input
+ .apply(Window.<KV<K, V>>into(new GlobalWindows()) // Everything into global window.
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))
+ .discardingFiredPanes())
+ .apply(String.format("Shuffle across %d shards", numShards),
+ ParDo.of(new EOSReshard<K, V>(numShards)))
+ .apply("Persist sharding", GroupByKey.<Integer, KV<K, V>>create())
+ .apply("Assign sequential ids", ParDo.of(new EOSSequencer<K, V>()))
+ .apply("Persist ids", GroupByKey.<Integer, KV<Long, KV<K, V>>>create())
+ .apply(String.format("Write to Kafka topic '%s'", spec.getTopic()),
+ ParDo.of(new KafkaEOWriter<>(spec, input.getCoder())));
+ }
+ }
+
+ /**
+ * Shuffle messages assigning each randomly to a shard.
+ */
+ private static class EOSReshard<K, V> extends DoFn<KV<K, V>, KV<Integer, KV<K, V>>> {
+ private final int numShards;
+ private transient int shardId;
+
+ EOSReshard(int numShards) {
+ this.numShards = numShards;
+ }
+
+ @Setup
+ public void setup() {
+ shardId = ThreadLocalRandom.current().nextInt(numShards);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext ctx) {
+ shardId = (shardId + 1) % numShards; // round-robin among shards.
+ ctx.output(KV.of(shardId, ctx.element()));
+ }
+ }
+
+ private static class EOSSequencer<K, V>
+ extends DoFn<KV<Integer, Iterable<KV<K, V>>>, KV<Integer, KV<Long, KV<K, V>>>> {
+ private static final String NEXT_ID = "nextId";
+ @StateId(NEXT_ID)
+ private final StateSpec<ValueState<Long>> nextIdSpec = StateSpecs.value();
+
+ @ProcessElement
+ public void processElement(@StateId(NEXT_ID) ValueState<Long> nextIdState, ProcessContext ctx) {
+ long nextId = MoreObjects.firstNonNull(nextIdState.read(), 0L);
+ int shard = ctx.element().getKey();
+ for (KV<K, V> value : ctx.element().getValue()) {
+ ctx.output(KV.of(shard, KV.of(nextId, value)));
+ nextId++;
+ }
+ nextIdState.write(nextId);
+ }
+ }
+
+ private static class KafkaEOWriter<K, V>
+ extends DoFn<KV<Integer, Iterable<KV<Long, KV<K, V>>>>, Void> {
+
+ private static final String NEXT_ID = "nextId";
+ private static final String MIN_BUFFERED_ID = "minBufferedId";
+ private static final String OUT_OF_ORDER_BUFFER = "outOfOrderBuffer";
+ private static final String WRITER_ID = "writerId";
+
+ private static final String METRIC_NAMESPACE = "KafkaEOSink";
+
+ // Not sure of a good limit. This applies only for large bundles.
+ private static final int MAX_RECORDS_PER_TXN = 1000;
+ private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
+
+ @StateId(NEXT_ID)
+ private final StateSpec<ValueState<Long>> sequenceIdSpec = StateSpecs.value();
+ @StateId(MIN_BUFFERED_ID)
+ private final StateSpec<ValueState<Long>> minBufferedId = StateSpecs.value();
+ @StateId(OUT_OF_ORDER_BUFFER)
+ private final StateSpec<BagState<KV<Long, KV<K, V>>>> outOfOrderBuffer;
+ // A random id assigned to each shard. Helps with detecting when multiple jobs are mistakenly
+ // started with same groupId used for storing state on Kafka side including the case where
+ // a job is restarted with same groupId, but the metadata from previous run is not removed.
+ // Better to be safe and error out with a clear message.
+ @StateId(WRITER_ID)
+ private final StateSpec<ValueState<String>> writerIdSpec = StateSpecs.value();
+
+ private final Write<K, V> spec;
+
+ // Metrics
+ private final Counter elementsWritten = SinkMetrics.elementsWritten();
+ // Elements buffered due to out of order arrivals.
+ private final Counter elementsBuffered = Metrics.counter(METRIC_NAMESPACE, "elementsBuffered");
+ private final Counter numTransactions = Metrics.counter(METRIC_NAMESPACE, "numTransactions");
+
+ KafkaEOWriter(Write<K, V> spec, Coder<KV<K, V>> elemCoder) {
+ this.spec = spec;
+ this.outOfOrderBuffer = StateSpecs.bag(KvCoder.of(BigEndianLongCoder.of(), elemCoder));
+ }
+
+ @Setup
+ public void setup() {
+ // This is on the worker. Ensure the runtime version is till compatible.
+ EOSWrite.ensureEOSSupport();
+ }
+
+ @ProcessElement
+ public void processElement(@StateId(NEXT_ID) ValueState<Long> nextIdState,
+ @StateId(MIN_BUFFERED_ID) ValueState<Long> minBufferedIdState,
+ @StateId(OUT_OF_ORDER_BUFFER)
+ BagState<KV<Long, KV<K, V>>> oooBufferState,
+ @StateId(WRITER_ID) ValueState<String> writerIdState,
+ ProcessContext ctx)
+ throws IOException {
+
+ int shard = ctx.element().getKey();
+
+ minBufferedIdState.readLater();
+ long nextId = MoreObjects.firstNonNull(nextIdState.read(), 0L);
+ long minBufferedId = MoreObjects.firstNonNull(minBufferedIdState.read(), Long.MAX_VALUE);
+
+ ShardWriterCache<K, V> cache =
+ (ShardWriterCache<K, V>) CACHE_BY_GROUP_ID.getUnchecked(spec.getSinkGroupId());
+ ShardWriter<K, V> writer = cache.removeIfPresent(shard);
+ if (writer == null) {
+ writer = initShardWriter(shard, writerIdState, nextId);
+ }
+
+ long committedId = writer.committedId;
+
+ if (committedId >= nextId) {
+ // This is a retry of an already committed batch.
+ LOG.info("{}: committed id {} is ahead of expected {}. {} records will be dropped "
+ + "(these are already written).",
+ shard, committedId, nextId - 1, committedId - nextId + 1);
+ nextId = committedId + 1;
+ }
+
+ try {
+ writer.beginTxn();
+ int txnSize = 0;
+
+ // Iterate in recordId order. The input iterator could be mostly sorted.
+ // There might be out of order messages buffered in earlier iterations. These
+ // will get merged if and when minBufferedId matches nextId.
+
+ Iterator<KV<Long, KV<K, V>>> iter = ctx.element().getValue().iterator();
+
+ while (iter.hasNext()) {
+ KV<Long, KV<K, V>> kv = iter.next();
+ long recordId = kv.getKey();
+
+ if (recordId < nextId) {
+ LOG.info("{}: dropping older record {}. Already committed till {}",
+ shard, recordId, committedId);
+ continue;
+ }
+
+ if (recordId > nextId) {
+ // Out of order delivery. Should be pretty rare (what about in a batch pipeline?)
+
+ LOG.info("{}: Saving out of order record {}, next record id to be written is {}",
+ shard, recordId, nextId);
+
+ // checkState(recordId - nextId < 10000, "records are way out of order");
+
+ oooBufferState.add(kv);
+ minBufferedId = Math.min(minBufferedId, recordId);
+ minBufferedIdState.write(minBufferedId);
+ elementsBuffered.inc();
+ continue;
+ }
+
+ // recordId and nextId match. Finally write record.
+
+ writer.sendRecord(kv.getValue(), elementsWritten);
+ nextId++;
+
+ if (++txnSize >= MAX_RECORDS_PER_TXN) {
+ writer.commitTxn(recordId, numTransactions);
+ txnSize = 0;
+ writer.beginTxn();
+ }
+
+ if (minBufferedId == nextId) {
+ // One or more of the buffered records can be committed now.
+ // Read all of them in to memory and sort them. Reading into memory
+ // might be problematic in extreme cases. Might need to improve it in future.
+
+ List<KV<Long, KV<K, V>>> buffered = Lists.newArrayList(oooBufferState.read());
+ Collections.sort(buffered, new KV.OrderByKey<Long, KV<K, V>>());
+
+ LOG.info("{} : merging {} buffered records (min buffered id is {}).",
+ shard, buffered.size(), minBufferedId);
+
+ oooBufferState.clear();
+ minBufferedIdState.clear();
+ minBufferedId = Long.MAX_VALUE;
+
+ iter = Iterators.mergeSorted(ImmutableList.of(iter, buffered.iterator()),
+ new KV.OrderByKey<Long, KV<K, V>>());
+ }
+ }
+
+ writer.commitTxn(nextId - 1, numTransactions);
+ nextIdState.write(nextId);
+
+ } catch (ProducerSpEL.UnrecoverableProducerException e) {
+ // Producer JavaDoc says these are not recoverable errors and producer should be closed.
+
+ // Close the producer and a new producer will be initialized in retry.
+ // It is possible that a rough worker keeps retrying and ends up fencing off
+ // active producers. How likely this might be or how well such a scenario is handled
+ // depends on the runner. For now we will leave it to upper layers, will need to revisit.
+
+ LOG.warn("{} : closing producer {} after unrecoverable error. The work might have migrated."
+ + " Committed id {}, current id {}.",
+ writer.shard, writer.producerName, writer.committedId, nextId - 1, e);
+
+ writer.producer.close();
+ writer = null; // No need to cache it.
+ throw e;
+ } finally {
+ if (writer != null) {
+ cache.insert(shard, writer);
+ }
+ }
+ }
+
+ private static class ShardMetadata {
+
+ @JsonProperty("seq")
+ public final long sequenceId;
+ @JsonProperty("id")
+ public final String writerId;
+
+ private ShardMetadata() { // for json deserializer
+ sequenceId = -1;
+ writerId = null;
+ }
+
+ ShardMetadata(long sequenceId, String writerId) {
+ this.sequenceId = sequenceId;
+ this.writerId = writerId;
+ }
+ }
+
+ /**
+ * A wrapper around Kafka producer. One for each of the shards.
+ */
+ private static class ShardWriter<K, V> {
+
+ private final int shard;
+ private final String writerId;
+ private final Producer<K, V> producer;
+ private final String producerName;
+ private final Write<K, V> spec;
+ private long committedId;
+
+ ShardWriter(int shard,
+ String writerId,
+ Producer<K, V> producer,
+ String producerName,
+ Write<K, V> spec,
+ long committedId) {
+ this.shard = shard;
+ this.writerId = writerId;
+ this.producer = producer;
+ this.producerName = producerName;
+ this.spec = spec;
+ this.committedId = committedId;
+ }
+
+ void beginTxn() {
+ ProducerSpEL.beginTransaction(producer);
+ }
+
+ void sendRecord(KV<K, V> record, Counter sendCounter) {
+ try {
+ producer.send(
+ new ProducerRecord<>(spec.getTopic(), record.getKey(), record.getValue()));
+ sendCounter.inc();
+ } catch (KafkaException e) {
+ ProducerSpEL.abortTransaction(producer);
+ throw e;
+ }
+ }
+
+ void commitTxn(long lastRecordId, Counter numTransactions) throws IOException {
+ try {
+ // Store id in consumer group metadata for the partition.
+ // NOTE: Kafka keeps this metadata for 24 hours since the last update. This limits
+ // how long the pipeline could be down before resuming it. It does not look like
+ // this TTL can be adjusted (asked about it on Kafka users list).
+ ProducerSpEL.sendOffsetsToTransaction(
+ producer,
+ ImmutableMap.of(new TopicPartition(spec.getTopic(), shard),
+ new OffsetAndMetadata(0L,
+ JSON_MAPPER.writeValueAsString(
+ new ShardMetadata(lastRecordId, writerId)))),
+ spec.getSinkGroupId());
+ ProducerSpEL.commitTransaction(producer);
+
+ numTransactions.inc();
+ LOG.debug("{} : committed {} records", shard, lastRecordId - committedId);
+
+ committedId = lastRecordId;
+ } catch (KafkaException e) {
+ ProducerSpEL.abortTransaction(producer);
+ throw e;
+ }
+ }
+ }
+
+ private ShardWriter<K, V> initShardWriter(int shard,
+ ValueState<String> writerIdState,
+ long nextId) throws IOException {
+
+ String producerName = String.format("producer_%d_for_%s", shard, spec.getSinkGroupId());
+ Producer<K, V> producer = initializeEosProducer(spec, producerName);
+
+ // Fetch latest committed metadata for the partition (if any). Checks committed sequence ids.
+ try {
+
+ String writerId = writerIdState.read();
+
+ OffsetAndMetadata committed;
+
+ try (Consumer<?, ?> consumer = openConsumer(spec)) {
+ committed = consumer.committed(new TopicPartition(spec.getTopic(), shard));
+ }
+
+ long committedSeqId = -1;
+
+ if (committed == null || committed.metadata() == null || committed.metadata().isEmpty()) {
+ checkState(nextId == 0 && writerId == null,
+ "State exists for shard %s (nextId %s, writerId '%s'), but there is no state "
+ + "stored with Kafka topic '%s' group id '%s'",
+ shard, nextId, writerId, spec.getTopic(), spec.getSinkGroupId());
+
+ writerId = String.format("%X - %s",
+ new Random().nextInt(Integer.MAX_VALUE),
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")
+ .withZone(DateTimeZone.UTC)
+ .print(DateTimeUtils.currentTimeMillis()));
+ writerIdState.write(writerId);
+ LOG.info("Assigned writer id '{}' to shard {}", writerId, shard);
+
+ } else {
+ ShardMetadata metadata = JSON_MAPPER.readValue(committed.metadata(),
+ ShardMetadata.class);
+
+ checkNotNull(metadata.writerId);
+
+ if (writerId == null) {
+ // a) This might be a restart of the job from scratch, in which case metatdata
+ // should be ignored and overwritten with new one.
+ // b) This job might be started with an incorrect group id which is an error.
+ // c) There is an extremely small chance that this is a retry of the first bundle
+ // where metatdate was committed to Kafka but the bundle results were not committed
+ // in Beam, in which case it should be treated as correct metadata.
+ // How can we tell these three cases apart? Be safe and throw an exception.
+ //
+ // We could let users explicitly an option to override the existing metadata.
+ //
+ throw new IllegalStateException(String.format(
+ "Kafka metadata exists for shard %s, but there is no stored state for it. "
+ + "This mostly indicates groupId '%s' is used else where or in earlier runs. "
+ + "Try another group id. Metadata for this shard on Kafka : '%s'",
+ shard, spec.getSinkGroupId(), committed.metadata()));
+ }
+
+ checkState(writerId.equals(metadata.writerId),
+ "Writer ids don't match. This is mostly a unintended misuse of groupId('%s')."
+ + "Beam '%s', Kafka '%s'",
+ spec.getSinkGroupId(), writerId, metadata.writerId);
+
+ committedSeqId = metadata.sequenceId;
+
+ checkState(committedSeqId >= (nextId - 1),
+ "Committed sequence id can not be lower than %s, partition metadata : %s",
+ nextId - 1, committed.metadata());
+ }
+
+ LOG.info("{} : initialized producer {} with committed sequence id {}",
+ shard, producerName, committedSeqId);
+
+ return new ShardWriter<>(shard, writerId, producer, producerName, spec, committedSeqId);
+
+ } catch (Exception e) {
+ producer.close();
+ throw e;
+ }
+ }
+
+ /**
+ * A wrapper around guava cache to provide insert()/remove() semantics. A ShardWriter will
+ * be closed if it is stays in cache for more than 1 minute, i.e. not used inside EOSWrite
+ * DoFn for a minute or more.
+ */
+ private static class ShardWriterCache<K, V> {
+
+ static final ScheduledExecutorService SCHEDULED_CLEAN_UP_THREAD =
+ Executors.newSingleThreadScheduledExecutor();
+
+ static final int CLEAN_UP_CHECK_INTERVAL_MS = 10 * 1000;
+ static final int IDLE_TIMEOUT_MS = 60 * 1000;
+
+ private final Cache<Integer, ShardWriter<K, V>> cache;
+
+ ShardWriterCache() {
+ this.cache = CacheBuilder
+ .newBuilder()
+ .expireAfterWrite(IDLE_TIMEOUT_MS, TimeUnit.MILLISECONDS)
+ .removalListener(new RemovalListener<Integer, ShardWriter<K, V>>() {
+ @Override
+ public void onRemoval(RemovalNotification<Integer, ShardWriter<K, V>> notification) {
+ if (notification.getCause() != RemovalCause.EXPLICIT) {
+ ShardWriter writer = notification.getValue();
+ LOG.info("{} : Closing idle shard writer {} after 1 minute of idle time.",
+ writer.shard, writer.producerName);
+ writer.producer.close();
+ }
+ }
+ }).build();
+
+ // run cache.cleanUp() every 10 seconds.
+ SCHEDULED_CLEAN_UP_THREAD.scheduleAtFixedRate(
+ new Runnable() {
+ @Override
+ public void run() {
+ cache.cleanUp();
+ }
+ },
+ CLEAN_UP_CHECK_INTERVAL_MS, CLEAN_UP_CHECK_INTERVAL_MS, TimeUnit.MILLISECONDS);
+ }
+
+ ShardWriter<K, V> removeIfPresent(int shard) {
+ return cache.asMap().remove(shard);
+ }
+
+ void insert(int shard, ShardWriter<K, V> writer) {
+ ShardWriter<K, V> existing = cache.asMap().putIfAbsent(shard, writer);
+ checkState(existing == null,
+ "Unexpected multiple instances of writers for shard %s", shard);
+ }
+ }
+
+ // One cache for each sink (usually there is only one sink per pipeline)
+ private static final LoadingCache<String, ShardWriterCache<?, ?>> CACHE_BY_GROUP_ID =
+ CacheBuilder.newBuilder()
+ .build(new CacheLoader<String, ShardWriterCache<?, ?>>() {
+ @Override
+ public ShardWriterCache<?, ?> load(String key) throws Exception {
+ return new ShardWriterCache<>();
+ }
+ });
+ }
+
+ /**
+ * Opens a generic consumer that is mainly meant for metadata operations like fetching
+ * number of partitions for a topic rather than for fetching messages.
+ */
+ private static Consumer<?, ?> openConsumer(Write<?, ?> spec) {
+ return spec.getConsumerFactoryFn().apply((ImmutableMap.of(
+ ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, spec
+ .getProducerConfig().get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG),
+ ConsumerConfig.GROUP_ID_CONFIG, spec.getSinkGroupId(),
+ ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class,
+ ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class
+ )));
+ }
+
+ private static <K, V> Producer<K, V> initializeEosProducer(Write<K, V> spec,
+ String producerName) {
+
+ Map<String, Object> producerConfig = new HashMap<>(spec.getProducerConfig());
+ producerConfig.putAll(ImmutableMap.of(
+ ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, spec.getKeySerializer(),
+ ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, spec.getValueSerializer(),
+ ProducerSpEL.ENABLE_IDEMPOTENCE_CONFIG, true,
+ ProducerSpEL.TRANSACTIONAL_ID_CONFIG, producerName));
+
+ Producer<K, V> producer = spec.getProducerFactoryFn() != null
+ ? spec.getProducerFactoryFn().apply((producerConfig))
+ : new KafkaProducer<K, V>(producerConfig);
+
+ ProducerSpEL.initTransactions(producer);
+ return producer;
+ }
}
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ProducerSpEL.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ProducerSpEL.java
new file mode 100644
index 0000000..08674e0
--- /dev/null
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ProducerSpEL.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kafka;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Map;
+
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.AuthorizationException;
+
+/**
+ * ProducerSpEL to handle newer versions Producer API. The API is updated in Kafka 0.11
+ * to support exactly-once semantics.
+ */
+class ProducerSpEL {
+
+ private static boolean supportsTransactions;
+
+ private static Method initTransactionsMethod;
+ private static Method beginTransactionMethod;
+ private static Method commitTransactionMethod;
+ private static Method abortTransactionMethod;
+ private static Method sendOffsetsToTransactionMethod;
+
+ static final String ENABLE_IDEMPOTENCE_CONFIG = "enable.idempotence";
+ static final String TRANSACTIONAL_ID_CONFIG = "transactional.id";
+
+ private static Class producerFencedExceptionClass;
+ private static Class outOfOrderSequenceExceptionClass;
+
+ static {
+ try {
+ initTransactionsMethod = Producer.class.getMethod("initTransactions");
+ beginTransactionMethod = Producer.class.getMethod("beginTransaction");
+ commitTransactionMethod = Producer.class.getMethod("commitTransaction");
+ abortTransactionMethod = Producer.class.getMethod("abortTransaction");
+ sendOffsetsToTransactionMethod = Producer.class.getMethod(
+ "sendOffsetsToTransaction", Map.class, String.class);
+
+ producerFencedExceptionClass = Class.forName(
+ "org.apache.kafka.common.errors.ProducerFencedException");
+ outOfOrderSequenceExceptionClass = Class.forName(
+ "org.apache.kafka.common.errors.OutOfOrderSequenceException");
+
+ supportsTransactions = true;
+ } catch (ClassNotFoundException | NoSuchMethodException e) {
+ supportsTransactions = false;
+ }
+ }
+
+ /**
+ * Wraps an unrecoverable producer exceptions, including the ones related transactions
+ * introduced in 0.11 (as described in documentation for {@link Producer}). The calller should
+ * close the producer when this exception is thrown.
+ */
+ static class UnrecoverableProducerException extends ApiException {
+ UnrecoverableProducerException(ApiException cause) {
+ super(cause);
+ }
+ }
+
+ static boolean supportsTransactions() {
+ return supportsTransactions;
+ }
+
+ private static void ensureTransactionsSupport() {
+ checkArgument(supportsTransactions(),
+ "This version of Kafka client library does not support transactions. ",
+ "Please used version 0.11 or later.");
+ }
+
+ private static Object invoke(Method method, Object obj, Object... args) {
+ try {
+ return method.invoke(obj, args);
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ return new RuntimeException(e);
+ } catch (ApiException e) {
+ Class eClass = e.getClass();
+ if (producerFencedExceptionClass.isAssignableFrom(eClass)
+ || outOfOrderSequenceExceptionClass.isAssignableFrom(eClass)
+ || AuthorizationException.class.isAssignableFrom(eClass)) {
+ throw new UnrecoverableProducerException(e);
+ }
+ throw e;
+ }
+ }
+
+ static void initTransactions(Producer<?, ?> producer) {
+ ensureTransactionsSupport();
+ invoke(initTransactionsMethod, producer);
+ }
+
+ static void beginTransaction(Producer<?, ?> producer) {
+ ensureTransactionsSupport();
+ invoke(beginTransactionMethod, producer);
+ }
+
+ static void commitTransaction(Producer<?, ?> producer) {
+ ensureTransactionsSupport();
+ invoke(commitTransactionMethod, producer);
+ }
+
+ static void abortTransaction(Producer<?, ?> producer) {
+ ensureTransactionsSupport();
+ invoke(abortTransactionMethod, producer);
+ }
+
+ static void sendOffsetsToTransaction(Producer<?, ?> producer,
+ Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId) {
+ ensureTransactionsSupport();
+ invoke(sendOffsetsToTransactionMethod, producer, offsets, consumerGroupId);
+ }
+}
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
index 691f7f4..2cbd448 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
@@ -24,11 +24,15 @@
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -36,8 +40,11 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -80,6 +87,7 @@
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.MockConsumer;
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
@@ -106,14 +114,19 @@
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests of {@link KafkaIO}.
* Run with 'mvn test -Dkafka.clients.version=0.10.1.1',
- * or 'mvn test -Dkafka.clients.version=0.9.0.1' for either Kafka client version
+ * or 'mvn test -Dkafka.clients.version=0.9.0.1' for either Kafka client version.
*/
@RunWith(JUnit4.class)
public class KafkaIOTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaIOTest.class);
+
/*
* The tests below borrow code and structure from CountingSourceTest. In addition verifies
* the reader interleaves the records from multiple partitions.
@@ -150,7 +163,7 @@
}
int numPartitions = partitions.size();
- long[] offsets = new long[numPartitions];
+ final long[] offsets = new long[numPartitions];
for (int i = 0; i < numElements; i++) {
int pIdx = i % numPartitions;
@@ -184,6 +197,36 @@
updateEndOffsets(ImmutableMap.of(tp, (long) records.get(tp).size()));
}
}
+ // Override offsetsForTimes() in order to look up the offsets by timestamp.
+ // Remove keyword '@Override' here, Kafka client 0.10.1.0 previous versions does not have
+ // this method.
+ // Should return Map<TopicPartition, OffsetAndTimestamp>, but 0.10.1.0 previous versions
+ // does not have the OffsetAndTimestamp class. So return a raw type and use reflection
+ // here.
+ @SuppressWarnings("unchecked")
+ public Map offsetsForTimes(Map<TopicPartition, Long> timestampsToSearch) {
+ HashMap<TopicPartition, Object> result = new HashMap<>();
+ try {
+ Class<?> cls = Class.forName("org.apache.kafka.clients.consumer.OffsetAndTimestamp");
+ // OffsetAndTimestamp(long offset, long timestamp)
+ Constructor constructor = cls.getDeclaredConstructor(long.class, long.class);
+
+ // In test scope, timestamp == offset.
+ for (Map.Entry<TopicPartition, Long> entry : timestampsToSearch.entrySet()) {
+ long maxOffset = offsets[partitions.indexOf(entry.getKey())];
+ Long offset = entry.getValue();
+ if (offset >= maxOffset) {
+ offset = null;
+ }
+ result.put(
+ entry.getKey(), constructor.newInstance(entry.getValue(), offset));
+ }
+ return result;
+ } catch (ClassNotFoundException | IllegalAccessException
+ | InstantiationException | NoSuchMethodException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ }
};
for (String topic : topics) {
@@ -239,12 +282,19 @@
}
}
+ private static KafkaIO.Read<Integer, Long> mkKafkaReadTransform(
+ int numElements,
+ @Nullable SerializableFunction<KV<Integer, Long>, Instant> timestampFn) {
+ return mkKafkaReadTransform(numElements, numElements, timestampFn);
+ }
+
/**
* Creates a consumer with two topics, with 10 partitions each.
* numElements are (round-robin) assigned all the 20 partitions.
*/
private static KafkaIO.Read<Integer, Long> mkKafkaReadTransform(
int numElements,
+ int maxNumRecords,
@Nullable SerializableFunction<KV<Integer, Long>, Instant> timestampFn) {
List<String> topics = ImmutableList.of("topic_a", "topic_b");
@@ -256,7 +306,7 @@
topics, 10, numElements, OffsetResetStrategy.EARLIEST)) // 20 partitions
.withKeyDeserializer(IntegerDeserializer.class)
.withValueDeserializer(LongDeserializer.class)
- .withMaxNumRecords(numElements);
+ .withMaxNumRecords(maxNumRecords);
if (timestampFn != null) {
return reader.withTimestampFn(timestampFn);
@@ -283,22 +333,31 @@
public static void addCountingAsserts(PCollection<Long> input, long numElements) {
// Count == numElements
- PAssert
- .thatSingleton(input.apply("Count", Count.<Long>globally()))
- .isEqualTo(numElements);
// Unique count == numElements
- PAssert
- .thatSingleton(input.apply(Distinct.<Long>create())
- .apply("UniqueCount", Count.<Long>globally()))
- .isEqualTo(numElements);
// Min == 0
- PAssert
- .thatSingleton(input.apply("Min", Min.<Long>globally()))
- .isEqualTo(0L);
// Max == numElements-1
+ addCountingAsserts(input, numElements, numElements, 0L, numElements - 1);
+ }
+
+ public static void addCountingAsserts(
+ PCollection<Long> input, long count, long uniqueCount, long min, long max) {
+
PAssert
- .thatSingleton(input.apply("Max", Max.<Long>globally()))
- .isEqualTo(numElements - 1);
+ .thatSingleton(input.apply("Count", Count.<Long>globally()))
+ .isEqualTo(count);
+
+ PAssert
+ .thatSingleton(input.apply(Distinct.<Long>create())
+ .apply("UniqueCount", Count.<Long>globally()))
+ .isEqualTo(uniqueCount);
+
+ PAssert
+ .thatSingleton(input.apply("Min", Min.<Long>globally()))
+ .isEqualTo(min);
+
+ PAssert
+ .thatSingleton(input.apply("Max", Max.<Long>globally()))
+ .isEqualTo(max);
}
@Test
@@ -315,6 +374,35 @@
}
@Test
+ public void testUnreachableKafkaBrokers() {
+ // Expect an exception when the Kafka brokers are not reachable on the workers.
+ // We specify partitions explicitly so that splitting does not involve server interaction.
+ // Set request timeout to 10ms so that test does not take long.
+
+ thrown.expect(Exception.class);
+ thrown.expectMessage("Reader-0: Timeout while initializing partition 'test-0'");
+
+ int numElements = 1000;
+ PCollection<Long> input = p
+ .apply(KafkaIO.<Integer, Long>read()
+ .withBootstrapServers("8.8.8.8:9092") // Google public DNS ip.
+ .withTopicPartitions(ImmutableList.of(new TopicPartition("test", 0)))
+ .withKeyDeserializer(IntegerDeserializer.class)
+ .withValueDeserializer(LongDeserializer.class)
+ .updateConsumerProperties(ImmutableMap.<String, Object>of(
+ ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10,
+ ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 5,
+ ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 8,
+ ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, 8))
+ .withMaxNumRecords(10)
+ .withoutMetadata())
+ .apply(Values.<Long>create());
+
+ addCountingAsserts(input, numElements);
+ p.run();
+ }
+
+ @Test
public void testUnboundedSourceWithSingleTopic() {
// same as testUnboundedSource, but with single topic
@@ -645,11 +733,10 @@
int numElements = 1000;
- synchronized (MOCK_PRODUCER_LOCK) {
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
- MOCK_PRODUCER.clear();
-
- ProducerSendCompletionThread completionThread = new ProducerSendCompletionThread().start();
+ ProducerSendCompletionThread completionThread =
+ new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
String topic = "test";
@@ -661,13 +748,13 @@
.withTopic(topic)
.withKeySerializer(IntegerSerializer.class)
.withValueSerializer(LongSerializer.class)
- .withProducerFactoryFn(new ProducerFactoryFn()));
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));
p.run();
completionThread.shutdown();
- verifyProducerRecords(topic, numElements, false);
+ verifyProducerRecords(producerWrapper.mockProducer, topic, numElements, false);
}
}
@@ -677,11 +764,10 @@
int numElements = 1000;
- synchronized (MOCK_PRODUCER_LOCK) {
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
- MOCK_PRODUCER.clear();
-
- ProducerSendCompletionThread completionThread = new ProducerSendCompletionThread().start();
+ ProducerSendCompletionThread completionThread =
+ new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
String topic = "test";
@@ -693,14 +779,58 @@
.withBootstrapServers("none")
.withTopic(topic)
.withValueSerializer(LongSerializer.class)
- .withProducerFactoryFn(new ProducerFactoryFn())
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey))
.values());
p.run();
completionThread.shutdown();
- verifyProducerRecords(topic, numElements, true);
+ verifyProducerRecords(producerWrapper.mockProducer, topic, numElements, true);
+ }
+ }
+
+ @Test
+ public void testEOSink() {
+ // testSink() with EOS enabled.
+ // This does not actually inject retries in a stage to test exactly-once-semantics.
+ // It mainly exercises the code in normal flow without retries.
+ // Ideally we should test EOS Sink by triggering replays of a messages between stages.
+ // It is not feasible to test such retries with direct runner. When DoFnTester supports
+ // state, we can test KafkaEOWriter DoFn directly to ensure it handles retries correctly.
+
+ if (!ProducerSpEL.supportsTransactions()) {
+ LOG.warn("testEOSink() is disabled as Kafka client version does not support transactions.");
+ return;
+ }
+
+ int numElements = 1000;
+
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+
+ ProducerSendCompletionThread completionThread =
+ new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
+
+ String topic = "test";
+
+ p
+ .apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn())
+ .withoutMetadata())
+ .apply(KafkaIO.<Integer, Long>write()
+ .withBootstrapServers("none")
+ .withTopic(topic)
+ .withKeySerializer(IntegerSerializer.class)
+ .withValueSerializer(LongSerializer.class)
+ .withEOS(1, "test")
+ .withConsumerFactoryFn(new ConsumerFactoryFn(
+ Lists.newArrayList(topic), 10, 10, OffsetResetStrategy.EARLIEST))
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));
+
+ p.run();
+
+ completionThread.shutdown();
+
+ verifyProducerRecords(producerWrapper.mockProducer, topic, numElements, false);
}
}
@@ -718,14 +848,12 @@
int numElements = 1000;
- synchronized (MOCK_PRODUCER_LOCK) {
-
- MOCK_PRODUCER.clear();
-
- String topic = "test";
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
ProducerSendCompletionThread completionThreadWithErrors =
- new ProducerSendCompletionThread(10, 100).start();
+ new ProducerSendCompletionThread(producerWrapper.mockProducer, 10, 100).start();
+
+ String topic = "test";
p
.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn())
@@ -735,7 +863,7 @@
.withTopic(topic)
.withKeySerializer(IntegerSerializer.class)
.withValueSerializer(LongSerializer.class)
- .withProducerFactoryFn(new ProducerFactoryFn()));
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));
try {
p.run();
@@ -749,6 +877,51 @@
}
@Test
+ public void testUnboundedSourceStartReadTime() {
+
+ assumeTrue(new ConsumerSpEL().hasOffsetsForTimes());
+
+ int numElements = 1000;
+ // In this MockConsumer, we let the elements of the time and offset equal and there are 20
+ // partitions. So set this startTime can read half elements.
+ int startTime = numElements / 20 / 2;
+ int maxNumRecords = numElements / 2;
+
+ PCollection<Long> input = p
+ .apply(mkKafkaReadTransform(numElements, maxNumRecords, new ValueAsTimestampFn())
+ .withStartReadTime(new Instant(startTime))
+ .withoutMetadata())
+ .apply(Values.<Long>create());
+
+ addCountingAsserts(input, maxNumRecords, maxNumRecords, maxNumRecords, numElements - 1);
+ p.run();
+
+ }
+
+ @Rule public ExpectedException noMessagesException = ExpectedException.none();
+
+ @Test
+ public void testUnboundedSourceStartReadTimeException() {
+
+ assumeTrue(new ConsumerSpEL().hasOffsetsForTimes());
+
+ noMessagesException.expect(RuntimeException.class);
+
+ int numElements = 1000;
+ // In this MockConsumer, we let the elements of the time and offset equal and there are 20
+ // partitions. So set this startTime can not read any element.
+ int startTime = numElements / 20;
+
+ p.apply(mkKafkaReadTransform(numElements, numElements, new ValueAsTimestampFn())
+ .withStartReadTime(new Instant(startTime))
+ .withoutMetadata())
+ .apply(Values.<Long>create());
+
+ p.run();
+
+ }
+
+ @Test
public void testSourceDisplayData() {
KafkaIO.Read<Integer, Long> read = mkKafkaReadTransform(10, null);
@@ -783,17 +956,19 @@
@Test
public void testSinkDisplayData() {
- KafkaIO.Write<Integer, Long> write = KafkaIO.<Integer, Long>write()
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+ KafkaIO.Write<Integer, Long> write = KafkaIO.<Integer, Long>write()
.withBootstrapServers("myServerA:9092,myServerB:9092")
.withTopic("myTopic")
.withValueSerializer(LongSerializer.class)
- .withProducerFactoryFn(new ProducerFactoryFn());
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey));
- DisplayData displayData = DisplayData.from(write);
+ DisplayData displayData = DisplayData.from(write);
- assertThat(displayData, hasDisplayItem("topic", "myTopic"));
- assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServerA:9092,myServerB:9092"));
- assertThat(displayData, hasDisplayItem("retries", 3));
+ assertThat(displayData, hasDisplayItem("topic", "myTopic"));
+ assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServerA:9092,myServerB:9092"));
+ assertThat(displayData, hasDisplayItem("retries", 3));
+ }
}
// interface for testing coder inference
@@ -879,11 +1054,10 @@
int numElements = 1000;
- synchronized (MOCK_PRODUCER_LOCK) {
+ try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
- MOCK_PRODUCER.clear();
-
- ProducerSendCompletionThread completionThread = new ProducerSendCompletionThread().start();
+ ProducerSendCompletionThread completionThread =
+ new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
String topic = "test";
@@ -895,7 +1069,7 @@
.withTopic(topic)
.withKeySerializer(IntegerSerializer.class)
.withValueSerializer(LongSerializer.class)
- .withProducerFactoryFn(new ProducerFactoryFn()));
+ .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));
PipelineResult result = p.run();
@@ -918,10 +1092,11 @@
}
}
- private static void verifyProducerRecords(String topic, int numElements, boolean keyIsAbsent) {
+ private static void verifyProducerRecords(MockProducer<Integer, Long> mockProducer,
+ String topic, int numElements, boolean keyIsAbsent) {
// verify that appropriate messages are written to kafka
- List<ProducerRecord<Integer, Long>> sent = MOCK_PRODUCER.history();
+ List<ProducerRecord<Integer, Long>> sent = mockProducer.history();
// sort by values
Collections.sort(sent, new Comparator<ProducerRecord<Integer, Long>>() {
@@ -944,63 +1119,104 @@
}
/**
- * Singleton MockProudcer. Using a singleton here since we need access to the object to fetch
- * the actual records published to the producer. This prohibits running the tests using
- * the producer in parallel, but there are only one or two tests.
+ * This wrapper over MockProducer. It also places the mock producer in global MOCK_PRODUCER_MAP.
+ * The map is needed so that the producer returned by ProducerFactoryFn during pipeline can be
+ * used in verification after the test. We also override {@code flush()} method in MockProducer
+ * so that test can control behavior of {@code send()} method (e.g. to inject errors).
*/
- private static final MockProducer<Integer, Long> MOCK_PRODUCER =
- new MockProducer<Integer, Long>(
- false, // disable synchronous completion of send. see ProducerSendCompletionThread below.
- new IntegerSerializer(),
- new LongSerializer()) {
+ private static class MockProducerWrapper implements AutoCloseable {
- // override flush() so that it does not complete all the waiting sends, giving a chance to
- // ProducerCompletionThread to inject errors.
+ final String producerKey;
+ final MockProducer<Integer, Long> mockProducer;
- @Override
- public void flush() {
- while (completeNext()) {
- // there are some uncompleted records. let the completion thread handle them.
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
+ // MockProducer has "closed" method starting version 0.11.
+ private static Method closedMethod;
+
+ static {
+ try {
+ closedMethod = MockProducer.class.getMethod("closed");
+ } catch (NoSuchMethodException e) {
+ closedMethod = null;
+ }
+ }
+
+
+ MockProducerWrapper() {
+ producerKey = String.valueOf(ThreadLocalRandom.current().nextLong());
+ mockProducer = new MockProducer<Integer, Long>(
+ false, // disable synchronous completion of send. see ProducerSendCompletionThread below.
+ new IntegerSerializer(),
+ new LongSerializer()) {
+
+ // override flush() so that it does not complete all the waiting sends, giving a chance to
+ // ProducerCompletionThread to inject errors.
+
+ @Override
+ public void flush() {
+ while (completeNext()) {
+ // there are some uncompleted records. let the completion thread handle them.
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ // ok to retry.
+ }
}
}
- }
- };
+ };
- // use a separate object serialize tests using MOCK_PRODUCER so that we don't interfere
- // with Kafka MockProducer locking itself.
- private static final Object MOCK_PRODUCER_LOCK = new Object();
+ // Add the producer to the global map so that producer factory function can access it.
+ assertNull(MOCK_PRODUCER_MAP.putIfAbsent(producerKey, mockProducer));
+ }
+
+ public void close() {
+ MOCK_PRODUCER_MAP.remove(producerKey);
+ try {
+ if (closedMethod == null || !((Boolean) closedMethod.invoke(mockProducer))) {
+ mockProducer.close();
+ }
+ } catch (Exception e) { // Not expected.
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static final ConcurrentMap<String, MockProducer<Integer, Long>> MOCK_PRODUCER_MAP =
+ new ConcurrentHashMap<>();
private static class ProducerFactoryFn
implements SerializableFunction<Map<String, Object>, Producer<Integer, Long>> {
+ final String producerKey;
+
+ ProducerFactoryFn(String producerKey) {
+ this.producerKey = producerKey;
+ }
@SuppressWarnings("unchecked")
@Override
public Producer<Integer, Long> apply(Map<String, Object> config) {
// Make sure the config is correctly set up for serializers.
-
- // There may not be a key serializer if we're interested only in values.
- if (config.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG) != null) {
- Utils.newInstance(
- ((Class<?>) config.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
- .asSubclass(Serializer.class)
- ).configure(config, true);
- }
+ Utils.newInstance(
+ ((Class<?>) config.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
+ .asSubclass(Serializer.class)
+ ).configure(config, true);
Utils.newInstance(
((Class<?>) config.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG))
.asSubclass(Serializer.class)
).configure(config, false);
- return MOCK_PRODUCER;
+ // Returning same producer in each instance in a pipeline seems to work fine currently.
+ // If DirectRunner creates multiple DoFn instances for sinks, we might need to handle
+ // it appropriately. I.e. allow multiple producers for each producerKey and concatenate
+ // all the messages written to each producer for verification after the pipeline finishes.
+
+ return MOCK_PRODUCER_MAP.get(producerKey);
}
}
private static class InjectedErrorException extends RuntimeException {
- public InjectedErrorException(String message) {
+ InjectedErrorException(String message) {
super(message);
}
}
@@ -1013,18 +1229,22 @@
*/
private static class ProducerSendCompletionThread {
+ private final MockProducer<Integer, Long> mockProducer;
private final int maxErrors;
private final int errorFrequency;
private final AtomicBoolean done = new AtomicBoolean(false);
private final ExecutorService injectorThread;
private int numCompletions = 0;
- ProducerSendCompletionThread() {
+ ProducerSendCompletionThread(MockProducer<Integer, Long> mockProducer) {
// complete everything successfully
- this(0, 0);
+ this(mockProducer, 0, 0);
}
- ProducerSendCompletionThread(final int maxErrors, final int errorFrequency) {
+ ProducerSendCompletionThread(MockProducer<Integer, Long> mockProducer,
+ int maxErrors,
+ int errorFrequency) {
+ this.mockProducer = mockProducer;
this.maxErrors = maxErrors;
this.errorFrequency = errorFrequency;
injectorThread = Executors.newSingleThreadExecutor();
@@ -1040,14 +1260,14 @@
boolean successful;
if (errorsInjected < maxErrors && ((numCompletions + 1) % errorFrequency) == 0) {
- successful = MOCK_PRODUCER.errorNext(
+ successful = mockProducer.errorNext(
new InjectedErrorException("Injected Error #" + (errorsInjected + 1)));
if (successful) {
errorsInjected++;
}
} else {
- successful = MOCK_PRODUCER.completeNext();
+ successful = mockProducer.completeNext();
}
if (successful) {
@@ -1057,6 +1277,7 @@
try {
Thread.sleep(1);
} catch (InterruptedException e) {
+ // ok to retry.
}
}
}
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index cb7064b..93dc2c5 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -31,16 +31,6 @@
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <beamUseDummyRunner>false</beamUseDummyRunner>
- </systemPropertyVariables>
- </configuration>
- </plugin>
-
<!-- Integration Tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -83,6 +73,12 @@
<dependency>
<groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-cloudwatch</artifactId>
+ <version>${aws.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-client</artifactId>
<version>1.6.1</version>
</dependency>
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/AWSClientsProvider.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/AWSClientsProvider.java
new file mode 100644
index 0000000..c82e4b1
--- /dev/null
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/AWSClientsProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kinesis;
+
+import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
+import com.amazonaws.services.kinesis.AmazonKinesis;
+
+import java.io.Serializable;
+
+/**
+ * Provides instances of AWS clients.
+ *
+ * <p>Please note, that any instance of {@link AWSClientsProvider} must be
+ * {@link Serializable} to ensure it can be sent to worker machines.
+ */
+public interface AWSClientsProvider extends Serializable {
+
+ AmazonKinesis getKinesisClient();
+
+ AmazonCloudWatch getCloudWatchClient();
+}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CheckpointGenerator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CheckpointGenerator.java
index 919d85a..2629c57 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CheckpointGenerator.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CheckpointGenerator.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import java.io.Serializable;
/**
@@ -25,6 +24,7 @@
* How exactly the checkpoint is generated is up to implementing class.
*/
interface CheckpointGenerator extends Serializable {
- KinesisReaderCheckpoint generate(SimplifiedKinesisClient client)
- throws TransientKinesisException;
+
+ KinesisReaderCheckpoint generate(SimplifiedKinesisClient client)
+ throws TransientKinesisException;
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
index 4bed0e3..5a28214 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
@@ -24,76 +24,79 @@
* Similar to Guava {@code Optional}, but throws {@link NoSuchElementException} for missing element.
*/
abstract class CustomOptional<T> {
- @SuppressWarnings("unchecked")
- public static <T> CustomOptional<T> absent() {
- return (Absent<T>) Absent.INSTANCE;
+
+ @SuppressWarnings("unchecked")
+ public static <T> CustomOptional<T> absent() {
+ return (Absent<T>) Absent.INSTANCE;
+ }
+
+ public static <T> CustomOptional<T> of(T v) {
+ return new Present<>(v);
+ }
+
+ public abstract boolean isPresent();
+
+ public abstract T get();
+
+ private static class Present<T> extends CustomOptional<T> {
+
+ private final T value;
+
+ private Present(T value) {
+ this.value = value;
}
- public static <T> CustomOptional<T> of(T v) {
- return new Present<>(v);
+ @Override
+ public boolean isPresent() {
+ return true;
}
- public abstract boolean isPresent();
-
- public abstract T get();
-
- private static class Present<T> extends CustomOptional<T> {
- private final T value;
-
- private Present(T value) {
- this.value = value;
- }
-
- @Override
- public boolean isPresent() {
- return true;
- }
-
- @Override
- public T get() {
- return value;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Present)) {
- return false;
- }
-
- Present<?> present = (Present<?>) o;
- return Objects.equals(value, present.value);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(value);
- }
+ @Override
+ public T get() {
+ return value;
}
- private static class Absent<T> extends CustomOptional<T> {
- private static final Absent<Object> INSTANCE = new Absent<>();
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Present)) {
+ return false;
+ }
- private Absent() {
- }
-
- @Override
- public boolean isPresent() {
- return false;
- }
-
- @Override
- public T get() {
- throw new NoSuchElementException();
- }
-
- @Override
- public boolean equals(Object o) {
- return o instanceof Absent;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
+ Present<?> present = (Present<?>) o;
+ return Objects.equals(value, present.value);
}
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value);
+ }
+ }
+
+ private static class Absent<T> extends CustomOptional<T> {
+
+ private static final Absent<Object> INSTANCE = new Absent<>();
+
+ private Absent() {
+ }
+
+ @Override
+ public boolean isPresent() {
+ return false;
+ }
+
+ @Override
+ public T get() {
+ throw new NoSuchElementException();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o instanceof Absent;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGenerator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGenerator.java
index 2ec293c..9933019 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGenerator.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGenerator.java
@@ -28,29 +28,31 @@
* List of shards is obtained dynamically on call to {@link #generate(SimplifiedKinesisClient)}.
*/
class DynamicCheckpointGenerator implements CheckpointGenerator {
- private final String streamName;
- private final StartingPoint startingPoint;
- public DynamicCheckpointGenerator(String streamName, StartingPoint startingPoint) {
- this.streamName = checkNotNull(streamName, "streamName");
- this.startingPoint = checkNotNull(startingPoint, "startingPoint");
- }
+ private final String streamName;
+ private final StartingPoint startingPoint;
- @Override
- public KinesisReaderCheckpoint generate(SimplifiedKinesisClient kinesis)
- throws TransientKinesisException {
- return new KinesisReaderCheckpoint(
- transform(kinesis.listShards(streamName), new Function<Shard, ShardCheckpoint>() {
- @Override
- public ShardCheckpoint apply(Shard shard) {
- return new ShardCheckpoint(streamName, shard.getShardId(), startingPoint);
- }
- })
- );
- }
+ public DynamicCheckpointGenerator(String streamName, StartingPoint startingPoint) {
+ this.streamName = checkNotNull(streamName, "streamName");
+ this.startingPoint = checkNotNull(startingPoint, "startingPoint");
+ }
- @Override
- public String toString() {
- return String.format("Checkpoint generator for %s: %s", streamName, startingPoint);
- }
+ @Override
+ public KinesisReaderCheckpoint generate(SimplifiedKinesisClient kinesis)
+ throws TransientKinesisException {
+ return new KinesisReaderCheckpoint(
+ transform(kinesis.listShards(streamName), new Function<Shard, ShardCheckpoint>() {
+
+ @Override
+ public ShardCheckpoint apply(Shard shard) {
+ return new ShardCheckpoint(streamName, shard.getShardId(), startingPoint);
+ }
+ })
+ );
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Checkpoint generator for %s: %s", streamName, startingPoint);
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/GetKinesisRecordsResult.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/GetKinesisRecordsResult.java
index 5a34d7d..bbbffed 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/GetKinesisRecordsResult.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/GetKinesisRecordsResult.java
@@ -21,6 +21,7 @@
import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
import com.google.common.base.Function;
+
import java.util.List;
import javax.annotation.Nullable;
@@ -28,27 +29,35 @@
* Represents the output of 'get' operation on Kinesis stream.
*/
class GetKinesisRecordsResult {
- private final List<KinesisRecord> records;
- private final String nextShardIterator;
- public GetKinesisRecordsResult(List<UserRecord> records, String nextShardIterator,
- final String streamName, final String shardId) {
- this.records = transform(records, new Function<UserRecord, KinesisRecord>() {
- @Nullable
- @Override
- public KinesisRecord apply(@Nullable UserRecord input) {
- assert input != null; // to make FindBugs happy
- return new KinesisRecord(input, streamName, shardId);
- }
- });
- this.nextShardIterator = nextShardIterator;
- }
+ private final List<KinesisRecord> records;
+ private final String nextShardIterator;
+ private final long millisBehindLatest;
- public List<KinesisRecord> getRecords() {
- return records;
- }
+ public GetKinesisRecordsResult(List<UserRecord> records, String nextShardIterator,
+ long millisBehindLatest, final String streamName, final String shardId) {
+ this.records = transform(records, new Function<UserRecord, KinesisRecord>() {
- public String getNextShardIterator() {
- return nextShardIterator;
- }
+ @Nullable
+ @Override
+ public KinesisRecord apply(@Nullable UserRecord input) {
+ assert input != null; // to make FindBugs happy
+ return new KinesisRecord(input, streamName, shardId);
+ }
+ });
+ this.nextShardIterator = nextShardIterator;
+ this.millisBehindLatest = millisBehindLatest;
+ }
+
+ public List<KinesisRecord> getRecords() {
+ return records;
+ }
+
+ public String getNextShardIterator() {
+ return nextShardIterator;
+ }
+
+ public long getMillisBehindLatest() {
+ return millisBehindLatest;
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisClientProvider.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisClientProvider.java
deleted file mode 100644
index c7fd7f6..0000000
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisClientProvider.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.kinesis;
-
-import com.amazonaws.services.kinesis.AmazonKinesis;
-import java.io.Serializable;
-
-/**
- * Provides instances of {@link AmazonKinesis} interface.
- *
- * <p>Please note, that any instance of {@link KinesisClientProvider} must be
- * {@link Serializable} to ensure it can be sent to worker machines.
- */
-interface KinesisClientProvider extends Serializable {
- AmazonKinesis get();
-}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
index c97316d..96f7a04 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
@@ -17,19 +17,21 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.regions.Regions;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.AmazonKinesisClient;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
import com.google.auto.value.AutoValue;
+
import javax.annotation.Nullable;
+
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
import org.apache.beam.sdk.transforms.PTransform;
@@ -46,8 +48,9 @@
*
* <pre>{@code
* p.apply(KinesisIO.read()
- * .from("streamName", InitialPositionInStream.LATEST)
- * .withClientProvider("AWS_KEY", _"AWS_SECRET", STREAM_REGION)
+ * .withStreamName("streamName")
+ * .withInitialPositionInStream(InitialPositionInStream.LATEST)
+ * .withAWSClientsProvider("AWS_KEY", _"AWS_SECRET", STREAM_REGION)
* .apply( ... ) // other transformations
* }</pre>
*
@@ -60,23 +63,28 @@
* <li>{@link InitialPositionInStream#TRIM_HORIZON} - reading will begin at
* the very beginning of the stream</li>
* </ul></li>
- * <li>data used to initialize {@link AmazonKinesis} client:
+ * <li>data used to initialize {@link AmazonKinesis} and {@link AmazonCloudWatch} clients:
* <ul>
* <li>credentials (aws key, aws secret)</li>
* <li>region where the stream is located</li>
* </ul></li>
* </ul>
*
- * <p>In case when you want to set up {@link AmazonKinesis} client by your own
- * (for example if you're using more sophisticated authorization methods like Amazon STS, etc.)
- * you can do it by implementing {@link KinesisClientProvider} class:
+ * <p>In case when you want to set up {@link AmazonKinesis} or {@link AmazonCloudWatch} client by
+ * your own (for example if you're using more sophisticated authorization methods like Amazon
+ * STS, etc.) you can do it by implementing {@link AWSClientsProvider} class:
*
* <pre>{@code
- * public class MyCustomKinesisClientProvider implements KinesisClientProvider {
+ * public class MyCustomKinesisClientProvider implements AWSClientsProvider {
* {@literal @}Override
- * public AmazonKinesis get() {
+ * public AmazonKinesis getKinesisClient() {
* // set up your client here
* }
+ *
+ * public AmazonCloudWatch getCloudWatchClient() {
+ * // set up your client here
+ * }
+ *
* }
* }</pre>
*
@@ -84,8 +92,9 @@
*
* <pre>{@code
* p.apply(KinesisIO.read()
- * .from("streamName", InitialPositionInStream.LATEST)
- * .withClientProvider(new MyCustomKinesisClientProvider())
+ * .withStreamName("streamName")
+ * .withInitialPositionInStream(InitialPositionInStream.LATEST)
+ * .withAWSClientsProvider(new MyCustomKinesisClientProvider())
* .apply( ... ) // other transformations
* }</pre>
*
@@ -94,150 +103,189 @@
*
* <pre>{@code
* p.apply(KinesisIO.read()
- * .from("streamName", instant)
- * .withClientProvider(new MyCustomKinesisClientProvider())
+ * .withStreamName("streamName")
+ * .withInitialTimestampInStream(instant)
+ * .withAWSClientsProvider(new MyCustomKinesisClientProvider())
* .apply( ... ) // other transformations
* }</pre>
*
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public final class KinesisIO {
- /** Returns a new {@link Read} transform for reading from Kinesis. */
- public static Read read() {
- return new AutoValue_KinesisIO_Read.Builder().setMaxNumRecords(-1).build();
+
+ /** Returns a new {@link Read} transform for reading from Kinesis. */
+ public static Read read() {
+ return new AutoValue_KinesisIO_Read.Builder()
+ .setMaxNumRecords(-1)
+ .setUpToDateThreshold(Duration.ZERO)
+ .build();
+ }
+
+ /** Implementation of {@link #read}. */
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<KinesisRecord>> {
+
+ @Nullable
+ abstract String getStreamName();
+
+ @Nullable
+ abstract StartingPoint getInitialPosition();
+
+ @Nullable
+ abstract AWSClientsProvider getAWSClientsProvider();
+
+ abstract int getMaxNumRecords();
+
+ @Nullable
+ abstract Duration getMaxReadTime();
+
+ abstract Duration getUpToDateThreshold();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ abstract Builder setStreamName(String streamName);
+
+ abstract Builder setInitialPosition(StartingPoint startingPoint);
+
+ abstract Builder setAWSClientsProvider(AWSClientsProvider clientProvider);
+
+ abstract Builder setMaxNumRecords(int maxNumRecords);
+
+ abstract Builder setMaxReadTime(Duration maxReadTime);
+
+ abstract Builder setUpToDateThreshold(Duration upToDateThreshold);
+
+ abstract Read build();
}
- /** Implementation of {@link #read}. */
- @AutoValue
- public abstract static class Read extends PTransform<PBegin, PCollection<KinesisRecord>> {
- @Nullable
- abstract String getStreamName();
-
- @Nullable
- abstract StartingPoint getInitialPosition();
-
- @Nullable
- abstract KinesisClientProvider getClientProvider();
-
- abstract int getMaxNumRecords();
-
- @Nullable
- abstract Duration getMaxReadTime();
-
- abstract Builder toBuilder();
-
- @AutoValue.Builder
- abstract static class Builder {
- abstract Builder setStreamName(String streamName);
- abstract Builder setInitialPosition(StartingPoint startingPoint);
- abstract Builder setClientProvider(KinesisClientProvider clientProvider);
- abstract Builder setMaxNumRecords(int maxNumRecords);
- abstract Builder setMaxReadTime(Duration maxReadTime);
-
- abstract Read build();
- }
-
- /**
- * Specify reading from streamName at some initial position.
- */
- public Read from(String streamName, InitialPositionInStream initialPosition) {
- return toBuilder()
- .setStreamName(streamName)
- .setInitialPosition(
- new StartingPoint(checkNotNull(initialPosition, "initialPosition")))
- .build();
- }
-
- /**
- * Specify reading from streamName beginning at given {@link Instant}.
- * This {@link Instant} must be in the past, i.e. before {@link Instant#now()}.
- */
- public Read from(String streamName, Instant initialTimestamp) {
- return toBuilder()
- .setStreamName(streamName)
- .setInitialPosition(
- new StartingPoint(checkNotNull(initialTimestamp, "initialTimestamp")))
- .build();
- }
-
- /**
- * Allows to specify custom {@link KinesisClientProvider}.
- * {@link KinesisClientProvider} provides {@link AmazonKinesis} instances which are later
- * used for communication with Kinesis.
- * You should use this method if {@link Read#withClientProvider(String, String, Regions)}
- * does not suit your needs.
- */
- public Read withClientProvider(KinesisClientProvider kinesisClientProvider) {
- return toBuilder().setClientProvider(kinesisClientProvider).build();
- }
-
- /**
- * Specify credential details and region to be used to read from Kinesis.
- * If you need more sophisticated credential protocol, then you should look at
- * {@link Read#withClientProvider(KinesisClientProvider)}.
- */
- public Read withClientProvider(String awsAccessKey, String awsSecretKey, Regions region) {
- return withClientProvider(new BasicKinesisProvider(awsAccessKey, awsSecretKey, region));
- }
-
- /** Specifies to read at most a given number of records. */
- public Read withMaxNumRecords(int maxNumRecords) {
- checkArgument(
- maxNumRecords > 0, "maxNumRecords must be positive, but was: %s", maxNumRecords);
- return toBuilder().setMaxNumRecords(maxNumRecords).build();
- }
-
- /** Specifies to read at most a given number of records. */
- public Read withMaxReadTime(Duration maxReadTime) {
- checkNotNull(maxReadTime, "maxReadTime");
- return toBuilder().setMaxReadTime(maxReadTime).build();
- }
-
- @Override
- public PCollection<KinesisRecord> expand(PBegin input) {
- org.apache.beam.sdk.io.Read.Unbounded<KinesisRecord> read =
- org.apache.beam.sdk.io.Read.from(
- new KinesisSource(getClientProvider(), getStreamName(), getInitialPosition()));
- if (getMaxNumRecords() > 0) {
- BoundedReadFromUnboundedSource<KinesisRecord> bounded =
- read.withMaxNumRecords(getMaxNumRecords());
- return getMaxReadTime() == null
- ? input.apply(bounded)
- : input.apply(bounded.withMaxReadTime(getMaxReadTime()));
- } else {
- return getMaxReadTime() == null
- ? input.apply(read)
- : input.apply(read.withMaxReadTime(getMaxReadTime()));
- }
- }
-
- private static final class BasicKinesisProvider implements KinesisClientProvider {
-
- private final String accessKey;
- private final String secretKey;
- private final Regions region;
-
- private BasicKinesisProvider(String accessKey, String secretKey, Regions region) {
- this.accessKey = checkNotNull(accessKey, "accessKey");
- this.secretKey = checkNotNull(secretKey, "secretKey");
- this.region = checkNotNull(region, "region");
- }
-
-
- private AWSCredentialsProvider getCredentialsProvider() {
- return new StaticCredentialsProvider(new BasicAWSCredentials(
- accessKey,
- secretKey
- ));
-
- }
-
- @Override
- public AmazonKinesis get() {
- AmazonKinesisClient client = new AmazonKinesisClient(getCredentialsProvider());
- client.withRegion(region);
- return client;
- }
- }
+ /**
+ * Specify reading from streamName.
+ */
+ public Read withStreamName(String streamName) {
+ return toBuilder().setStreamName(streamName).build();
}
+
+ /**
+ * Specify reading from some initial position in stream.
+ */
+ public Read withInitialPositionInStream(InitialPositionInStream initialPosition) {
+ return toBuilder()
+ .setInitialPosition(new StartingPoint(initialPosition))
+ .build();
+ }
+
+ /**
+ * Specify reading beginning at given {@link Instant}.
+ * This {@link Instant} must be in the past, i.e. before {@link Instant#now()}.
+ */
+ public Read withInitialTimestampInStream(Instant initialTimestamp) {
+ return toBuilder()
+ .setInitialPosition(new StartingPoint(initialTimestamp))
+ .build();
+ }
+
+ /**
+ * Allows to specify custom {@link AWSClientsProvider}.
+ * {@link AWSClientsProvider} provides {@link AmazonKinesis} and {@link AmazonCloudWatch}
+ * instances which are later used for communication with Kinesis.
+ * You should use this method if {@link Read#withAWSClientsProvider(String, String, Regions)}
+ * does not suit your needs.
+ */
+ public Read withAWSClientsProvider(AWSClientsProvider awsClientsProvider) {
+ return toBuilder().setAWSClientsProvider(awsClientsProvider).build();
+ }
+
+ /**
+ * Specify credential details and region to be used to read from Kinesis.
+ * If you need more sophisticated credential protocol, then you should look at
+ * {@link Read#withAWSClientsProvider(AWSClientsProvider)}.
+ */
+ public Read withAWSClientsProvider(String awsAccessKey, String awsSecretKey, Regions region) {
+ return withAWSClientsProvider(new BasicKinesisProvider(awsAccessKey, awsSecretKey, region));
+ }
+
+ /** Specifies to read at most a given number of records. */
+ public Read withMaxNumRecords(int maxNumRecords) {
+ checkArgument(
+ maxNumRecords > 0, "maxNumRecords must be positive, but was: %s", maxNumRecords);
+ return toBuilder().setMaxNumRecords(maxNumRecords).build();
+ }
+
+ /** Specifies to read at most a given number of records. */
+ public Read withMaxReadTime(Duration maxReadTime) {
+ checkArgument(maxReadTime != null, "maxReadTime can not be null");
+ return toBuilder().setMaxReadTime(maxReadTime).build();
+ }
+
+ /**
+ * Specifies how late records consumed by this source can be to still be considered on time.
+ * When this limit is exceeded the actual backlog size will be evaluated and the runner might
+ * decide to scale the amount of resources allocated to the pipeline in order to
+ * speed up ingestion.
+ */
+ public Read withUpToDateThreshold(Duration upToDateThreshold) {
+ checkArgument(upToDateThreshold != null, "upToDateThreshold can not be null");
+ return toBuilder().setUpToDateThreshold(upToDateThreshold).build();
+ }
+
+ @Override
+ public PCollection<KinesisRecord> expand(PBegin input) {
+ org.apache.beam.sdk.io.Read.Unbounded<KinesisRecord> read =
+ org.apache.beam.sdk.io.Read.from(
+ new KinesisSource(getAWSClientsProvider(), getStreamName(),
+ getInitialPosition(), getUpToDateThreshold()));
+ if (getMaxNumRecords() > 0) {
+ BoundedReadFromUnboundedSource<KinesisRecord> bounded =
+ read.withMaxNumRecords(getMaxNumRecords());
+ return getMaxReadTime() == null
+ ? input.apply(bounded)
+ : input.apply(bounded.withMaxReadTime(getMaxReadTime()));
+ } else {
+ return getMaxReadTime() == null
+ ? input.apply(read)
+ : input.apply(read.withMaxReadTime(getMaxReadTime()));
+ }
+ }
+
+ private static final class BasicKinesisProvider implements AWSClientsProvider {
+
+ private final String accessKey;
+ private final String secretKey;
+ private final Regions region;
+
+ private BasicKinesisProvider(String accessKey, String secretKey, Regions region) {
+ checkArgument(accessKey != null, "accessKey can not be null");
+ checkArgument(secretKey != null, "secretKey can not be null");
+ checkArgument(region != null, "region can not be null");
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+ this.region = region;
+ }
+
+ private AWSCredentialsProvider getCredentialsProvider() {
+ return new StaticCredentialsProvider(new BasicAWSCredentials(
+ accessKey,
+ secretKey
+ ));
+
+ }
+
+ @Override
+ public AmazonKinesis getKinesisClient() {
+ AmazonKinesisClient client = new AmazonKinesisClient(getCredentialsProvider());
+ client.withRegion(region);
+ return client;
+ }
+
+ @Override
+ public AmazonCloudWatch getCloudWatchClient() {
+ AmazonCloudWatchClient client = new AmazonCloudWatchClient(getCredentialsProvider());
+ client.withRegion(region);
+ return client;
+ }
+ }
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java
index 2138094..665b897 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java
@@ -17,129 +17,197 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.Lists.newArrayList;
import java.io.IOException;
-import java.util.List;
import java.util.NoSuchElementException;
+
import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.transforms.Min;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.MovingFunction;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Reads data from multiple kinesis shards in a single thread.
* It uses simple round robin algorithm when fetching data from shards.
*/
class KinesisReader extends UnboundedSource.UnboundedReader<KinesisRecord> {
- private static final Logger LOG = LoggerFactory.getLogger(KinesisReader.class);
- private final SimplifiedKinesisClient kinesis;
- private final UnboundedSource<KinesisRecord, ?> source;
- private final CheckpointGenerator initialCheckpointGenerator;
- private RoundRobin<ShardRecordsIterator> shardIterators;
- private CustomOptional<KinesisRecord> currentRecord = CustomOptional.absent();
+ private static final Logger LOG = LoggerFactory.getLogger(KinesisReader.class);
+ /**
+ * Period of samples to determine watermark.
+ */
+ private static final Duration SAMPLE_PERIOD = Duration.standardMinutes(1);
- public KinesisReader(SimplifiedKinesisClient kinesis,
- CheckpointGenerator initialCheckpointGenerator,
- UnboundedSource<KinesisRecord, ?> source) {
- this.kinesis = checkNotNull(kinesis, "kinesis");
- this.initialCheckpointGenerator =
- checkNotNull(initialCheckpointGenerator, "initialCheckpointGenerator");
- this.source = source;
+ /**
+ * Period of updates to determine watermark.
+ */
+ private static final Duration SAMPLE_UPDATE = Duration.standardSeconds(5);
+
+ /**
+ * Minimum number of unread messages required before considering updating watermark.
+ */
+ static final int MIN_WATERMARK_MESSAGES = 10;
+
+ /**
+ * Minimum number of SAMPLE_UPDATE periods over which unread messages should be spread
+ * before considering updating watermark.
+ */
+ private static final int MIN_WATERMARK_SPREAD = 2;
+
+ private final SimplifiedKinesisClient kinesis;
+ private final KinesisSource source;
+ private final CheckpointGenerator initialCheckpointGenerator;
+ private CustomOptional<KinesisRecord> currentRecord = CustomOptional.absent();
+ private MovingFunction minReadTimestampMsSinceEpoch;
+ private Instant lastWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ private long lastBacklogBytes;
+ private Instant backlogBytesLastCheckTime = new Instant(0L);
+ private Duration upToDateThreshold;
+ private Duration backlogBytesCheckThreshold;
+ private ShardReadersPool shardReadersPool;
+
+ KinesisReader(SimplifiedKinesisClient kinesis,
+ CheckpointGenerator initialCheckpointGenerator,
+ KinesisSource source,
+ Duration upToDateThreshold) {
+ this(kinesis, initialCheckpointGenerator, source, upToDateThreshold,
+ Duration.standardSeconds(30));
+ }
+
+ KinesisReader(SimplifiedKinesisClient kinesis,
+ CheckpointGenerator initialCheckpointGenerator,
+ KinesisSource source,
+ Duration upToDateThreshold,
+ Duration backlogBytesCheckThreshold) {
+ this.kinesis = checkNotNull(kinesis, "kinesis");
+ this.initialCheckpointGenerator = checkNotNull(initialCheckpointGenerator,
+ "initialCheckpointGenerator");
+ this.source = source;
+ this.minReadTimestampMsSinceEpoch = new MovingFunction(SAMPLE_PERIOD.getMillis(),
+ SAMPLE_UPDATE.getMillis(),
+ MIN_WATERMARK_SPREAD,
+ MIN_WATERMARK_MESSAGES,
+ Min.ofLongs());
+ this.upToDateThreshold = upToDateThreshold;
+ this.backlogBytesCheckThreshold = backlogBytesCheckThreshold;
+ }
+
+ /**
+ * Generates initial checkpoint and instantiates iterators for shards.
+ */
+ @Override
+ public boolean start() throws IOException {
+ LOG.info("Starting reader using {}", initialCheckpointGenerator);
+
+ try {
+ shardReadersPool = createShardReadersPool();
+ shardReadersPool.start();
+ } catch (TransientKinesisException e) {
+ throw new IOException(e);
}
- /**
- * Generates initial checkpoint and instantiates iterators for shards.
- */
- @Override
- public boolean start() throws IOException {
- LOG.info("Starting reader using {}", initialCheckpointGenerator);
+ return advance();
+ }
- try {
- KinesisReaderCheckpoint initialCheckpoint =
- initialCheckpointGenerator.generate(kinesis);
- List<ShardRecordsIterator> iterators = newArrayList();
- for (ShardCheckpoint checkpoint : initialCheckpoint) {
- iterators.add(checkpoint.getShardRecordsIterator(kinesis));
- }
- shardIterators = new RoundRobin<>(iterators);
- } catch (TransientKinesisException e) {
- throw new IOException(e);
- }
-
- return advance();
+ /**
+ * Moves to the next record in one of the shards.
+ * If current shard iterator can be move forward (i.e. there's a record present) then we do it.
+ * If not, we iterate over shards in a round-robin manner.
+ */
+ @Override
+ public boolean advance() throws IOException {
+ currentRecord = shardReadersPool.nextRecord();
+ if (currentRecord.isPresent()) {
+ Instant approximateArrivalTimestamp = currentRecord.get().getApproximateArrivalTimestamp();
+ minReadTimestampMsSinceEpoch.add(Instant.now().getMillis(),
+ approximateArrivalTimestamp.getMillis());
+ return true;
}
+ return false;
+ }
- /**
- * Moves to the next record in one of the shards.
- * If current shard iterator can be move forward (i.e. there's a record present) then we do it.
- * If not, we iterate over shards in a round-robin manner.
- */
- @Override
- public boolean advance() throws IOException {
- try {
- for (int i = 0; i < shardIterators.size(); ++i) {
- currentRecord = shardIterators.getCurrent().next();
- if (currentRecord.isPresent()) {
- return true;
- } else {
- shardIterators.moveForward();
- }
- }
- } catch (TransientKinesisException e) {
- LOG.warn("Transient exception occurred", e);
- }
- return false;
+ @Override
+ public byte[] getCurrentRecordId() throws NoSuchElementException {
+ return currentRecord.get().getUniqueId();
+ }
+
+ @Override
+ public KinesisRecord getCurrent() throws NoSuchElementException {
+ return currentRecord.get();
+ }
+
+ /**
+ * Returns the approximate time that the current record was inserted into the stream.
+ * It is not guaranteed to be accurate - this could lead to mark some records as "late"
+ * even if they were not. Beware of this when setting
+ * {@link org.apache.beam.sdk.values.WindowingStrategy#withAllowedLateness}
+ */
+ @Override
+ public Instant getCurrentTimestamp() throws NoSuchElementException {
+ return currentRecord.get().getApproximateArrivalTimestamp();
+ }
+
+ @Override
+ public void close() throws IOException {
+ shardReadersPool.stop();
+ }
+
+ @Override
+ public Instant getWatermark() {
+ Instant now = Instant.now();
+ long readMin = minReadTimestampMsSinceEpoch.get(now.getMillis());
+ if (readMin == Long.MAX_VALUE && shardReadersPool.allShardsUpToDate()) {
+ lastWatermark = now;
+ } else if (minReadTimestampMsSinceEpoch.isSignificant()) {
+ Instant minReadTime = new Instant(readMin);
+ if (minReadTime.isAfter(lastWatermark)) {
+ lastWatermark = minReadTime;
+ }
}
+ return lastWatermark;
+ }
- @Override
- public byte[] getCurrentRecordId() throws NoSuchElementException {
- return currentRecord.get().getUniqueId();
+ @Override
+ public UnboundedSource.CheckpointMark getCheckpointMark() {
+ return shardReadersPool.getCheckpointMark();
+ }
+
+ @Override
+ public UnboundedSource<KinesisRecord, ?> getCurrentSource() {
+ return source;
+ }
+
+ /**
+ * Returns total size of all records that remain in Kinesis stream after current watermark.
+ * When currently processed record is not further behind than {@link #upToDateThreshold}
+ * then this method returns 0.
+ */
+ @Override
+ public long getTotalBacklogBytes() {
+ Instant watermark = getWatermark();
+ if (watermark.plus(upToDateThreshold).isAfterNow()) {
+ return 0L;
}
-
- @Override
- public KinesisRecord getCurrent() throws NoSuchElementException {
- return currentRecord.get();
+ if (backlogBytesLastCheckTime.plus(backlogBytesCheckThreshold).isAfterNow()) {
+ return lastBacklogBytes;
}
-
- /**
- * When {@link KinesisReader} was advanced to the current record.
- * We cannot use approximate arrival timestamp given for each record by Kinesis as it
- * is not guaranteed to be accurate - this could lead to mark some records as "late"
- * even if they were not.
- */
- @Override
- public Instant getCurrentTimestamp() throws NoSuchElementException {
- return currentRecord.get().getReadTime();
+ try {
+ lastBacklogBytes = kinesis.getBacklogBytes(source.getStreamName(), watermark);
+ backlogBytesLastCheckTime = Instant.now();
+ } catch (TransientKinesisException e) {
+ LOG.warn("Transient exception occurred.", e);
}
+ LOG.info("Total backlog bytes for {} stream with {} watermark: {}", source.getStreamName(),
+ watermark, lastBacklogBytes);
+ return lastBacklogBytes;
+ }
- @Override
- public void close() throws IOException {
- }
-
- /**
- * Current time.
- * We cannot give better approximation of the watermark with current semantics of
- * {@link KinesisReader#getCurrentTimestamp()}, because we don't know when the next
- * {@link KinesisReader#advance()} will be called.
- */
- @Override
- public Instant getWatermark() {
- return Instant.now();
- }
-
- @Override
- public UnboundedSource.CheckpointMark getCheckpointMark() {
- return KinesisReaderCheckpoint.asCurrentStateOf(shardIterators);
- }
-
- @Override
- public UnboundedSource<KinesisRecord, ?> getCurrentSource() {
- return source;
- }
-
+ ShardReadersPool createShardReadersPool() throws TransientKinesisException {
+ return new ShardReadersPool(kinesis, initialCheckpointGenerator.generate(kinesis));
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java
index f0fa45d..eca8791 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java
@@ -17,17 +17,16 @@
*/
package org.apache.beam.sdk.io.kinesis;
-import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.partition;
-import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.Iterator;
import java.util.List;
-import javax.annotation.Nullable;
+
import org.apache.beam.sdk.io.UnboundedSource;
/**
@@ -37,60 +36,46 @@
* This class is immutable.
*/
class KinesisReaderCheckpoint implements Iterable<ShardCheckpoint>, UnboundedSource
- .CheckpointMark, Serializable {
- private final List<ShardCheckpoint> shardCheckpoints;
+ .CheckpointMark, Serializable {
- public KinesisReaderCheckpoint(Iterable<ShardCheckpoint> shardCheckpoints) {
- this.shardCheckpoints = ImmutableList.copyOf(shardCheckpoints);
+ private final List<ShardCheckpoint> shardCheckpoints;
+
+ public KinesisReaderCheckpoint(Iterable<ShardCheckpoint> shardCheckpoints) {
+ this.shardCheckpoints = ImmutableList.copyOf(shardCheckpoints);
+ }
+
+ /**
+ * Splits given multi-shard checkpoint into partitions of approximately equal size.
+ *
+ * @param desiredNumSplits - upper limit for number of partitions to generate.
+ * @return list of checkpoints covering consecutive partitions of current checkpoint.
+ */
+ public List<KinesisReaderCheckpoint> splitInto(int desiredNumSplits) {
+ int partitionSize = divideAndRoundUp(shardCheckpoints.size(), desiredNumSplits);
+
+ List<KinesisReaderCheckpoint> checkpoints = newArrayList();
+ for (List<ShardCheckpoint> shardPartition : partition(shardCheckpoints, partitionSize)) {
+ checkpoints.add(new KinesisReaderCheckpoint(shardPartition));
}
+ return checkpoints;
+ }
- public static KinesisReaderCheckpoint asCurrentStateOf(Iterable<ShardRecordsIterator>
- iterators) {
- return new KinesisReaderCheckpoint(transform(iterators,
- new Function<ShardRecordsIterator, ShardCheckpoint>() {
+ private int divideAndRoundUp(int nominator, int denominator) {
+ return (nominator + denominator - 1) / denominator;
+ }
- @Nullable
- @Override
- public ShardCheckpoint apply(@Nullable
- ShardRecordsIterator shardRecordsIterator) {
- assert shardRecordsIterator != null;
- return shardRecordsIterator.getCheckpoint();
- }
- }));
- }
+ @Override
+ public void finalizeCheckpoint() throws IOException {
- /**
- * Splits given multi-shard checkpoint into partitions of approximately equal size.
- *
- * @param desiredNumSplits - upper limit for number of partitions to generate.
- * @return list of checkpoints covering consecutive partitions of current checkpoint.
- */
- public List<KinesisReaderCheckpoint> splitInto(int desiredNumSplits) {
- int partitionSize = divideAndRoundUp(shardCheckpoints.size(), desiredNumSplits);
+ }
- List<KinesisReaderCheckpoint> checkpoints = newArrayList();
- for (List<ShardCheckpoint> shardPartition : partition(shardCheckpoints, partitionSize)) {
- checkpoints.add(new KinesisReaderCheckpoint(shardPartition));
- }
- return checkpoints;
- }
+ @Override
+ public String toString() {
+ return shardCheckpoints.toString();
+ }
- private int divideAndRoundUp(int nominator, int denominator) {
- return (nominator + denominator - 1) / denominator;
- }
-
- @Override
- public void finalizeCheckpoint() throws IOException {
-
- }
-
- @Override
- public String toString() {
- return shardCheckpoints.toString();
- }
-
- @Override
- public Iterator<ShardCheckpoint> iterator() {
- return shardCheckpoints.iterator();
- }
+ @Override
+ public Iterator<ShardCheckpoint> iterator() {
+ return shardCheckpoints.iterator();
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
index 02b5370..057b7bb 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
@@ -22,7 +22,9 @@
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
import com.google.common.base.Charsets;
+
import java.nio.ByteBuffer;
+
import org.apache.commons.lang.builder.EqualsBuilder;
import org.joda.time.Instant;
@@ -30,91 +32,92 @@
* {@link UserRecord} enhanced with utility methods.
*/
public class KinesisRecord {
- private Instant readTime;
- private String streamName;
- private String shardId;
- private long subSequenceNumber;
- private String sequenceNumber;
- private Instant approximateArrivalTimestamp;
- private ByteBuffer data;
- private String partitionKey;
- public KinesisRecord(UserRecord record, String streamName, String shardId) {
- this(record.getData(), record.getSequenceNumber(), record.getSubSequenceNumber(),
- record.getPartitionKey(),
- new Instant(record.getApproximateArrivalTimestamp()),
- Instant.now(),
- streamName, shardId);
- }
+ private Instant readTime;
+ private String streamName;
+ private String shardId;
+ private long subSequenceNumber;
+ private String sequenceNumber;
+ private Instant approximateArrivalTimestamp;
+ private ByteBuffer data;
+ private String partitionKey;
- public KinesisRecord(ByteBuffer data, String sequenceNumber, long subSequenceNumber,
- String partitionKey, Instant approximateArrivalTimestamp,
- Instant readTime,
- String streamName, String shardId) {
- this.data = data;
- this.sequenceNumber = sequenceNumber;
- this.subSequenceNumber = subSequenceNumber;
- this.partitionKey = partitionKey;
- this.approximateArrivalTimestamp = approximateArrivalTimestamp;
- this.readTime = readTime;
- this.streamName = streamName;
- this.shardId = shardId;
- }
+ public KinesisRecord(UserRecord record, String streamName, String shardId) {
+ this(record.getData(), record.getSequenceNumber(), record.getSubSequenceNumber(),
+ record.getPartitionKey(),
+ new Instant(record.getApproximateArrivalTimestamp()),
+ Instant.now(),
+ streamName, shardId);
+ }
- public ExtendedSequenceNumber getExtendedSequenceNumber() {
- return new ExtendedSequenceNumber(getSequenceNumber(), getSubSequenceNumber());
- }
+ public KinesisRecord(ByteBuffer data, String sequenceNumber, long subSequenceNumber,
+ String partitionKey, Instant approximateArrivalTimestamp,
+ Instant readTime,
+ String streamName, String shardId) {
+ this.data = data;
+ this.sequenceNumber = sequenceNumber;
+ this.subSequenceNumber = subSequenceNumber;
+ this.partitionKey = partitionKey;
+ this.approximateArrivalTimestamp = approximateArrivalTimestamp;
+ this.readTime = readTime;
+ this.streamName = streamName;
+ this.shardId = shardId;
+ }
- /***
- * @return unique id of the record based on its position in the stream
- */
- public byte[] getUniqueId() {
- return getExtendedSequenceNumber().toString().getBytes(Charsets.UTF_8);
- }
+ public ExtendedSequenceNumber getExtendedSequenceNumber() {
+ return new ExtendedSequenceNumber(getSequenceNumber(), getSubSequenceNumber());
+ }
- public Instant getReadTime() {
- return readTime;
- }
+ /***
+ * @return unique id of the record based on its position in the stream
+ */
+ public byte[] getUniqueId() {
+ return getExtendedSequenceNumber().toString().getBytes(Charsets.UTF_8);
+ }
- public String getStreamName() {
- return streamName;
- }
+ public Instant getReadTime() {
+ return readTime;
+ }
- public String getShardId() {
- return shardId;
- }
+ public String getStreamName() {
+ return streamName;
+ }
- public byte[] getDataAsBytes() {
- return getData().array();
- }
+ public String getShardId() {
+ return shardId;
+ }
- @Override
- public boolean equals(Object obj) {
- return EqualsBuilder.reflectionEquals(this, obj);
- }
+ public byte[] getDataAsBytes() {
+ return getData().array();
+ }
- @Override
- public int hashCode() {
- return reflectionHashCode(this);
- }
+ @Override
+ public boolean equals(Object obj) {
+ return EqualsBuilder.reflectionEquals(this, obj);
+ }
- public long getSubSequenceNumber() {
- return subSequenceNumber;
- }
+ @Override
+ public int hashCode() {
+ return reflectionHashCode(this);
+ }
- public String getSequenceNumber() {
- return sequenceNumber;
- }
+ public long getSubSequenceNumber() {
+ return subSequenceNumber;
+ }
- public Instant getApproximateArrivalTimestamp() {
- return approximateArrivalTimestamp;
- }
+ public String getSequenceNumber() {
+ return sequenceNumber;
+ }
- public ByteBuffer getData() {
- return data;
- }
+ public Instant getApproximateArrivalTimestamp() {
+ return approximateArrivalTimestamp;
+ }
- public String getPartitionKey() {
- return partitionKey;
- }
+ public ByteBuffer getData() {
+ return data;
+ }
+
+ public String getPartitionKey() {
+ return partitionKey;
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
index f233e27..dcf564d 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
@@ -21,6 +21,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
@@ -33,40 +34,41 @@
* A {@link Coder} for {@link KinesisRecord}.
*/
class KinesisRecordCoder extends AtomicCoder<KinesisRecord> {
- private static final StringUtf8Coder STRING_CODER = StringUtf8Coder.of();
- private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
- private static final InstantCoder INSTANT_CODER = InstantCoder.of();
- private static final VarLongCoder VAR_LONG_CODER = VarLongCoder.of();
- public static KinesisRecordCoder of() {
- return new KinesisRecordCoder();
- }
+ private static final StringUtf8Coder STRING_CODER = StringUtf8Coder.of();
+ private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
+ private static final InstantCoder INSTANT_CODER = InstantCoder.of();
+ private static final VarLongCoder VAR_LONG_CODER = VarLongCoder.of();
- @Override
- public void encode(KinesisRecord value, OutputStream outStream) throws
- IOException {
- BYTE_ARRAY_CODER.encode(value.getData().array(), outStream);
- STRING_CODER.encode(value.getSequenceNumber(), outStream);
- STRING_CODER.encode(value.getPartitionKey(), outStream);
- INSTANT_CODER.encode(value.getApproximateArrivalTimestamp(), outStream);
- VAR_LONG_CODER.encode(value.getSubSequenceNumber(), outStream);
- INSTANT_CODER.encode(value.getReadTime(), outStream);
- STRING_CODER.encode(value.getStreamName(), outStream);
- STRING_CODER.encode(value.getShardId(), outStream);
- }
+ public static KinesisRecordCoder of() {
+ return new KinesisRecordCoder();
+ }
- @Override
- public KinesisRecord decode(InputStream inStream) throws IOException {
- ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream));
- String sequenceNumber = STRING_CODER.decode(inStream);
- String partitionKey = STRING_CODER.decode(inStream);
- Instant approximateArrivalTimestamp = INSTANT_CODER.decode(inStream);
- long subSequenceNumber = VAR_LONG_CODER.decode(inStream);
- Instant readTimestamp = INSTANT_CODER.decode(inStream);
- String streamName = STRING_CODER.decode(inStream);
- String shardId = STRING_CODER.decode(inStream);
- return new KinesisRecord(data, sequenceNumber, subSequenceNumber, partitionKey,
- approximateArrivalTimestamp, readTimestamp, streamName, shardId
- );
- }
+ @Override
+ public void encode(KinesisRecord value, OutputStream outStream) throws
+ IOException {
+ BYTE_ARRAY_CODER.encode(value.getData().array(), outStream);
+ STRING_CODER.encode(value.getSequenceNumber(), outStream);
+ STRING_CODER.encode(value.getPartitionKey(), outStream);
+ INSTANT_CODER.encode(value.getApproximateArrivalTimestamp(), outStream);
+ VAR_LONG_CODER.encode(value.getSubSequenceNumber(), outStream);
+ INSTANT_CODER.encode(value.getReadTime(), outStream);
+ STRING_CODER.encode(value.getStreamName(), outStream);
+ STRING_CODER.encode(value.getShardId(), outStream);
+ }
+
+ @Override
+ public KinesisRecord decode(InputStream inStream) throws IOException {
+ ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream));
+ String sequenceNumber = STRING_CODER.decode(inStream);
+ String partitionKey = STRING_CODER.decode(inStream);
+ Instant approximateArrivalTimestamp = INSTANT_CODER.decode(inStream);
+ long subSequenceNumber = VAR_LONG_CODER.decode(inStream);
+ Instant readTimestamp = INSTANT_CODER.decode(inStream);
+ String streamName = STRING_CODER.decode(inStream);
+ String shardId = STRING_CODER.decode(inStream);
+ return new KinesisRecord(data, sequenceNumber, subSequenceNumber, partitionKey,
+ approximateArrivalTimestamp, readTimestamp, streamName, shardId
+ );
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java
index 7e67d07..b1a6c19 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java
@@ -21,92 +21,107 @@
import static com.google.common.collect.Lists.newArrayList;
import java.util.List;
+
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Represents source for single stream in Kinesis.
*/
class KinesisSource extends UnboundedSource<KinesisRecord, KinesisReaderCheckpoint> {
- private static final Logger LOG = LoggerFactory.getLogger(KinesisSource.class);
- private final KinesisClientProvider kinesis;
- private CheckpointGenerator initialCheckpointGenerator;
+ private static final Logger LOG = LoggerFactory.getLogger(KinesisSource.class);
- public KinesisSource(KinesisClientProvider kinesis, String streamName,
- StartingPoint startingPoint) {
- this(kinesis, new DynamicCheckpointGenerator(streamName, startingPoint));
+ private final AWSClientsProvider awsClientsProvider;
+ private final String streamName;
+ private final Duration upToDateThreshold;
+ private CheckpointGenerator initialCheckpointGenerator;
+
+ KinesisSource(AWSClientsProvider awsClientsProvider, String streamName,
+ StartingPoint startingPoint, Duration upToDateThreshold) {
+ this(awsClientsProvider, new DynamicCheckpointGenerator(streamName, startingPoint), streamName,
+ upToDateThreshold);
+ }
+
+ private KinesisSource(AWSClientsProvider awsClientsProvider,
+ CheckpointGenerator initialCheckpoint, String streamName,
+ Duration upToDateThreshold) {
+ this.awsClientsProvider = awsClientsProvider;
+ this.initialCheckpointGenerator = initialCheckpoint;
+ this.streamName = streamName;
+ this.upToDateThreshold = upToDateThreshold;
+ validate();
+ }
+
+ /**
+ * Generate splits for reading from the stream.
+ * Basically, it'll try to evenly split set of shards in the stream into
+ * {@code desiredNumSplits} partitions. Each partition is then a split.
+ */
+ @Override
+ public List<KinesisSource> split(int desiredNumSplits,
+ PipelineOptions options) throws Exception {
+ KinesisReaderCheckpoint checkpoint =
+ initialCheckpointGenerator.generate(SimplifiedKinesisClient.from(awsClientsProvider));
+
+ List<KinesisSource> sources = newArrayList();
+
+ for (KinesisReaderCheckpoint partition : checkpoint.splitInto(desiredNumSplits)) {
+ sources.add(new KinesisSource(
+ awsClientsProvider,
+ new StaticCheckpointGenerator(partition),
+ streamName,
+ upToDateThreshold));
+ }
+ return sources;
+ }
+
+ /**
+ * Creates reader based on given {@link KinesisReaderCheckpoint}.
+ * If {@link KinesisReaderCheckpoint} is not given, then we use
+ * {@code initialCheckpointGenerator} to generate new checkpoint.
+ */
+ @Override
+ public UnboundedReader<KinesisRecord> createReader(PipelineOptions options,
+ KinesisReaderCheckpoint checkpointMark) {
+
+ CheckpointGenerator checkpointGenerator = initialCheckpointGenerator;
+
+ if (checkpointMark != null) {
+ checkpointGenerator = new StaticCheckpointGenerator(checkpointMark);
}
- private KinesisSource(KinesisClientProvider kinesisClientProvider,
- CheckpointGenerator initialCheckpoint) {
- this.kinesis = kinesisClientProvider;
- this.initialCheckpointGenerator = initialCheckpoint;
- validate();
- }
+ LOG.info("Creating new reader using {}", checkpointGenerator);
- /**
- * Generate splits for reading from the stream.
- * Basically, it'll try to evenly split set of shards in the stream into
- * {@code desiredNumSplits} partitions. Each partition is then a split.
- */
- @Override
- public List<KinesisSource> split(int desiredNumSplits,
- PipelineOptions options) throws Exception {
- KinesisReaderCheckpoint checkpoint =
- initialCheckpointGenerator.generate(SimplifiedKinesisClient.from(kinesis));
+ return new KinesisReader(
+ SimplifiedKinesisClient.from(awsClientsProvider),
+ checkpointGenerator,
+ this,
+ upToDateThreshold);
+ }
- List<KinesisSource> sources = newArrayList();
+ @Override
+ public Coder<KinesisReaderCheckpoint> getCheckpointMarkCoder() {
+ return SerializableCoder.of(KinesisReaderCheckpoint.class);
+ }
- for (KinesisReaderCheckpoint partition : checkpoint.splitInto(desiredNumSplits)) {
- sources.add(new KinesisSource(
- kinesis,
- new StaticCheckpointGenerator(partition)));
- }
- return sources;
- }
+ @Override
+ public void validate() {
+ checkNotNull(awsClientsProvider);
+ checkNotNull(initialCheckpointGenerator);
+ }
- /**
- * Creates reader based on given {@link KinesisReaderCheckpoint}.
- * If {@link KinesisReaderCheckpoint} is not given, then we use
- * {@code initialCheckpointGenerator} to generate new checkpoint.
- */
- @Override
- public UnboundedReader<KinesisRecord> createReader(PipelineOptions options,
- KinesisReaderCheckpoint checkpointMark) {
+ @Override
+ public Coder<KinesisRecord> getOutputCoder() {
+ return KinesisRecordCoder.of();
+ }
- CheckpointGenerator checkpointGenerator = initialCheckpointGenerator;
-
- if (checkpointMark != null) {
- checkpointGenerator = new StaticCheckpointGenerator(checkpointMark);
- }
-
- LOG.info("Creating new reader using {}", checkpointGenerator);
-
- return new KinesisReader(
- SimplifiedKinesisClient.from(kinesis),
- checkpointGenerator,
- this);
- }
-
- @Override
- public Coder<KinesisReaderCheckpoint> getCheckpointMarkCoder() {
- return SerializableCoder.of(KinesisReaderCheckpoint.class);
- }
-
- @Override
- public void validate() {
- checkNotNull(kinesis);
- checkNotNull(initialCheckpointGenerator);
- }
-
- @Override
- public Coder<KinesisRecord> getDefaultOutputCoder() {
- return KinesisRecordCoder.of();
- }
+ String getStreamName() {
+ return streamName;
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java
index 40e65fc..eca725c 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java
@@ -21,7 +21,6 @@
import java.util.List;
-
/**
* Filters out records, which were already processed and checkpointed.
*
@@ -29,13 +28,14 @@
* accuracy, not with "subSequenceNumber" accuracy.
*/
class RecordFilter {
- public List<KinesisRecord> apply(List<KinesisRecord> records, ShardCheckpoint checkpoint) {
- List<KinesisRecord> filteredRecords = newArrayList();
- for (KinesisRecord record : records) {
- if (checkpoint.isBeforeOrAt(record)) {
- filteredRecords.add(record);
- }
- }
- return filteredRecords;
+
+ public List<KinesisRecord> apply(List<KinesisRecord> records, ShardCheckpoint checkpoint) {
+ List<KinesisRecord> filteredRecords = newArrayList();
+ for (KinesisRecord record : records) {
+ if (checkpoint.isBeforeOrAt(record)) {
+ filteredRecords.add(record);
+ }
}
+ return filteredRecords;
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RoundRobin.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RoundRobin.java
deleted file mode 100644
index e4ff541..0000000
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RoundRobin.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.kinesis;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.Queues.newArrayDeque;
-
-import java.util.Deque;
-import java.util.Iterator;
-
-/**
- * Very simple implementation of round robin algorithm.
- */
-class RoundRobin<T> implements Iterable<T> {
- private final Deque<T> deque;
-
- public RoundRobin(Iterable<T> collection) {
- this.deque = newArrayDeque(collection);
- checkArgument(!deque.isEmpty(), "Tried to initialize RoundRobin with empty collection");
- }
-
- public T getCurrent() {
- return deque.getFirst();
- }
-
- public void moveForward() {
- deque.addLast(deque.removeFirst());
- }
-
- public int size() {
- return deque.size();
- }
-
- @Override
- public Iterator<T> iterator() {
- return deque.iterator();
- }
-}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java
index 6aa3504..94e3b96 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import static com.amazonaws.services.kinesis.model.ShardIteratorType.AFTER_SEQUENCE_NUMBER;
import static com.amazonaws.services.kinesis.model.ShardIteratorType.AT_SEQUENCE_NUMBER;
import static com.amazonaws.services.kinesis.model.ShardIteratorType.AT_TIMESTAMP;
@@ -27,9 +26,10 @@
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.model.Record;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
-import java.io.Serializable;
-import org.joda.time.Instant;
+import java.io.Serializable;
+
+import org.joda.time.Instant;
/**
* Checkpoint mark for single shard in the stream.
@@ -45,131 +45,126 @@
* This class is immutable.
*/
class ShardCheckpoint implements Serializable {
- private final String streamName;
- private final String shardId;
- private final String sequenceNumber;
- private final ShardIteratorType shardIteratorType;
- private final Long subSequenceNumber;
- private final Instant timestamp;
- public ShardCheckpoint(String streamName, String shardId, StartingPoint
- startingPoint) {
- this(streamName, shardId,
- ShardIteratorType.fromValue(startingPoint.getPositionName()),
- startingPoint.getTimestamp());
+ private final String streamName;
+ private final String shardId;
+ private final String sequenceNumber;
+ private final ShardIteratorType shardIteratorType;
+ private final Long subSequenceNumber;
+ private final Instant timestamp;
+
+ public ShardCheckpoint(String streamName, String shardId, StartingPoint
+ startingPoint) {
+ this(streamName, shardId,
+ ShardIteratorType.fromValue(startingPoint.getPositionName()),
+ startingPoint.getTimestamp());
+ }
+
+ public ShardCheckpoint(String streamName, String shardId, ShardIteratorType
+ shardIteratorType, Instant timestamp) {
+ this(streamName, shardId, shardIteratorType, null, null, timestamp);
+ }
+
+ public ShardCheckpoint(String streamName, String shardId, ShardIteratorType
+ shardIteratorType, String sequenceNumber, Long subSequenceNumber) {
+ this(streamName, shardId, shardIteratorType, sequenceNumber, subSequenceNumber, null);
+ }
+
+ private ShardCheckpoint(String streamName, String shardId, ShardIteratorType shardIteratorType,
+ String sequenceNumber, Long subSequenceNumber, Instant timestamp) {
+ this.shardIteratorType = checkNotNull(shardIteratorType, "shardIteratorType");
+ this.streamName = checkNotNull(streamName, "streamName");
+ this.shardId = checkNotNull(shardId, "shardId");
+ if (shardIteratorType == AT_SEQUENCE_NUMBER || shardIteratorType == AFTER_SEQUENCE_NUMBER) {
+ checkNotNull(sequenceNumber,
+ "You must provide sequence number for AT_SEQUENCE_NUMBER"
+ + " or AFTER_SEQUENCE_NUMBER");
+ } else {
+ checkArgument(sequenceNumber == null,
+ "Sequence number must be null for LATEST, TRIM_HORIZON or AT_TIMESTAMP");
+ }
+ if (shardIteratorType == AT_TIMESTAMP) {
+ checkNotNull(timestamp,
+ "You must provide timestamp for AT_TIMESTAMP");
+ } else {
+ checkArgument(timestamp == null,
+ "Timestamp must be null for an iterator type other than AT_TIMESTAMP");
}
- public ShardCheckpoint(String streamName, String shardId, ShardIteratorType
- shardIteratorType, Instant timestamp) {
- this(streamName, shardId, shardIteratorType, null, null, timestamp);
- }
+ this.subSequenceNumber = subSequenceNumber;
+ this.sequenceNumber = sequenceNumber;
+ this.timestamp = timestamp;
+ }
- public ShardCheckpoint(String streamName, String shardId, ShardIteratorType
- shardIteratorType, String sequenceNumber, Long subSequenceNumber) {
- this(streamName, shardId, shardIteratorType, sequenceNumber, subSequenceNumber, null);
+ /**
+ * Used to compare {@link ShardCheckpoint} object to {@link KinesisRecord}. Depending
+ * on the the underlying shardIteratorType, it will either compare the timestamp or the
+ * {@link ExtendedSequenceNumber}.
+ *
+ * @param other
+ * @return if current checkpoint mark points before or at given {@link ExtendedSequenceNumber}
+ */
+ public boolean isBeforeOrAt(KinesisRecord other) {
+ if (shardIteratorType == AT_TIMESTAMP) {
+ return timestamp.compareTo(other.getApproximateArrivalTimestamp()) <= 0;
}
-
- private ShardCheckpoint(String streamName, String shardId, ShardIteratorType shardIteratorType,
- String sequenceNumber, Long subSequenceNumber, Instant timestamp) {
- this.shardIteratorType = checkNotNull(shardIteratorType, "shardIteratorType");
- this.streamName = checkNotNull(streamName, "streamName");
- this.shardId = checkNotNull(shardId, "shardId");
- if (shardIteratorType == AT_SEQUENCE_NUMBER || shardIteratorType == AFTER_SEQUENCE_NUMBER) {
- checkNotNull(sequenceNumber,
- "You must provide sequence number for AT_SEQUENCE_NUMBER"
- + " or AFTER_SEQUENCE_NUMBER");
- } else {
- checkArgument(sequenceNumber == null,
- "Sequence number must be null for LATEST, TRIM_HORIZON or AT_TIMESTAMP");
- }
- if (shardIteratorType == AT_TIMESTAMP) {
- checkNotNull(timestamp,
- "You must provide timestamp for AT_SEQUENCE_NUMBER"
- + " or AFTER_SEQUENCE_NUMBER");
- } else {
- checkArgument(timestamp == null,
- "Timestamp must be null for an iterator type other than AT_TIMESTAMP");
- }
-
- this.subSequenceNumber = subSequenceNumber;
- this.sequenceNumber = sequenceNumber;
- this.timestamp = timestamp;
+ int result = extendedSequenceNumber().compareTo(other.getExtendedSequenceNumber());
+ if (result == 0) {
+ return shardIteratorType == AT_SEQUENCE_NUMBER;
}
+ return result < 0;
+ }
- /**
- * Used to compare {@link ShardCheckpoint} object to {@link KinesisRecord}. Depending
- * on the the underlying shardIteratorType, it will either compare the timestamp or the
- * {@link ExtendedSequenceNumber}.
- *
- * @param other
- * @return if current checkpoint mark points before or at given {@link ExtendedSequenceNumber}
- */
- public boolean isBeforeOrAt(KinesisRecord other) {
- if (shardIteratorType == AT_TIMESTAMP) {
- return timestamp.compareTo(other.getApproximateArrivalTimestamp()) <= 0;
- }
- int result = extendedSequenceNumber().compareTo(other.getExtendedSequenceNumber());
- if (result == 0) {
- return shardIteratorType == AT_SEQUENCE_NUMBER;
- }
- return result < 0;
+ private ExtendedSequenceNumber extendedSequenceNumber() {
+ String fullSequenceNumber = sequenceNumber;
+ if (fullSequenceNumber == null) {
+ fullSequenceNumber = shardIteratorType.toString();
}
+ return new ExtendedSequenceNumber(fullSequenceNumber, subSequenceNumber);
+ }
- private ExtendedSequenceNumber extendedSequenceNumber() {
- String fullSequenceNumber = sequenceNumber;
- if (fullSequenceNumber == null) {
- fullSequenceNumber = shardIteratorType.toString();
- }
- return new ExtendedSequenceNumber(fullSequenceNumber, subSequenceNumber);
- }
+ @Override
+ public String toString() {
+ return String.format("Checkpoint %s for stream %s, shard %s: %s", shardIteratorType,
+ streamName, shardId,
+ sequenceNumber);
+ }
- @Override
- public String toString() {
- return String.format("Checkpoint %s for stream %s, shard %s: %s", shardIteratorType,
- streamName, shardId,
- sequenceNumber);
+ public String getShardIterator(SimplifiedKinesisClient kinesisClient)
+ throws TransientKinesisException {
+ if (checkpointIsInTheMiddleOfAUserRecord()) {
+ return kinesisClient.getShardIterator(streamName,
+ shardId, AT_SEQUENCE_NUMBER,
+ sequenceNumber, null);
}
+ return kinesisClient.getShardIterator(streamName,
+ shardId, shardIteratorType,
+ sequenceNumber, timestamp);
+ }
- public ShardRecordsIterator getShardRecordsIterator(SimplifiedKinesisClient kinesis)
- throws TransientKinesisException {
- return new ShardRecordsIterator(this, kinesis);
- }
+ private boolean checkpointIsInTheMiddleOfAUserRecord() {
+ return shardIteratorType == AFTER_SEQUENCE_NUMBER && subSequenceNumber != null;
+ }
- public String getShardIterator(SimplifiedKinesisClient kinesisClient)
- throws TransientKinesisException {
- if (checkpointIsInTheMiddleOfAUserRecord()) {
- return kinesisClient.getShardIterator(streamName,
- shardId, AT_SEQUENCE_NUMBER,
- sequenceNumber, null);
- }
- return kinesisClient.getShardIterator(streamName,
- shardId, shardIteratorType,
- sequenceNumber, timestamp);
- }
+ /**
+ * Used to advance checkpoint mark to position after given {@link Record}.
+ *
+ * @param record
+ * @return new checkpoint object pointing directly after given {@link Record}
+ */
+ public ShardCheckpoint moveAfter(KinesisRecord record) {
+ return new ShardCheckpoint(
+ streamName, shardId,
+ AFTER_SEQUENCE_NUMBER,
+ record.getSequenceNumber(),
+ record.getSubSequenceNumber());
+ }
- private boolean checkpointIsInTheMiddleOfAUserRecord() {
- return shardIteratorType == AFTER_SEQUENCE_NUMBER && subSequenceNumber != null;
- }
+ public String getStreamName() {
+ return streamName;
+ }
- /**
- * Used to advance checkpoint mark to position after given {@link Record}.
- *
- * @param record
- * @return new checkpoint object pointing directly after given {@link Record}
- */
- public ShardCheckpoint moveAfter(KinesisRecord record) {
- return new ShardCheckpoint(
- streamName, shardId,
- AFTER_SEQUENCE_NUMBER,
- record.getSequenceNumber(),
- record.getSubSequenceNumber());
- }
-
- public String getStreamName() {
- return streamName;
- }
-
- public String getShardId() {
- return shardId;
- }
+ public String getShardId() {
+ return shardId;
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java
new file mode 100644
index 0000000..83e3081
--- /dev/null
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kinesis;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Iterables.transform;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Internal shard iterators pool.
+ * It maintains the thread pool for reading Kinesis shards in separate threads.
+ * Read records are stored in a blocking queue of limited capacity.
+ */
+class ShardReadersPool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ShardReadersPool.class);
+ private static final int DEFAULT_CAPACITY_PER_SHARD = 10_000;
+ private ExecutorService executorService;
+ private BlockingQueue<KinesisRecord> recordsQueue;
+ private Map<String, ShardRecordsIterator> shardIteratorsMap;
+ private SimplifiedKinesisClient kinesis;
+ private KinesisReaderCheckpoint initialCheckpoint;
+ private final int queueCapacityPerShard;
+ private AtomicBoolean poolOpened = new AtomicBoolean(true);
+
+ ShardReadersPool(SimplifiedKinesisClient kinesis, KinesisReaderCheckpoint initialCheckpoint) {
+ this(kinesis, initialCheckpoint, DEFAULT_CAPACITY_PER_SHARD);
+ }
+
+ ShardReadersPool(SimplifiedKinesisClient kinesis, KinesisReaderCheckpoint initialCheckpoint,
+ int queueCapacityPerShard) {
+ this.kinesis = kinesis;
+ this.initialCheckpoint = initialCheckpoint;
+ this.queueCapacityPerShard = queueCapacityPerShard;
+ }
+
+ void start() throws TransientKinesisException {
+ ImmutableMap.Builder<String, ShardRecordsIterator> shardsMap = ImmutableMap.builder();
+ for (ShardCheckpoint checkpoint : initialCheckpoint) {
+ shardsMap.put(checkpoint.getShardId(), createShardIterator(kinesis, checkpoint));
+ }
+ shardIteratorsMap = shardsMap.build();
+ executorService = Executors.newFixedThreadPool(shardIteratorsMap.size());
+ recordsQueue = new LinkedBlockingQueue<>(queueCapacityPerShard * shardIteratorsMap.size());
+ for (final ShardRecordsIterator shardRecordsIterator : shardIteratorsMap.values()) {
+ executorService.submit(new Runnable() {
+
+ @Override
+ public void run() {
+ readLoop(shardRecordsIterator);
+ }
+ });
+ }
+ }
+
+ private void readLoop(ShardRecordsIterator shardRecordsIterator) {
+ while (poolOpened.get()) {
+ try {
+ List<KinesisRecord> kinesisRecords = shardRecordsIterator.readNextBatch();
+ for (KinesisRecord kinesisRecord : kinesisRecords) {
+ recordsQueue.put(kinesisRecord);
+ }
+ } catch (TransientKinesisException e) {
+ LOG.warn("Transient exception occurred.", e);
+ } catch (InterruptedException e) {
+ LOG.warn("Thread was interrupted, finishing the read loop", e);
+ break;
+ } catch (Throwable e) {
+ LOG.error("Unexpected exception occurred", e);
+ }
+ }
+ LOG.info("Kinesis Shard read loop has finished");
+ }
+
+ CustomOptional<KinesisRecord> nextRecord() {
+ try {
+ KinesisRecord record = recordsQueue.poll(1, TimeUnit.SECONDS);
+ if (record == null) {
+ return CustomOptional.absent();
+ }
+ shardIteratorsMap.get(record.getShardId()).ackRecord(record);
+ return CustomOptional.of(record);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for KinesisRecord from the buffer");
+ return CustomOptional.absent();
+ }
+ }
+
+ void stop() {
+ LOG.info("Closing shard iterators pool");
+ poolOpened.set(false);
+ executorService.shutdownNow();
+ boolean isShutdown = false;
+ int attemptsLeft = 3;
+ while (!isShutdown && attemptsLeft-- > 0) {
+ try {
+ isShutdown = executorService.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while waiting for the executor service to shutdown");
+ throw new RuntimeException(e);
+ }
+ if (!isShutdown && attemptsLeft > 0) {
+ LOG.warn("Executor service is taking long time to shutdown, will retry. {} attempts left",
+ attemptsLeft);
+ }
+ }
+ }
+
+ boolean allShardsUpToDate() {
+ boolean shardsUpToDate = true;
+ for (ShardRecordsIterator shardRecordsIterator : shardIteratorsMap.values()) {
+ shardsUpToDate &= shardRecordsIterator.isUpToDate();
+ }
+ return shardsUpToDate;
+ }
+
+ KinesisReaderCheckpoint getCheckpointMark() {
+ return new KinesisReaderCheckpoint(transform(shardIteratorsMap.values(),
+ new Function<ShardRecordsIterator, ShardCheckpoint>() {
+ @Override
+ public ShardCheckpoint apply(ShardRecordsIterator shardRecordsIterator) {
+ checkArgument(shardRecordsIterator != null, "shardRecordsIterator can not be null");
+ return shardRecordsIterator.getCheckpoint();
+ }
+ }));
+ }
+
+ ShardRecordsIterator createShardIterator(SimplifiedKinesisClient kinesis,
+ ShardCheckpoint checkpoint) throws TransientKinesisException {
+ return new ShardRecordsIterator(checkpoint, kinesis);
+ }
+
+}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java
index 872f604..c1450be 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java
@@ -18,81 +18,81 @@
package org.apache.beam.sdk.io.kinesis;
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.Queues.newArrayDeque;
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
-import java.util.Deque;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Iterates over records in a single shard.
- * Under the hood records are retrieved from Kinesis in batches and stored in the in-memory queue.
- * Then the caller of {@link ShardRecordsIterator#next()} can read from queue one by one.
+ * Records are retrieved in batches via calls to {@link ShardRecordsIterator#readNextBatch()}.
+ * Client has to confirm processed records by calling
+ * {@link ShardRecordsIterator#ackRecord(KinesisRecord)} method.
*/
class ShardRecordsIterator {
- private static final Logger LOG = LoggerFactory.getLogger(ShardRecordsIterator.class);
- private final SimplifiedKinesisClient kinesis;
- private final RecordFilter filter;
- private ShardCheckpoint checkpoint;
- private String shardIterator;
- private Deque<KinesisRecord> data = newArrayDeque();
+ private static final Logger LOG = LoggerFactory.getLogger(ShardRecordsIterator.class);
- public ShardRecordsIterator(final ShardCheckpoint initialCheckpoint,
- SimplifiedKinesisClient simplifiedKinesisClient) throws
- TransientKinesisException {
- this(initialCheckpoint, simplifiedKinesisClient, new RecordFilter());
+ private final SimplifiedKinesisClient kinesis;
+ private final RecordFilter filter;
+ private final String streamName;
+ private final String shardId;
+ private AtomicReference<ShardCheckpoint> checkpoint;
+ private String shardIterator;
+ private AtomicLong millisBehindLatest = new AtomicLong(Long.MAX_VALUE);
+
+ ShardRecordsIterator(final ShardCheckpoint initialCheckpoint,
+ SimplifiedKinesisClient simplifiedKinesisClient) throws TransientKinesisException {
+ this(initialCheckpoint, simplifiedKinesisClient, new RecordFilter());
+ }
+
+ ShardRecordsIterator(final ShardCheckpoint initialCheckpoint,
+ SimplifiedKinesisClient simplifiedKinesisClient,
+ RecordFilter filter) throws TransientKinesisException {
+ this.checkpoint = new AtomicReference<>(checkNotNull(initialCheckpoint, "initialCheckpoint"));
+ this.filter = checkNotNull(filter, "filter");
+ this.kinesis = checkNotNull(simplifiedKinesisClient, "simplifiedKinesisClient");
+ this.streamName = initialCheckpoint.getStreamName();
+ this.shardId = initialCheckpoint.getShardId();
+ this.shardIterator = initialCheckpoint.getShardIterator(kinesis);
+ }
+
+ List<KinesisRecord> readNextBatch() throws TransientKinesisException {
+ GetKinesisRecordsResult response = fetchRecords();
+ LOG.debug("Fetched {} new records", response.getRecords().size());
+
+ List<KinesisRecord> filteredRecords = filter.apply(response.getRecords(), checkpoint.get());
+ millisBehindLatest.set(response.getMillisBehindLatest());
+ return filteredRecords;
+ }
+
+ private GetKinesisRecordsResult fetchRecords() throws TransientKinesisException {
+ try {
+ GetKinesisRecordsResult response = kinesis.getRecords(shardIterator, streamName, shardId);
+ shardIterator = response.getNextShardIterator();
+ return response;
+ } catch (ExpiredIteratorException e) {
+ LOG.info("Refreshing expired iterator", e);
+ shardIterator = checkpoint.get().getShardIterator(kinesis);
+ return fetchRecords();
}
+ }
- public ShardRecordsIterator(final ShardCheckpoint initialCheckpoint,
- SimplifiedKinesisClient simplifiedKinesisClient,
- RecordFilter filter) throws
- TransientKinesisException {
+ ShardCheckpoint getCheckpoint() {
+ return checkpoint.get();
+ }
- this.checkpoint = checkNotNull(initialCheckpoint, "initialCheckpoint");
- this.filter = checkNotNull(filter, "filter");
- this.kinesis = checkNotNull(simplifiedKinesisClient, "simplifiedKinesisClient");
- shardIterator = checkpoint.getShardIterator(kinesis);
- }
+ boolean isUpToDate() {
+ return millisBehindLatest.get() == 0L;
+ }
- /**
- * Returns record if there's any present.
- * Returns absent() if there are no new records at this time in the shard.
- */
- public CustomOptional<KinesisRecord> next() throws TransientKinesisException {
- readMoreIfNecessary();
-
- if (data.isEmpty()) {
- return CustomOptional.absent();
- } else {
- KinesisRecord record = data.removeFirst();
- checkpoint = checkpoint.moveAfter(record);
- return CustomOptional.of(record);
- }
- }
-
- private void readMoreIfNecessary() throws TransientKinesisException {
- if (data.isEmpty()) {
- GetKinesisRecordsResult response;
- try {
- response = kinesis.getRecords(shardIterator, checkpoint.getStreamName(),
- checkpoint.getShardId());
- } catch (ExpiredIteratorException e) {
- LOG.info("Refreshing expired iterator", e);
- shardIterator = checkpoint.getShardIterator(kinesis);
- response = kinesis.getRecords(shardIterator, checkpoint.getStreamName(),
- checkpoint.getShardId());
- }
- LOG.debug("Fetched {} new records", response.getRecords().size());
- shardIterator = response.getNextShardIterator();
- data.addAll(filter.apply(response.getRecords(), checkpoint));
- }
- }
-
- public ShardCheckpoint getCheckpoint() {
- return checkpoint;
- }
-
+ void ackRecord(KinesisRecord record) {
+ checkpoint.set(checkpoint.get().moveAfter(record));
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java
index 3e3984a..74605e5 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java
@@ -17,8 +17,15 @@
*/
package org.apache.beam.sdk.io.kinesis;
+import static com.google.common.base.Preconditions.checkNotNull;
+import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
+import com.amazonaws.services.cloudwatch.model.Datapoint;
+import com.amazonaws.services.cloudwatch.model.Dimension;
+import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsRequest;
+import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsResult;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
@@ -31,127 +38,205 @@
import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.google.common.collect.Lists;
+
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
+
import org.joda.time.Instant;
+import org.joda.time.Minutes;
/**
* Wraps {@link AmazonKinesis} class providing much simpler interface and
* proper error handling.
*/
class SimplifiedKinesisClient {
- private final AmazonKinesis kinesis;
- public SimplifiedKinesisClient(AmazonKinesis kinesis) {
- this.kinesis = kinesis;
- }
+ private static final String KINESIS_NAMESPACE = "AWS/Kinesis";
+ private static final String INCOMING_RECORDS_METRIC = "IncomingBytes";
+ private static final int PERIOD_GRANULARITY_IN_SECONDS = 60;
+ private static final String SUM_STATISTIC = "Sum";
+ private static final String STREAM_NAME_DIMENSION = "StreamName";
+ private final AmazonKinesis kinesis;
+ private final AmazonCloudWatch cloudWatch;
- public static SimplifiedKinesisClient from(KinesisClientProvider provider) {
- return new SimplifiedKinesisClient(provider.get());
- }
+ public SimplifiedKinesisClient(AmazonKinesis kinesis, AmazonCloudWatch cloudWatch) {
+ this.kinesis = checkNotNull(kinesis, "kinesis");
+ this.cloudWatch = checkNotNull(cloudWatch, "cloudWatch");
+ }
- public String getShardIterator(final String streamName, final String shardId,
- final ShardIteratorType shardIteratorType,
- final String startingSequenceNumber, final Instant timestamp)
- throws TransientKinesisException {
- final Date date = timestamp != null ? timestamp.toDate() : null;
- return wrapExceptions(new Callable<String>() {
- @Override
- public String call() throws Exception {
- return kinesis.getShardIterator(new GetShardIteratorRequest()
- .withStreamName(streamName)
- .withShardId(shardId)
- .withShardIteratorType(shardIteratorType)
- .withStartingSequenceNumber(startingSequenceNumber)
- .withTimestamp(date)
- ).getShardIterator();
- }
- });
- }
+ public static SimplifiedKinesisClient from(AWSClientsProvider provider) {
+ return new SimplifiedKinesisClient(provider.getKinesisClient(),
+ provider.getCloudWatchClient());
+ }
- public List<Shard> listShards(final String streamName) throws TransientKinesisException {
- return wrapExceptions(new Callable<List<Shard>>() {
- @Override
- public List<Shard> call() throws Exception {
- List<Shard> shards = Lists.newArrayList();
- String lastShardId = null;
+ public String getShardIterator(final String streamName, final String shardId,
+ final ShardIteratorType shardIteratorType,
+ final String startingSequenceNumber, final Instant timestamp)
+ throws TransientKinesisException {
+ final Date date = timestamp != null ? timestamp.toDate() : null;
+ return wrapExceptions(new Callable<String>() {
- StreamDescription description;
- do {
- description = kinesis.describeStream(streamName, lastShardId)
- .getStreamDescription();
+ @Override
+ public String call() throws Exception {
+ return kinesis.getShardIterator(new GetShardIteratorRequest()
+ .withStreamName(streamName)
+ .withShardId(shardId)
+ .withShardIteratorType(shardIteratorType)
+ .withStartingSequenceNumber(startingSequenceNumber)
+ .withTimestamp(date)
+ ).getShardIterator();
+ }
+ });
+ }
- shards.addAll(description.getShards());
- lastShardId = shards.get(shards.size() - 1).getShardId();
- } while (description.getHasMoreShards());
+ public List<Shard> listShards(final String streamName) throws TransientKinesisException {
+ return wrapExceptions(new Callable<List<Shard>>() {
- return shards;
- }
- });
- }
+ @Override
+ public List<Shard> call() throws Exception {
+ List<Shard> shards = Lists.newArrayList();
+ String lastShardId = null;
- /**
- * Gets records from Kinesis and deaggregates them if needed.
- *
- * @return list of deaggregated records
- * @throws TransientKinesisException - in case of recoverable situation
- */
- public GetKinesisRecordsResult getRecords(String shardIterator, String streamName,
- String shardId) throws TransientKinesisException {
- return getRecords(shardIterator, streamName, shardId, null);
- }
+ StreamDescription description;
+ do {
+ description = kinesis.describeStream(streamName, lastShardId)
+ .getStreamDescription();
- /**
- * Gets records from Kinesis and deaggregates them if needed.
- *
- * @return list of deaggregated records
- * @throws TransientKinesisException - in case of recoverable situation
- */
- public GetKinesisRecordsResult getRecords(final String shardIterator, final String streamName,
- final String shardId, final Integer limit)
- throws
- TransientKinesisException {
- return wrapExceptions(new Callable<GetKinesisRecordsResult>() {
- @Override
- public GetKinesisRecordsResult call() throws Exception {
- GetRecordsResult response = kinesis.getRecords(new GetRecordsRequest()
- .withShardIterator(shardIterator)
- .withLimit(limit));
- return new GetKinesisRecordsResult(
- UserRecord.deaggregate(response.getRecords()),
- response.getNextShardIterator(),
- streamName, shardId);
- }
- });
- }
+ shards.addAll(description.getShards());
+ lastShardId = shards.get(shards.size() - 1).getShardId();
+ } while (description.getHasMoreShards());
- /**
- * Wraps Amazon specific exceptions into more friendly format.
- *
- * @throws TransientKinesisException - in case of recoverable situation, i.e.
- * the request rate is too high, Kinesis remote service
- * failed, network issue, etc.
- * @throws ExpiredIteratorException - if iterator needs to be refreshed
- * @throws RuntimeException - in all other cases
- */
- private <T> T wrapExceptions(Callable<T> callable) throws TransientKinesisException {
- try {
- return callable.call();
- } catch (ExpiredIteratorException e) {
- throw e;
- } catch (LimitExceededException | ProvisionedThroughputExceededException e) {
- throw new TransientKinesisException(
- "Too many requests to Kinesis. Wait some time and retry.", e);
- } catch (AmazonServiceException e) {
- if (e.getErrorType() == AmazonServiceException.ErrorType.Service) {
- throw new TransientKinesisException(
- "Kinesis backend failed. Wait some time and retry.", e);
- }
- throw new RuntimeException("Kinesis client side failure", e);
- } catch (Exception e) {
- throw new RuntimeException("Unknown kinesis failure, when trying to reach kinesis", e);
+ return shards;
+ }
+ });
+ }
+
+ /**
+ * Gets records from Kinesis and deaggregates them if needed.
+ *
+ * @return list of deaggregated records
+ * @throws TransientKinesisException - in case of recoverable situation
+ */
+ public GetKinesisRecordsResult getRecords(String shardIterator, String streamName,
+ String shardId) throws TransientKinesisException {
+ return getRecords(shardIterator, streamName, shardId, null);
+ }
+
+ /**
+ * Gets records from Kinesis and deaggregates them if needed.
+ *
+ * @return list of deaggregated records
+ * @throws TransientKinesisException - in case of recoverable situation
+ */
+ public GetKinesisRecordsResult getRecords(final String shardIterator, final String streamName,
+ final String shardId, final Integer limit)
+ throws
+ TransientKinesisException {
+ return wrapExceptions(new Callable<GetKinesisRecordsResult>() {
+
+ @Override
+ public GetKinesisRecordsResult call() throws Exception {
+ GetRecordsResult response = kinesis.getRecords(new GetRecordsRequest()
+ .withShardIterator(shardIterator)
+ .withLimit(limit));
+ return new GetKinesisRecordsResult(
+ UserRecord.deaggregate(response.getRecords()),
+ response.getNextShardIterator(),
+ response.getMillisBehindLatest(),
+ streamName, shardId);
+ }
+ });
+ }
+
+ /**
+ * Gets total size in bytes of all events that remain in Kinesis stream after specified instant.
+ *
+ * @return total size in bytes of all Kinesis events after specified instant
+ */
+ public long getBacklogBytes(String streamName, Instant countSince)
+ throws TransientKinesisException {
+ return getBacklogBytes(streamName, countSince, new Instant());
+ }
+
+ /**
+ * Gets total size in bytes of all events that remain in Kinesis stream between specified
+ * instants.
+ *
+ * @return total size in bytes of all Kinesis events after specified instant
+ */
+ public long getBacklogBytes(final String streamName, final Instant countSince,
+ final Instant countTo) throws TransientKinesisException {
+ return wrapExceptions(new Callable<Long>() {
+
+ @Override
+ public Long call() throws Exception {
+ Minutes period = Minutes.minutesBetween(countSince, countTo);
+ if (period.isLessThan(Minutes.ONE)) {
+ return 0L;
}
+
+ GetMetricStatisticsRequest request = createMetricStatisticsRequest(streamName,
+ countSince, countTo, period);
+
+ long totalSizeInBytes = 0;
+ GetMetricStatisticsResult result = cloudWatch.getMetricStatistics(request);
+ for (Datapoint point : result.getDatapoints()) {
+ totalSizeInBytes += point
+ .getSum()
+ .longValue();
+ }
+ return totalSizeInBytes;
+ }
+ });
+ }
+
+ GetMetricStatisticsRequest createMetricStatisticsRequest(String streamName, Instant countSince,
+ Instant countTo, Minutes period) {
+ return new GetMetricStatisticsRequest()
+ .withNamespace(KINESIS_NAMESPACE)
+ .withMetricName(INCOMING_RECORDS_METRIC)
+ .withPeriod(period.getMinutes() * PERIOD_GRANULARITY_IN_SECONDS)
+ .withStartTime(countSince.toDate())
+ .withEndTime(countTo.toDate())
+ .withStatistics(Collections.singletonList(SUM_STATISTIC))
+ .withDimensions(Collections.singletonList(new Dimension()
+ .withName(STREAM_NAME_DIMENSION)
+ .withValue(streamName)));
+ }
+
+ /**
+ * Wraps Amazon specific exceptions into more friendly format.
+ *
+ * @throws TransientKinesisException - in case of recoverable situation, i.e.
+ * the request rate is too high, Kinesis remote service
+ * failed, network issue, etc.
+ * @throws ExpiredIteratorException - if iterator needs to be refreshed
+ * @throws RuntimeException - in all other cases
+ */
+ private <T> T wrapExceptions(Callable<T> callable) throws TransientKinesisException {
+ try {
+ return callable.call();
+ } catch (ExpiredIteratorException e) {
+ throw e;
+ } catch (LimitExceededException | ProvisionedThroughputExceededException e) {
+ throw new TransientKinesisException(
+ "Too many requests to Kinesis. Wait some time and retry.", e);
+ } catch (AmazonServiceException e) {
+ if (e.getErrorType() == AmazonServiceException.ErrorType.Service) {
+ throw new TransientKinesisException(
+ "Kinesis backend failed. Wait some time and retry.", e);
+ }
+ throw new RuntimeException("Kinesis client side failure", e);
+ } catch (AmazonClientException e) {
+ if (e.isRetryable()) {
+ throw new TransientKinesisException("Retryable client failure", e);
+ }
+ throw new RuntimeException("Not retryable client failure", e);
+ } catch (Exception e) {
+ throw new RuntimeException("Unknown kinesis failure, when trying to reach kinesis", e);
}
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java
index d8842c4..f9298fa 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java
@@ -17,13 +17,14 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import static com.google.common.base.Preconditions.checkNotNull;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
+
import java.io.Serializable;
import java.util.Objects;
+
import org.joda.time.Instant;
/**
@@ -32,54 +33,55 @@
* in which case the reader will start reading at the specified point in time.
*/
class StartingPoint implements Serializable {
- private final InitialPositionInStream position;
- private final Instant timestamp;
- public StartingPoint(InitialPositionInStream position) {
- this.position = checkNotNull(position, "position");
- this.timestamp = null;
- }
+ private final InitialPositionInStream position;
+ private final Instant timestamp;
- public StartingPoint(Instant timestamp) {
- this.timestamp = checkNotNull(timestamp, "timestamp");
- this.position = null;
- }
+ public StartingPoint(InitialPositionInStream position) {
+ this.position = checkNotNull(position, "position");
+ this.timestamp = null;
+ }
- public InitialPositionInStream getPosition() {
- return position;
- }
+ public StartingPoint(Instant timestamp) {
+ this.timestamp = checkNotNull(timestamp, "timestamp");
+ this.position = null;
+ }
- public String getPositionName() {
- return position != null ? position.name() : ShardIteratorType.AT_TIMESTAMP.name();
- }
+ public InitialPositionInStream getPosition() {
+ return position;
+ }
- public Instant getTimestamp() {
- return timestamp != null ? timestamp : null;
- }
+ public String getPositionName() {
+ return position != null ? position.name() : ShardIteratorType.AT_TIMESTAMP.name();
+ }
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- StartingPoint that = (StartingPoint) o;
- return position == that.position && Objects.equals(timestamp, that.timestamp);
- }
+ public Instant getTimestamp() {
+ return timestamp != null ? timestamp : null;
+ }
- @Override
- public int hashCode() {
- return Objects.hash(position, timestamp);
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
}
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ StartingPoint that = (StartingPoint) o;
+ return position == that.position && Objects.equals(timestamp, that.timestamp);
+ }
- @Override
- public String toString() {
- if (timestamp == null) {
- return position.toString();
- } else {
- return "Starting at timestamp " + timestamp;
- }
+ @Override
+ public int hashCode() {
+ return Objects.hash(position, timestamp);
+ }
+
+ @Override
+ public String toString() {
+ if (timestamp == null) {
+ return position.toString();
+ } else {
+ return "Starting at timestamp " + timestamp;
}
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java
index 22dc973..1ec865d 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java
@@ -23,20 +23,21 @@
* Always returns the same instance of checkpoint.
*/
class StaticCheckpointGenerator implements CheckpointGenerator {
- private final KinesisReaderCheckpoint checkpoint;
- public StaticCheckpointGenerator(KinesisReaderCheckpoint checkpoint) {
- checkNotNull(checkpoint, "checkpoint");
- this.checkpoint = checkpoint;
- }
+ private final KinesisReaderCheckpoint checkpoint;
- @Override
- public KinesisReaderCheckpoint generate(SimplifiedKinesisClient client) {
- return checkpoint;
- }
+ public StaticCheckpointGenerator(KinesisReaderCheckpoint checkpoint) {
+ checkNotNull(checkpoint, "checkpoint");
+ this.checkpoint = checkpoint;
+ }
- @Override
- public String toString() {
- return checkpoint.toString();
- }
+ @Override
+ public KinesisReaderCheckpoint generate(SimplifiedKinesisClient client) {
+ return checkpoint;
+ }
+
+ @Override
+ public String toString() {
+ return checkpoint.toString();
+ }
}
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/TransientKinesisException.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/TransientKinesisException.java
index 57ad8a8..0ea37ec 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/TransientKinesisException.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/TransientKinesisException.java
@@ -17,13 +17,14 @@
*/
package org.apache.beam.sdk.io.kinesis;
-import com.amazonaws.AmazonServiceException;
+import com.amazonaws.AmazonClientException;
/**
* A transient exception thrown by Kinesis.
*/
class TransientKinesisException extends Exception {
- public TransientKinesisException(String s, AmazonServiceException e) {
- super(s, e);
- }
+
+ public TransientKinesisException(String s, AmazonClientException e) {
+ super(s, e);
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java
index 046c9d9..d6e8817 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java
@@ -26,6 +26,7 @@
import com.amazonaws.AmazonWebServiceRequest;
import com.amazonaws.ResponseMetadata;
import com.amazonaws.regions.Region;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.model.AddTagsToStreamRequest;
import com.amazonaws.services.kinesis.model.AddTagsToStreamResult;
@@ -66,310 +67,323 @@
import com.amazonaws.services.kinesis.model.SplitShardResult;
import com.amazonaws.services.kinesis.model.StreamDescription;
import com.google.common.base.Function;
+
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.List;
import javax.annotation.Nullable;
+
import org.apache.commons.lang.builder.EqualsBuilder;
import org.joda.time.Instant;
+import org.mockito.Mockito;
/**
* Mock implemenation of {@link AmazonKinesis} for testing.
*/
class AmazonKinesisMock implements AmazonKinesis {
- static class TestData implements Serializable {
- private final String data;
- private final Instant arrivalTimestamp;
- private final String sequenceNumber;
+ static class TestData implements Serializable {
- public TestData(KinesisRecord record) {
- this(new String(record.getData().array()),
- record.getApproximateArrivalTimestamp(),
- record.getSequenceNumber());
- }
+ private final String data;
+ private final Instant arrivalTimestamp;
+ private final String sequenceNumber;
- public TestData(String data, Instant arrivalTimestamp, String sequenceNumber) {
- this.data = data;
- this.arrivalTimestamp = arrivalTimestamp;
- this.sequenceNumber = sequenceNumber;
- }
-
- public Record convertToRecord() {
- return new Record().
- withApproximateArrivalTimestamp(arrivalTimestamp.toDate()).
- withData(ByteBuffer.wrap(data.getBytes())).
- withSequenceNumber(sequenceNumber).
- withPartitionKey("");
- }
-
- @Override
- public boolean equals(Object obj) {
- return EqualsBuilder.reflectionEquals(this, obj);
- }
-
- @Override
- public int hashCode() {
- return reflectionHashCode(this);
- }
+ public TestData(KinesisRecord record) {
+ this(new String(record.getData().array()),
+ record.getApproximateArrivalTimestamp(),
+ record.getSequenceNumber());
}
- static class Provider implements KinesisClientProvider {
-
- private final List<List<TestData>> shardedData;
- private final int numberOfRecordsPerGet;
-
- public Provider(List<List<TestData>> shardedData, int numberOfRecordsPerGet) {
- this.shardedData = shardedData;
- this.numberOfRecordsPerGet = numberOfRecordsPerGet;
- }
-
- @Override
- public AmazonKinesis get() {
- return new AmazonKinesisMock(transform(shardedData,
- new Function<List<TestData>, List<Record>>() {
- @Override
- public List<Record> apply(@Nullable List<TestData> testDatas) {
- return transform(testDatas, new Function<TestData, Record>() {
- @Override
- public Record apply(@Nullable TestData testData) {
- return testData.convertToRecord();
- }
- });
- }
- }), numberOfRecordsPerGet);
- }
+ public TestData(String data, Instant arrivalTimestamp, String sequenceNumber) {
+ this.data = data;
+ this.arrivalTimestamp = arrivalTimestamp;
+ this.sequenceNumber = sequenceNumber;
}
- private final List<List<Record>> shardedData;
+ public Record convertToRecord() {
+ return new Record().
+ withApproximateArrivalTimestamp(arrivalTimestamp.toDate()).
+ withData(ByteBuffer.wrap(data.getBytes())).
+ withSequenceNumber(sequenceNumber).
+ withPartitionKey("");
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return EqualsBuilder.reflectionEquals(this, obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return reflectionHashCode(this);
+ }
+ }
+
+ static class Provider implements AWSClientsProvider {
+
+ private final List<List<TestData>> shardedData;
private final int numberOfRecordsPerGet;
- public AmazonKinesisMock(List<List<Record>> shardedData, int numberOfRecordsPerGet) {
- this.shardedData = shardedData;
- this.numberOfRecordsPerGet = numberOfRecordsPerGet;
+ public Provider(List<List<TestData>> shardedData, int numberOfRecordsPerGet) {
+ this.shardedData = shardedData;
+ this.numberOfRecordsPerGet = numberOfRecordsPerGet;
}
@Override
- public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) {
- String[] shardIteratorParts = getRecordsRequest.getShardIterator().split(":");
- int shardId = parseInt(shardIteratorParts[0]);
- int startingRecord = parseInt(shardIteratorParts[1]);
- List<Record> shardData = shardedData.get(shardId);
+ public AmazonKinesis getKinesisClient() {
+ return new AmazonKinesisMock(transform(shardedData,
+ new Function<List<TestData>, List<Record>>() {
- int toIndex = min(startingRecord + numberOfRecordsPerGet, shardData.size());
- int fromIndex = min(startingRecord, toIndex);
- return new GetRecordsResult().
- withRecords(shardData.subList(fromIndex, toIndex)).
- withNextShardIterator(String.format("%s:%s", shardId, toIndex));
- }
+ @Override
+ public List<Record> apply(@Nullable List<TestData> testDatas) {
+ return transform(testDatas, new Function<TestData, Record>() {
- @Override
- public GetShardIteratorResult getShardIterator(
- GetShardIteratorRequest getShardIteratorRequest) {
- ShardIteratorType shardIteratorType = ShardIteratorType.fromValue(
- getShardIteratorRequest.getShardIteratorType());
-
- String shardIterator;
- if (shardIteratorType == ShardIteratorType.TRIM_HORIZON) {
- shardIterator = String.format("%s:%s", getShardIteratorRequest.getShardId(), 0);
- } else {
- throw new RuntimeException("Not implemented");
- }
-
- return new GetShardIteratorResult().withShardIterator(shardIterator);
- }
-
- @Override
- public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
- int nextShardId = 0;
- if (exclusiveStartShardId != null) {
- nextShardId = parseInt(exclusiveStartShardId) + 1;
- }
- boolean hasMoreShards = nextShardId + 1 < shardedData.size();
-
- List<Shard> shards = newArrayList();
- if (nextShardId < shardedData.size()) {
- shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
- }
-
- return new DescribeStreamResult().withStreamDescription(
- new StreamDescription().withHasMoreShards(hasMoreShards).withShards(shards)
- );
- }
-
- @Override
- public void setEndpoint(String endpoint) {
+ @Override
+ public Record apply(@Nullable TestData testData) {
+ return testData.convertToRecord();
+ }
+ });
+ }
+ }), numberOfRecordsPerGet);
}
@Override
- public void setRegion(Region region) {
+ public AmazonCloudWatch getCloudWatchClient() {
+ return Mockito.mock(AmazonCloudWatch.class);
+ }
+ }
+ private final List<List<Record>> shardedData;
+ private final int numberOfRecordsPerGet;
+
+ public AmazonKinesisMock(List<List<Record>> shardedData, int numberOfRecordsPerGet) {
+ this.shardedData = shardedData;
+ this.numberOfRecordsPerGet = numberOfRecordsPerGet;
+ }
+
+ @Override
+ public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) {
+ String[] shardIteratorParts = getRecordsRequest.getShardIterator().split(":");
+ int shardId = parseInt(shardIteratorParts[0]);
+ int startingRecord = parseInt(shardIteratorParts[1]);
+ List<Record> shardData = shardedData.get(shardId);
+
+ int toIndex = min(startingRecord + numberOfRecordsPerGet, shardData.size());
+ int fromIndex = min(startingRecord, toIndex);
+ return new GetRecordsResult()
+ .withRecords(shardData.subList(fromIndex, toIndex))
+ .withNextShardIterator(String.format("%s:%s", shardId, toIndex))
+ .withMillisBehindLatest(0L);
+ }
+
+ @Override
+ public GetShardIteratorResult getShardIterator(
+ GetShardIteratorRequest getShardIteratorRequest) {
+ ShardIteratorType shardIteratorType = ShardIteratorType.fromValue(
+ getShardIteratorRequest.getShardIteratorType());
+
+ String shardIterator;
+ if (shardIteratorType == ShardIteratorType.TRIM_HORIZON) {
+ shardIterator = String.format("%s:%s", getShardIteratorRequest.getShardId(), 0);
+ } else {
+ throw new RuntimeException("Not implemented");
}
- @Override
- public AddTagsToStreamResult addTagsToStream(AddTagsToStreamRequest addTagsToStreamRequest) {
- throw new RuntimeException("Not implemented");
+ return new GetShardIteratorResult().withShardIterator(shardIterator);
+ }
+
+ @Override
+ public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
+ int nextShardId = 0;
+ if (exclusiveStartShardId != null) {
+ nextShardId = parseInt(exclusiveStartShardId) + 1;
+ }
+ boolean hasMoreShards = nextShardId + 1 < shardedData.size();
+
+ List<Shard> shards = newArrayList();
+ if (nextShardId < shardedData.size()) {
+ shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
}
- @Override
- public CreateStreamResult createStream(CreateStreamRequest createStreamRequest) {
- throw new RuntimeException("Not implemented");
- }
+ return new DescribeStreamResult().withStreamDescription(
+ new StreamDescription().withHasMoreShards(hasMoreShards).withShards(shards)
+ );
+ }
- @Override
- public CreateStreamResult createStream(String streamName, Integer shardCount) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public void setEndpoint(String endpoint) {
- @Override
- public DecreaseStreamRetentionPeriodResult decreaseStreamRetentionPeriod(
- DecreaseStreamRetentionPeriodRequest decreaseStreamRetentionPeriodRequest) {
- throw new RuntimeException("Not implemented");
- }
+ }
- @Override
- public DeleteStreamResult deleteStream(DeleteStreamRequest deleteStreamRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public void setRegion(Region region) {
- @Override
- public DeleteStreamResult deleteStream(String streamName) {
- throw new RuntimeException("Not implemented");
- }
+ }
- @Override
- public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public AddTagsToStreamResult addTagsToStream(AddTagsToStreamRequest addTagsToStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public DescribeStreamResult describeStream(String streamName) {
+ @Override
+ public CreateStreamResult createStream(CreateStreamRequest createStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public CreateStreamResult createStream(String streamName, Integer shardCount) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public DescribeStreamResult describeStream(String streamName,
- Integer limit, String exclusiveStartShardId) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DecreaseStreamRetentionPeriodResult decreaseStreamRetentionPeriod(
+ DecreaseStreamRetentionPeriodRequest decreaseStreamRetentionPeriodRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public DisableEnhancedMonitoringResult disableEnhancedMonitoring(
- DisableEnhancedMonitoringRequest disableEnhancedMonitoringRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DeleteStreamResult deleteStream(DeleteStreamRequest deleteStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public EnableEnhancedMonitoringResult enableEnhancedMonitoring(
- EnableEnhancedMonitoringRequest enableEnhancedMonitoringRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DeleteStreamResult deleteStream(String streamName) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public GetShardIteratorResult getShardIterator(String streamName,
- String shardId,
- String shardIteratorType) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public GetShardIteratorResult getShardIterator(String streamName,
- String shardId,
- String shardIteratorType,
- String startingSequenceNumber) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DescribeStreamResult describeStream(String streamName) {
- @Override
- public IncreaseStreamRetentionPeriodResult increaseStreamRetentionPeriod(
- IncreaseStreamRetentionPeriodRequest increaseStreamRetentionPeriodRequest) {
- throw new RuntimeException("Not implemented");
- }
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ListStreamsResult listStreams(ListStreamsRequest listStreamsRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DescribeStreamResult describeStream(String streamName,
+ Integer limit, String exclusiveStartShardId) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ListStreamsResult listStreams() {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public DisableEnhancedMonitoringResult disableEnhancedMonitoring(
+ DisableEnhancedMonitoringRequest disableEnhancedMonitoringRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ListStreamsResult listStreams(String exclusiveStartStreamName) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public EnableEnhancedMonitoringResult enableEnhancedMonitoring(
+ EnableEnhancedMonitoringRequest enableEnhancedMonitoringRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ListStreamsResult listStreams(Integer limit, String exclusiveStartStreamName) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public GetShardIteratorResult getShardIterator(String streamName,
+ String shardId,
+ String shardIteratorType) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ListTagsForStreamResult listTagsForStream(
- ListTagsForStreamRequest listTagsForStreamRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public GetShardIteratorResult getShardIterator(String streamName,
+ String shardId,
+ String shardIteratorType,
+ String startingSequenceNumber) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public MergeShardsResult mergeShards(MergeShardsRequest mergeShardsRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public IncreaseStreamRetentionPeriodResult increaseStreamRetentionPeriod(
+ IncreaseStreamRetentionPeriodRequest increaseStreamRetentionPeriodRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public MergeShardsResult mergeShards(String streamName,
- String shardToMerge, String adjacentShardToMerge) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public ListStreamsResult listStreams(ListStreamsRequest listStreamsRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public PutRecordResult putRecord(PutRecordRequest putRecordRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public ListStreamsResult listStreams() {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public PutRecordResult putRecord(String streamName, ByteBuffer data, String partitionKey) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public ListStreamsResult listStreams(String exclusiveStartStreamName) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public PutRecordResult putRecord(String streamName, ByteBuffer data,
- String partitionKey, String sequenceNumberForOrdering) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public ListStreamsResult listStreams(Integer limit, String exclusiveStartStreamName) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public ListTagsForStreamResult listTagsForStream(
+ ListTagsForStreamRequest listTagsForStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public RemoveTagsFromStreamResult removeTagsFromStream(
- RemoveTagsFromStreamRequest removeTagsFromStreamRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public MergeShardsResult mergeShards(MergeShardsRequest mergeShardsRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public SplitShardResult splitShard(SplitShardRequest splitShardRequest) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public MergeShardsResult mergeShards(String streamName,
+ String shardToMerge, String adjacentShardToMerge) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public SplitShardResult splitShard(String streamName,
- String shardToSplit, String newStartingHashKey) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public PutRecordResult putRecord(PutRecordRequest putRecordRequest) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public void shutdown() {
+ @Override
+ public PutRecordResult putRecord(String streamName, ByteBuffer data, String partitionKey) {
+ throw new RuntimeException("Not implemented");
+ }
- }
+ @Override
+ public PutRecordResult putRecord(String streamName, ByteBuffer data,
+ String partitionKey, String sequenceNumberForOrdering) {
+ throw new RuntimeException("Not implemented");
+ }
- @Override
- public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) {
- throw new RuntimeException("Not implemented");
- }
+ @Override
+ public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ @Override
+ public RemoveTagsFromStreamResult removeTagsFromStream(
+ RemoveTagsFromStreamRequest removeTagsFromStreamRequest) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ @Override
+ public SplitShardResult splitShard(SplitShardRequest splitShardRequest) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ @Override
+ public SplitShardResult splitShard(String streamName,
+ String shardToSplit, String newStartingHashKey) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ @Override
+ public void shutdown() {
+
+ }
+
+ @Override
+ public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) {
+ throw new RuntimeException("Not implemented");
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
index 00acffe..0b16bb7 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
@@ -18,24 +18,27 @@
package org.apache.beam.sdk.io.kinesis;
import com.google.common.testing.EqualsTester;
+
import java.util.NoSuchElementException;
+
import org.junit.Test;
/**
* Tests {@link CustomOptional}.
*/
public class CustomOptionalTest {
- @Test(expected = NoSuchElementException.class)
- public void absentThrowsNoSuchElementExceptionOnGet() {
- CustomOptional.absent().get();
- }
- @Test
- public void testEqualsAndHashCode() {
- new EqualsTester()
- .addEqualityGroup(CustomOptional.absent(), CustomOptional.absent())
- .addEqualityGroup(CustomOptional.of(3), CustomOptional.of(3))
- .addEqualityGroup(CustomOptional.of(11))
- .addEqualityGroup(CustomOptional.of("3")).testEquals();
- }
+ @Test(expected = NoSuchElementException.class)
+ public void absentThrowsNoSuchElementExceptionOnGet() {
+ CustomOptional.absent().get();
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ new EqualsTester()
+ .addEqualityGroup(CustomOptional.absent(), CustomOptional.absent())
+ .addEqualityGroup(CustomOptional.of(3), CustomOptional.of(3))
+ .addEqualityGroup(CustomOptional.of(11))
+ .addEqualityGroup(CustomOptional.of("3")).testEquals();
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java
index c92ac9a..1bb9717 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java
@@ -28,30 +28,29 @@
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
-
/***
*/
@RunWith(MockitoJUnitRunner.class)
public class DynamicCheckpointGeneratorTest {
- @Mock
- private SimplifiedKinesisClient kinesisClient;
- @Mock
- private Shard shard1, shard2, shard3;
+ @Mock
+ private SimplifiedKinesisClient kinesisClient;
+ @Mock
+ private Shard shard1, shard2, shard3;
- @Test
- public void shouldMapAllShardsToCheckpoints() throws Exception {
- given(shard1.getShardId()).willReturn("shard-01");
- given(shard2.getShardId()).willReturn("shard-02");
- given(shard3.getShardId()).willReturn("shard-03");
- given(kinesisClient.listShards("stream")).willReturn(asList(shard1, shard2, shard3));
+ @Test
+ public void shouldMapAllShardsToCheckpoints() throws Exception {
+ given(shard1.getShardId()).willReturn("shard-01");
+ given(shard2.getShardId()).willReturn("shard-02");
+ given(shard3.getShardId()).willReturn("shard-03");
+ given(kinesisClient.listShards("stream")).willReturn(asList(shard1, shard2, shard3));
- StartingPoint startingPoint = new StartingPoint(InitialPositionInStream.LATEST);
- DynamicCheckpointGenerator underTest = new DynamicCheckpointGenerator("stream",
- startingPoint);
+ StartingPoint startingPoint = new StartingPoint(InitialPositionInStream.LATEST);
+ DynamicCheckpointGenerator underTest = new DynamicCheckpointGenerator("stream",
+ startingPoint);
- KinesisReaderCheckpoint checkpoint = underTest.generate(kinesisClient);
+ KinesisReaderCheckpoint checkpoint = underTest.generate(kinesisClient);
- assertThat(checkpoint).hasSize(3);
- }
+ assertThat(checkpoint).hasSize(3);
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java
index 567e25f..73554bb 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java
@@ -21,7 +21,9 @@
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
import com.google.common.collect.Iterables;
+
import java.util.List;
+
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.DoFn;
@@ -36,59 +38,61 @@
*/
public class KinesisMockReadTest {
- @Rule
- public final transient TestPipeline p = TestPipeline.create();
+ @Rule
+ public final transient TestPipeline p = TestPipeline.create();
- @Test
- public void readsDataFromMockKinesis() {
- int noOfShards = 3;
- int noOfEventsPerShard = 100;
- List<List<AmazonKinesisMock.TestData>> testData =
- provideTestData(noOfShards, noOfEventsPerShard);
+ @Test
+ public void readsDataFromMockKinesis() {
+ int noOfShards = 3;
+ int noOfEventsPerShard = 100;
+ List<List<AmazonKinesisMock.TestData>> testData =
+ provideTestData(noOfShards, noOfEventsPerShard);
- PCollection<AmazonKinesisMock.TestData> result = p
- .apply(
- KinesisIO.read()
- .from("stream", InitialPositionInStream.TRIM_HORIZON)
- .withClientProvider(new AmazonKinesisMock.Provider(testData, 10))
- .withMaxNumRecords(noOfShards * noOfEventsPerShard))
- .apply(ParDo.of(new KinesisRecordToTestData()));
- PAssert.that(result).containsInAnyOrder(Iterables.concat(testData));
- p.run();
+ PCollection<AmazonKinesisMock.TestData> result = p
+ .apply(
+ KinesisIO.read()
+ .withStreamName("stream")
+ .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON)
+ .withAWSClientsProvider(new AmazonKinesisMock.Provider(testData, 10))
+ .withMaxNumRecords(noOfShards * noOfEventsPerShard))
+ .apply(ParDo.of(new KinesisRecordToTestData()));
+ PAssert.that(result).containsInAnyOrder(Iterables.concat(testData));
+ p.run();
+ }
+
+ private static class KinesisRecordToTestData extends
+ DoFn<KinesisRecord, AmazonKinesisMock.TestData> {
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ c.output(new AmazonKinesisMock.TestData(c.element()));
+ }
+ }
+
+ private List<List<AmazonKinesisMock.TestData>> provideTestData(
+ int noOfShards,
+ int noOfEventsPerShard) {
+
+ int seqNumber = 0;
+
+ List<List<AmazonKinesisMock.TestData>> shardedData = newArrayList();
+ for (int i = 0; i < noOfShards; ++i) {
+ List<AmazonKinesisMock.TestData> shardData = newArrayList();
+ shardedData.add(shardData);
+
+ DateTime arrival = DateTime.now();
+ for (int j = 0; j < noOfEventsPerShard; ++j) {
+ arrival = arrival.plusSeconds(1);
+
+ seqNumber++;
+ shardData.add(new AmazonKinesisMock.TestData(
+ Integer.toString(seqNumber),
+ arrival.toInstant(),
+ Integer.toString(seqNumber))
+ );
+ }
}
- private static class KinesisRecordToTestData extends
- DoFn<KinesisRecord, AmazonKinesisMock.TestData> {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- c.output(new AmazonKinesisMock.TestData(c.element()));
- }
- }
-
- private List<List<AmazonKinesisMock.TestData>> provideTestData(
- int noOfShards,
- int noOfEventsPerShard) {
-
- int seqNumber = 0;
-
- List<List<AmazonKinesisMock.TestData>> shardedData = newArrayList();
- for (int i = 0; i < noOfShards; ++i) {
- List<AmazonKinesisMock.TestData> shardData = newArrayList();
- shardedData.add(shardData);
-
- DateTime arrival = DateTime.now();
- for (int j = 0; j < noOfEventsPerShard; ++j) {
- arrival = arrival.plusSeconds(1);
-
- seqNumber++;
- shardData.add(new AmazonKinesisMock.TestData(
- Integer.toString(seqNumber),
- arrival.toInstant(),
- Integer.toString(seqNumber))
- );
- }
- }
-
- return shardedData;
- }
+ return shardedData;
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java
index 8c8da64..1038a47 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java
@@ -17,13 +17,14 @@
*/
package org.apache.beam.sdk.io.kinesis;
-
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
import com.google.common.collect.Iterables;
+
import java.util.Iterator;
import java.util.List;
+
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -35,33 +36,34 @@
*/
@RunWith(MockitoJUnitRunner.class)
public class KinesisReaderCheckpointTest {
- @Mock
- private ShardCheckpoint a, b, c;
- private KinesisReaderCheckpoint checkpoint;
+ @Mock
+ private ShardCheckpoint a, b, c;
- @Before
- public void setUp() {
- checkpoint = new KinesisReaderCheckpoint(asList(a, b, c));
- }
+ private KinesisReaderCheckpoint checkpoint;
- @Test
- public void splitsCheckpointAccordingly() {
- verifySplitInto(1);
- verifySplitInto(2);
- verifySplitInto(3);
- verifySplitInto(4);
- }
+ @Before
+ public void setUp() {
+ checkpoint = new KinesisReaderCheckpoint(asList(a, b, c));
+ }
- @Test(expected = UnsupportedOperationException.class)
- public void isImmutable() {
- Iterator<ShardCheckpoint> iterator = checkpoint.iterator();
- iterator.remove();
- }
+ @Test
+ public void splitsCheckpointAccordingly() {
+ verifySplitInto(1);
+ verifySplitInto(2);
+ verifySplitInto(3);
+ verifySplitInto(4);
+ }
- private void verifySplitInto(int size) {
- List<KinesisReaderCheckpoint> split = checkpoint.splitInto(size);
- assertThat(Iterables.concat(split)).containsOnly(a, b, c);
- assertThat(split).hasSize(Math.min(size, 3));
- }
+ @Test(expected = UnsupportedOperationException.class)
+ public void isImmutable() {
+ Iterator<ShardCheckpoint> iterator = checkpoint.iterator();
+ iterator.remove();
+ }
+
+ private void verifySplitInto(int size) {
+ List<KinesisReaderCheckpoint> split = checkpoint.splitInto(size);
+ assertThat(Iterables.concat(split)).containsOnly(a, b, c);
+ assertThat(split).hasSize(Math.min(size, 3));
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderIT.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderIT.java
index 8eb6546..7126594 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderIT.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderIT.java
@@ -23,6 +23,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import com.amazonaws.regions.Regions;
+
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -31,6 +32,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.PAssert;
@@ -50,72 +52,76 @@
* You need to provide all {@link KinesisTestOptions} in order to run this.
*/
public class KinesisReaderIT {
- private static final long PIPELINE_STARTUP_TIME = TimeUnit.SECONDS.toMillis(10);
- private ExecutorService singleThreadExecutor = newSingleThreadExecutor();
- @Rule
- public final transient TestPipeline p = TestPipeline.create();
+ private static final long PIPELINE_STARTUP_TIME = TimeUnit.SECONDS.toMillis(10);
+ private ExecutorService singleThreadExecutor = newSingleThreadExecutor();
- @Ignore
- @Test
- public void readsDataFromRealKinesisStream()
- throws IOException, InterruptedException, ExecutionException {
- KinesisTestOptions options = readKinesisOptions();
- List<String> testData = prepareTestData(1000);
+ @Rule
+ public final transient TestPipeline p = TestPipeline.create();
- Future<?> future = startTestPipeline(testData, options);
- KinesisUploader.uploadAll(testData, options);
- future.get();
+ @Ignore
+ @Test
+ public void readsDataFromRealKinesisStream()
+ throws IOException, InterruptedException, ExecutionException {
+ KinesisTestOptions options = readKinesisOptions();
+ List<String> testData = prepareTestData(1000);
+
+ Future<?> future = startTestPipeline(testData, options);
+ KinesisUploader.uploadAll(testData, options);
+ future.get();
+ }
+
+ private List<String> prepareTestData(int count) {
+ List<String> data = newArrayList();
+ for (int i = 0; i < count; ++i) {
+ data.add(RandomStringUtils.randomAlphabetic(32));
}
+ return data;
+ }
- private List<String> prepareTestData(int count) {
- List<String> data = newArrayList();
- for (int i = 0; i < count; ++i) {
- data.add(RandomStringUtils.randomAlphabetic(32));
+ private Future<?> startTestPipeline(List<String> testData, KinesisTestOptions options)
+ throws InterruptedException {
+
+ PCollection<String> result = p.
+ apply(KinesisIO.read()
+ .withStreamName(options.getAwsKinesisStream())
+ .withInitialTimestampInStream(Instant.now())
+ .withAWSClientsProvider(options.getAwsAccessKey(), options.getAwsSecretKey(),
+ Regions.fromName(options.getAwsKinesisRegion()))
+ .withMaxReadTime(Duration.standardMinutes(3))
+ ).
+ apply(ParDo.of(new RecordDataToString()));
+ PAssert.that(result).containsInAnyOrder(testData);
+
+ Future<?> future = singleThreadExecutor.submit(new Callable<Void>() {
+
+ @Override
+ public Void call() throws Exception {
+ PipelineResult result = p.run();
+ PipelineResult.State state = result.getState();
+ while (state != PipelineResult.State.DONE && state != PipelineResult.State.FAILED) {
+ Thread.sleep(1000);
+ state = result.getState();
}
- return data;
+ assertThat(state).isEqualTo(PipelineResult.State.DONE);
+ return null;
+ }
+ });
+ Thread.sleep(PIPELINE_STARTUP_TIME);
+ return future;
+ }
+
+ private KinesisTestOptions readKinesisOptions() {
+ PipelineOptionsFactory.register(KinesisTestOptions.class);
+ return TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
+ }
+
+ private static class RecordDataToString extends DoFn<KinesisRecord, String> {
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ checkNotNull(c.element(), "Null record given");
+ c.output(new String(c.element().getData().array(), StandardCharsets.UTF_8));
}
-
- private Future<?> startTestPipeline(List<String> testData, KinesisTestOptions options)
- throws InterruptedException {
-
- PCollection<String> result = p.
- apply(KinesisIO.read()
- .from(options.getAwsKinesisStream(), Instant.now())
- .withClientProvider(options.getAwsAccessKey(), options.getAwsSecretKey(),
- Regions.fromName(options.getAwsKinesisRegion()))
- .withMaxReadTime(Duration.standardMinutes(3))
- ).
- apply(ParDo.of(new RecordDataToString()));
- PAssert.that(result).containsInAnyOrder(testData);
-
- Future<?> future = singleThreadExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- PipelineResult result = p.run();
- PipelineResult.State state = result.getState();
- while (state != PipelineResult.State.DONE && state != PipelineResult.State.FAILED) {
- Thread.sleep(1000);
- state = result.getState();
- }
- assertThat(state).isEqualTo(PipelineResult.State.DONE);
- return null;
- }
- });
- Thread.sleep(PIPELINE_STARTUP_TIME);
- return future;
- }
-
- private KinesisTestOptions readKinesisOptions() {
- PipelineOptionsFactory.register(KinesisTestOptions.class);
- return TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
- }
-
- private static class RecordDataToString extends DoFn<KinesisRecord, String> {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- checkNotNull(c.element(), "Null record given");
- c.output(new String(c.element().getData().array(), StandardCharsets.UTF_8));
- }
- }
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderTest.java
index 3111029..11ae011 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderTest.java
@@ -19,102 +19,206 @@
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.NoSuchElementException;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.OngoingStubbing;
/**
* Tests {@link KinesisReader}.
*/
@RunWith(MockitoJUnitRunner.class)
public class KinesisReaderTest {
- @Mock
- private SimplifiedKinesisClient kinesis;
- @Mock
- private CheckpointGenerator generator;
- @Mock
- private ShardCheckpoint firstCheckpoint, secondCheckpoint;
- @Mock
- private ShardRecordsIterator firstIterator, secondIterator;
- @Mock
- private KinesisRecord a, b, c, d;
- private KinesisReader reader;
+ @Mock
+ private SimplifiedKinesisClient kinesis;
+ @Mock
+ private CheckpointGenerator generator;
+ @Mock
+ private ShardCheckpoint firstCheckpoint, secondCheckpoint;
+ @Mock
+ private KinesisRecord a, b, c, d;
+ @Mock
+ private KinesisSource kinesisSource;
+ @Mock
+ private ShardReadersPool shardReadersPool;
- @Before
- public void setUp() throws IOException, TransientKinesisException {
- when(generator.generate(kinesis)).thenReturn(new KinesisReaderCheckpoint(
- asList(firstCheckpoint, secondCheckpoint)
- ));
- when(firstCheckpoint.getShardRecordsIterator(kinesis)).thenReturn(firstIterator);
- when(secondCheckpoint.getShardRecordsIterator(kinesis)).thenReturn(secondIterator);
- when(firstIterator.next()).thenReturn(CustomOptional.<KinesisRecord>absent());
- when(secondIterator.next()).thenReturn(CustomOptional.<KinesisRecord>absent());
+ private KinesisReader reader;
- reader = new KinesisReader(kinesis, generator, null);
+ @Before
+ public void setUp() throws IOException, TransientKinesisException {
+ when(generator.generate(kinesis)).thenReturn(new KinesisReaderCheckpoint(
+ asList(firstCheckpoint, secondCheckpoint)
+ ));
+ when(shardReadersPool.nextRecord()).thenReturn(CustomOptional.<KinesisRecord>absent());
+ when(a.getApproximateArrivalTimestamp()).thenReturn(Instant.now());
+ when(b.getApproximateArrivalTimestamp()).thenReturn(Instant.now());
+ when(c.getApproximateArrivalTimestamp()).thenReturn(Instant.now());
+ when(d.getApproximateArrivalTimestamp()).thenReturn(Instant.now());
+
+ reader = createReader(Duration.ZERO);
+ }
+
+ private KinesisReader createReader(Duration backlogBytesCheckThreshold)
+ throws TransientKinesisException {
+ KinesisReader kinesisReader = spy(new KinesisReader(kinesis, generator, kinesisSource,
+ Duration.ZERO, backlogBytesCheckThreshold));
+ doReturn(shardReadersPool).when(kinesisReader)
+ .createShardReadersPool();
+ return kinesisReader;
+ }
+
+ @Test
+ public void startReturnsFalseIfNoDataAtTheBeginning() throws IOException {
+ assertThat(reader.start()).isFalse();
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void throwsNoSuchElementExceptionIfNoData() throws IOException {
+ reader.start();
+ reader.getCurrent();
+ }
+
+ @Test
+ public void startReturnsTrueIfSomeDataAvailable() throws IOException,
+ TransientKinesisException {
+ when(shardReadersPool.nextRecord()).
+ thenReturn(CustomOptional.of(a)).
+ thenReturn(CustomOptional.<KinesisRecord>absent());
+
+ assertThat(reader.start()).isTrue();
+ }
+
+ @Test
+ public void readsThroughAllDataAvailable() throws IOException, TransientKinesisException {
+ when(shardReadersPool.nextRecord()).
+ thenReturn(CustomOptional.of(c)).
+ thenReturn(CustomOptional.<KinesisRecord>absent()).
+ thenReturn(CustomOptional.of(a)).
+ thenReturn(CustomOptional.<KinesisRecord>absent()).
+ thenReturn(CustomOptional.of(d)).
+ thenReturn(CustomOptional.of(b)).
+ thenReturn(CustomOptional.<KinesisRecord>absent());
+
+ assertThat(reader.start()).isTrue();
+ assertThat(reader.getCurrent()).isEqualTo(c);
+ assertThat(reader.advance()).isFalse();
+ assertThat(reader.advance()).isTrue();
+ assertThat(reader.getCurrent()).isEqualTo(a);
+ assertThat(reader.advance()).isFalse();
+ assertThat(reader.advance()).isTrue();
+ assertThat(reader.getCurrent()).isEqualTo(d);
+ assertThat(reader.advance()).isTrue();
+ assertThat(reader.getCurrent()).isEqualTo(b);
+ assertThat(reader.advance()).isFalse();
+ }
+
+ @Test
+ public void watermarkDoesNotChangeWhenToFewSampleRecords()
+ throws IOException, TransientKinesisException {
+ final long timestampMs = 1000L;
+
+ prepareRecordsWithArrivalTimestamps(timestampMs, 1, KinesisReader.MIN_WATERMARK_MESSAGES / 2);
+
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ assertThat(reader.getWatermark()).isEqualTo(BoundedWindow.TIMESTAMP_MIN_VALUE);
}
+ }
- @Test
- public void startReturnsFalseIfNoDataAtTheBeginning() throws IOException {
- assertThat(reader.start()).isFalse();
+ @Test
+ public void watermarkAdvancesWhenEnoughRecordsReadRecently()
+ throws IOException, TransientKinesisException {
+ long timestampMs = 1000L;
+
+ prepareRecordsWithArrivalTimestamps(timestampMs, 1, KinesisReader.MIN_WATERMARK_MESSAGES);
+
+ int recordsNeededForWatermarkAdvancing = KinesisReader.MIN_WATERMARK_MESSAGES;
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ if (--recordsNeededForWatermarkAdvancing > 0) {
+ assertThat(reader.getWatermark()).isEqualTo(BoundedWindow.TIMESTAMP_MIN_VALUE);
+ } else {
+ assertThat(reader.getWatermark()).isEqualTo(new Instant(timestampMs));
+ }
}
+ }
- @Test(expected = NoSuchElementException.class)
- public void throwsNoSuchElementExceptionIfNoData() throws IOException {
- reader.start();
- reader.getCurrent();
+ @Test
+ public void watermarkMonotonicallyIncreases()
+ throws IOException, TransientKinesisException {
+ long timestampMs = 1000L;
+
+ prepareRecordsWithArrivalTimestamps(timestampMs, -1, KinesisReader.MIN_WATERMARK_MESSAGES * 2);
+
+ Instant lastWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ for (boolean more = reader.start(); more; more = reader.advance()) {
+ Instant currentWatermark = reader.getWatermark();
+ assertThat(currentWatermark).isGreaterThanOrEqualTo(lastWatermark);
+ lastWatermark = currentWatermark;
}
+ assertThat(reader.advance()).isFalse();
+ }
- @Test
- public void startReturnsTrueIfSomeDataAvailable() throws IOException,
- TransientKinesisException {
- when(firstIterator.next()).
- thenReturn(CustomOptional.of(a)).
- thenReturn(CustomOptional.<KinesisRecord>absent());
-
- assertThat(reader.start()).isTrue();
+ private void prepareRecordsWithArrivalTimestamps(long initialTimestampMs, int increment,
+ int count) throws TransientKinesisException {
+ long timestampMs = initialTimestampMs;
+ KinesisRecord firstRecord = prepareRecordMockWithArrivalTimestamp(timestampMs);
+ OngoingStubbing<CustomOptional<KinesisRecord>> shardReadersPoolStubbing =
+ when(shardReadersPool.nextRecord()).thenReturn(CustomOptional.of(firstRecord));
+ for (int i = 0; i < count; i++) {
+ timestampMs += increment;
+ KinesisRecord record = prepareRecordMockWithArrivalTimestamp(timestampMs);
+ shardReadersPoolStubbing = shardReadersPoolStubbing.thenReturn(CustomOptional.of(record));
}
+ shardReadersPoolStubbing.thenReturn(CustomOptional.<KinesisRecord>absent());
+ }
- @Test
- public void advanceReturnsFalseIfThereIsTransientExceptionInKinesis()
- throws IOException, TransientKinesisException {
- reader.start();
+ private KinesisRecord prepareRecordMockWithArrivalTimestamp(long timestampMs) {
+ KinesisRecord record = mock(KinesisRecord.class);
+ when(record.getApproximateArrivalTimestamp()).thenReturn(new Instant(timestampMs));
+ return record;
+ }
- when(firstIterator.next()).thenThrow(TransientKinesisException.class);
+ @Test
+ public void getTotalBacklogBytesShouldReturnLastSeenValueWhenKinesisExceptionsOccur()
+ throws TransientKinesisException, IOException {
+ reader.start();
+ when(kinesisSource.getStreamName()).thenReturn("stream1");
+ when(kinesis.getBacklogBytes(eq("stream1"), any(Instant.class)))
+ .thenReturn(10L)
+ .thenThrow(TransientKinesisException.class)
+ .thenReturn(20L);
- assertThat(reader.advance()).isFalse();
- }
+ assertThat(reader.getTotalBacklogBytes()).isEqualTo(10);
+ assertThat(reader.getTotalBacklogBytes()).isEqualTo(10);
+ assertThat(reader.getTotalBacklogBytes()).isEqualTo(20);
+ }
- @Test
- public void readsThroughAllDataAvailable() throws IOException, TransientKinesisException {
- when(firstIterator.next()).
- thenReturn(CustomOptional.<KinesisRecord>absent()).
- thenReturn(CustomOptional.of(a)).
- thenReturn(CustomOptional.<KinesisRecord>absent()).
- thenReturn(CustomOptional.of(b)).
- thenReturn(CustomOptional.<KinesisRecord>absent());
+ @Test
+ public void getTotalBacklogBytesShouldReturnLastSeenValueWhenCalledFrequently()
+ throws TransientKinesisException, IOException {
+ KinesisReader backlogCachingReader = createReader(Duration.standardSeconds(30));
+ backlogCachingReader.start();
+ when(kinesisSource.getStreamName()).thenReturn("stream1");
+ when(kinesis.getBacklogBytes(eq("stream1"), any(Instant.class)))
+ .thenReturn(10L)
+ .thenReturn(20L);
- when(secondIterator.next()).
- thenReturn(CustomOptional.of(c)).
- thenReturn(CustomOptional.<KinesisRecord>absent()).
- thenReturn(CustomOptional.of(d)).
- thenReturn(CustomOptional.<KinesisRecord>absent());
-
- assertThat(reader.start()).isTrue();
- assertThat(reader.getCurrent()).isEqualTo(c);
- assertThat(reader.advance()).isTrue();
- assertThat(reader.getCurrent()).isEqualTo(a);
- assertThat(reader.advance()).isTrue();
- assertThat(reader.getCurrent()).isEqualTo(d);
- assertThat(reader.advance()).isTrue();
- assertThat(reader.getCurrent()).isEqualTo(b);
- assertThat(reader.advance()).isFalse();
- }
-
+ assertThat(backlogCachingReader.getTotalBacklogBytes()).isEqualTo(10);
+ assertThat(backlogCachingReader.getTotalBacklogBytes()).isEqualTo(10);
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoderTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoderTest.java
index 8771c86..c9f01bb 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoderTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoderTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.kinesis;
import java.nio.ByteBuffer;
+
import org.apache.beam.sdk.testing.CoderProperties;
import org.joda.time.Instant;
import org.junit.Test;
@@ -26,20 +27,21 @@
* Tests {@link KinesisRecordCoder}.
*/
public class KinesisRecordCoderTest {
- @Test
- public void encodingAndDecodingWorks() throws Exception {
- KinesisRecord record = new KinesisRecord(
- ByteBuffer.wrap("data".getBytes()),
- "sequence",
- 128L,
- "partition",
- Instant.now(),
- Instant.now(),
- "stream",
- "shard"
- );
- CoderProperties.coderDecodeEncodeEqual(
- new KinesisRecordCoder(), record
- );
- }
+
+ @Test
+ public void encodingAndDecodingWorks() throws Exception {
+ KinesisRecord record = new KinesisRecord(
+ ByteBuffer.wrap("data".getBytes()),
+ "sequence",
+ 128L,
+ "partition",
+ Instant.now(),
+ Instant.now(),
+ "stream",
+ "shard"
+ );
+ CoderProperties.coderDecodeEncodeEqual(
+ new KinesisRecordCoder(), record
+ );
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisTestOptions.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisTestOptions.java
index 324de46..76bcb27 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisTestOptions.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisTestOptions.java
@@ -25,23 +25,28 @@
* Options for Kinesis integration tests.
*/
public interface KinesisTestOptions extends TestPipelineOptions {
- @Description("AWS region where Kinesis stream resided")
- @Default.String("aws-kinesis-region")
- String getAwsKinesisRegion();
- void setAwsKinesisRegion(String value);
- @Description("Kinesis stream name")
- @Default.String("aws-kinesis-stream")
- String getAwsKinesisStream();
- void setAwsKinesisStream(String value);
+ @Description("AWS region where Kinesis stream resided")
+ @Default.String("aws-kinesis-region")
+ String getAwsKinesisRegion();
- @Description("AWS secret key")
- @Default.String("aws-secret-key")
- String getAwsSecretKey();
- void setAwsSecretKey(String value);
+ void setAwsKinesisRegion(String value);
- @Description("AWS access key")
- @Default.String("aws-access-key")
- String getAwsAccessKey();
- void setAwsAccessKey(String value);
+ @Description("Kinesis stream name")
+ @Default.String("aws-kinesis-stream")
+ String getAwsKinesisStream();
+
+ void setAwsKinesisStream(String value);
+
+ @Description("AWS secret key")
+ @Default.String("aws-secret-key")
+ String getAwsSecretKey();
+
+ void setAwsSecretKey(String value);
+
+ @Description("AWS access key")
+ @Default.String("aws-access-key")
+ String getAwsAccessKey();
+
+ void setAwsAccessKey(String value);
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
index 7518ff7..7a7cb02 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
@@ -29,6 +29,7 @@
import com.amazonaws.services.kinesis.model.PutRecordsResultEntry;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
+
import java.nio.ByteBuffer;
import java.util.List;
@@ -37,47 +38,46 @@
*/
public class KinesisUploader {
- public static final int MAX_NUMBER_OF_RECORDS_IN_BATCH = 499;
+ public static final int MAX_NUMBER_OF_RECORDS_IN_BATCH = 499;
- public static void uploadAll(List<String> data, KinesisTestOptions options) {
- AmazonKinesisClient client = new AmazonKinesisClient(
- new StaticCredentialsProvider(
- new BasicAWSCredentials(
- options.getAwsAccessKey(), options.getAwsSecretKey()))
- ).withRegion(Regions.fromName(options.getAwsKinesisRegion()));
+ public static void uploadAll(List<String> data, KinesisTestOptions options) {
+ AmazonKinesisClient client = new AmazonKinesisClient(
+ new StaticCredentialsProvider(
+ new BasicAWSCredentials(
+ options.getAwsAccessKey(), options.getAwsSecretKey()))
+ ).withRegion(Regions.fromName(options.getAwsKinesisRegion()));
- List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH);
+ List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH);
+ for (List<String> partition : partitions) {
+ List<PutRecordsRequestEntry> allRecords = newArrayList();
+ for (String row : partition) {
+ allRecords.add(new PutRecordsRequestEntry().
+ withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8))).
+ withPartitionKey(Integer.toString(row.hashCode()))
- for (List<String> partition : partitions) {
- List<PutRecordsRequestEntry> allRecords = newArrayList();
- for (String row : partition) {
- allRecords.add(new PutRecordsRequestEntry().
- withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8))).
- withPartitionKey(Integer.toString(row.hashCode()))
+ );
+ }
- );
- }
-
- PutRecordsResult result;
- do {
- result = client.putRecords(
- new PutRecordsRequest().
- withStreamName(options.getAwsKinesisStream()).
- withRecords(allRecords));
- List<PutRecordsRequestEntry> failedRecords = newArrayList();
- int i = 0;
- for (PutRecordsResultEntry row : result.getRecords()) {
- if (row.getErrorCode() != null) {
- failedRecords.add(allRecords.get(i));
- }
- ++i;
- }
- allRecords = failedRecords;
- }
-
- while (result.getFailedRecordCount() > 0);
+ PutRecordsResult result;
+ do {
+ result = client.putRecords(
+ new PutRecordsRequest().
+ withStreamName(options.getAwsKinesisStream()).
+ withRecords(allRecords));
+ List<PutRecordsRequestEntry> failedRecords = newArrayList();
+ int i = 0;
+ for (PutRecordsResultEntry row : result.getRecords()) {
+ if (row.getErrorCode() != null) {
+ failedRecords.add(allRecords.get(i));
+ }
+ ++i;
}
+ allRecords = failedRecords;
+ }
+
+ while (result.getFailedRecordCount() > 0);
}
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java
index f979c01..cb32562 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java
@@ -20,47 +20,49 @@
import static org.mockito.BDDMockito.given;
import com.google.common.collect.Lists;
+
import java.util.Collections;
import java.util.List;
+
import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
-
/***
*/
@RunWith(MockitoJUnitRunner.class)
public class RecordFilterTest {
- @Mock
- private ShardCheckpoint checkpoint;
- @Mock
- private KinesisRecord record1, record2, record3, record4, record5;
- @Test
- public void shouldFilterOutRecordsBeforeOrAtCheckpoint() {
- given(checkpoint.isBeforeOrAt(record1)).willReturn(false);
- given(checkpoint.isBeforeOrAt(record2)).willReturn(true);
- given(checkpoint.isBeforeOrAt(record3)).willReturn(true);
- given(checkpoint.isBeforeOrAt(record4)).willReturn(false);
- given(checkpoint.isBeforeOrAt(record5)).willReturn(true);
- List<KinesisRecord> records = Lists.newArrayList(record1, record2,
- record3, record4, record5);
- RecordFilter underTest = new RecordFilter();
+ @Mock
+ private ShardCheckpoint checkpoint;
+ @Mock
+ private KinesisRecord record1, record2, record3, record4, record5;
- List<KinesisRecord> retainedRecords = underTest.apply(records, checkpoint);
+ @Test
+ public void shouldFilterOutRecordsBeforeOrAtCheckpoint() {
+ given(checkpoint.isBeforeOrAt(record1)).willReturn(false);
+ given(checkpoint.isBeforeOrAt(record2)).willReturn(true);
+ given(checkpoint.isBeforeOrAt(record3)).willReturn(true);
+ given(checkpoint.isBeforeOrAt(record4)).willReturn(false);
+ given(checkpoint.isBeforeOrAt(record5)).willReturn(true);
+ List<KinesisRecord> records = Lists.newArrayList(record1, record2,
+ record3, record4, record5);
+ RecordFilter underTest = new RecordFilter();
- Assertions.assertThat(retainedRecords).containsOnly(record2, record3, record5);
- }
+ List<KinesisRecord> retainedRecords = underTest.apply(records, checkpoint);
- @Test
- public void shouldNotFailOnEmptyList() {
- List<KinesisRecord> records = Collections.emptyList();
- RecordFilter underTest = new RecordFilter();
+ Assertions.assertThat(retainedRecords).containsOnly(record2, record3, record5);
+ }
- List<KinesisRecord> retainedRecords = underTest.apply(records, checkpoint);
+ @Test
+ public void shouldNotFailOnEmptyList() {
+ List<KinesisRecord> records = Collections.emptyList();
+ RecordFilter underTest = new RecordFilter();
- Assertions.assertThat(retainedRecords).isEmpty();
- }
+ List<KinesisRecord> retainedRecords = underTest.apply(records, checkpoint);
+
+ Assertions.assertThat(retainedRecords).isEmpty();
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RoundRobinTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RoundRobinTest.java
deleted file mode 100644
index f032eea..0000000
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RoundRobinTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.kinesis;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.util.Collections;
-import java.util.List;
-import org.junit.Test;
-
-/**
- * Tests {@link RoundRobin}.
- */
-public class RoundRobinTest {
- @Test(expected = IllegalArgumentException.class)
- public void doesNotAllowCreationWithEmptyCollection() {
- new RoundRobin<>(Collections.emptyList());
- }
-
- @Test
- public void goesThroughElementsInCycle() {
- List<String> input = newArrayList("a", "b", "c");
-
- RoundRobin<String> roundRobin = new RoundRobin<>(newArrayList(input));
-
- input.addAll(input); // duplicate the input
- for (String element : input) {
- assertThat(roundRobin.getCurrent()).isEqualTo(element);
- assertThat(roundRobin.getCurrent()).isEqualTo(element);
- roundRobin.moveForward();
- }
- }
-
- @Test
- public void usualIteratorGoesThroughElementsOnce() {
- List<String> input = newArrayList("a", "b", "c");
-
- RoundRobin<String> roundRobin = new RoundRobin<>(input);
- assertThat(roundRobin).hasSize(3).containsOnly(input.toArray(new String[0]));
- }
-}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardCheckpointTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardCheckpointTest.java
index 39ab36f..d4784c4 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardCheckpointTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardCheckpointTest.java
@@ -32,7 +32,9 @@
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
+
import java.io.IOException;
+
import org.joda.time.DateTime;
import org.joda.time.Instant;
import org.junit.Before;
@@ -46,104 +48,105 @@
*/
@RunWith(MockitoJUnitRunner.class)
public class ShardCheckpointTest {
- private static final String AT_SEQUENCE_SHARD_IT = "AT_SEQUENCE_SHARD_IT";
- private static final String AFTER_SEQUENCE_SHARD_IT = "AFTER_SEQUENCE_SHARD_IT";
- private static final String STREAM_NAME = "STREAM";
- private static final String SHARD_ID = "SHARD_ID";
- @Mock
- private SimplifiedKinesisClient client;
- @Before
- public void setUp() throws IOException, TransientKinesisException {
- when(client.getShardIterator(
- eq(STREAM_NAME), eq(SHARD_ID), eq(AT_SEQUENCE_NUMBER),
- anyString(), isNull(Instant.class))).
- thenReturn(AT_SEQUENCE_SHARD_IT);
- when(client.getShardIterator(
- eq(STREAM_NAME), eq(SHARD_ID), eq(AFTER_SEQUENCE_NUMBER),
- anyString(), isNull(Instant.class))).
- thenReturn(AFTER_SEQUENCE_SHARD_IT);
- }
+ private static final String AT_SEQUENCE_SHARD_IT = "AT_SEQUENCE_SHARD_IT";
+ private static final String AFTER_SEQUENCE_SHARD_IT = "AFTER_SEQUENCE_SHARD_IT";
+ private static final String STREAM_NAME = "STREAM";
+ private static final String SHARD_ID = "SHARD_ID";
+ @Mock
+ private SimplifiedKinesisClient client;
- @Test
- public void testProvidingShardIterator() throws IOException, TransientKinesisException {
- assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", null).getShardIterator(client))
- .isEqualTo(AT_SEQUENCE_SHARD_IT);
- assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", null).getShardIterator(client))
- .isEqualTo(AFTER_SEQUENCE_SHARD_IT);
- assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 10L).getShardIterator(client)).isEqualTo
- (AT_SEQUENCE_SHARD_IT);
- assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 10L).getShardIterator(client))
- .isEqualTo(AT_SEQUENCE_SHARD_IT);
- }
+ @Before
+ public void setUp() throws IOException, TransientKinesisException {
+ when(client.getShardIterator(
+ eq(STREAM_NAME), eq(SHARD_ID), eq(AT_SEQUENCE_NUMBER),
+ anyString(), isNull(Instant.class))).
+ thenReturn(AT_SEQUENCE_SHARD_IT);
+ when(client.getShardIterator(
+ eq(STREAM_NAME), eq(SHARD_ID), eq(AFTER_SEQUENCE_NUMBER),
+ anyString(), isNull(Instant.class))).
+ thenReturn(AFTER_SEQUENCE_SHARD_IT);
+ }
- @Test
- public void testComparisonWithExtendedSequenceNumber() {
- assertThat(new ShardCheckpoint("", "", new StartingPoint(LATEST)).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isTrue();
+ @Test
+ public void testProvidingShardIterator() throws IOException, TransientKinesisException {
+ assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", null).getShardIterator(client))
+ .isEqualTo(AT_SEQUENCE_SHARD_IT);
+ assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", null).getShardIterator(client))
+ .isEqualTo(AFTER_SEQUENCE_SHARD_IT);
+ assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 10L).getShardIterator(client)).isEqualTo
+ (AT_SEQUENCE_SHARD_IT);
+ assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 10L).getShardIterator(client))
+ .isEqualTo(AT_SEQUENCE_SHARD_IT);
+ }
- assertThat(new ShardCheckpoint("", "", new StartingPoint(TRIM_HORIZON)).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isTrue();
+ @Test
+ public void testComparisonWithExtendedSequenceNumber() {
+ assertThat(new ShardCheckpoint("", "", new StartingPoint(LATEST)).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isTrue();
- assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "10", 1L).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isTrue();
+ assertThat(new ShardCheckpoint("", "", new StartingPoint(TRIM_HORIZON)).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isTrue();
- assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isTrue();
+ assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "10", 1L).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isTrue();
- assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isFalse();
+ assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isTrue();
- assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 1L).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("100", 0L))
- )).isFalse();
+ assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isFalse();
- assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
- recordWith(new ExtendedSequenceNumber("99", 1L))
- )).isFalse();
- }
+ assertThat(checkpoint(AT_SEQUENCE_NUMBER, "100", 1L).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("100", 0L))
+ )).isFalse();
- @Test
- public void testComparisonWithTimestamp() {
- DateTime referenceTimestamp = DateTime.now();
+ assertThat(checkpoint(AFTER_SEQUENCE_NUMBER, "100", 0L).isBeforeOrAt(
+ recordWith(new ExtendedSequenceNumber("99", 1L))
+ )).isFalse();
+ }
- assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
- .isBeforeOrAt(recordWith(referenceTimestamp.minusMillis(10).toInstant()))
- ).isFalse();
+ @Test
+ public void testComparisonWithTimestamp() {
+ DateTime referenceTimestamp = DateTime.now();
- assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
- .isBeforeOrAt(recordWith(referenceTimestamp.toInstant()))
- ).isTrue();
+ assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
+ .isBeforeOrAt(recordWith(referenceTimestamp.minusMillis(10).toInstant()))
+ ).isFalse();
- assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
- .isBeforeOrAt(recordWith(referenceTimestamp.plusMillis(10).toInstant()))
- ).isTrue();
- }
+ assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
+ .isBeforeOrAt(recordWith(referenceTimestamp.toInstant()))
+ ).isTrue();
- private KinesisRecord recordWith(ExtendedSequenceNumber extendedSequenceNumber) {
- KinesisRecord record = mock(KinesisRecord.class);
- given(record.getExtendedSequenceNumber()).willReturn(extendedSequenceNumber);
- return record;
- }
+ assertThat(checkpoint(AT_TIMESTAMP, referenceTimestamp.toInstant())
+ .isBeforeOrAt(recordWith(referenceTimestamp.plusMillis(10).toInstant()))
+ ).isTrue();
+ }
- private ShardCheckpoint checkpoint(ShardIteratorType iteratorType, String sequenceNumber,
- Long subSequenceNumber) {
- return new ShardCheckpoint(STREAM_NAME, SHARD_ID, iteratorType, sequenceNumber,
- subSequenceNumber);
- }
+ private KinesisRecord recordWith(ExtendedSequenceNumber extendedSequenceNumber) {
+ KinesisRecord record = mock(KinesisRecord.class);
+ given(record.getExtendedSequenceNumber()).willReturn(extendedSequenceNumber);
+ return record;
+ }
- private KinesisRecord recordWith(Instant approximateArrivalTimestamp) {
- KinesisRecord record = mock(KinesisRecord.class);
- given(record.getApproximateArrivalTimestamp()).willReturn(approximateArrivalTimestamp);
- return record;
- }
+ private ShardCheckpoint checkpoint(ShardIteratorType iteratorType, String sequenceNumber,
+ Long subSequenceNumber) {
+ return new ShardCheckpoint(STREAM_NAME, SHARD_ID, iteratorType, sequenceNumber,
+ subSequenceNumber);
+ }
- private ShardCheckpoint checkpoint(ShardIteratorType iteratorType, Instant timestamp) {
- return new ShardCheckpoint(STREAM_NAME, SHARD_ID, iteratorType, timestamp);
- }
+ private KinesisRecord recordWith(Instant approximateArrivalTimestamp) {
+ KinesisRecord record = mock(KinesisRecord.class);
+ given(record.getApproximateArrivalTimestamp()).willReturn(approximateArrivalTimestamp);
+ return record;
+ }
+
+ private ShardCheckpoint checkpoint(ShardIteratorType iteratorType, Instant timestamp) {
+ return new ShardCheckpoint(STREAM_NAME, SHARD_ID, iteratorType, timestamp);
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java
new file mode 100644
index 0000000..03cc428
--- /dev/null
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.kinesis;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.base.Stopwatch;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Tests {@link ShardReadersPool}.
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class ShardReadersPoolTest {
+
+ @Mock
+ private ShardRecordsIterator firstIterator, secondIterator;
+ @Mock
+ private ShardCheckpoint firstCheckpoint, secondCheckpoint;
+ @Mock
+ private SimplifiedKinesisClient kinesis;
+ @Mock
+ private KinesisRecord a, b, c, d;
+
+ private ShardReadersPool shardReadersPool;
+
+ @Before
+ public void setUp() throws TransientKinesisException {
+ when(a.getShardId()).thenReturn("shard1");
+ when(b.getShardId()).thenReturn("shard1");
+ when(c.getShardId()).thenReturn("shard2");
+ when(d.getShardId()).thenReturn("shard2");
+ when(firstCheckpoint.getShardId()).thenReturn("shard1");
+ when(secondCheckpoint.getShardId()).thenReturn("shard2");
+ KinesisReaderCheckpoint checkpoint = new KinesisReaderCheckpoint(
+ Arrays.asList(firstCheckpoint, secondCheckpoint));
+ shardReadersPool = Mockito.spy(new ShardReadersPool(kinesis, checkpoint));
+ doReturn(firstIterator).when(shardReadersPool).createShardIterator(kinesis, firstCheckpoint);
+ doReturn(secondIterator).when(shardReadersPool).createShardIterator(kinesis, secondCheckpoint);
+ }
+
+ @Test
+ public void shouldReturnAllRecords() throws TransientKinesisException {
+ when(firstIterator.readNextBatch())
+ .thenReturn(Collections.<KinesisRecord>emptyList())
+ .thenReturn(asList(a, b))
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+ when(secondIterator.readNextBatch())
+ .thenReturn(singletonList(c))
+ .thenReturn(singletonList(d))
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+
+ shardReadersPool.start();
+ List<KinesisRecord> fetchedRecords = new ArrayList<>();
+ while (fetchedRecords.size() < 4) {
+ CustomOptional<KinesisRecord> nextRecord = shardReadersPool.nextRecord();
+ if (nextRecord.isPresent()) {
+ fetchedRecords.add(nextRecord.get());
+ }
+ }
+ assertThat(fetchedRecords).containsExactlyInAnyOrder(a, b, c, d);
+ }
+
+ @Test
+ public void shouldReturnAbsentOptionalWhenNoRecords() throws TransientKinesisException {
+ when(firstIterator.readNextBatch())
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+ when(secondIterator.readNextBatch())
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+
+ shardReadersPool.start();
+ CustomOptional<KinesisRecord> nextRecord = shardReadersPool.nextRecord();
+ assertThat(nextRecord.isPresent()).isFalse();
+ }
+
+ @Test
+ public void shouldCheckpointReadRecords() throws TransientKinesisException {
+ when(firstIterator.readNextBatch())
+ .thenReturn(asList(a, b))
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+ when(secondIterator.readNextBatch())
+ .thenReturn(singletonList(c))
+ .thenReturn(singletonList(d))
+ .thenReturn(Collections.<KinesisRecord>emptyList());
+
+ shardReadersPool.start();
+ int recordsFound = 0;
+ while (recordsFound < 4) {
+ CustomOptional<KinesisRecord> nextRecord = shardReadersPool.nextRecord();
+ if (nextRecord.isPresent()) {
+ recordsFound++;
+ KinesisRecord kinesisRecord = nextRecord.get();
+ if (kinesisRecord.getShardId().equals("shard1")) {
+ verify(firstIterator).ackRecord(kinesisRecord);
+ } else {
+ verify(secondIterator).ackRecord(kinesisRecord);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void shouldInterruptKinesisReadingAndStopShortly() throws TransientKinesisException {
+ when(firstIterator.readNextBatch()).thenAnswer(new Answer<List<KinesisRecord>>() {
+
+ @Override
+ public List<KinesisRecord> answer(InvocationOnMock invocation) throws Throwable {
+ Thread.sleep(TimeUnit.MINUTES.toMillis(1));
+ return Collections.emptyList();
+ }
+ });
+ shardReadersPool.start();
+
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ shardReadersPool.stop();
+ assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(TimeUnit.SECONDS.toMillis(1));
+ }
+
+ @Test
+ public void shouldInterruptPuttingRecordsToQueueAndStopShortly()
+ throws TransientKinesisException {
+ when(firstIterator.readNextBatch()).thenReturn(asList(a, b, c));
+ KinesisReaderCheckpoint checkpoint = new KinesisReaderCheckpoint(
+ Arrays.asList(firstCheckpoint, secondCheckpoint));
+ ShardReadersPool shardReadersPool = new ShardReadersPool(kinesis, checkpoint, 2);
+ shardReadersPool.start();
+
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ shardReadersPool.stop();
+ assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(TimeUnit.SECONDS.toMillis(1));
+
+ }
+
+ @Test
+ public void shouldDetectThatNotAllShardsAreUpToDate() throws TransientKinesisException {
+ when(firstIterator.isUpToDate()).thenReturn(true);
+ when(secondIterator.isUpToDate()).thenReturn(false);
+ shardReadersPool.start();
+
+ assertThat(shardReadersPool.allShardsUpToDate()).isFalse();
+ }
+
+ @Test
+ public void shouldDetectThatAllShardsAreUpToDate() throws TransientKinesisException {
+ when(firstIterator.isUpToDate()).thenReturn(true);
+ when(secondIterator.isUpToDate()).thenReturn(true);
+ shardReadersPool.start();
+
+ assertThat(shardReadersPool.allShardsUpToDate()).isTrue();
+ }
+}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIteratorTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIteratorTest.java
index 49e806d..a77eafa 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIteratorTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIteratorTest.java
@@ -25,8 +25,10 @@
import static org.mockito.Mockito.when;
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
+
import java.io.IOException;
import java.util.Collections;
+
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -40,112 +42,119 @@
*/
@RunWith(MockitoJUnitRunner.class)
public class ShardRecordsIteratorTest {
- private static final String INITIAL_ITERATOR = "INITIAL_ITERATOR";
- private static final String SECOND_ITERATOR = "SECOND_ITERATOR";
- private static final String SECOND_REFRESHED_ITERATOR = "SECOND_REFRESHED_ITERATOR";
- private static final String THIRD_ITERATOR = "THIRD_ITERATOR";
- private static final String STREAM_NAME = "STREAM_NAME";
- private static final String SHARD_ID = "SHARD_ID";
- @Mock
- private SimplifiedKinesisClient kinesisClient;
- @Mock
- private ShardCheckpoint firstCheckpoint, aCheckpoint, bCheckpoint, cCheckpoint, dCheckpoint;
- @Mock
- private GetKinesisRecordsResult firstResult, secondResult, thirdResult;
- @Mock
- private KinesisRecord a, b, c, d;
- @Mock
- private RecordFilter recordFilter;
+ private static final String INITIAL_ITERATOR = "INITIAL_ITERATOR";
+ private static final String SECOND_ITERATOR = "SECOND_ITERATOR";
+ private static final String SECOND_REFRESHED_ITERATOR = "SECOND_REFRESHED_ITERATOR";
+ private static final String THIRD_ITERATOR = "THIRD_ITERATOR";
+ private static final String STREAM_NAME = "STREAM_NAME";
+ private static final String SHARD_ID = "SHARD_ID";
- private ShardRecordsIterator iterator;
+ @Mock
+ private SimplifiedKinesisClient kinesisClient;
+ @Mock
+ private ShardCheckpoint firstCheckpoint, aCheckpoint, bCheckpoint, cCheckpoint, dCheckpoint;
+ @Mock
+ private GetKinesisRecordsResult firstResult, secondResult, thirdResult;
+ @Mock
+ private KinesisRecord a, b, c, d;
+ @Mock
+ private RecordFilter recordFilter;
- @Before
- public void setUp() throws IOException, TransientKinesisException {
- when(firstCheckpoint.getShardIterator(kinesisClient)).thenReturn(INITIAL_ITERATOR);
- when(firstCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
- when(firstCheckpoint.getShardId()).thenReturn(SHARD_ID);
+ private ShardRecordsIterator iterator;
- when(firstCheckpoint.moveAfter(a)).thenReturn(aCheckpoint);
- when(aCheckpoint.moveAfter(b)).thenReturn(bCheckpoint);
- when(aCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
- when(aCheckpoint.getShardId()).thenReturn(SHARD_ID);
- when(bCheckpoint.moveAfter(c)).thenReturn(cCheckpoint);
- when(bCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
- when(bCheckpoint.getShardId()).thenReturn(SHARD_ID);
- when(cCheckpoint.moveAfter(d)).thenReturn(dCheckpoint);
- when(cCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
- when(cCheckpoint.getShardId()).thenReturn(SHARD_ID);
- when(dCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
- when(dCheckpoint.getShardId()).thenReturn(SHARD_ID);
+ @Before
+ public void setUp() throws IOException, TransientKinesisException {
+ when(firstCheckpoint.getShardIterator(kinesisClient)).thenReturn(INITIAL_ITERATOR);
+ when(firstCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
+ when(firstCheckpoint.getShardId()).thenReturn(SHARD_ID);
- when(kinesisClient.getRecords(INITIAL_ITERATOR, STREAM_NAME, SHARD_ID))
- .thenReturn(firstResult);
- when(kinesisClient.getRecords(SECOND_ITERATOR, STREAM_NAME, SHARD_ID))
- .thenReturn(secondResult);
- when(kinesisClient.getRecords(THIRD_ITERATOR, STREAM_NAME, SHARD_ID))
- .thenReturn(thirdResult);
+ when(firstCheckpoint.moveAfter(a)).thenReturn(aCheckpoint);
+ when(aCheckpoint.moveAfter(b)).thenReturn(bCheckpoint);
+ when(aCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
+ when(aCheckpoint.getShardId()).thenReturn(SHARD_ID);
+ when(bCheckpoint.moveAfter(c)).thenReturn(cCheckpoint);
+ when(bCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
+ when(bCheckpoint.getShardId()).thenReturn(SHARD_ID);
+ when(cCheckpoint.moveAfter(d)).thenReturn(dCheckpoint);
+ when(cCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
+ when(cCheckpoint.getShardId()).thenReturn(SHARD_ID);
+ when(dCheckpoint.getStreamName()).thenReturn(STREAM_NAME);
+ when(dCheckpoint.getShardId()).thenReturn(SHARD_ID);
- when(firstResult.getNextShardIterator()).thenReturn(SECOND_ITERATOR);
- when(secondResult.getNextShardIterator()).thenReturn(THIRD_ITERATOR);
- when(thirdResult.getNextShardIterator()).thenReturn(THIRD_ITERATOR);
+ when(kinesisClient.getRecords(INITIAL_ITERATOR, STREAM_NAME, SHARD_ID))
+ .thenReturn(firstResult);
+ when(kinesisClient.getRecords(SECOND_ITERATOR, STREAM_NAME, SHARD_ID))
+ .thenReturn(secondResult);
+ when(kinesisClient.getRecords(THIRD_ITERATOR, STREAM_NAME, SHARD_ID))
+ .thenReturn(thirdResult);
- when(firstResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
- when(secondResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
- when(thirdResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
+ when(firstResult.getNextShardIterator()).thenReturn(SECOND_ITERATOR);
+ when(secondResult.getNextShardIterator()).thenReturn(THIRD_ITERATOR);
+ when(thirdResult.getNextShardIterator()).thenReturn(THIRD_ITERATOR);
- when(recordFilter.apply(anyListOf(KinesisRecord.class), any(ShardCheckpoint
- .class))).thenAnswer(new IdentityAnswer());
+ when(firstResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
+ when(secondResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
+ when(thirdResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
- iterator = new ShardRecordsIterator(firstCheckpoint, kinesisClient, recordFilter);
+ when(recordFilter.apply(anyListOf(KinesisRecord.class), any(ShardCheckpoint
+ .class))).thenAnswer(new IdentityAnswer());
+
+ iterator = new ShardRecordsIterator(firstCheckpoint, kinesisClient, recordFilter);
+ }
+
+ @Test
+ public void goesThroughAvailableRecords() throws IOException, TransientKinesisException {
+ when(firstResult.getRecords()).thenReturn(asList(a, b, c));
+ when(secondResult.getRecords()).thenReturn(singletonList(d));
+ when(thirdResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
+
+ assertThat(iterator.getCheckpoint()).isEqualTo(firstCheckpoint);
+ assertThat(iterator.readNextBatch()).isEqualTo(asList(a, b, c));
+ assertThat(iterator.readNextBatch()).isEqualTo(singletonList(d));
+ assertThat(iterator.readNextBatch()).isEqualTo(Collections.emptyList());
+
+ }
+
+ @Test
+ public void conformingRecordsMovesCheckpoint() throws IOException, TransientKinesisException {
+ when(firstResult.getRecords()).thenReturn(asList(a, b, c));
+ when(secondResult.getRecords()).thenReturn(singletonList(d));
+ when(thirdResult.getRecords()).thenReturn(Collections.<KinesisRecord>emptyList());
+
+ iterator.ackRecord(a);
+ assertThat(iterator.getCheckpoint()).isEqualTo(aCheckpoint);
+ iterator.ackRecord(b);
+ assertThat(iterator.getCheckpoint()).isEqualTo(bCheckpoint);
+ iterator.ackRecord(c);
+ assertThat(iterator.getCheckpoint()).isEqualTo(cCheckpoint);
+ iterator.ackRecord(d);
+ assertThat(iterator.getCheckpoint()).isEqualTo(dCheckpoint);
+ }
+
+ @Test
+ public void refreshesExpiredIterator() throws IOException, TransientKinesisException {
+ when(firstResult.getRecords()).thenReturn(singletonList(a));
+ when(secondResult.getRecords()).thenReturn(singletonList(b));
+
+ when(kinesisClient.getRecords(SECOND_ITERATOR, STREAM_NAME, SHARD_ID))
+ .thenThrow(ExpiredIteratorException.class);
+ when(aCheckpoint.getShardIterator(kinesisClient))
+ .thenReturn(SECOND_REFRESHED_ITERATOR);
+ when(kinesisClient.getRecords(SECOND_REFRESHED_ITERATOR, STREAM_NAME, SHARD_ID))
+ .thenReturn(secondResult);
+
+ assertThat(iterator.readNextBatch()).isEqualTo(singletonList(a));
+ iterator.ackRecord(a);
+ assertThat(iterator.readNextBatch()).isEqualTo(singletonList(b));
+ assertThat(iterator.readNextBatch()).isEqualTo(Collections.emptyList());
+ }
+
+ private static class IdentityAnswer implements Answer<Object> {
+
+ @Override
+ public Object answer(InvocationOnMock invocation) throws Throwable {
+ return invocation.getArguments()[0];
}
-
- @Test
- public void returnsAbsentIfNoRecordsPresent() throws IOException, TransientKinesisException {
- assertThat(iterator.next()).isEqualTo(CustomOptional.absent());
- assertThat(iterator.next()).isEqualTo(CustomOptional.absent());
- assertThat(iterator.next()).isEqualTo(CustomOptional.absent());
- }
-
- @Test
- public void goesThroughAvailableRecords() throws IOException, TransientKinesisException {
- when(firstResult.getRecords()).thenReturn(asList(a, b, c));
- when(secondResult.getRecords()).thenReturn(singletonList(d));
-
- assertThat(iterator.getCheckpoint()).isEqualTo(firstCheckpoint);
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(a));
- assertThat(iterator.getCheckpoint()).isEqualTo(aCheckpoint);
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(b));
- assertThat(iterator.getCheckpoint()).isEqualTo(bCheckpoint);
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(c));
- assertThat(iterator.getCheckpoint()).isEqualTo(cCheckpoint);
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(d));
- assertThat(iterator.getCheckpoint()).isEqualTo(dCheckpoint);
- assertThat(iterator.next()).isEqualTo(CustomOptional.absent());
- assertThat(iterator.getCheckpoint()).isEqualTo(dCheckpoint);
- }
-
- @Test
- public void refreshesExpiredIterator() throws IOException, TransientKinesisException {
- when(firstResult.getRecords()).thenReturn(singletonList(a));
- when(secondResult.getRecords()).thenReturn(singletonList(b));
-
- when(kinesisClient.getRecords(SECOND_ITERATOR, STREAM_NAME, SHARD_ID))
- .thenThrow(ExpiredIteratorException.class);
- when(aCheckpoint.getShardIterator(kinesisClient))
- .thenReturn(SECOND_REFRESHED_ITERATOR);
- when(kinesisClient.getRecords(SECOND_REFRESHED_ITERATOR, STREAM_NAME, SHARD_ID))
- .thenReturn(secondResult);
-
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(a));
- assertThat(iterator.next()).isEqualTo(CustomOptional.of(b));
- assertThat(iterator.next()).isEqualTo(CustomOptional.absent());
- }
-
- private static class IdentityAnswer implements Answer<Object> {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- return invocation.getArguments()[0];
- }
- }
+ }
}
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClientTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClientTest.java
index 96434fd..75c0ae0 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClientTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClientTest.java
@@ -21,9 +21,14 @@
import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verifyZeroInteractions;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.AmazonServiceException.ErrorType;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
+import com.amazonaws.services.cloudwatch.model.Datapoint;
+import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsRequest;
+import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsResult;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.model.DescribeStreamResult;
import com.amazonaws.services.kinesis.model.ExpiredIteratorException;
@@ -34,8 +39,11 @@
import com.amazonaws.services.kinesis.model.Shard;
import com.amazonaws.services.kinesis.model.ShardIteratorType;
import com.amazonaws.services.kinesis.model.StreamDescription;
+
import java.util.List;
+
import org.joda.time.Instant;
+import org.joda.time.Minutes;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
@@ -46,179 +54,281 @@
*/
@RunWith(MockitoJUnitRunner.class)
public class SimplifiedKinesisClientTest {
- private static final String STREAM = "stream";
- private static final String SHARD_1 = "shard-01";
- private static final String SHARD_2 = "shard-02";
- private static final String SHARD_3 = "shard-03";
- private static final String SHARD_ITERATOR = "iterator";
- private static final String SEQUENCE_NUMBER = "abc123";
- @Mock
- private AmazonKinesis kinesis;
- @InjectMocks
- private SimplifiedKinesisClient underTest;
+ private static final String STREAM = "stream";
+ private static final String SHARD_1 = "shard-01";
+ private static final String SHARD_2 = "shard-02";
+ private static final String SHARD_3 = "shard-03";
+ private static final String SHARD_ITERATOR = "iterator";
+ private static final String SEQUENCE_NUMBER = "abc123";
- @Test
- public void shouldReturnIteratorStartingWithSequenceNumber() throws Exception {
- given(kinesis.getShardIterator(new GetShardIteratorRequest()
- .withStreamName(STREAM)
- .withShardId(SHARD_1)
- .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
- .withStartingSequenceNumber(SEQUENCE_NUMBER)
- )).willReturn(new GetShardIteratorResult()
- .withShardIterator(SHARD_ITERATOR));
+ @Mock
+ private AmazonKinesis kinesis;
+ @Mock
+ private AmazonCloudWatch cloudWatch;
+ @InjectMocks
+ private SimplifiedKinesisClient underTest;
- String stream = underTest.getShardIterator(STREAM, SHARD_1,
- ShardIteratorType.AT_SEQUENCE_NUMBER, SEQUENCE_NUMBER, null);
+ @Test
+ public void shouldReturnIteratorStartingWithSequenceNumber() throws Exception {
+ given(kinesis.getShardIterator(new GetShardIteratorRequest()
+ .withStreamName(STREAM)
+ .withShardId(SHARD_1)
+ .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
+ .withStartingSequenceNumber(SEQUENCE_NUMBER)
+ )).willReturn(new GetShardIteratorResult()
+ .withShardIterator(SHARD_ITERATOR));
- assertThat(stream).isEqualTo(SHARD_ITERATOR);
+ String stream = underTest.getShardIterator(STREAM, SHARD_1,
+ ShardIteratorType.AT_SEQUENCE_NUMBER, SEQUENCE_NUMBER, null);
+
+ assertThat(stream).isEqualTo(SHARD_ITERATOR);
+ }
+
+ @Test
+ public void shouldReturnIteratorStartingWithTimestamp() throws Exception {
+ Instant timestamp = Instant.now();
+ given(kinesis.getShardIterator(new GetShardIteratorRequest()
+ .withStreamName(STREAM)
+ .withShardId(SHARD_1)
+ .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
+ .withTimestamp(timestamp.toDate())
+ )).willReturn(new GetShardIteratorResult()
+ .withShardIterator(SHARD_ITERATOR));
+
+ String stream = underTest.getShardIterator(STREAM, SHARD_1,
+ ShardIteratorType.AT_SEQUENCE_NUMBER, null, timestamp);
+
+ assertThat(stream).isEqualTo(SHARD_ITERATOR);
+ }
+
+ @Test
+ public void shouldHandleExpiredIterationExceptionForGetShardIterator() {
+ shouldHandleGetShardIteratorError(new ExpiredIteratorException(""),
+ ExpiredIteratorException.class);
+ }
+
+ @Test
+ public void shouldHandleLimitExceededExceptionForGetShardIterator() {
+ shouldHandleGetShardIteratorError(new LimitExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleProvisionedThroughputExceededExceptionForGetShardIterator() {
+ shouldHandleGetShardIteratorError(new ProvisionedThroughputExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleServiceErrorForGetShardIterator() {
+ shouldHandleGetShardIteratorError(newAmazonServiceException(ErrorType.Service),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleClientErrorForGetShardIterator() {
+ shouldHandleGetShardIteratorError(newAmazonServiceException(ErrorType.Client),
+ RuntimeException.class);
+ }
+
+ @Test
+ public void shouldHandleUnexpectedExceptionForGetShardIterator() {
+ shouldHandleGetShardIteratorError(new NullPointerException(),
+ RuntimeException.class);
+ }
+
+ private void shouldHandleGetShardIteratorError(
+ Exception thrownException,
+ Class<? extends Exception> expectedExceptionClass) {
+ GetShardIteratorRequest request = new GetShardIteratorRequest()
+ .withStreamName(STREAM)
+ .withShardId(SHARD_1)
+ .withShardIteratorType(ShardIteratorType.LATEST);
+
+ given(kinesis.getShardIterator(request)).willThrow(thrownException);
+
+ try {
+ underTest.getShardIterator(STREAM, SHARD_1, ShardIteratorType.LATEST, null, null);
+ failBecauseExceptionWasNotThrown(expectedExceptionClass);
+ } catch (Exception e) {
+ assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
+ } finally {
+ reset(kinesis);
}
+ }
- @Test
- public void shouldReturnIteratorStartingWithTimestamp() throws Exception {
- Instant timestamp = Instant.now();
- given(kinesis.getShardIterator(new GetShardIteratorRequest()
- .withStreamName(STREAM)
- .withShardId(SHARD_1)
- .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
- .withTimestamp(timestamp.toDate())
- )).willReturn(new GetShardIteratorResult()
- .withShardIterator(SHARD_ITERATOR));
+ @Test
+ public void shouldListAllShards() throws Exception {
+ Shard shard1 = new Shard().withShardId(SHARD_1);
+ Shard shard2 = new Shard().withShardId(SHARD_2);
+ Shard shard3 = new Shard().withShardId(SHARD_3);
+ given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult()
+ .withStreamDescription(new StreamDescription()
+ .withShards(shard1, shard2)
+ .withHasMoreShards(true)));
+ given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult()
+ .withStreamDescription(new StreamDescription()
+ .withShards(shard3)
+ .withHasMoreShards(false)));
- String stream = underTest.getShardIterator(STREAM, SHARD_1,
- ShardIteratorType.AT_SEQUENCE_NUMBER, null, timestamp);
+ List<Shard> shards = underTest.listShards(STREAM);
- assertThat(stream).isEqualTo(SHARD_ITERATOR);
+ assertThat(shards).containsOnly(shard1, shard2, shard3);
+ }
+
+ @Test
+ public void shouldHandleExpiredIterationExceptionForShardListing() {
+ shouldHandleShardListingError(new ExpiredIteratorException(""),
+ ExpiredIteratorException.class);
+ }
+
+ @Test
+ public void shouldHandleLimitExceededExceptionForShardListing() {
+ shouldHandleShardListingError(new LimitExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleProvisionedThroughputExceededExceptionForShardListing() {
+ shouldHandleShardListingError(new ProvisionedThroughputExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleServiceErrorForShardListing() {
+ shouldHandleShardListingError(newAmazonServiceException(ErrorType.Service),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleClientErrorForShardListing() {
+ shouldHandleShardListingError(newAmazonServiceException(ErrorType.Client),
+ RuntimeException.class);
+ }
+
+ @Test
+ public void shouldHandleUnexpectedExceptionForShardListing() {
+ shouldHandleShardListingError(new NullPointerException(),
+ RuntimeException.class);
+ }
+
+ private void shouldHandleShardListingError(
+ Exception thrownException,
+ Class<? extends Exception> expectedExceptionClass) {
+ given(kinesis.describeStream(STREAM, null)).willThrow(thrownException);
+ try {
+ underTest.listShards(STREAM);
+ failBecauseExceptionWasNotThrown(expectedExceptionClass);
+ } catch (Exception e) {
+ assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
+ } finally {
+ reset(kinesis);
}
+ }
- @Test
- public void shouldHandleExpiredIterationExceptionForGetShardIterator() {
- shouldHandleGetShardIteratorError(new ExpiredIteratorException(""),
- ExpiredIteratorException.class);
+ @Test
+ public void shouldCountBytesWhenSingleDataPointReturned() throws Exception {
+ Instant countSince = new Instant("2017-04-06T10:00:00.000Z");
+ Instant countTo = new Instant("2017-04-06T11:00:00.000Z");
+ Minutes periodTime = Minutes.minutesBetween(countSince, countTo);
+ GetMetricStatisticsRequest metricStatisticsRequest =
+ underTest.createMetricStatisticsRequest(STREAM, countSince, countTo, periodTime);
+ GetMetricStatisticsResult result = new GetMetricStatisticsResult()
+ .withDatapoints(new Datapoint().withSum(1.0));
+
+ given(cloudWatch.getMetricStatistics(metricStatisticsRequest)).willReturn(result);
+
+ long backlogBytes = underTest.getBacklogBytes(STREAM, countSince, countTo);
+
+ assertThat(backlogBytes).isEqualTo(1L);
+ }
+
+ @Test
+ public void shouldCountBytesWhenMultipleDataPointsReturned() throws Exception {
+ Instant countSince = new Instant("2017-04-06T10:00:00.000Z");
+ Instant countTo = new Instant("2017-04-06T11:00:00.000Z");
+ Minutes periodTime = Minutes.minutesBetween(countSince, countTo);
+ GetMetricStatisticsRequest metricStatisticsRequest =
+ underTest.createMetricStatisticsRequest(STREAM, countSince, countTo, periodTime);
+ GetMetricStatisticsResult result = new GetMetricStatisticsResult()
+ .withDatapoints(
+ new Datapoint().withSum(1.0),
+ new Datapoint().withSum(3.0),
+ new Datapoint().withSum(2.0)
+ );
+
+ given(cloudWatch.getMetricStatistics(metricStatisticsRequest)).willReturn(result);
+
+ long backlogBytes = underTest.getBacklogBytes(STREAM, countSince, countTo);
+
+ assertThat(backlogBytes).isEqualTo(6L);
+ }
+
+ @Test
+ public void shouldNotCallCloudWatchWhenSpecifiedPeriodTooShort() throws Exception {
+ Instant countSince = new Instant("2017-04-06T10:00:00.000Z");
+ Instant countTo = new Instant("2017-04-06T10:00:02.000Z");
+
+ long backlogBytes = underTest.getBacklogBytes(STREAM, countSince, countTo);
+
+ assertThat(backlogBytes).isEqualTo(0L);
+ verifyZeroInteractions(cloudWatch);
+ }
+
+ @Test
+ public void shouldHandleLimitExceededExceptionForGetBacklogBytes() {
+ shouldHandleGetBacklogBytesError(new LimitExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleProvisionedThroughputExceededExceptionForGetBacklogBytes() {
+ shouldHandleGetBacklogBytesError(new ProvisionedThroughputExceededException(""),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleServiceErrorForGetBacklogBytes() {
+ shouldHandleGetBacklogBytesError(newAmazonServiceException(ErrorType.Service),
+ TransientKinesisException.class);
+ }
+
+ @Test
+ public void shouldHandleClientErrorForGetBacklogBytes() {
+ shouldHandleGetBacklogBytesError(newAmazonServiceException(ErrorType.Client),
+ RuntimeException.class);
+ }
+
+ @Test
+ public void shouldHandleUnexpectedExceptionForGetBacklogBytes() {
+ shouldHandleGetBacklogBytesError(new NullPointerException(),
+ RuntimeException.class);
+ }
+
+ private void shouldHandleGetBacklogBytesError(
+ Exception thrownException,
+ Class<? extends Exception> expectedExceptionClass) {
+ Instant countSince = new Instant("2017-04-06T10:00:00.000Z");
+ Instant countTo = new Instant("2017-04-06T11:00:00.000Z");
+ Minutes periodTime = Minutes.minutesBetween(countSince, countTo);
+ GetMetricStatisticsRequest metricStatisticsRequest =
+ underTest.createMetricStatisticsRequest(STREAM, countSince, countTo, periodTime);
+
+ given(cloudWatch.getMetricStatistics(metricStatisticsRequest)).willThrow(thrownException);
+ try {
+ underTest.getBacklogBytes(STREAM, countSince, countTo);
+ failBecauseExceptionWasNotThrown(expectedExceptionClass);
+ } catch (Exception e) {
+ assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
+ } finally {
+ reset(kinesis);
}
+ }
- @Test
- public void shouldHandleLimitExceededExceptionForGetShardIterator() {
- shouldHandleGetShardIteratorError(new LimitExceededException(""),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleProvisionedThroughputExceededExceptionForGetShardIterator() {
- shouldHandleGetShardIteratorError(new ProvisionedThroughputExceededException(""),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleServiceErrorForGetShardIterator() {
- shouldHandleGetShardIteratorError(newAmazonServiceException(ErrorType.Service),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleClientErrorForGetShardIterator() {
- shouldHandleGetShardIteratorError(newAmazonServiceException(ErrorType.Client),
- RuntimeException.class);
- }
-
- @Test
- public void shouldHandleUnexpectedExceptionForGetShardIterator() {
- shouldHandleGetShardIteratorError(new NullPointerException(),
- RuntimeException.class);
- }
-
- private void shouldHandleGetShardIteratorError(
- Exception thrownException,
- Class<? extends Exception> expectedExceptionClass) {
- GetShardIteratorRequest request = new GetShardIteratorRequest()
- .withStreamName(STREAM)
- .withShardId(SHARD_1)
- .withShardIteratorType(ShardIteratorType.LATEST);
-
- given(kinesis.getShardIterator(request)).willThrow(thrownException);
-
- try {
- underTest.getShardIterator(STREAM, SHARD_1, ShardIteratorType.LATEST, null, null);
- failBecauseExceptionWasNotThrown(expectedExceptionClass);
- } catch (Exception e) {
- assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
- } finally {
- reset(kinesis);
- }
- }
-
- @Test
- public void shouldListAllShards() throws Exception {
- Shard shard1 = new Shard().withShardId(SHARD_1);
- Shard shard2 = new Shard().withShardId(SHARD_2);
- Shard shard3 = new Shard().withShardId(SHARD_3);
- given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult()
- .withStreamDescription(new StreamDescription()
- .withShards(shard1, shard2)
- .withHasMoreShards(true)));
- given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult()
- .withStreamDescription(new StreamDescription()
- .withShards(shard3)
- .withHasMoreShards(false)));
-
- List<Shard> shards = underTest.listShards(STREAM);
-
- assertThat(shards).containsOnly(shard1, shard2, shard3);
- }
-
- @Test
- public void shouldHandleExpiredIterationExceptionForShardListing() {
- shouldHandleShardListingError(new ExpiredIteratorException(""),
- ExpiredIteratorException.class);
- }
-
- @Test
- public void shouldHandleLimitExceededExceptionForShardListing() {
- shouldHandleShardListingError(new LimitExceededException(""),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleProvisionedThroughputExceededExceptionForShardListing() {
- shouldHandleShardListingError(new ProvisionedThroughputExceededException(""),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleServiceErrorForShardListing() {
- shouldHandleShardListingError(newAmazonServiceException(ErrorType.Service),
- TransientKinesisException.class);
- }
-
- @Test
- public void shouldHandleClientErrorForShardListing() {
- shouldHandleShardListingError(newAmazonServiceException(ErrorType.Client),
- RuntimeException.class);
- }
-
- @Test
- public void shouldHandleUnexpectedExceptionForShardListing() {
- shouldHandleShardListingError(new NullPointerException(),
- RuntimeException.class);
- }
-
- private void shouldHandleShardListingError(
- Exception thrownException,
- Class<? extends Exception> expectedExceptionClass) {
- given(kinesis.describeStream(STREAM, null)).willThrow(thrownException);
- try {
- underTest.listShards(STREAM);
- failBecauseExceptionWasNotThrown(expectedExceptionClass);
- } catch (Exception e) {
- assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
- } finally {
- reset(kinesis);
- }
- }
-
- private AmazonServiceException newAmazonServiceException(ErrorType errorType) {
- AmazonServiceException exception = new AmazonServiceException("");
- exception.setErrorType(errorType);
- return exception;
- }
+ private AmazonServiceException newAmazonServiceException(ErrorType errorType) {
+ AmazonServiceException exception = new AmazonServiceException("");
+ exception.setErrorType(errorType);
+ return exception;
+ }
}
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index 912e20c..2504c59 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
index b63775d..4dc2405 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
@@ -117,7 +117,7 @@
* to the file separated with line feeds.
* </p>
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class MongoDbGridFSIO {
/**
@@ -431,16 +431,12 @@
}
@Override
- public void validate() {
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
spec.populateDisplayData(builder);
}
@Override
- public Coder<ObjectId> getDefaultOutputCoder() {
+ public Coder<ObjectId> getOutputCoder() {
return SerializableCoder.of(ObjectId.class);
}
diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
index 620df74..9007051 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
@@ -18,12 +18,12 @@
package org.apache.beam.sdk.io.mongodb;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
@@ -93,19 +93,27 @@
*
* }</pre>
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class MongoDbIO {
private static final Logger LOG = LoggerFactory.getLogger(MongoDbIO.class);
/** Read data from MongoDB. */
public static Read read() {
- return new AutoValue_MongoDbIO_Read.Builder().setNumSplits(0).build();
+ return new AutoValue_MongoDbIO_Read.Builder()
+ .setKeepAlive(true)
+ .setMaxConnectionIdleTime(60000)
+ .setNumSplits(0)
+ .build();
}
/** Write data to MongoDB. */
public static Write write() {
- return new AutoValue_MongoDbIO_Write.Builder().setBatchSize(1024L).build();
+ return new AutoValue_MongoDbIO_Write.Builder()
+ .setKeepAlive(true)
+ .setMaxConnectionIdleTime(60000)
+ .setBatchSize(1024L)
+ .build();
}
private MongoDbIO() {
@@ -117,16 +125,20 @@
@AutoValue
public abstract static class Read extends PTransform<PBegin, PCollection<Document>> {
@Nullable abstract String uri();
+ abstract boolean keepAlive();
+ abstract int maxConnectionIdleTime();
@Nullable abstract String database();
@Nullable abstract String collection();
@Nullable abstract String filter();
abstract int numSplits();
- abstract Builder toBuilder();
+ abstract Builder builder();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setUri(String uri);
+ abstract Builder setKeepAlive(boolean keepAlive);
+ abstract Builder setMaxConnectionIdleTime(int maxConnectionIdleTime);
abstract Builder setDatabase(String database);
abstract Builder setCollection(String collection);
abstract Builder setFilter(String filter);
@@ -135,49 +147,105 @@
}
/**
- * Example documentation for withUri.
+ * Define the location of the MongoDB instances using an URI. The URI describes the hosts to
+ * be used and some options.
+ *
+ * <p>The format of the URI is:
+ *
+ * <pre>{@code
+ * mongodb://[username:password@]host1[:port1]...[,hostN[:portN]]][/[database][?options]]
+ * }</pre>
+ *
+ * <p>Where:
+ * <ul>
+ * <li>{@code mongodb://} is a required prefix to identify that this is a string in the
+ * standard connection format.</li>
+ * <li>{@code username:password@} are optional. If given, the driver will attempt to
+ * login to a database after connecting to a database server. For some authentication
+ * mechanisms, only the username is specified and the password is not, in which case
+ * the ":" after the username is left off as well.</li>
+ * <li>{@code host1} is the only required part of the URI. It identifies a server
+ * address to connect to.</li>
+ * <li>{@code :portX} is optional and defaults to {@code :27017} if not provided.</li>
+ * <li>{@code /database} is the name of the database to login to and thus is only
+ * relevant if the {@code username:password@} syntax is used. If not specified, the
+ * "admin" database will be used by default. It has to be equivalent with the database
+ * you specific with {@link Read#withDatabase(String)}.</li>
+ * <li>{@code ?options} are connection options. Note that if {@code database} is absent
+ * there is still a {@code /} required between the last {@code host} and the {@code ?}
+ * introducing the options. Options are name=value pairs and the pairs are separated by
+ * "{@code &}". The {@code KeepAlive} connection option can't be passed via the URI,
+ * instead you have to use {@link Read#withKeepAlive(boolean)}. Same for the
+ * {@code MaxConnectionIdleTime} connection option via
+ * {@link Read#withMaxConnectionIdleTime(int)}.
+ * </li>
+ * </ul>
*/
public Read withUri(String uri) {
- checkNotNull(uri);
- return toBuilder().setUri(uri).build();
+ checkArgument(uri != null, "MongoDbIO.read().withUri(uri) called with null uri");
+ return builder().setUri(uri).build();
}
+ /**
+ * Sets whether socket keep alive is enabled.
+ */
+ public Read withKeepAlive(boolean keepAlive) {
+ return builder().setKeepAlive(keepAlive).build();
+ }
+
+ /**
+ * Sets the maximum idle time for a pooled connection.
+ */
+ public Read withMaxConnectionIdleTime(int maxConnectionIdleTime) {
+ return builder().setMaxConnectionIdleTime(maxConnectionIdleTime).build();
+ }
+
+ /**
+ * Sets the database to use.
+ */
public Read withDatabase(String database) {
- checkNotNull(database);
- return toBuilder().setDatabase(database).build();
+ checkArgument(database != null, "database can not be null");
+ return builder().setDatabase(database).build();
}
+ /**
+ * Sets the collection to consider in the database.
+ */
public Read withCollection(String collection) {
- checkNotNull(collection);
- return toBuilder().setCollection(collection).build();
+ checkArgument(collection != null, "collection can not be null");
+ return builder().setCollection(collection).build();
}
+ /**
+ * Sets a filter on the documents in a collection.
+ */
public Read withFilter(String filter) {
- checkNotNull(filter);
- return toBuilder().setFilter(filter).build();
+ checkArgument(filter != null, "filter can not be null");
+ return builder().setFilter(filter).build();
}
+ /**
+ * Sets the user defined number of splits.
+ */
public Read withNumSplits(int numSplits) {
- checkArgument(numSplits >= 0);
- return toBuilder().setNumSplits(numSplits).build();
+ checkArgument(numSplits >= 0, "invalid num_splits: must be >= 0, but was %d", numSplits);
+ return builder().setNumSplits(numSplits).build();
}
@Override
public PCollection<Document> expand(PBegin input) {
+ checkArgument(uri() != null, "withUri() is required");
+ checkArgument(database() != null, "withDatabase() is required");
+ checkArgument(collection() != null, "withCollection() is required");
return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedMongoDbSource(this)));
}
@Override
- public void validate(PipelineOptions options) {
- checkNotNull(uri(), "uri");
- checkNotNull(database(), "database");
- checkNotNull(collection(), "collection");
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder.add(DisplayData.item("uri", uri()));
+ builder.add(DisplayData.item("keepAlive", keepAlive()));
+ builder.add(DisplayData.item("maxConnectionIdleTime", maxConnectionIdleTime()));
builder.add(DisplayData.item("database", database()));
builder.add(DisplayData.item("collection", collection()));
builder.addIfNotNull(DisplayData.item("filter", filter()));
@@ -197,16 +265,11 @@
}
@Override
- public Coder<Document> getDefaultOutputCoder() {
+ public Coder<Document> getOutputCoder() {
return SerializableCoder.of(Document.class);
}
@Override
- public void validate() {
- spec.validate(null);
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
spec.populateDisplayData(builder);
}
@@ -218,61 +281,71 @@
@Override
public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) {
- MongoClient mongoClient = new MongoClient(new MongoClientURI(spec.uri()));
- MongoDatabase mongoDatabase = mongoClient.getDatabase(spec.database());
+ try (MongoClient mongoClient = new MongoClient(new MongoClientURI(spec.uri()))) {
+ return getEstimatedSizeBytes(mongoClient, spec.database(), spec.collection());
+ }
+ }
+
+ private long getEstimatedSizeBytes(MongoClient mongoClient,
+ String database,
+ String collection) {
+ MongoDatabase mongoDatabase = mongoClient.getDatabase(database);
// get the Mongo collStats object
// it gives the size for the entire collection
BasicDBObject stat = new BasicDBObject();
- stat.append("collStats", spec.collection());
+ stat.append("collStats", collection);
Document stats = mongoDatabase.runCommand(stat);
+
return stats.get("size", Number.class).longValue();
}
@Override
public List<BoundedSource<Document>> split(long desiredBundleSizeBytes,
PipelineOptions options) {
- MongoClient mongoClient = new MongoClient(new MongoClientURI(spec.uri()));
- MongoDatabase mongoDatabase = mongoClient.getDatabase(spec.database());
+ try (MongoClient mongoClient = new MongoClient(new MongoClientURI(spec.uri()))) {
+ MongoDatabase mongoDatabase = mongoClient.getDatabase(spec.database());
- List<Document> splitKeys;
- if (spec.numSplits() > 0) {
- // the user defines his desired number of splits
- // calculate the batch size
- long estimatedSizeBytes = getEstimatedSizeBytes(options);
- desiredBundleSizeBytes = estimatedSizeBytes / spec.numSplits();
- }
+ List<Document> splitKeys;
+ if (spec.numSplits() > 0) {
+ // the user defines his desired number of splits
+ // calculate the batch size
+ long estimatedSizeBytes = getEstimatedSizeBytes(mongoClient,
+ spec.database(), spec.collection());
+ desiredBundleSizeBytes = estimatedSizeBytes / spec.numSplits();
+ }
- // the desired batch size is small, using default chunk size of 1MB
- if (desiredBundleSizeBytes < 1024 * 1024) {
- desiredBundleSizeBytes = 1 * 1024 * 1024;
- }
+ // the desired batch size is small, using default chunk size of 1MB
+ if (desiredBundleSizeBytes < 1024 * 1024) {
+ desiredBundleSizeBytes = 1 * 1024 * 1024;
+ }
- // now we have the batch size (provided by user or provided by the runner)
- // we use Mongo splitVector command to get the split keys
- BasicDBObject splitVectorCommand = new BasicDBObject();
- splitVectorCommand.append("splitVector", spec.database() + "." + spec.collection());
- splitVectorCommand.append("keyPattern", new BasicDBObject().append("_id", 1));
- splitVectorCommand.append("force", false);
- // maxChunkSize is the Mongo partition size in MB
- LOG.debug("Splitting in chunk of {} MB", desiredBundleSizeBytes / 1024 / 1024);
- splitVectorCommand.append("maxChunkSize", desiredBundleSizeBytes / 1024 / 1024);
- Document splitVectorCommandResult = mongoDatabase.runCommand(splitVectorCommand);
- splitKeys = (List<Document>) splitVectorCommandResult.get("splitKeys");
+ // now we have the batch size (provided by user or provided by the runner)
+ // we use Mongo splitVector command to get the split keys
+ BasicDBObject splitVectorCommand = new BasicDBObject();
+ splitVectorCommand.append("splitVector", spec.database() + "." + spec.collection());
+ splitVectorCommand.append("keyPattern", new BasicDBObject().append("_id", 1));
+ splitVectorCommand.append("force", false);
+ // maxChunkSize is the Mongo partition size in MB
+ LOG.debug("Splitting in chunk of {} MB", desiredBundleSizeBytes / 1024 / 1024);
+ splitVectorCommand.append("maxChunkSize", desiredBundleSizeBytes / 1024 / 1024);
+ Document splitVectorCommandResult = mongoDatabase.runCommand(splitVectorCommand);
+ splitKeys = (List<Document>) splitVectorCommandResult.get("splitKeys");
- List<BoundedSource<Document>> sources = new ArrayList<>();
- if (splitKeys.size() < 1) {
- LOG.debug("Split keys is low, using an unique source");
- sources.add(this);
+ List<BoundedSource<Document>> sources = new ArrayList<>();
+ if (splitKeys.size() < 1) {
+ LOG.debug("Split keys is low, using an unique source");
+ sources.add(this);
+ return sources;
+ }
+
+ LOG.debug("Number of splits is {}", splitKeys.size());
+ for (String shardFilter : splitKeysToFilters(splitKeys, spec.filter())) {
+ sources.add(new BoundedMongoDbSource(spec.withFilter(shardFilter)));
+ }
+
return sources;
}
-
- LOG.debug("Number of splits is {}", splitKeys.size());
- for (String shardFilter : splitKeysToFilters(splitKeys, spec.filter())) {
- sources.add(new BoundedMongoDbSource(spec.withFilter(shardFilter)));
- }
-
- return sources;
}
/**
@@ -367,7 +440,10 @@
@Override
public boolean start() {
Read spec = source.spec;
- client = new MongoClient(new MongoClientURI(spec.uri()));
+ MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
+ optionsBuilder.maxConnectionIdleTime(spec.maxConnectionIdleTime());
+ optionsBuilder.socketKeepAlive(spec.keepAlive());
+ client = new MongoClient(new MongoClientURI(spec.uri(), optionsBuilder));
MongoDatabase mongoDatabase = client.getDatabase(spec.database());
@@ -426,53 +502,126 @@
*/
@AutoValue
public abstract static class Write extends PTransform<PCollection<Document>, PDone> {
+
@Nullable abstract String uri();
+ abstract boolean keepAlive();
+ abstract int maxConnectionIdleTime();
@Nullable abstract String database();
@Nullable abstract String collection();
abstract long batchSize();
- abstract Builder toBuilder();
+ abstract Builder builder();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setUri(String uri);
+ abstract Builder setKeepAlive(boolean keepAlive);
+ abstract Builder setMaxConnectionIdleTime(int maxConnectionIdleTime);
abstract Builder setDatabase(String database);
abstract Builder setCollection(String collection);
abstract Builder setBatchSize(long batchSize);
abstract Write build();
}
+ /**
+ * Define the location of the MongoDB instances using an URI. The URI describes the hosts to
+ * be used and some options.
+ *
+ * <p>The format of the URI is:
+ *
+ * <pre>{@code
+ * mongodb://[username:password@]host1[:port1],...[,hostN[:portN]]][/[database][?options]]
+ * }</pre>
+ *
+ * <p>Where:
+ * <ul>
+ * <li>{@code mongodb://} is a required prefix to identify that this is a string in the
+ * standard connection format.</li>
+ * <li>{@code username:password@} are optional. If given, the driver will attempt to
+ * login to a database after connecting to a database server. For some authentication
+ * mechanisms, only the username is specified and the password is not, in which case
+ * the ":" after the username is left off as well.</li>
+ * <li>{@code host1} is the only required part of the URI. It identifies a server
+ * address to connect to.</li>
+ * <li>{@code :portX} is optional and defaults to {@code :27017} if not provided.</li>
+ * <li>{@code /database} is the name of the database to login to and thus is only
+ * relevant if the {@code username:password@} syntax is used. If not specified, the
+ * "admin" database will be used by default. It has to be equivalent with the database
+ * you specific with {@link Write#withDatabase(String)}.</li>
+ * <li>{@code ?options} are connection options. Note that if {@code database} is absent
+ * there is still a {@code /} required between the last {@code host} and the {@code ?}
+ * introducing the options. Options are name=value pairs and the pairs are separated by
+ * "{@code &}". The {@code KeepAlive} connection option can't be passed via the URI, instead
+ * you have to use {@link Write#withKeepAlive(boolean)}. Same for the
+ * {@code MaxConnectionIdleTime} connection option via
+ * {@link Write#withMaxConnectionIdleTime(int)}.
+ * </li>
+ * </ul>
+ */
public Write withUri(String uri) {
- return toBuilder().setUri(uri).build();
+ checkArgument(uri != null, "uri can not be null");
+ return builder().setUri(uri).build();
}
+ /**
+ * Sets whether socket keep alive is enabled.
+ */
+ public Write withKeepAlive(boolean keepAlive) {
+ return builder().setKeepAlive(keepAlive).build();
+ }
+
+ /**
+ * Sets the maximum idle time for a pooled connection.
+ */
+ public Write withMaxConnectionIdleTime(int maxConnectionIdleTime) {
+ return builder().setMaxConnectionIdleTime(maxConnectionIdleTime).build();
+ }
+
+ /**
+ * Sets the database to use.
+ */
public Write withDatabase(String database) {
- return toBuilder().setDatabase(database).build();
+ checkArgument(database != null, "database can not be null");
+ return builder().setDatabase(database).build();
}
+ /**
+ * Sets the collection where to write data in the database.
+ */
public Write withCollection(String collection) {
- return toBuilder().setCollection(collection).build();
+ checkArgument(collection != null, "collection can not be null");
+ return builder().setCollection(collection).build();
}
+ /**
+ * Define the size of the batch to group write operations.
+ */
public Write withBatchSize(long batchSize) {
- return toBuilder().setBatchSize(batchSize).build();
+ checkArgument(batchSize >= 0, "Batch size must be >= 0, but was %d", batchSize);
+ return builder().setBatchSize(batchSize).build();
}
@Override
public PDone expand(PCollection<Document> input) {
+ checkArgument(uri() != null, "withUri() is required");
+ checkArgument(database() != null, "withDatabase() is required");
+ checkArgument(collection() != null, "withCollection() is required");
+
input.apply(ParDo.of(new WriteFn(this)));
return PDone.in(input.getPipeline());
}
@Override
- public void validate(PipelineOptions options) {
- checkNotNull(uri(), "uri");
- checkNotNull(database(), "database");
- checkNotNull(collection(), "collection");
- checkNotNull(batchSize(), "batchSize");
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("uri", uri()));
+ builder.add(DisplayData.item("keepAlive", keepAlive()));
+ builder.add(DisplayData.item("maxConnectionIdleTime", maxConnectionIdleTime()));
+ builder.add(DisplayData.item("database", database()));
+ builder.add(DisplayData.item("collection", collection()));
+ builder.add(DisplayData.item("batchSize", batchSize()));
}
- private static class WriteFn extends DoFn<Document, Void> {
+ static class WriteFn extends DoFn<Document, Void> {
private final Write spec;
private transient MongoClient client;
private List<Document> batch;
@@ -483,7 +632,10 @@
@Setup
public void createMongoClient() throws Exception {
- client = new MongoClient(new MongoClientURI(spec.uri()));
+ MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
+ builder.socketKeepAlive(spec.keepAlive());
+ builder.maxConnectionIdleTime(spec.maxConnectionIdleTime());
+ client = new MongoClient(new MongoClientURI(spec.uri(), builder));
}
@StartBundle
@@ -507,11 +659,12 @@
}
private void flush() {
+ if (batch.isEmpty()) {
+ return;
+ }
MongoDatabase mongoDatabase = client.getDatabase(spec.database());
MongoCollection<Document> mongoCollection = mongoDatabase.getCollection(spec.collection());
-
mongoCollection.insertMany(batch);
-
batch.clear();
}
diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
index 826af1c..19f8d87 100644
--- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
+++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
@@ -233,7 +233,7 @@
}
}
})
- .withSkew(new Duration(3601000L))
+ .withSkew(new Duration(3610000L))
.withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())));
PAssert.thatSingleton(output.apply("Count All", Count.<KV<String, Integer>>globally()))
diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
index cd26b48..a3fe063 100644
--- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
+++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.mongodb;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
@@ -44,6 +45,7 @@
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFnTester;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.SimpleFunction;
@@ -189,6 +191,42 @@
}
@Test
+ public void testReadWithCustomConnectionOptions() throws Exception {
+ MongoDbIO.Read read = MongoDbIO.read()
+ .withUri("mongodb://localhost:" + port)
+ .withKeepAlive(false)
+ .withMaxConnectionIdleTime(10)
+ .withDatabase(DATABASE)
+ .withCollection(COLLECTION);
+ assertFalse(read.keepAlive());
+ assertEquals(10, read.maxConnectionIdleTime());
+
+ PCollection<Document> documents = pipeline.apply(read);
+
+ PAssert.thatSingleton(documents.apply("Count All", Count.<Document>globally()))
+ .isEqualTo(1000L);
+
+ PAssert.that(documents
+ .apply("Map Scientist", MapElements.via(new SimpleFunction<Document, KV<String, Void>>() {
+ public KV<String, Void> apply(Document input) {
+ return KV.of(input.getString("scientist"), null);
+ }
+ }))
+ .apply("Count Scientist", Count.<String, Void>perKey())
+ ).satisfies(new SerializableFunction<Iterable<KV<String, Long>>, Void>() {
+ @Override
+ public Void apply(Iterable<KV<String, Long>> input) {
+ for (KV<String, Long> element : input) {
+ assertEquals(100L, element.getValue().longValue());
+ }
+ return null;
+ }
+ });
+
+ pipeline.run();
+ }
+
+ @Test
public void testReadWithFilter() throws Exception {
PCollection<Document> output = pipeline.apply(
@@ -233,4 +271,14 @@
}
+ @Test
+ public void testWriteEmptyCollection() throws Exception {
+ MongoDbIO.Write write =
+ MongoDbIO.write()
+ .withUri("mongodb://localhost:" + port)
+ .withDatabase("test")
+ .withCollection("empty");
+ DoFnTester<Document, Void> fnTester = DoFnTester.of(new MongoDbIO.Write.WriteFn(write));
+ fnTester.processBundle(new ArrayList<Document>());
+ }
}
diff --git a/sdks/java/io/mqtt/pom.xml b/sdks/java/io/mqtt/pom.xml
index baaf771..9a29eba 100644
--- a/sdks/java/io/mqtt/pom.xml
+++ b/sdks/java/io/mqtt/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
index 228a85d..f9083bb 100644
--- a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
+++ b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
@@ -97,7 +97,7 @@
*
* }</pre>
*/
-@Experimental
+@Experimental(Experimental.Kind.SOURCE_SINK)
public class MqttIO {
private static final Logger LOG = LoggerFactory.getLogger(MqttIO.class);
@@ -149,12 +149,8 @@
* @return A connection configuration to the MQTT broker.
*/
public static ConnectionConfiguration create(String serverUri, String topic) {
- checkArgument(serverUri != null,
- "MqttIO.ConnectionConfiguration.create(serverUri, topic) called with null "
- + "serverUri");
- checkArgument(topic != null,
- "MqttIO.ConnectionConfiguration.create(serverUri, topic) called with null "
- + "topic");
+ checkArgument(serverUri != null, "serverUri can not be null");
+ checkArgument(topic != null, "topic can not be null");
return new AutoValue_MqttIO_ConnectionConfiguration.Builder().setServerUri(serverUri)
.setTopic(topic).build();
}
@@ -168,14 +164,9 @@
* @return A connection configuration to the MQTT broker.
*/
public static ConnectionConfiguration create(String serverUri, String topic, String clientId) {
- checkArgument(serverUri != null,
- "MqttIO.ConnectionConfiguration.create(serverUri, topic) called with null "
- + "serverUri");
- checkArgument(topic != null,
- "MqttIO.ConnectionConfiguration.create(serverUri, topic) called with null "
- + "topic");
- checkArgument(clientId != null, "MqttIO.ConnectionConfiguration.create(serverUri,"
- + "topic, clientId) called with null clientId");
+ checkArgument(serverUri != null, "serverUri can not be null");
+ checkArgument(topic != null, "topic can not be null");
+ checkArgument(clientId != null, "clientId can not be null");
return new AutoValue_MqttIO_ConnectionConfiguration.Builder().setServerUri(serverUri)
.setTopic(topic).setClientId(clientId).build();
}
@@ -242,9 +233,7 @@
* Define the MQTT connection configuration used to connect to the MQTT broker.
*/
public Read withConnectionConfiguration(ConnectionConfiguration configuration) {
- checkArgument(configuration != null,
- "MqttIO.read().withConnectionConfiguration(configuration) called with null "
- + "configuration or not called at all");
+ checkArgument(configuration != null, "configuration can not be null");
return builder().setConnectionConfiguration(configuration).build();
}
@@ -254,8 +243,6 @@
* will provide a bounded {@link PCollection}.
*/
public Read withMaxNumRecords(long maxNumRecords) {
- checkArgument(maxReadTime() == null,
- "maxNumRecord and maxReadTime are exclusive");
return builder().setMaxNumRecords(maxNumRecords).build();
}
@@ -265,13 +252,14 @@
* {@link PCollection}.
*/
public Read withMaxReadTime(Duration maxReadTime) {
- checkArgument(maxNumRecords() == Long.MAX_VALUE,
- "maxNumRecord and maxReadTime are exclusive");
return builder().setMaxReadTime(maxReadTime).build();
}
@Override
public PCollection<byte[]> expand(PBegin input) {
+ checkArgument(
+ maxReadTime() == null || maxNumRecords() == Long.MAX_VALUE,
+ "withMaxNumRecords() and withMaxReadTime() are exclusive");
org.apache.beam.sdk.io.Read.Unbounded<byte[]> unbounded =
org.apache.beam.sdk.io.Read.from(new UnboundedMqttSource(this));
@@ -288,11 +276,6 @@
}
@Override
- public void validate(PipelineOptions options) {
- // validation is performed in the ConnectionConfiguration create()
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
connectionConfiguration().populateDisplayData(builder);
@@ -372,11 +355,6 @@
}
@Override
- public void validate() {
- spec.validate(null);
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
spec.populateDisplayData(builder);
}
@@ -387,7 +365,7 @@
}
@Override
- public Coder<byte[]> getDefaultOutputCoder() {
+ public Coder<byte[]> getOutputCoder() {
return ByteArrayCoder.of();
}
}
@@ -511,9 +489,7 @@
* Define MQTT connection configuration used to connect to the MQTT broker.
*/
public Write withConnectionConfiguration(ConnectionConfiguration configuration) {
- checkArgument(configuration != null,
- "MqttIO.write().withConnectionConfiguration(configuration) called with null "
- + "configuration or not called at all");
+ checkArgument(configuration != null, "configuration can not be null");
return builder().setConnectionConfiguration(configuration).build();
}
@@ -538,11 +514,6 @@
}
@Override
- public void validate(PipelineOptions options) {
- // validate is done in connection configuration
- }
-
- @Override
public void populateDisplayData(DisplayData.Builder builder) {
connectionConfiguration().populateDisplayData(builder);
builder.add(DisplayData.item("retained", retained()));
diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml
index 9657612..99936a2 100644
--- a/sdks/java/io/pom.xml
+++ b/sdks/java/io/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -33,50 +33,33 @@
(sources and sinks) to consume and produce data from systems.</description>
<properties>
- <!--
- This is the version of Hadoop used to compile the hadoop-common module.
- This dependency is defined with a provided scope.
- Users must supply their own Hadoop version at runtime.
- -->
- <hadoop.version>2.7.3</hadoop.version>
+ <!-- Necessary to make sure that integration with perfkit in io-it-suite works -->
+ <integrationTestPipelineOptions />
+ <pkbBeamRunnerProfile />
+ <pkbBeamRunnerOption />
</properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<modules>
+ <module>amqp</module>
+ <module>cassandra</module>
<module>common</module>
<module>elasticsearch</module>
+ <module>elasticsearch-tests</module>
<module>google-cloud-platform</module>
<module>hadoop-common</module>
<module>hadoop-file-system</module>
<module>hadoop</module>
<module>hbase</module>
+ <module>hcatalog</module>
<module>jdbc</module>
<module>jms</module>
<module>kafka</module>
<module>kinesis</module>
<module>mongodb</module>
<module>mqtt</module>
+ <module>redis</module>
+ <module>solr</module>
+ <module>tika</module>
<module>xml</module>
</modules>
@@ -109,11 +92,36 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire-plugin.version}</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
</plugins>
</build>
<properties>
<skipITs>false</skipITs>
</properties>
</profile>
+
+ <!-- this profile is for use with io-it-suite -->
+ <profile>
+ <id>forceDirectRunner</id>
+ <activation>
+ <property>
+ <name>forceDirectRunner</name>
+ </property>
+ </activation>
+ <properties>
+ <!-- These intentionally have a hanging equals sign so that an empty
+ string is passed to pkb. Passing "" will cause e.g. -P"" to
+ get added to the mvn command line -->
+ <pkbBeamRunnerProfile>-beam_runner_profile=</pkbBeamRunnerProfile>
+ <pkbBeamRunnerOption>-beam_runner_option=</pkbBeamRunnerOption>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/sdks/java/io/redis/pom.xml b/sdks/java/io/redis/pom.xml
new file mode 100644
index 0000000..d89e627
--- /dev/null
+++ b/sdks/java/io/redis/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-redis</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Redis</name>
+ <description>IO to read and write on a Redis keystore.</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
+ <version>2.9.0</version>
+ </dependency>
+
+ <!-- compile dependency -->
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.kstyrc</groupId>
+ <artifactId>embedded-redis</artifactId>
+ <version>0.6</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java
new file mode 100644
index 0000000..efcc77b
--- /dev/null
+++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.redis;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+
+import java.io.Serializable;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.Protocol;
+
+/**
+ * {@code RedisConnectionConfiguration} describes and wraps a connectionConfiguration to Redis
+ * server or cluster.
+ */
+@AutoValue
+public abstract class RedisConnectionConfiguration implements Serializable {
+
+ abstract String host();
+ abstract int port();
+ @Nullable abstract String auth();
+ abstract int timeout();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setHost(String host);
+ abstract Builder setPort(int port);
+ abstract Builder setAuth(String auth);
+ abstract Builder setTimeout(int timeout);
+ abstract RedisConnectionConfiguration build();
+ }
+
+ public static RedisConnectionConfiguration create() {
+ return new AutoValue_RedisConnectionConfiguration.Builder()
+ .setHost(Protocol.DEFAULT_HOST)
+ .setPort(Protocol.DEFAULT_PORT)
+ .setTimeout(Protocol.DEFAULT_TIMEOUT).build();
+ }
+
+ public static RedisConnectionConfiguration create(String host, int port) {
+ return new AutoValue_RedisConnectionConfiguration.Builder()
+ .setHost(host)
+ .setPort(port)
+ .setTimeout(Protocol.DEFAULT_TIMEOUT).build();
+ }
+
+ /**
+ * Define the host name of the Redis server.
+ */
+ public RedisConnectionConfiguration withHost(String host) {
+ checkArgument(host != null, "host can not be null");
+ return builder().setHost(host).build();
+ }
+
+ /**
+ * Define the port number of the Redis server.
+ */
+ public RedisConnectionConfiguration withPort(int port) {
+ checkArgument(port > 0, "port can not be negative or 0");
+ return builder().setPort(port).build();
+ }
+
+ /**
+ * Define the password to authenticate on the Redis server.
+ */
+ public RedisConnectionConfiguration withAuth(String auth) {
+ checkArgument(auth != null, "auth can not be null");
+ return builder().setAuth(auth).build();
+ }
+
+ /**
+ * Define the Redis connection timeout. A timeout of zero is interpreted as an infinite timeout.
+ */
+ public RedisConnectionConfiguration withTimeout(int timeout) {
+ checkArgument(timeout >= 0, "timeout can not be negative");
+ return builder().setTimeout(timeout).build();
+ }
+
+ /**
+ * Connect to the Redis instance.
+ */
+ public Jedis connect() {
+ Jedis jedis = new Jedis(host(), port(), timeout());
+ if (auth() != null) {
+ jedis.auth(auth());
+ }
+ return jedis;
+ }
+
+ /**
+ * Populate the display data with connectionConfiguration details.
+ */
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("host", host()));
+ builder.add(DisplayData.item("port", port()));
+ builder.addIfNotNull(DisplayData.item("timeout", timeout()));
+ }
+
+}
diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java
new file mode 100644
index 0000000..bfbad13
--- /dev/null
+++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.redis;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.SerializableFunctions;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.Pipeline;
+import redis.clients.jedis.ScanParams;
+import redis.clients.jedis.ScanResult;
+
+/**
+ * An IO to manipulate Redis key/value database.
+ *
+ * <h3>Reading Redis key/value pairs</h3>
+ *
+ * <p>{@link #read()} provides a source which returns a bounded {@link PCollection} containing
+ * key/value pairs as {@code KV<String, String>}.
+ *
+ * <p>To configure a Redis source, you have to provide Redis server hostname and port number.
+ * Optionally, you can provide a key pattern (to filter the keys). The following example
+ * illustrates how to configure a source:
+ *
+ * <pre>{@code
+ *
+ * pipeline.apply(RedisIO.read()
+ * .withEndpoint("::1", 6379)
+ * .withKeyPattern("foo*"))
+ *
+ * }</pre>
+ *
+ * <p>It's also possible to specify Redis authentication and connection timeout with the
+ * corresponding methods:
+ *
+ * <pre>{@code
+ *
+ * pipeline.apply(RedisIO.read()
+ * .withEndpoint("::1", 6379)
+ * .withAuth("authPassword")
+ * .withTimeout(60000)
+ * .withKeyPattern("foo*"))
+ *
+ * }</pre>
+ *
+ * <p>{@link #readAll()} can be used to request Redis server using input PCollection elements as key
+ * pattern (as String).
+ *
+ * <pre>{@code
+ *
+ * pipeline.apply(...)
+ * // here we have a PCollection<String> with the key patterns
+ * .apply(RedisIO.readAll().withEndpoint("::1", 6379))
+ * // here we have a PCollection<KV<String,String>>
+ *
+ * }</pre>
+ *
+ * <h3>Writing Redis key/value pairs</h3>
+ *
+ * <p>{@link #write()} provides a sink to write key/value pairs represented as
+ * {@link KV} from an incoming {@link PCollection}.
+ *
+ * <p>To configure the target Redis server, you have to provide Redis server hostname and port
+ * number. The following example illustrates how to configure a sink:
+ *
+ * <pre>{@code
+ *
+ * pipeline.apply(...)
+ * // here we a have a PCollection<String, String> with key/value pairs
+ * .apply(RedisIO.write().withEndpoint("::1", 6379))
+ *
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RedisIO {
+
+ /**
+ * Read data from a Redis server.
+ */
+ public static Read read() {
+ return new AutoValue_RedisIO_Read.Builder()
+ .setConnectionConfiguration(RedisConnectionConfiguration.create())
+ .setKeyPattern("*").build();
+ }
+
+ /**
+ * Like {@link #read()} but executes multiple instances of the Redis query substituting each
+ * element of a {@link PCollection} as key pattern.
+ */
+ public static ReadAll readAll() {
+ return new AutoValue_RedisIO_ReadAll.Builder()
+ .setConnectionConfiguration(RedisConnectionConfiguration.create())
+ .build();
+ }
+
+ /**
+ * Write data to a Redis server.
+ */
+ public static Write write() {
+ return new AutoValue_RedisIO_Write.Builder()
+ .setConnectionConfiguration(RedisConnectionConfiguration.create())
+ .build();
+ }
+
+ private RedisIO() {
+ }
+
+ /**
+ * Implementation of {@link #read()}.
+ */
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<KV<String, String>>> {
+
+ @Nullable abstract RedisConnectionConfiguration connectionConfiguration();
+ @Nullable abstract String keyPattern();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ @Nullable abstract Builder setConnectionConfiguration(
+ RedisConnectionConfiguration connection);
+ @Nullable abstract Builder setKeyPattern(String keyPattern);
+ abstract Read build();
+ }
+
+ public Read withEndpoint(String host, int port) {
+ checkArgument(host != null, "host can not be null");
+ checkArgument(port > 0, "port can not be negative or 0");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withHost(host))
+ .setConnectionConfiguration(connectionConfiguration().withPort(port))
+ .build();
+ }
+
+ public Read withAuth(String auth) {
+ checkArgument(auth != null, "auth can not be null");
+ return builder().setConnectionConfiguration(connectionConfiguration().withAuth(auth)).build();
+ }
+
+ public Read withTimeout(int timeout) {
+ checkArgument(timeout >= 0, "timeout can not be negative");
+ return builder().setConnectionConfiguration(connectionConfiguration().withTimeout(timeout))
+ .build();
+ }
+
+ public Read withKeyPattern(String keyPattern) {
+ checkArgument(keyPattern != null, "keyPattern can not be null");
+ return builder().setKeyPattern(keyPattern).build();
+ }
+
+ public Read withConnectionConfiguration(RedisConnectionConfiguration connection) {
+ checkArgument(connection != null, "connection can not be null");
+ return builder().setConnectionConfiguration(connection).build();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ connectionConfiguration().populateDisplayData(builder);
+ }
+
+ @Override
+ public PCollection<KV<String, String>> expand(PBegin input) {
+ checkArgument(connectionConfiguration() != null,
+ "withConnectionConfiguration() is required");
+
+ return input
+ .apply(Create.of(keyPattern()))
+ .apply(RedisIO.readAll().withConnectionConfiguration(connectionConfiguration()));
+ }
+
+ }
+
+ /**
+ * Implementation of {@link #readAll()}.
+ */
+ @AutoValue
+ public abstract static class ReadAll
+ extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
+
+ @Nullable abstract RedisConnectionConfiguration connectionConfiguration();
+
+ abstract ReadAll.Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ @Nullable abstract ReadAll.Builder setConnectionConfiguration(
+ RedisConnectionConfiguration connection);
+ abstract ReadAll build();
+ }
+
+ public ReadAll withEndpoint(String host, int port) {
+ checkArgument(host != null, "host can not be null");
+ checkArgument(port > 0, "port can not be negative or 0");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withHost(host))
+ .setConnectionConfiguration(connectionConfiguration().withPort(port))
+ .build();
+ }
+
+ public ReadAll withAuth(String auth) {
+ checkArgument(auth != null, "auth can not be null");
+ return builder().setConnectionConfiguration(connectionConfiguration().withAuth(auth)).build();
+ }
+
+ public ReadAll withTimeout(int timeout) {
+ checkArgument(timeout >= 0, "timeout can not be negative");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withTimeout(timeout)).build();
+ }
+
+ public ReadAll withConnectionConfiguration(RedisConnectionConfiguration connection) {
+ checkArgument(connection != null, "connection can not be null");
+ return builder().setConnectionConfiguration(connection).build();
+ }
+
+ @Override
+ public PCollection<KV<String, String>> expand(PCollection<String> input) {
+ checkArgument(connectionConfiguration() != null,
+ "withConnectionConfiguration() is required");
+
+ return input.apply(ParDo.of(new ReadFn(connectionConfiguration())))
+ .setCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
+ .apply(new Reparallelize());
+ }
+
+ }
+
+ /**
+ * A {@link DoFn} requesting Redis server to get key/value pairs.
+ */
+ private static class ReadFn extends DoFn<String, KV<String, String>> {
+
+ private final RedisConnectionConfiguration connectionConfiguration;
+
+ private transient Jedis jedis;
+
+ public ReadFn(RedisConnectionConfiguration connectionConfiguration) {
+ this.connectionConfiguration = connectionConfiguration;
+ }
+
+ @Setup
+ public void setup() {
+ jedis = connectionConfiguration.connect();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext processContext) throws Exception {
+ ScanParams scanParams = new ScanParams();
+ scanParams.match(processContext.element());
+
+ String cursor = ScanParams.SCAN_POINTER_START;
+ boolean finished = false;
+ while (!finished) {
+ ScanResult<String> scanResult = jedis.scan(cursor, scanParams);
+ List<String> keys = scanResult.getResult();
+
+ Pipeline pipeline = jedis.pipelined();
+ if (keys != null) {
+ for (String key : keys) {
+ pipeline.get(key);
+ }
+ List<Object> values = pipeline.syncAndReturnAll();
+ for (int i = 0; i < values.size(); i++) {
+ processContext.output(KV.of(keys.get(i), (String) values.get(i)));
+ }
+ }
+
+ cursor = scanResult.getStringCursor();
+ if (cursor.equals("0")) {
+ finished = true;
+ }
+ }
+ }
+
+ @Teardown
+ public void teardown() {
+ jedis.close();
+ }
+
+ }
+
+ private static class Reparallelize
+ extends PTransform<PCollection<KV<String, String>>, PCollection<KV<String, String>>> {
+
+ @Override public PCollection<KV<String, String>> expand(PCollection<KV<String, String>> input) {
+ // reparallelize mimics the same behavior as in JdbcIO
+ // breaking fusion
+ PCollectionView<Iterable<KV<String, String>>> empty = input
+ .apply("Consume",
+ Filter.by(SerializableFunctions.<KV<String, String>, Boolean>constant(false)))
+ .apply(View.<KV<String, String>>asIterable());
+ PCollection<KV<String, String>> materialized = input
+ .apply("Identity", ParDo.of(new DoFn<KV<String, String>, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext context) {
+ context.output(context.element());
+ }
+ }).withSideInputs(empty));
+ return materialized.apply(Reshuffle.<KV<String, String>>viaRandomKey());
+ }
+ }
+
+ /**
+ * A {@link PTransform} to write to a Redis server.
+ */
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<KV<String, String>>, PDone> {
+
+ @Nullable abstract RedisConnectionConfiguration connectionConfiguration();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ abstract Builder setConnectionConfiguration(
+ RedisConnectionConfiguration connectionConfiguration);
+
+ abstract Write build();
+
+ }
+
+ public Write withEndpoint(String host, int port) {
+ checkArgument(host != null, "host can not be null");
+ checkArgument(port > 0, "port can not be negative or 0");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withHost(host))
+ .setConnectionConfiguration(connectionConfiguration().withPort(port))
+ .build();
+ }
+
+ public Write withAuth(String auth) {
+ checkArgument(auth != null, "auth can not be null");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withAuth(auth))
+ .build();
+ }
+
+ public Write withTimeout(int timeout) {
+ checkArgument(timeout >= 0, "timeout can not be negative");
+ return builder()
+ .setConnectionConfiguration(connectionConfiguration().withTimeout(timeout))
+ .build();
+ }
+
+ public Write withConnectionConfiguration(RedisConnectionConfiguration connection) {
+ checkArgument(connection != null, "connection can not be null");
+ return builder().setConnectionConfiguration(connection).build();
+ }
+
+ @Override
+ public PDone expand(PCollection<KV<String, String>> input) {
+ checkArgument(connectionConfiguration() != null, "withConnectionConfiguration() is required");
+
+ input.apply(ParDo.of(new WriteFn(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ private static class WriteFn extends DoFn<KV<String, String>, Void> {
+
+ private static final int DEFAULT_BATCH_SIZE = 1000;
+
+ private final Write spec;
+
+ private transient Jedis jedis;
+ private transient Pipeline pipeline;
+
+ private int batchCount;
+
+ public WriteFn(Write spec) {
+ this.spec = spec;
+ }
+
+ @Setup
+ public void setup() {
+ jedis = spec.connectionConfiguration().connect();
+ }
+
+ @StartBundle
+ public void startBundle() {
+ pipeline = jedis.pipelined();
+ pipeline.multi();
+ batchCount = 0;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext processContext) {
+ KV<String, String> record = processContext.element();
+ pipeline.append(record.getKey(), record.getValue());
+
+ batchCount++;
+
+ if (batchCount >= DEFAULT_BATCH_SIZE) {
+ pipeline.exec();
+ batchCount = 0;
+ }
+ }
+
+ @FinishBundle
+ public void finishBundle() {
+ pipeline.exec();
+ batchCount = 0;
+ }
+
+ @Teardown
+ public void teardown() {
+ jedis.close();
+ }
+ }
+
+ }
+
+}
diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/package-info.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/package-info.java
new file mode 100644
index 0000000..a650acc
--- /dev/null
+++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transforms for reading and writing from Redis.
+ */
+package org.apache.beam.sdk.io.redis;
diff --git a/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java b/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java
new file mode 100644
index 0000000..b5ba847
--- /dev/null
+++ b/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.redis;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.ArrayList;
+
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import redis.embedded.RedisServer;
+
+/**
+ * Test on the Redis IO.
+ */
+public class RedisIOTest {
+
+ @Rule public TestPipeline writePipeline = TestPipeline.create();
+ @Rule public TestPipeline readPipeline = TestPipeline.create();
+
+ private EmbeddedRedis embeddedRedis;
+
+ @Before
+ public void before() throws Exception {
+ embeddedRedis = new EmbeddedRedis();
+ }
+
+ @After
+ public void after() throws Exception {
+ embeddedRedis.close();
+ }
+
+ @Test
+ public void testWriteRead() throws Exception {
+ ArrayList<KV<String, String>> data = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ KV<String, String> kv = KV.of("key " + i, "value " + i);
+ data.add(kv);
+ }
+ PCollection<KV<String, String>> write = writePipeline.apply(Create.of(data));
+ write.apply(RedisIO.write().withEndpoint("::1", embeddedRedis.getPort()));
+
+ writePipeline.run();
+
+ PCollection<KV<String, String>> read = readPipeline.apply("Read",
+ RedisIO.read().withEndpoint("::1", embeddedRedis.getPort())
+ .withKeyPattern("key*"));
+ PAssert.that(read).containsInAnyOrder(data);
+
+ PCollection<KV<String, String>> readNotMatch = readPipeline.apply("ReadNotMatch",
+ RedisIO.read().withEndpoint("::1", embeddedRedis.getPort())
+ .withKeyPattern("foobar*"));
+ PAssert.thatSingleton(readNotMatch.apply(Count.<KV<String, String>>globally())).isEqualTo(0L);
+
+ readPipeline.run();
+ }
+
+ /**
+ * Simple embedded Redis instance wrapper to control Redis server.
+ */
+ private static class EmbeddedRedis implements AutoCloseable {
+
+ private final int port;
+ private final RedisServer redisServer;
+
+ public EmbeddedRedis() throws IOException {
+ try (ServerSocket serverSocket = new ServerSocket(0)) {
+ port = serverSocket.getLocalPort();
+ }
+ redisServer = new RedisServer(port);
+ redisServer.start();
+ }
+
+ public int getPort() {
+ return this.port;
+ }
+
+ @Override
+ public void close() {
+ redisServer.stop();
+ }
+
+ }
+
+}
diff --git a/sdks/java/io/solr/pom.xml b/sdks/java/io/solr/pom.xml
new file mode 100644
index 0000000..0c5d6b0
--- /dev/null
+++ b/sdks/java/io/solr/pom.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <groupId>org.apache.beam</groupId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>beam-sdks-java-io-solr</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Solr</name>
+ <description>IO to read and write from/to Solr.</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>5.5.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+
+ <!-- compile dependencies -->
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.4.1</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-test-framework</artifactId>
+ <version>5.5.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>5.5.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.carrotsearch.randomizedtesting</groupId>
+ <artifactId>randomizedtesting-runner</artifactId>
+ <version>2.3.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java
new file mode 100644
index 0000000..44d7b88
--- /dev/null
+++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.beam.sdk.io.solr.SolrIO.ConnectionConfiguration;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.params.SolrParams;
+
+/**
+ * Client for interact with Solr.
+ * @param <ClientT> type of SolrClient
+ */
+class AuthorizedSolrClient<ClientT extends SolrClient> implements Closeable {
+ private final ClientT solrClient;
+ private final String username;
+ private final String password;
+
+ AuthorizedSolrClient(ClientT solrClient, ConnectionConfiguration configuration) {
+ checkArgument(
+ solrClient != null,
+ "solrClient can not be null");
+ checkArgument(
+ configuration != null,
+ "configuration can not be null");
+ this.solrClient = solrClient;
+ this.username = configuration.getUsername();
+ this.password = configuration.getPassword();
+ }
+
+ QueryResponse query(String collection, SolrParams solrParams)
+ throws IOException, SolrServerException {
+ QueryRequest query = new QueryRequest(solrParams);
+ return process(collection, query);
+ }
+
+ <ResponseT extends SolrResponse> ResponseT process(String collection,
+ SolrRequest<ResponseT> request) throws IOException, SolrServerException {
+ request.setBasicAuthCredentials(username, password);
+ return request.process(solrClient, collection);
+ }
+
+ CoreAdminResponse process(CoreAdminRequest request)
+ throws IOException, SolrServerException {
+ return process(null, request);
+ }
+
+ SolrResponse process(CollectionAdminRequest request)
+ throws IOException, SolrServerException {
+ return process(null, request);
+ }
+
+ static ClusterState getClusterState(
+ AuthorizedSolrClient<CloudSolrClient> authorizedSolrClient) {
+ authorizedSolrClient.solrClient.connect();
+ return authorizedSolrClient.solrClient.getZkStateReader().getClusterState();
+ }
+
+ @Override public void close() throws IOException {
+ solrClient.close();
+ }
+}
diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoder.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoder.java
new file mode 100644
index 0000000..aef3c4b
--- /dev/null
+++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoder.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import com.google.auto.service.AutoService;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.util.VarInt;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.commons.compress.utils.BoundedInputStream;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.JavaBinCodec;
+
+/** A {@link Coder} that encodes using {@link JavaBinCodec}. */
+class JavaBinCodecCoder<T> extends AtomicCoder<T> {
+ private final Class<T> clazz;
+
+ private JavaBinCodecCoder(Class<T> clazz) {
+ this.clazz = clazz;
+ }
+
+ public static <T> JavaBinCodecCoder<T> of(Class<T> clazz) {
+ return new JavaBinCodecCoder<>(clazz);
+ }
+
+ @Override
+ public void encode(T value, OutputStream outStream) throws IOException {
+ if (value == null) {
+ throw new CoderException("cannot encode a null SolrDocument");
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ JavaBinCodec codec = new JavaBinCodec();
+ codec.marshal(value, baos);
+
+ byte[] bytes = baos.toByteArray();
+ VarInt.encode(bytes.length, outStream);
+ outStream.write(bytes);
+ }
+
+ @Override
+ public T decode(InputStream inStream) throws IOException {
+ DataInputStream in = new DataInputStream(inStream);
+
+ int len = VarInt.decodeInt(in);
+ if (len < 0) {
+ throw new CoderException("Invalid encoded SolrDocument length: " + len);
+ }
+
+ JavaBinCodec codec = new JavaBinCodec();
+ return (T) codec.unmarshal(new BoundedInputStream(in, len));
+ }
+
+ @Override
+ public TypeDescriptor<T> getEncodedTypeDescriptor() {
+ return TypeDescriptor.of(clazz);
+ }
+
+ @AutoService(CoderProviderRegistrar.class)
+ public static class Provider implements CoderProviderRegistrar {
+ @Override
+ public List<CoderProvider> getCoderProviders() {
+ return Arrays.asList(
+ CoderProviders.forCoder(
+ TypeDescriptor.of(SolrDocument.class), JavaBinCodecCoder.of(SolrDocument.class)),
+ CoderProviders.forCoder(
+ TypeDescriptor.of(SolrInputDocument.class),
+ JavaBinCodecCoder.of(SolrInputDocument.class)));
+ }
+ }
+}
diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java
new file mode 100644
index 0000000..f811139
--- /dev/null
+++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java
@@ -0,0 +1,705 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ * Transforms for reading and writing data from/to Solr.
+ *
+ * <h3>Reading from Solr</h3>
+ *
+ * <p>{@link SolrIO#read SolrIO.read()} returns a bounded {@link PCollection
+ * PCollection<SolrDocument>} representing Solr documents.
+ *
+ * <p>To configure the {@link SolrIO#read}, you have to provide a connection configuration
+ * containing the Zookeeper address of the Solr cluster, and the collection name. The following
+ * example illustrates options for configuring the source:
+ *
+ * <pre>{@code
+ * SolrIO.ConnectionConfiguration conn = SolrIO.ConnectionConfiguration.create("127.0.0.1:9983");
+ * // Optionally: .withBasicCredentials(username, password)
+ *
+ * PCollection<SolrDocument> docs = p.apply(
+ * SolrIO.read().from("my-collection").withConnectionConfiguration(conn));
+ *
+ * }</pre>
+ *
+ * <p>You can specify a query on the {@code read()} using {@code withQuery()}.
+ *
+ * <h3>Writing to Solr</h3>
+ *
+ * <p>To write documents to Solr, use {@link SolrIO#write SolrIO.write()}, which writes Solr
+ * documents from a {@link PCollection PCollection<SolrInputDocument>} (which can be bounded
+ * or unbounded).
+ *
+ * <p>To configure {@link SolrIO#write SolrIO.write()}, similar to the read, you have to provide a
+ * connection configuration, and a collection name. For instance:
+ *
+ * <pre>{@code
+ * PCollection<SolrInputDocument> inputDocs = ...;
+ * inputDocs.apply(SolrIO.write().to("my-collection").withConnectionConfiguration(conn));
+ *
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class SolrIO {
+
+ public static Read read() {
+ // 1000 for batch size is good enough in many cases,
+ // ex: if document size is large, around 10KB, the response's size will be around 10MB
+ // if document seize is small, around 1KB, the response's size will be around 1MB
+ return new AutoValue_SolrIO_Read.Builder().setBatchSize(1000).setQuery("*:*").build();
+ }
+
+ public static Write write() {
+ // 1000 for batch size is good enough in many cases,
+ // ex: if document size is large, around 10KB, the request's size will be around 10MB
+ // if document seize is small, around 1KB, the request's size will be around 1MB
+ return new AutoValue_SolrIO_Write.Builder().setMaxBatchSize(1000).build();
+ }
+
+ private SolrIO() {}
+
+ /** A POJO describing a connection configuration to Solr. */
+ @AutoValue
+ public abstract static class ConnectionConfiguration implements Serializable {
+
+ abstract String getZkHost();
+
+ @Nullable
+ abstract String getUsername();
+
+ @Nullable
+ abstract String getPassword();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setZkHost(String zkHost);
+
+ abstract Builder setUsername(String username);
+
+ abstract Builder setPassword(String password);
+
+ abstract ConnectionConfiguration build();
+ }
+
+ /**
+ * Creates a new Solr connection configuration.
+ *
+ * @param zkHost host of zookeeper
+ * @return the connection configuration object
+ */
+ public static ConnectionConfiguration create(String zkHost) {
+ checkArgument(zkHost != null, "zkHost can not be null");
+ return new AutoValue_SolrIO_ConnectionConfiguration.Builder().setZkHost(zkHost).build();
+ }
+
+ /** If Solr basic authentication is enabled, provide the username and password. */
+ public ConnectionConfiguration withBasicCredentials(String username, String password) {
+ checkArgument(username != null, "username can not be null");
+ checkArgument(!username.isEmpty(), "username can not be empty");
+ checkArgument(password != null, "password can not be null");
+ checkArgument(!password.isEmpty(), "password can not be empty");
+ return builder().setUsername(username).setPassword(password).build();
+ }
+
+ private void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("zkHost", getZkHost()));
+ builder.addIfNotNull(DisplayData.item("username", getUsername()));
+ }
+
+ private HttpClient createHttpClient() {
+ // This is bug in Solr, if we don't create a customize HttpClient,
+ // UpdateRequest with commit flag will throw an authentication error.
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(HttpClientUtil.PROP_BASIC_AUTH_USER, getUsername());
+ params.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, getPassword());
+ return HttpClientUtil.createClient(params);
+ }
+
+ AuthorizedSolrClient<CloudSolrClient> createClient() throws MalformedURLException {
+ CloudSolrClient solrClient = new CloudSolrClient(getZkHost(), createHttpClient());
+ return new AuthorizedSolrClient<>(solrClient, this);
+ }
+
+ AuthorizedSolrClient<HttpSolrClient> createClient(String shardUrl) {
+ HttpSolrClient solrClient = new HttpSolrClient(shardUrl, createHttpClient());
+ return new AuthorizedSolrClient<>(solrClient, this);
+ }
+ }
+
+ /** A {@link PTransform} reading data from Solr. */
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<SolrDocument>> {
+ private static final long MAX_BATCH_SIZE = 10000L;
+
+ @Nullable
+ abstract ConnectionConfiguration getConnectionConfiguration();
+
+ @Nullable
+ abstract String getCollection();
+
+ abstract String getQuery();
+
+ abstract int getBatchSize();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConnectionConfiguration(ConnectionConfiguration connectionConfiguration);
+
+ abstract Builder setQuery(String query);
+
+ abstract Builder setBatchSize(int batchSize);
+
+ abstract Builder setCollection(String collection);
+
+ abstract Read build();
+ }
+
+ /** Provide the Solr connection configuration object. */
+ public Read withConnectionConfiguration(ConnectionConfiguration connectionConfiguration) {
+ checkArgument(connectionConfiguration != null, "connectionConfiguration can not be null");
+ return builder().setConnectionConfiguration(connectionConfiguration).build();
+ }
+
+ /**
+ * Provide name of collection while reading from Solr.
+ *
+ * @param collection the collection toward which the requests will be issued
+ */
+ public Read from(String collection) {
+ checkArgument(collection != null, "collection can not be null");
+ return builder().setCollection(collection).build();
+ }
+
+ /**
+ * Provide a query used while reading from Solr.
+ *
+ * @param query the query. See <a
+ * href="https://cwiki.apache.org/confluence/display/solr/The+Standard+Query+Parser">Solr
+ * Query </a>
+ */
+ public Read withQuery(String query) {
+ checkArgument(query != null, "query can not be null");
+ checkArgument(!query.isEmpty(), "query can not be empty");
+ return builder().setQuery(query).build();
+ }
+
+ /**
+ * Provide a size for the cursor read. See <a
+ * href="https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results">cursor API</a>
+ * Default is 100. Maximum is 10 000. If documents are small, increasing batch size might
+ * improve read performance. If documents are big, you might need to decrease batchSize
+ *
+ * @param batchSize number of documents read in each scroll read
+ */
+ @VisibleForTesting
+ Read withBatchSize(int batchSize) {
+ // TODO remove this configuration, we can figure out the best number
+ // by tuning batchSize when pipelines run.
+ checkArgument(
+ batchSize > 0 && batchSize < MAX_BATCH_SIZE,
+ "Valid values for batchSize are 1 (inclusize) to %s (exclusive), but was: %s ",
+ MAX_BATCH_SIZE,
+ batchSize);
+ return builder().setBatchSize(batchSize).build();
+ }
+
+ @Override
+ public PCollection<SolrDocument> expand(PBegin input) {
+ checkArgument(
+ getConnectionConfiguration() != null, "withConnectionConfiguration() is required");
+ checkArgument(getCollection() != null, "from() is required");
+
+ return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedSolrSource(this, null)));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.addIfNotNull(DisplayData.item("query", getQuery()));
+ getConnectionConfiguration().populateDisplayData(builder);
+ }
+ }
+
+ /** A POJO describing a replica of Solr. */
+ @AutoValue
+ abstract static class ReplicaInfo implements Serializable {
+ public abstract String coreName();
+
+ public abstract String coreUrl();
+
+ public abstract String baseUrl();
+
+ static ReplicaInfo create(Replica replica) {
+ return new AutoValue_SolrIO_ReplicaInfo(
+ replica.getStr(ZkStateReader.CORE_NAME_PROP),
+ replica.getCoreUrl(),
+ replica.getStr(ZkStateReader.BASE_URL_PROP));
+ }
+ }
+
+ /** A {@link BoundedSource} reading from Solr. */
+ @VisibleForTesting
+ static class BoundedSolrSource extends BoundedSource<SolrDocument> {
+
+ private final SolrIO.Read spec;
+ // replica is the info of the shard where the source will read the documents
+ @Nullable private final ReplicaInfo replica;
+
+ BoundedSolrSource(Read spec, @Nullable Replica replica) {
+ this.spec = spec;
+ this.replica = replica == null ? null : ReplicaInfo.create(replica);
+ }
+
+ @Override
+ public List<? extends BoundedSource<SolrDocument>> split(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration();
+ List<BoundedSolrSource> sources = new ArrayList<>();
+ try (AuthorizedSolrClient<CloudSolrClient> client = connectionConfig.createClient()) {
+ String collection = spec.getCollection();
+ final ClusterState clusterState = AuthorizedSolrClient.getClusterState(client);
+ DocCollection docCollection = clusterState.getCollection(collection);
+ for (Slice slice : docCollection.getSlices()) {
+ ArrayList<Replica> replicas = new ArrayList<>(slice.getReplicas());
+ Collections.shuffle(replicas);
+ // Load balancing by randomly picking an active replica
+ Replica randomActiveReplica = null;
+ for (Replica replica : replicas) {
+ // We need to check both state of the replica and live nodes
+ // to make sure that the replica is alive
+ if (replica.getState() == Replica.State.ACTIVE
+ && clusterState.getLiveNodes().contains(replica.getNodeName())) {
+ randomActiveReplica = replica;
+ break;
+ }
+ }
+ // TODO in case of this replica goes inactive while the pipeline runs.
+ // We should pick another active replica of this shard.
+ checkState(
+ randomActiveReplica != null,
+ "Can not found an active replica for slice %s",
+ slice.getName());
+ sources.add(new BoundedSolrSource(spec, randomActiveReplica));
+ }
+ }
+ return sources;
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions options) throws IOException {
+ if (replica != null) {
+ return getEstimatedSizeOfShard(replica);
+ } else {
+ return getEstimatedSizeOfCollection();
+ }
+ }
+
+ private long getEstimatedSizeOfShard(ReplicaInfo replica) throws IOException {
+ try (AuthorizedSolrClient solrClient =
+ spec.getConnectionConfiguration().createClient(replica.baseUrl())) {
+ CoreAdminRequest req = new CoreAdminRequest();
+ req.setAction(CoreAdminParams.CoreAdminAction.STATUS);
+ req.setIndexInfoNeeded(true);
+ CoreAdminResponse response;
+ try {
+ response = solrClient.process(req);
+ } catch (SolrServerException e) {
+ throw new IOException("Can not get core status from " + replica, e);
+ }
+ NamedList<Object> coreStatus = response.getCoreStatus(replica.coreName());
+ NamedList<Object> indexStats = (NamedList<Object>) coreStatus.get("index");
+ return (long) indexStats.get("sizeInBytes");
+ }
+ }
+
+ private long getEstimatedSizeOfCollection() throws IOException {
+ long sizeInBytes = 0;
+ ConnectionConfiguration config = spec.getConnectionConfiguration();
+ try (AuthorizedSolrClient<CloudSolrClient> solrClient = config.createClient()) {
+ DocCollection docCollection =
+ AuthorizedSolrClient.getClusterState(solrClient).getCollection(spec.getCollection());
+ if (docCollection.getSlices().isEmpty()) {
+ return 0;
+ }
+
+ ArrayList<Slice> slices = new ArrayList<>(docCollection.getSlices());
+ Collections.shuffle(slices);
+ ExecutorService executor =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder()
+ .setThreadFactory(MoreExecutors.platformThreadFactory())
+ .setDaemon(true)
+ .setNameFormat("solrio-size-of-collection-estimation")
+ .build());
+ try {
+ ArrayList<Future<Long>> futures = new ArrayList<>();
+ for (int i = 0; i < 100 && i < slices.size(); i++) {
+ Slice slice = slices.get(i);
+ final Replica replica = slice.getLeader();
+ Future<Long> future =
+ executor.submit(
+ new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ return getEstimatedSizeOfShard(ReplicaInfo.create(replica));
+ }
+ });
+ futures.add(future);
+ }
+ for (Future<Long> future : futures) {
+ try {
+ sizeInBytes += future.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ } catch (ExecutionException e) {
+ throw new IOException("Can not estimate size of shard", e.getCause());
+ }
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+
+ if (slices.size() <= 100) {
+ return sizeInBytes;
+ }
+ return (sizeInBytes / 100) * slices.size();
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ spec.populateDisplayData(builder);
+ if (replica != null) {
+ builder.addIfNotNull(DisplayData.item("shardUrl", replica.coreUrl()));
+ }
+ }
+
+ @Override
+ public BoundedReader<SolrDocument> createReader(PipelineOptions options) throws IOException {
+ return new BoundedSolrReader(this);
+ }
+
+ @Override
+ public Coder<SolrDocument> getOutputCoder() {
+ return JavaBinCodecCoder.of(SolrDocument.class);
+ }
+ }
+
+ private static class BoundedSolrReader extends BoundedSource.BoundedReader<SolrDocument> {
+
+ private final BoundedSolrSource source;
+
+ private AuthorizedSolrClient solrClient;
+ private SolrDocument current;
+ private String cursorMark;
+ private Iterator<SolrDocument> batchIterator;
+ private boolean done;
+ private String uniqueKey;
+
+ private BoundedSolrReader(BoundedSolrSource source) {
+ this.source = source;
+ this.cursorMark = CursorMarkParams.CURSOR_MARK_START;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ if (source.replica != null) {
+ solrClient =
+ source.spec.getConnectionConfiguration().createClient(source.replica.baseUrl());
+ } else {
+ solrClient = source.spec.getConnectionConfiguration().createClient();
+ }
+ SchemaRequest.UniqueKey uniqueKeyRequest = new SchemaRequest.UniqueKey();
+ try {
+ String collection = source.spec.getCollection();
+ SchemaResponse.UniqueKeyResponse uniqueKeyResponse =
+ (SchemaResponse.UniqueKeyResponse) solrClient.process(collection, uniqueKeyRequest);
+ uniqueKey = uniqueKeyResponse.getUniqueKey();
+ } catch (SolrServerException e) {
+ throw new IOException("Can not get unique key from solr", e);
+ }
+ return advance();
+ }
+
+ private SolrQuery getQueryParams(BoundedSolrSource source) {
+ String query = source.spec.getQuery();
+ if (query == null) {
+ query = "*:*";
+ }
+ SolrQuery solrQuery = new SolrQuery(query);
+ solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ solrQuery.setRows(source.spec.getBatchSize());
+ solrQuery.addSort(uniqueKey, SolrQuery.ORDER.asc);
+ if (source.replica != null) {
+ solrQuery.setDistrib(false);
+ }
+ return solrQuery;
+ }
+
+ private void updateCursorMark(QueryResponse response) {
+ if (cursorMark.equals(response.getNextCursorMark())) {
+ done = true;
+ }
+ cursorMark = response.getNextCursorMark();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ if (batchIterator != null && batchIterator.hasNext()) {
+ current = batchIterator.next();
+ return true;
+ } else {
+ SolrQuery solrQuery = getQueryParams(source);
+ try {
+ QueryResponse response;
+ if (source.replica != null) {
+ response = solrClient.query(source.replica.coreName(), solrQuery);
+ } else {
+ response = solrClient.query(source.spec.getCollection(), solrQuery);
+ }
+ updateCursorMark(response);
+ return readNextBatchAndReturnFirstDocument(response);
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ private boolean readNextBatchAndReturnFirstDocument(QueryResponse response) {
+ if (done) {
+ current = null;
+ batchIterator = null;
+ return false;
+ }
+
+ batchIterator = response.getResults().iterator();
+ current = batchIterator.next();
+ return true;
+ }
+
+ @Override
+ public SolrDocument getCurrent() throws NoSuchElementException {
+ if (current == null) {
+ throw new NoSuchElementException();
+ }
+ return current;
+ }
+
+ @Override
+ public void close() throws IOException {
+ solrClient.close();
+ }
+
+ @Override
+ public BoundedSource<SolrDocument> getCurrentSource() {
+ return source;
+ }
+ }
+
+ /** A {@link PTransform} writing data to Solr. */
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<SolrInputDocument>, PDone> {
+
+ @Nullable
+ abstract ConnectionConfiguration getConnectionConfiguration();
+
+ @Nullable
+ abstract String getCollection();
+
+ abstract int getMaxBatchSize();
+
+ abstract Builder builder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConnectionConfiguration(ConnectionConfiguration connectionConfiguration);
+
+ abstract Builder setCollection(String collection);
+
+ abstract Builder setMaxBatchSize(int maxBatchSize);
+
+ abstract Write build();
+ }
+
+ /** Provide the Solr connection configuration object. */
+ public Write withConnectionConfiguration(ConnectionConfiguration connectionConfiguration) {
+ checkArgument(connectionConfiguration != null, "connectionConfiguration can not be null");
+ return builder().setConnectionConfiguration(connectionConfiguration).build();
+ }
+
+ /**
+ * Provide name of collection while reading from Solr.
+ *
+ * @param collection the collection toward which the requests will be issued
+ */
+ public Write to(String collection) {
+ checkArgument(collection != null, "collection can not be null");
+ return builder().setCollection(collection).build();
+ }
+
+ /**
+ * Provide a maximum size in number of documents for the batch. Depending on the execution
+ * engine, size of bundles may vary, this sets the maximum size. Change this if you need to have
+ * smaller batch.
+ *
+ * @param batchSize maximum batch size in number of documents
+ */
+ @VisibleForTesting
+ Write withMaxBatchSize(int batchSize) {
+ // TODO remove this configuration, we can figure out the best number
+ // by tuning batchSize when pipelines run.
+ checkArgument(batchSize > 0, "batchSize must be larger than 0, but was: %s", batchSize);
+ return builder().setMaxBatchSize(batchSize).build();
+ }
+
+ @Override
+ public PDone expand(PCollection<SolrInputDocument> input) {
+ checkState(
+ getConnectionConfiguration() != null,
+ "withConnectionConfiguration() is required");
+ checkState(getCollection() != null, "to() is required");
+
+ input.apply(ParDo.of(new WriteFn(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ @VisibleForTesting
+ static class WriteFn extends DoFn<SolrInputDocument, Void> {
+
+ private final Write spec;
+
+ private transient AuthorizedSolrClient solrClient;
+ private Collection<SolrInputDocument> batch;
+
+ WriteFn(Write spec) {
+ this.spec = spec;
+ }
+
+ @Setup
+ public void createClient() throws Exception {
+ solrClient = spec.getConnectionConfiguration().createClient();
+ }
+
+ @StartBundle
+ public void startBundle(StartBundleContext context) throws Exception {
+ batch = new ArrayList<>();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) throws Exception {
+ SolrInputDocument document = context.element();
+ batch.add(document);
+ if (batch.size() >= spec.getMaxBatchSize()) {
+ flushBatch();
+ }
+ }
+
+ @FinishBundle
+ public void finishBundle(FinishBundleContext context) throws Exception {
+ flushBatch();
+ }
+
+ private void flushBatch() throws IOException {
+ if (batch.isEmpty()) {
+ return;
+ }
+ try {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.add(batch);
+ solrClient.process(spec.getCollection(), updateRequest);
+ } catch (SolrServerException e) {
+ throw new IOException("Error writing to Solr", e);
+ } finally {
+ batch.clear();
+ }
+ }
+
+ @Teardown
+ public void closeClient() throws Exception {
+ if (solrClient != null) {
+ solrClient.close();
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/package-info.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/package-info.java
new file mode 100644
index 0000000..83867ed
--- /dev/null
+++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Transforms for reading and writing from/to Solr. */
+package org.apache.beam.sdk.io.solr;
diff --git a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoderTest.java b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoderTest.java
new file mode 100644
index 0000000..1fb435d
--- /dev/null
+++ b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoderTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.testing.CoderProperties;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.solr.common.SolrDocument;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test case for {@link JavaBinCodecCoder}. */
+@RunWith(JUnit4.class)
+public class JavaBinCodecCoderTest {
+ private static final Coder<SolrDocument> TEST_CODER = JavaBinCodecCoder.of(SolrDocument.class);
+ private static final List<SolrDocument> TEST_VALUES = new ArrayList<>();
+
+ static {
+ SolrDocument doc = new SolrDocument();
+ doc.put("id", "1");
+ doc.put("content", "wheel on the bus");
+ doc.put("_version_", 1573597324260671488L);
+ TEST_VALUES.add(doc);
+
+ doc = new SolrDocument();
+ doc.put("id", "2");
+ doc.put("content", "goes round and round");
+ doc.put("_version_", 1573597324260671489L);
+ TEST_VALUES.add(doc);
+ }
+
+ @Test
+ public void testDecodeEncodeEqual() throws Exception {
+ for (SolrDocument value : TEST_VALUES) {
+ CoderProperties.coderDecodeEncodeContentsInSameOrder(TEST_CODER, value);
+ CoderProperties.structuralValueDecodeEncodeEqual(TEST_CODER, value);
+ }
+ }
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void encodeNullThrowsCoderException() throws Exception {
+ thrown.expect(CoderException.class);
+ thrown.expectMessage("cannot encode a null SolrDocument");
+
+ CoderUtils.encodeToBase64(TEST_CODER, null);
+ }
+
+ @Test
+ public void testEncodedTypeDescriptor() throws Exception {
+ assertThat(
+ TEST_CODER.getEncodedTypeDescriptor(), equalTo(TypeDescriptor.of(SolrDocument.class)));
+ }
+}
diff --git a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java
new file mode 100644
index 0000000..4358ce4
--- /dev/null
+++ b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.google.common.io.BaseEncoding;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.security.Sha256AuthenticationProvider;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A test of {@link SolrIO} on an independent Solr instance. */
+@ThreadLeakScope(value = ThreadLeakScope.Scope.NONE)
+@SolrTestCaseJ4.SuppressSSL
+public class SolrIOTest extends SolrCloudTestCase {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrIOTest.class);
+
+ private static final String SOLR_COLLECTION = "beam";
+ private static final int NUM_SHARDS = 3;
+ private static final long NUM_DOCS = 400L;
+ private static final int NUM_SCIENTISTS = 10;
+ private static final int BATCH_SIZE = 200;
+
+ private static AuthorizedSolrClient<CloudSolrClient> solrClient;
+ private static SolrIO.ConnectionConfiguration connectionConfiguration;
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ // setup credential for solr user,
+ // See https://cwiki.apache.org/confluence/display/solr/Basic+Authentication+Plugin
+ String password = "SolrRocks";
+ // salt's size can be arbitrary
+ byte[] salt = new byte[random().nextInt(30) + 1];
+ random().nextBytes(salt);
+ String base64Salt = BaseEncoding.base64().encode(salt);
+ String sha56 = Sha256AuthenticationProvider.sha256(password, base64Salt);
+ String credential = sha56 + " " + base64Salt;
+ String securityJson =
+ "{"
+ + "'authentication':{"
+ + " 'blockUnknown': true,"
+ + " 'class':'solr.BasicAuthPlugin',"
+ + " 'credentials':{'solr':'"
+ + credential
+ + "'}}"
+ + "}";
+
+ configureCluster(3).addConfig("conf", getFile("cloud-minimal/conf").toPath()).configure();
+ ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
+ zkStateReader
+ .getZkClient()
+ .setData("/security.json", securityJson.getBytes(Charset.defaultCharset()), true);
+ String zkAddress = cluster.getZkServer().getZkAddress();
+ connectionConfiguration =
+ SolrIO.ConnectionConfiguration.create(zkAddress).withBasicCredentials("solr", password);
+ solrClient = connectionConfiguration.createClient();
+ SolrIOTestUtils.createCollection(SOLR_COLLECTION, NUM_SHARDS, 1, solrClient);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ solrClient.close();
+ }
+
+ @Before
+ public void before() throws Exception {
+ SolrIOTestUtils.clearCollection(SOLR_COLLECTION, solrClient);
+ }
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ public void testBadCredentials() throws IOException {
+ thrown.expect(SolrException.class);
+
+ String zkAddress = cluster.getZkServer().getZkAddress();
+ SolrIO.ConnectionConfiguration connectionConfiguration =
+ SolrIO.ConnectionConfiguration.create(zkAddress)
+ .withBasicCredentials("solr", "wrongpassword");
+ try (AuthorizedSolrClient solrClient = connectionConfiguration.createClient()) {
+ SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient);
+ }
+ }
+
+ @Test
+ public void testSizes() throws Exception {
+ SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient);
+
+ PipelineOptions options = PipelineOptionsFactory.create();
+ SolrIO.Read read =
+ SolrIO.read().withConnectionConfiguration(connectionConfiguration).from(SOLR_COLLECTION);
+ SolrIO.BoundedSolrSource initialSource = new SolrIO.BoundedSolrSource(read, null);
+ // can't use equal assert as Solr collections never have same size
+ // (due to internal Lucene implementation)
+ long estimatedSize = initialSource.getEstimatedSizeBytes(options);
+ LOG.info("Estimated size: {}", estimatedSize);
+ assertThat(
+ "Wrong estimated size bellow minimum",
+ estimatedSize,
+ greaterThan(SolrIOTestUtils.MIN_DOC_SIZE * NUM_DOCS));
+ assertThat(
+ "Wrong estimated size beyond maximum",
+ estimatedSize,
+ lessThan(SolrIOTestUtils.MAX_DOC_SIZE * NUM_DOCS));
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient);
+
+ PCollection<SolrDocument> output =
+ pipeline.apply(
+ SolrIO.read()
+ .withConnectionConfiguration(connectionConfiguration)
+ .from(SOLR_COLLECTION)
+ .withBatchSize(101));
+ PAssert.thatSingleton(output.apply("Count", Count.<SolrDocument>globally()))
+ .isEqualTo(NUM_DOCS);
+ pipeline.run();
+ }
+
+ @Test
+ public void testReadWithQuery() throws Exception {
+ SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient);
+
+ PCollection<SolrDocument> output =
+ pipeline.apply(
+ SolrIO.read()
+ .withConnectionConfiguration(connectionConfiguration)
+ .from(SOLR_COLLECTION)
+ .withQuery("scientist:Franklin"));
+ PAssert.thatSingleton(output.apply("Count", Count.<SolrDocument>globally()))
+ .isEqualTo(NUM_DOCS / NUM_SCIENTISTS);
+ pipeline.run();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+ List<SolrInputDocument> data = SolrIOTestUtils.createDocuments(NUM_DOCS);
+ SolrIO.Write write =
+ SolrIO.write().withConnectionConfiguration(connectionConfiguration).to(SOLR_COLLECTION);
+ pipeline.apply(Create.of(data)).apply(write);
+ pipeline.run();
+
+ long currentNumDocs = SolrIOTestUtils.commitAndGetCurrentNumDocs(SOLR_COLLECTION, solrClient);
+ assertEquals(NUM_DOCS, currentNumDocs);
+
+ QueryResponse response = solrClient.query(SOLR_COLLECTION, new SolrQuery("scientist:Lovelace"));
+ assertEquals(NUM_DOCS / NUM_SCIENTISTS, response.getResults().getNumFound());
+ }
+
+ @Test
+ public void testWriteWithMaxBatchSize() throws Exception {
+ SolrIO.Write write =
+ SolrIO.write()
+ .withConnectionConfiguration(connectionConfiguration)
+ .to(SOLR_COLLECTION)
+ .withMaxBatchSize(BATCH_SIZE);
+ // write bundles size is the runner decision, we cannot force a bundle size,
+ // so we test the Writer as a DoFn outside of a runner.
+ try (DoFnTester<SolrInputDocument, Void> fnTester =
+ DoFnTester.of(new SolrIO.Write.WriteFn(write))) {
+ List<SolrInputDocument> input = SolrIOTestUtils.createDocuments(NUM_DOCS);
+ long numDocsProcessed = 0;
+ long numDocsInserted = 0;
+ for (SolrInputDocument document : input) {
+ fnTester.processElement(document);
+ numDocsProcessed++;
+ // test every 100 docs to avoid overloading Solr
+ if ((numDocsProcessed % 100) == 0) {
+ // force the index to upgrade after inserting for the inserted docs
+ // to be searchable immediately
+ long currentNumDocs =
+ SolrIOTestUtils.commitAndGetCurrentNumDocs(SOLR_COLLECTION, solrClient);
+ if ((numDocsProcessed % BATCH_SIZE) == 0) {
+ /* bundle end */
+ assertEquals(
+ "we are at the end of a bundle, we should have inserted all processed documents",
+ numDocsProcessed,
+ currentNumDocs);
+ numDocsInserted = currentNumDocs;
+ } else {
+ /* not bundle end */
+ assertEquals(
+ "we are not at the end of a bundle, we should have inserted no more documents",
+ numDocsInserted,
+ currentNumDocs);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSplit() throws Exception {
+ SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient);
+
+ PipelineOptions options = PipelineOptionsFactory.create();
+ SolrIO.Read read =
+ SolrIO.read().withConnectionConfiguration(connectionConfiguration).from(SOLR_COLLECTION);
+ SolrIO.BoundedSolrSource initialSource = new SolrIO.BoundedSolrSource(read, null);
+ //desiredBundleSize is ignored for now
+ int desiredBundleSizeBytes = 0;
+ List<? extends BoundedSource<SolrDocument>> splits =
+ initialSource.split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options);
+
+ int expectedNumSplits = NUM_SHARDS;
+ assertEquals(expectedNumSplits, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<SolrDocument> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ // docs are hashed by id to shards, in this test, NUM_DOCS >> NUM_SHARDS
+ // therefore, can not exist an empty shard.
+ assertEquals("Wrong number of empty splits", expectedNumSplits, nonEmptySplits);
+ }
+}
diff --git a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTestUtils.java b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTestUtils.java
new file mode 100644
index 0000000..fb99d55
--- /dev/null
+++ b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTestUtils.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.solr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrInputDocument;
+
+/** Test utilities to use with {@link SolrIO}. */
+public class SolrIOTestUtils {
+ public static final long MIN_DOC_SIZE = 30L;
+ public static final long MAX_DOC_SIZE = 150L;
+
+ static void createCollection(
+ String collection, int numShards, int replicationFactor, AuthorizedSolrClient client)
+ throws Exception {
+ CollectionAdminRequest.Create create =
+ new CollectionAdminRequest.Create()
+ .setCollectionName(collection)
+ .setNumShards(numShards)
+ .setReplicationFactor(replicationFactor)
+ .setMaxShardsPerNode(2);
+ client.process(create);
+ }
+
+ /** Inserts the given number of test documents into Solr. */
+ static void insertTestDocuments(String collection, long numDocs, AuthorizedSolrClient client)
+ throws IOException {
+ List<SolrInputDocument> data = createDocuments(numDocs);
+ try {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setAction(UpdateRequest.ACTION.COMMIT, true, true);
+ updateRequest.add(data);
+ client.process(collection, updateRequest);
+ } catch (SolrServerException e) {
+ throw new IOException("Failed to insert test documents to collection", e);
+ }
+ }
+
+ /** Delete given collection. */
+ static void deleteCollection(String collection, AuthorizedSolrClient client) throws IOException {
+ try {
+ CollectionAdminRequest.Delete delete =
+ new CollectionAdminRequest.Delete().setCollectionName(collection);
+ client.process(delete);
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** Clear given collection. */
+ static void clearCollection(String collection, AuthorizedSolrClient client) throws IOException {
+ try {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setAction(UpdateRequest.ACTION.COMMIT, true, true);
+ updateRequest.deleteByQuery("*:*");
+ client.process(collection, updateRequest);
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Forces a commit of the given collection to make recently inserted documents available for
+ * search.
+ *
+ * @return The number of docs in the index
+ */
+ static long commitAndGetCurrentNumDocs(String collection, AuthorizedSolrClient client)
+ throws IOException {
+ SolrQuery solrQuery = new SolrQuery("*:*");
+ solrQuery.setRows(0);
+ try {
+ UpdateRequest update = new UpdateRequest();
+ update.setAction(UpdateRequest.ACTION.COMMIT, true, true);
+ client.process(collection, update);
+
+ return client.query(collection, new SolrQuery("*:*")).getResults().getNumFound();
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Generates a list of test documents for insertion.
+ *
+ * @return the list of json String representing the documents
+ */
+ static List<SolrInputDocument> createDocuments(long numDocs) {
+ String[] scientists = {
+ "Lovelace",
+ "Franklin",
+ "Meitner",
+ "Hopper",
+ "Curie",
+ "Faraday",
+ "Newton",
+ "Bohr",
+ "Galilei",
+ "Maxwell"
+ };
+ ArrayList<SolrInputDocument> data = new ArrayList<>();
+ for (int i = 0; i < numDocs; i++) {
+ int index = i % scientists.length;
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", String.valueOf(i));
+ doc.setField("scientist", scientists[index]);
+ data.add(doc);
+ }
+ return data;
+ }
+}
diff --git a/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/schema.xml b/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/schema.xml
new file mode 100644
index 0000000..08a1716
--- /dev/null
+++ b/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/schema.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+ <fieldType name="string" class="solr.StrField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <dynamicField name="*" type="string" indexed="true" stored="true"/>
+ <!-- for versioning -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="id" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true" />
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/solrconfig.xml b/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/solrconfig.xml
new file mode 100644
index 0000000..8da7d28
--- /dev/null
+++ b/sdks/java/io/solr/src/test/resources/cloud-minimal/conf/solrconfig.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+</config>
+
diff --git a/sdks/java/io/tika/pom.xml b/sdks/java/io/tika/pom.xml
new file mode 100644
index 0000000..5e48099
--- /dev/null
+++ b/sdks/java/io/tika/pom.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-tika</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: Tika</name>
+ <description>Tika Input to parse files.</description>
+
+
+ <properties>
+ <tika.version>1.16</tika.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${tika.version}</version>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parsers</artifactId>
+ <version>${tika.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
new file mode 100644
index 0000000..f78d603
--- /dev/null
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Throwables;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.util.SerializableThrowable;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * The result of parsing a single file with Tika: contains the file's location, metadata, extracted
+ * text, and optionally an error. If there is an error, the metadata and extracted text may be
+ * partial (i.e. not represent the entire file).
+ */
+public class ParseResult implements Serializable {
+ private final String fileLocation;
+ private final String content;
+ private final Metadata metadata;
+ private final String[] metadataNames;
+ @Nullable private final SerializableThrowable error;
+
+ public static ParseResult success(String fileLocation, String content, Metadata metadata) {
+ return new ParseResult(fileLocation, content, metadata, null);
+ }
+
+ public static ParseResult success(String fileLocation, String content) {
+ return new ParseResult(fileLocation, content, new Metadata(), null);
+ }
+
+ public static ParseResult failure(
+ String fileLocation, String partialContent, Metadata partialMetadata, Throwable error) {
+ return new ParseResult(fileLocation, partialContent, partialMetadata, error);
+ }
+
+ private ParseResult(String fileLocation, String content, Metadata metadata, Throwable error) {
+ checkArgument(fileLocation != null, "fileLocation can not be null");
+ checkArgument(content != null, "content can not be null");
+ checkArgument(metadata != null, "metadata can not be null");
+ this.fileLocation = fileLocation;
+ this.content = content;
+ this.metadata = metadata;
+ this.metadataNames = metadata.names();
+ this.error = (error == null) ? null : new SerializableThrowable(error);
+ }
+
+ /** Returns the absolute path to the input file. */
+ public String getFileLocation() {
+ return fileLocation;
+ }
+
+ /** Returns whether this file was parsed successfully. */
+ public boolean isSuccess() {
+ return error == null;
+ }
+
+ /** Returns the parse error, if the file was parsed unsuccessfully. */
+ public Throwable getError() {
+ checkState(error != null, "This is a successful ParseResult");
+ return error.getThrowable();
+ }
+
+ /**
+ * Same as {@link #getError}, but returns the complete stack trace of the error as a {@link
+ * String}.
+ */
+ public String getErrorAsString() {
+ return Throwables.getStackTraceAsString(getError());
+ }
+
+ /** Returns the extracted text. May be partial, if this parse result contains a failure. */
+ public String getContent() {
+ return content;
+ }
+
+ /** Returns the extracted metadata. May be partial, if this parse result contains a failure. */
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ getFileLocation(),
+ getContent(),
+ getMetadataHashCode(),
+ isSuccess() ? "" : Throwables.getStackTraceAsString(getError()));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ParseResult)) {
+ return false;
+ }
+
+ ParseResult other = (ParseResult) obj;
+ return Objects.equals(getFileLocation(), other.getFileLocation())
+ && Objects.equals(getContent(), other.getContent())
+ && Objects.equals(getMetadata(), other.getMetadata())
+ && (isSuccess()
+ ? other.isSuccess()
+ : (!other.isSuccess() && Objects.equals(getErrorAsString(), other.getErrorAsString())));
+ }
+
+ // TODO: Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
+ private int getMetadataHashCode() {
+ int hashCode = 0;
+ for (String name : metadataNames) {
+ hashCode += name.hashCode() ^ Arrays.hashCode(metadata.getValues(name));
+ }
+ return hashCode;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("fileLocation", fileLocation)
+ .add("content", "<" + content.length() + " chars>")
+ .add("metadata", metadata)
+ .add("error", getError() == null ? null : Throwables.getStackTraceAsString(getError()))
+ .toString();
+ }
+}
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
new file mode 100644
index 0000000..26f116d
--- /dev/null
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.auto.value.AutoValue;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.xml.sax.ContentHandler;
+
+/**
+ * Transforms for parsing arbitrary files using <a href="https://tika.apache.org/">Apache Tika</a>.
+ *
+ * <p>Tika is able to extract text and metadata from files in many well known text, binary and
+ * scientific formats.
+ *
+ * <p>The entry points are {@link #parse} and {@link #parseFiles}. They parse a set of files and
+ * return a {@link PCollection} containing one {@link ParseResult} per each file. {@link #parse}
+ * implements the common case of parsing all files matching a single filepattern, while {@link
+ * #parseFiles} should be used for all use cases requiring more control, in combination with {@link
+ * FileIO#match} and {@link FileIO#readMatches} (see their respective documentation).
+ *
+ * <p>{@link #parse} does not automatically uncompress compressed files: they are passed to Tika
+ * as-is.
+ *
+ * <p>It's possible that some files will partially or completely fail to parse. In that case, the
+ * respective {@link ParseResult} will be marked unsuccessful (see {@link ParseResult#isSuccess})
+ * and will contain the error, available via {@link ParseResult#getError}.
+ *
+ * <p>Example: using {@link #parse} to parse all PDF files in a directory on GCS.
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<ParseResult> results =
+ * p.apply(TikaIO.parse().filepattern("gs://my-bucket/files/*.pdf"));
+ * }</pre>
+ *
+ * <p>Example: using {@link #parseFiles} in combination with {@link FileIO} to continuously parse
+ * new PDF files arriving into the directory.
+ *
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * PCollection<ParseResult> results =
+ * p.apply(FileIO.match().filepattern("gs://my-bucket/files/*.pdf")
+ * .continuously(...))
+ * .apply(FileIO.readMatches())
+ * .apply(TikaIO.parseFiles());
+ * }</pre>
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class TikaIO {
+ /** Parses files matching a given filepattern. */
+ public static Parse parse() {
+ return new AutoValue_TikaIO_Parse.Builder().build();
+ }
+
+ /** Parses files in a {@link PCollection} of {@link ReadableFile}. */
+ public static ParseFiles parseFiles() {
+ return new AutoValue_TikaIO_ParseFiles.Builder().build();
+ }
+
+ /** Implementation of {@link #parse}. */
+ @AutoValue
+ public abstract static class Parse extends PTransform<PBegin, PCollection<ParseResult>> {
+ @Nullable
+ abstract ValueProvider<String> getFilepattern();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setFilepattern(ValueProvider<String> filepattern);
+
+ abstract Parse build();
+ }
+
+ /** Matches the given filepattern. */
+ public Parse filepattern(String filepattern) {
+ return this.filepattern(ValueProvider.StaticValueProvider.of(filepattern));
+ }
+
+ /** Like {@link #filepattern(String)} but using a {@link ValueProvider}. */
+ public Parse filepattern(ValueProvider<String> filepattern) {
+ return toBuilder().setFilepattern(filepattern).build();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder.addIfNotNull(
+ DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"));
+ }
+
+ @Override
+ public PCollection<ParseResult> expand(PBegin input) {
+ return input
+ .apply(FileIO.match().filepattern(getFilepattern()))
+ .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+ .apply(parseFiles());
+ }
+ }
+
+ /** Implementation of {@link #parseFiles}. */
+ @AutoValue
+ public abstract static class ParseFiles
+ extends PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+
+ @Nullable
+ abstract ValueProvider<String> getTikaConfigPath();
+
+ @Nullable
+ abstract String getContentTypeHint();
+
+ @Nullable
+ abstract Metadata getInputMetadata();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
+
+ abstract Builder setContentTypeHint(String contentTypeHint);
+
+ abstract Builder setInputMetadata(Metadata metadata);
+
+ abstract ParseFiles build();
+ }
+
+ /**
+ * Uses the given <a
+ * href="https://tika.apache.org/1.16/configuring.html#Using_a_Tika_Configuration_XML_file">Tika
+ * Configuration XML file</a>.
+ */
+ public ParseFiles withTikaConfigPath(String tikaConfigPath) {
+ checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
+ return withTikaConfigPath(StaticValueProvider.of(tikaConfigPath));
+ }
+
+ /** Like {@code with(tikaConfigPath)}. */
+ public ParseFiles withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
+ checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
+ return toBuilder().setTikaConfigPath(tikaConfigPath).build();
+ }
+
+ /**
+ * Sets a content type hint to make the file parser detection more efficient. Overrides the
+ * content type hint in {@link #withInputMetadata}, if any.
+ */
+ public ParseFiles withContentTypeHint(String contentTypeHint) {
+ checkNotNull(contentTypeHint, "contentTypeHint can not be null.");
+ return toBuilder().setContentTypeHint(contentTypeHint).build();
+ }
+
+ /** Sets the input metadata for {@link Parser#parse}. */
+ public ParseFiles withInputMetadata(Metadata metadata) {
+ Metadata inputMetadata = this.getInputMetadata();
+ if (inputMetadata != null) {
+ for (String name : metadata.names()) {
+ inputMetadata.set(name, metadata.get(name));
+ }
+ } else {
+ inputMetadata = metadata;
+ }
+ return toBuilder().setInputMetadata(inputMetadata).build();
+ }
+
+ @Override
+ public PCollection<ParseResult> expand(PCollection<ReadableFile> input) {
+ return input.apply(ParDo.of(new ParseToStringFn(this)));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ if (getTikaConfigPath() != null) {
+ builder.add(
+ DisplayData.item("tikaConfigPath", getTikaConfigPath()).withLabel("TikaConfig Path"));
+ }
+ Metadata metadata = getInputMetadata();
+ if (metadata != null) {
+ //TODO: use metadata.toString() only without a trim() once Apache Tika 1.17 gets released
+ builder.add(
+ DisplayData.item("inputMetadata", metadata.toString().trim())
+ .withLabel("Input Metadata"));
+ }
+ builder.addIfNotNull(
+ DisplayData.item("contentTypeHint", getContentTypeHint()).withLabel("Content type hint"));
+ }
+
+ private static class ParseToStringFn extends DoFn<ReadableFile, ParseResult> {
+ private final ParseFiles spec;
+ private transient TikaConfig tikaConfig;
+
+ ParseToStringFn(ParseFiles spec) {
+ this.spec = spec;
+ }
+
+ @Setup
+ public void setup() throws Exception {
+ if (spec.getTikaConfigPath() != null) {
+ ResourceId configResource =
+ FileSystems.matchSingleFileSpec(spec.getTikaConfigPath().get()).resourceId();
+ tikaConfig = new TikaConfig(Channels.newInputStream(FileSystems.open(configResource)));
+ }
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ ReadableFile file = c.element();
+ InputStream stream = Channels.newInputStream(file.open());
+ try (InputStream tikaStream = TikaInputStream.get(stream)) {
+ Parser parser =
+ tikaConfig == null ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
+
+ ParseContext context = new ParseContext();
+ context.set(Parser.class, parser);
+ Metadata tikaMetadata =
+ spec.getInputMetadata() != null
+ ? spec.getInputMetadata()
+ : new org.apache.tika.metadata.Metadata();
+ if (spec.getContentTypeHint() != null) {
+ tikaMetadata.set(Metadata.CONTENT_TYPE, spec.getContentTypeHint());
+ }
+
+ String location = file.getMetadata().resourceId().toString();
+ ParseResult res;
+ ContentHandler tikaHandler = new ToTextContentHandler();
+ try {
+ parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
+ res = ParseResult.success(location, tikaHandler.toString(), tikaMetadata);
+ } catch (Exception e) {
+ res = ParseResult.failure(location, tikaHandler.toString(), tikaMetadata, e);
+ }
+
+ c.output(res);
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/package-info.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/package-info.java
new file mode 100644
index 0000000..972d69f
--- /dev/null
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transform for reading and parsing files with Apache Tika.
+ */
+package org.apache.beam.sdk.io.tika;
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
new file mode 100644
index 0000000..95bcee0
--- /dev/null
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import org.apache.tika.metadata.Metadata;
+import org.junit.Test;
+
+/** Tests {@link ParseResult}. */
+public class ParseResultTest {
+ @Test
+ public void testEqualsAndHashCode() {
+ ParseResult successBase = ParseResult.success("a.txt", "hello", getMetadata());
+ ParseResult successSame = ParseResult.success("a.txt", "hello", getMetadata());
+ ParseResult successDifferentName = ParseResult.success("b.txt", "hello", getMetadata());
+ ParseResult successDifferentContent = ParseResult.success("a.txt", "goodbye", getMetadata());
+ ParseResult successDifferentMetadata = ParseResult.success("a.txt", "hello", new Metadata());
+
+ RuntimeException oops = new RuntimeException("oops");
+ ParseResult failureBase = ParseResult.failure("a.txt", "", new Metadata(), oops);
+ ParseResult failureSame = ParseResult.failure("a.txt", "", new Metadata(), oops);
+ ParseResult failureDifferentName = ParseResult.failure("b.txt", "", new Metadata(), oops);
+ ParseResult failureDifferentContent =
+ ParseResult.failure("b.txt", "partial", new Metadata(), oops);
+ ParseResult failureDifferentMetadata = ParseResult.failure("b.txt", "", getMetadata(), oops);
+ ParseResult failureDifferentError =
+ ParseResult.failure("a.txt", "", new Metadata(), new RuntimeException("eek"));
+
+ assertEquals(successBase, successSame);
+ assertEquals(successBase.hashCode(), successSame.hashCode());
+
+ assertThat(successDifferentName, not(equalTo(successBase)));
+ assertThat(successDifferentContent, not(equalTo(successBase)));
+ assertThat(successDifferentMetadata, not(equalTo(successBase)));
+
+ assertThat(successDifferentName.hashCode(), not(equalTo(successBase.hashCode())));
+ assertThat(successDifferentContent.hashCode(), not(equalTo(successBase.hashCode())));
+ assertThat(successDifferentMetadata.hashCode(), not(equalTo(successBase.hashCode())));
+
+ assertThat(failureBase, not(equalTo(successBase)));
+ assertThat(successBase, not(equalTo(failureBase)));
+
+ assertEquals(failureBase, failureSame);
+ assertEquals(failureBase.hashCode(), failureSame.hashCode());
+
+ assertThat(failureDifferentName, not(equalTo(failureBase)));
+ assertThat(failureDifferentError, not(equalTo(failureBase)));
+ assertThat(failureDifferentContent, not(equalTo(failureBase)));
+ assertThat(failureDifferentMetadata, not(equalTo(failureBase)));
+
+ assertThat(failureDifferentName.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentError.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentContent.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentMetadata.hashCode(), not(equalTo(failureBase.hashCode())));
+ }
+
+ static Metadata getMetadata() {
+ Metadata m = new Metadata();
+ m.add("Author", "BeamTikaUser");
+ m.add("Author", "BeamTikaUser2");
+ m.add("Date", "2017-09-01");
+ return m;
+ }
+}
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
new file mode 100644
index 0000000..1c95e9f
--- /dev/null
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.junit.Rule;
+import org.junit.Test;
+
+/** Tests for {@link TikaIO}. */
+public class TikaIOTest implements Serializable {
+ private static final String PDF_ZIP_FILE =
+ "\n\n\n\n\n\n\n\napache-beam-tika.pdf\n\n\nCombining\n\n\nApache Beam\n\n\n"
+ + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
+ + "in most known formats.\n\n\n\n\n\n\n";
+
+ private static final String ODT_FILE =
+ "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+ + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
+ + " files\nin most known formats.\n";
+
+ @Rule
+ public transient TestPipeline p = TestPipeline.create();
+
+ @Test
+ public void testParseAndParseFiles() throws IOException {
+ Path root =
+ Paths.get(getClass().getResource("/valid/apache-beam-tika.odt").getPath()).getParent();
+
+ List<ParseResult> expected =
+ Arrays.asList(
+ ParseResult.success(
+ root.resolve("apache-beam-tika.odt").toString(), ODT_FILE, getOdtMetadata()),
+ ParseResult.success(root.resolve("apache-beam-tika-pdf.zip").toString(), PDF_ZIP_FILE));
+
+ PCollection<ParseResult> parse =
+ p.apply("Parse", TikaIO.parse().filepattern(root.resolve("*").toString()))
+ .apply("FilterParse", ParDo.of(new FilterMetadataFn()));
+ PAssert.that(parse).containsInAnyOrder(expected);
+
+ PCollection<ParseResult> parseFiles =
+ p.apply("ParseFiles", FileIO.match().filepattern(root.resolve("*").toString()))
+ .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+ .apply(TikaIO.parseFiles())
+ .apply("FilterParseFiles", ParDo.of(new FilterMetadataFn()));
+ PAssert.that(parseFiles).containsInAnyOrder(expected);
+ p.run();
+ }
+
+ private static Metadata getOdtMetadata() {
+ Metadata m = new Metadata();
+ m.set("Author", "BeamTikaUser");
+ return m;
+ }
+
+ private static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ ParseResult result = c.element();
+ Metadata m = new Metadata();
+ // Files contain many metadata properties. This function drops all but the "Author"
+ // property manually added to "apache-beam-tika.odt" resource only to make
+ // the tests simpler
+ if (result.getFileLocation().endsWith("valid/apache-beam-tika.odt")) {
+ m.set("Author", result.getMetadata().get("Author"));
+ }
+ ParseResult newResult = ParseResult.success(result.getFileLocation(), result.getContent(), m);
+ c.output(newResult);
+ }
+ }
+
+ @Test
+ public void testParseDamagedPdfFile() throws IOException {
+ String path = getClass().getResource("/damaged.pdf").getPath();
+ PCollection<ParseResult> res = p.apply("ParseInvalidPdfFile", TikaIO.parse().filepattern(path));
+
+ PAssert.thatSingleton(res)
+ .satisfies(
+ new SerializableFunction<ParseResult, Void>() {
+ @Override
+ public Void apply(ParseResult input) {
+ assertEquals(path, input.getFileLocation());
+ assertFalse(input.isSuccess());
+ assertTrue(input.getError() instanceof TikaException);
+ return null;
+ }
+ });
+ p.run();
+ }
+
+ @Test
+ public void testParseDisplayData() {
+ TikaIO.Parse parse = TikaIO.parse().filepattern("file.pdf");
+
+ DisplayData displayData = DisplayData.from(parse);
+
+ assertThat(displayData, hasDisplayItem("filePattern", "file.pdf"));
+ assertEquals(1, displayData.items().size());
+ }
+
+ @Test
+ public void testParseFilesDisplayData() {
+ TikaIO.ParseFiles parseFiles =
+ TikaIO.parseFiles()
+ .withTikaConfigPath("/tikaConfigPath")
+ .withContentTypeHint("application/pdf");
+
+ DisplayData displayData = DisplayData.from(parseFiles);
+
+ assertThat(displayData, hasDisplayItem("tikaConfigPath", "/tikaConfigPath"));
+ assertThat(displayData, hasDisplayItem("contentTypeHint", "application/pdf"));
+ }
+}
diff --git a/sdks/java/io/tika/src/test/resources/damaged.pdf b/sdks/java/io/tika/src/test/resources/damaged.pdf
new file mode 100644
index 0000000..7653b4b
--- /dev/null
+++ b/sdks/java/io/tika/src/test/resources/damaged.pdf
@@ -0,0 +1,2 @@
+%PDF-1.4
+
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
new file mode 100644
index 0000000..4c0e0ef
--- /dev/null
+++ b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
Binary files differ
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
new file mode 100644
index 0000000..87c5577
--- /dev/null
+++ b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
Binary files differ
diff --git a/sdks/java/io/xml/pom.xml b/sdks/java/io/xml/pom.xml
index cf7dd33..9633e61 100644
--- a/sdks/java/io/xml/pom.xml
+++ b/sdks/java/io/xml/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
index 7255a94..47626cd 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
@@ -17,26 +17,31 @@
*/
package org.apache.beam.sdk.io.xml;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
import java.nio.charset.Charset;
import javax.annotation.Nullable;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.ValidationEventHandler;
-import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.CompressedSource;
+import org.apache.beam.sdk.io.Compression;
import org.apache.beam.sdk.io.FileBasedSink;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
import org.apache.beam.sdk.io.OffsetBasedSource;
+import org.apache.beam.sdk.io.ReadAllViaFileBasedSource;
import org.apache.beam.sdk.io.fs.ResourceId;
-import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
@@ -45,10 +50,9 @@
public class XmlIO {
// CHECKSTYLE.OFF: JavadocStyle
/**
- * Reads XML files. This source reads one or more XML files and creates a {@link PCollection} of a
- * given type. Please note the example given below.
+ * Reads XML files as a {@link PCollection} of a given type mapped via JAXB.
*
- * <p>The XML file must be of the following form, where {@code root} and {@code record} are XML
+ * <p>The XML files must be of the following form, where {@code root} and {@code record} are XML
* element names that are defined by the user:
*
* <pre>{@code
@@ -73,7 +77,7 @@
* Reading the source will generate a {@code PCollection} of the given JAXB annotated Java type.
* Optionally users may provide a minimum size of a bundle that should be created for the source.
*
- * <p>The following example shows how to use this method in a Beam pipeline:
+ * <p>Example:
*
* <pre>{@code
* PCollection<String> output = p.apply(XmlIO.<Record>read()
@@ -83,38 +87,48 @@
* .withRecordClass(Record.class));
* }</pre>
*
- * <p>By default, UTF-8 charset is used. If your file is using a different charset, you have to
- * specify the following:
- *
- * <pre>{@code
- * PCollection<String> output = p.apply(XmlIO.<Record>read()
- * .from(file.toPath().toString())
- * .withRooElement("root")
- * .withRecordElement("record")
- * .withRecordClass(Record.class)
- * .withCharset(StandardCharsets.ISO_8859_1));
- * }</pre>
- *
- * <p>{@link java.nio.charset.StandardCharsets} provides static references to common charsets.
+ * <p>By default, UTF-8 charset is used. To specify a different charset, use {@link
+ * Read#withCharset}.
*
* <p>Currently, only XML files that use single-byte characters are supported. Using a file that
* contains multi-byte characters may result in data loss or duplication.
*
- * <h3>Permissions</h3>
- *
- * <p>Permission requirements depend on the {@link PipelineRunner
- * PipelineRunner} that is used to execute the Beam pipeline. Please refer to the documentation of
- * corresponding {@link PipelineRunner PipelineRunners} for more details.
- *
* @param <T> Type of the objects that represent the records of the XML file. The {@code
* PCollection} generated by this source will be of this type.
*/
// CHECKSTYLE.ON: JavadocStyle
public static <T> Read<T> read() {
return new AutoValue_XmlIO_Read.Builder<T>()
- .setMinBundleSize(Read.DEFAULT_MIN_BUNDLE_SIZE)
- .setCompressionType(Read.CompressionType.AUTO)
- .setCharset("UTF-8")
+ .setConfiguration(
+ new AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset("UTF-8").build())
+ .setMinBundleSize(1L)
+ .setCompression(Compression.AUTO)
+ .build();
+ }
+
+ /**
+ * Like {@link #read}, but reads each file in a {@link PCollection} of {@link ReadableFile}, which
+ * allows more flexible usage via different configuration options of {@link FileIO#match} and
+ * {@link FileIO#readMatches} that are not explicitly provided for {@link #read}.
+ *
+ * <p>For example:
+ *
+ * <pre>{@code
+ * PCollection<ReadableFile> files = p
+ * .apply(FileIO.match().filepattern(options.getInputFilepatternProvider()).continuously(
+ * Duration.standardSeconds(30), afterTimeSinceNewOutput(Duration.standardMinutes(5))))
+ * .apply(FileIO.readMatches().withCompression(GZIP));
+ *
+ * PCollection<String> output = files.apply(XmlIO.<Record>readFiles()
+ * .withRootElement("root")
+ * .withRecordElement("record")
+ * .withRecordClass(Record.class));
+ * }</pre>
+ */
+ public static <T> ReadFiles<T> readFiles() {
+ return new AutoValue_XmlIO_ReadFiles.Builder<T>()
+ .setConfiguration(
+ new AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset("UTF-8").build())
.build();
}
@@ -230,85 +244,118 @@
return new AutoValue_XmlIO_Write.Builder<T>().setCharset("UTF-8").build();
}
- /** Implementation of {@link #read}. */
@AutoValue
- public abstract static class Read<T> extends PTransform<PBegin, PCollection<T>> {
- private static final int DEFAULT_MIN_BUNDLE_SIZE = 8 * 1024;
-
- @Nullable
- abstract String getFileOrPatternSpec();
-
- @Nullable
- abstract String getRootElement();
-
- @Nullable
- abstract String getRecordElement();
-
- @Nullable
- abstract Class<T> getRecordClass();
-
- abstract CompressionType getCompressionType();
-
- abstract long getMinBundleSize();
-
- @Nullable
- abstract String getCharset();
+ abstract static class MappingConfiguration<T> implements HasDisplayData, Serializable {
+ @Nullable abstract String getRootElement();
+ @Nullable abstract String getRecordElement();
+ @Nullable abstract Class<T> getRecordClass();
+ @Nullable abstract String getCharset();
+ @Nullable abstract ValidationEventHandler getValidationEventHandler();
abstract Builder<T> toBuilder();
- @Nullable
- abstract ValidationEventHandler getValidationEventHandler();
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setRootElement(String rootElement);
+ abstract Builder<T> setRecordElement(String recordElement);
+ abstract Builder<T> setRecordClass(Class<T> recordClass);
+ abstract Builder<T> setCharset(String charset);
+ abstract Builder<T> setValidationEventHandler(ValidationEventHandler validationEventHandler);
+
+ abstract MappingConfiguration<T> build();
+ }
+
+ private MappingConfiguration<T> withRootElement(String rootElement) {
+ return toBuilder().setRootElement(rootElement).build();
+ }
+
+ private MappingConfiguration<T> withRecordElement(String recordElement) {
+ return toBuilder().setRecordElement(recordElement).build();
+ }
+
+ private MappingConfiguration<T> withRecordClass(Class<T> recordClass) {
+ return toBuilder().setRecordClass(recordClass).build();
+ }
+
+ private MappingConfiguration<T> withCharset(Charset charset) {
+ return toBuilder().setCharset(charset.name()).build();
+ }
+
+ private MappingConfiguration<T> withValidationEventHandler(
+ ValidationEventHandler validationEventHandler) {
+ return toBuilder().setValidationEventHandler(validationEventHandler).build();
+ }
+
+ private void validate() {
+ checkArgument(getRootElement() != null, "withRootElement() is required");
+ checkArgument(getRecordElement() != null, "withRecordElement() is required");
+ checkArgument(getRecordClass() != null, "withRecordClass() is required");
+ checkArgument(getCharset() != null, "withCharset() is required");
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder
+ .addIfNotNull(
+ DisplayData.item("rootElement", getRootElement()).withLabel("XML Root Element"))
+ .addIfNotNull(
+ DisplayData.item("recordElement", getRecordElement()).withLabel("XML Record Element"))
+ .addIfNotNull(
+ DisplayData.item("recordClass", getRecordClass()).withLabel("XML Record Class"))
+ .addIfNotNull(DisplayData.item("charset", getCharset()).withLabel("Charset"));
+ }
+ }
+
+ /** Implementation of {@link #read}. */
+ @AutoValue
+ public abstract static class Read<T> extends PTransform<PBegin, PCollection<T>> {
+ abstract MappingConfiguration<T> getConfiguration();
+ @Nullable abstract String getFileOrPatternSpec();
+ abstract Compression getCompression();
+ abstract long getMinBundleSize();
+
+ abstract Builder<T> toBuilder();
@AutoValue.Builder
abstract static class Builder<T> {
+ abstract Builder<T> setConfiguration(MappingConfiguration<T> configuration);
abstract Builder<T> setFileOrPatternSpec(String fileOrPatternSpec);
-
- abstract Builder<T> setRootElement(String rootElement);
-
- abstract Builder<T> setRecordElement(String recordElement);
-
- abstract Builder<T> setRecordClass(Class<T> recordClass);
-
+ abstract Builder<T> setCompression(Compression compression);
abstract Builder<T> setMinBundleSize(long minBundleSize);
- abstract Builder<T> setCompressionType(CompressionType compressionType);
-
- abstract Builder<T> setCharset(String charset);
-
- abstract Builder<T> setValidationEventHandler(ValidationEventHandler validationEventHandler);
-
abstract Read<T> build();
}
- /** Strategy for determining the compression type of XML files being read. */
+ /** @deprecated Use {@link Compression} instead. */
+ @Deprecated
public enum CompressionType {
- /** Automatically determine the compression type based on filename extension. */
- AUTO(""),
- /** Uncompressed (i.e., may be split). */
- UNCOMPRESSED(""),
- /** GZipped. */
- GZIP(".gz"),
- /** BZipped. */
- BZIP2(".bz2"),
- /** Zipped. */
- ZIP(".zip"),
- /** Deflate compressed. */
- DEFLATE(".deflate");
+ /** @see Compression#AUTO */
+ AUTO(Compression.AUTO),
- private String filenameSuffix;
+ /** @see Compression#UNCOMPRESSED */
+ UNCOMPRESSED(Compression.UNCOMPRESSED),
- CompressionType(String suffix) {
- this.filenameSuffix = suffix;
+ /** @see Compression#GZIP */
+ GZIP(Compression.GZIP),
+
+ /** @see Compression#BZIP2 */
+ BZIP2(Compression.BZIP2),
+
+ /** @see Compression#ZIP */
+ ZIP(Compression.ZIP),
+
+ /** @see Compression#DEFLATE */
+ DEFLATE(Compression.DEFLATE);
+
+ private Compression canonical;
+
+ CompressionType(Compression canonical) {
+ this.canonical = canonical;
}
- /**
- * Determine if a given filename matches a compression type based on its extension.
- *
- * @param filename the filename to match
- * @return true iff the filename ends with the compression type's known extension.
- */
+ /** @see Compression#matches */
public boolean matches(String filename) {
- return filename.toLowerCase().endsWith(filenameSuffix.toLowerCase());
+ return canonical.matches(filename);
}
}
@@ -320,13 +367,17 @@
return toBuilder().setFileOrPatternSpec(fileOrPatternSpec).build();
}
+ private Read<T> withConfiguration(MappingConfiguration<T> configuration) {
+ return toBuilder().setConfiguration(configuration).build();
+ }
+
/**
* Sets name of the root element of the XML document. This will be used to create a valid
* starting root element when initiating a bundle of records created from an XML document. This
* is a required parameter.
*/
public Read<T> withRootElement(String rootElement) {
- return toBuilder().setRootElement(rootElement).build();
+ return withConfiguration(getConfiguration().withRootElement(rootElement));
}
/**
@@ -334,7 +385,7 @@
* the first record of a bundle created from the XML document. This is a required parameter.
*/
public Read<T> withRecordElement(String recordElement) {
- return toBuilder().setRecordElement(recordElement).build();
+ return withConfiguration(getConfiguration().withRecordElement(recordElement));
}
/**
@@ -343,7 +394,7 @@
* parameter.
*/
public Read<T> withRecordClass(Class<T> recordClass) {
- return toBuilder().setRecordClass(recordClass).build();
+ return withConfiguration(getConfiguration().withRecordClass(recordClass));
}
/**
@@ -355,22 +406,22 @@
return toBuilder().setMinBundleSize(minBundleSize).build();
}
- /**
- * Decompresses all input files using the specified compression type.
- *
- * <p>If no compression type is specified, the default is {@link CompressionType#AUTO}. In this
- * mode, the compression type of the file is determined by its extension. Supports .gz, .bz2,
- * .zip and .deflate compression.
- */
+ /** @deprecated use {@link #withCompression}. */
+ @Deprecated
public Read<T> withCompressionType(CompressionType compressionType) {
- return toBuilder().setCompressionType(compressionType).build();
+ return withCompression(compressionType.canonical);
+ }
+
+ /** Decompresses all input files using the specified compression type. */
+ public Read<T> withCompression(Compression compression) {
+ return toBuilder().setCompression(compression).build();
}
/**
* Sets the XML file charset.
*/
public Read<T> withCharset(Charset charset) {
- return toBuilder().setCharset(charset.name()).build();
+ return withConfiguration(getConfiguration().withCharset(charset));
}
/**
@@ -378,23 +429,8 @@
* parameter will cause the JAXB unmarshaller event handler to be unspecified.
*/
public Read<T> withValidationEventHandler(ValidationEventHandler validationEventHandler) {
- return toBuilder().setValidationEventHandler(validationEventHandler).build();
- }
-
- @Override
- public void validate(PipelineOptions options) {
- checkNotNull(
- getRootElement(),
- "rootElement is null. Use builder method withRootElement() to set this.");
- checkNotNull(
- getRecordElement(),
- "recordElement is null. Use builder method withRecordElement() to set this.");
- checkNotNull(
- getRecordClass(),
- "recordClass is null. Use builder method withRecordClass() to set this.");
- checkNotNull(
- getCharset(),
- "charset is null. Use builder method withCharset() to set this.");
+ return withConfiguration(
+ getConfiguration().withValidationEventHandler(validationEventHandler));
}
@Override
@@ -405,47 +441,90 @@
.withLabel("Minimum Bundle Size"),
1L)
.add(DisplayData.item("filePattern", getFileOrPatternSpec()).withLabel("File Pattern"))
- .addIfNotNull(
- DisplayData.item("rootElement", getRootElement()).withLabel("XML Root Element"))
- .addIfNotNull(
- DisplayData.item("recordElement", getRecordElement()).withLabel("XML Record Element"))
- .addIfNotNull(
- DisplayData.item("recordClass", getRecordClass()).withLabel("XML Record Class"))
- .addIfNotNull(
- DisplayData.item("charset", getCharset()).withLabel("Charset"));
+ .include("configuration", getConfiguration());
}
@VisibleForTesting
BoundedSource<T> createSource() {
- XmlSource<T> source = new XmlSource<>(this);
- switch (getCompressionType()) {
- case UNCOMPRESSED:
- return source;
- case AUTO:
- return CompressedSource.from(source);
- case BZIP2:
- return CompressedSource.from(source)
- .withDecompression(CompressedSource.CompressionMode.BZIP2);
- case GZIP:
- return CompressedSource.from(source)
- .withDecompression(CompressedSource.CompressionMode.GZIP);
- case ZIP:
- return CompressedSource.from(source)
- .withDecompression(CompressedSource.CompressionMode.ZIP);
- case DEFLATE:
- return CompressedSource.from(source)
- .withDecompression(CompressedSource.CompressionMode.DEFLATE);
- default:
- throw new IllegalArgumentException("Unknown compression type: " + getCompressionType());
- }
+ return CompressedSource.from(
+ new XmlSource<>(
+ StaticValueProvider.of(getFileOrPatternSpec()), getConfiguration(), 1L))
+ .withCompression(getCompression());
}
@Override
public PCollection<T> expand(PBegin input) {
+ getConfiguration().validate();
return input.apply(org.apache.beam.sdk.io.Read.from(createSource()));
}
}
+ /** Implementation of {@link #readFiles}. */
+ @AutoValue
+ public abstract static class ReadFiles<T>
+ extends PTransform<PCollection<ReadableFile>, PCollection<T>> {
+ abstract MappingConfiguration<T> getConfiguration();
+ abstract Builder<T> toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder<T> {
+ abstract Builder<T> setConfiguration(MappingConfiguration<T> configuration);
+ abstract ReadFiles<T> build();
+ }
+
+ private ReadFiles<T> withConfiguration(MappingConfiguration<T> configuration) {
+ return toBuilder().setConfiguration(configuration).build();
+ }
+
+ /** Like {@link Read#withRootElement}. */
+ public ReadFiles<T> withRootElement(String rootElement) {
+ return withConfiguration(getConfiguration().withRootElement(rootElement));
+ }
+
+ /** Like {@link Read#withRecordElement}. */
+ public ReadFiles<T> withRecordElement(String recordElement) {
+ return withConfiguration(getConfiguration().withRecordElement(recordElement));
+ }
+
+ /** Like {@link Read#withRecordClass}. */
+ public ReadFiles<T> withRecordClass(Class<T> recordClass) {
+ return withConfiguration(getConfiguration().withRecordClass(recordClass));
+ }
+
+ /** Like {@link Read#withCharset}. */
+ public ReadFiles<T> withCharset(Charset charset) {
+ return withConfiguration(getConfiguration().withCharset(charset));
+ }
+
+ /** Like {@link Read#withValidationEventHandler}. */
+ public ReadFiles<T> withValidationEventHandler(ValidationEventHandler validationEventHandler) {
+ return withConfiguration(
+ getConfiguration().withValidationEventHandler(validationEventHandler));
+ }
+
+ @Override
+ public PCollection<T> expand(PCollection<ReadableFile> input) {
+ return input.apply(
+ new ReadAllViaFileBasedSource<T>(
+ 64 * 1024L * 1024L,
+ new CreateSourceFn<>(getConfiguration()),
+ JAXBCoder.of(getConfiguration().getRecordClass())));
+ }
+ }
+
+ private static class CreateSourceFn<T> implements SerializableFunction<String, XmlSource<T>> {
+ private final MappingConfiguration<T> configuration;
+
+ public CreateSourceFn(MappingConfiguration<T> configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public XmlSource<T> apply(String input) {
+ return new XmlSource<>(StaticValueProvider.of(input), configuration, 1L);
+ }
+ }
+
/** Implementation of {@link #write}. */
@AutoValue
public abstract static class Write<T> extends PTransform<PCollection<T>, PDone> {
@@ -507,21 +586,19 @@
}
@Override
- public void validate(PipelineOptions options) {
- checkNotNull(getRecordClass(), "Missing a class to bind to a JAXB context.");
- checkNotNull(getRootElement(), "Missing a root element name.");
- checkNotNull(getFilenamePrefix(), "Missing a filename to write to.");
- checkNotNull(getCharset(), "Missing charset");
+ public PDone expand(PCollection<T> input) {
+ checkArgument(getRecordClass() != null, "withRecordClass() is required");
+ checkArgument(getRootElement() != null, "withRootElement() is required");
+ checkArgument(getFilenamePrefix() != null, "to() is required");
+ checkArgument(getCharset() != null, "withCharset() is required");
try {
JAXBContext.newInstance(getRecordClass());
} catch (JAXBException e) {
throw new RuntimeException("Error binding classes to a JAXB Context.", e);
}
- }
- @Override
- public PDone expand(PCollection<T> input) {
- return input.apply(org.apache.beam.sdk.io.WriteFiles.to(createSink()));
+ input.apply(org.apache.beam.sdk.io.WriteFiles.to(createSink()));
+ return PDone.in(input.getPipeline());
}
@VisibleForTesting
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
index 60075a7..b54d95b 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
@@ -25,40 +25,31 @@
import javax.xml.bind.Marshaller;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.DefaultFilenamePolicy;
+import org.apache.beam.sdk.io.DynamicFileDestinations;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.ShardNameTemplate;
import org.apache.beam.sdk.io.fs.ResourceId;
-import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.MimeTypes;
/** Implementation of {@link XmlIO#write}. */
-class XmlSink<T> extends FileBasedSink<T> {
+class XmlSink<T> extends FileBasedSink<T, Void, T> {
private static final String XML_EXTENSION = ".xml";
private final XmlIO.Write<T> spec;
- private static DefaultFilenamePolicy makeFilenamePolicy(XmlIO.Write<?> spec) {
- return DefaultFilenamePolicy.constructUsingStandardParameters(
- spec.getFilenamePrefix(), ShardNameTemplate.INDEX_OF_MAX, XML_EXTENSION);
+ private static <T> DefaultFilenamePolicy makeFilenamePolicy(XmlIO.Write<T> spec) {
+ return DefaultFilenamePolicy.fromStandardParameters(
+ spec.getFilenamePrefix(), ShardNameTemplate.INDEX_OF_MAX, XML_EXTENSION, false);
}
XmlSink(XmlIO.Write<T> spec) {
- super(spec.getFilenamePrefix(), makeFilenamePolicy(spec));
+ super(spec.getFilenamePrefix(), DynamicFileDestinations.<T>constant(makeFilenamePolicy(spec)));
this.spec = spec;
}
/**
- * Validates that the root element, class to bind to a JAXB context, and filenamePrefix have
- * been set and that the class can be bound in a JAXB context.
- */
- @Override
- public void validate(PipelineOptions options) {
- spec.validate(null);
- }
-
- /**
* Creates an {@link XmlWriteOperation}.
*/
@Override
@@ -75,10 +66,8 @@
super.populateDisplayData(builder);
}
- /**
- * {@link WriteOperation} for XML {@link FileBasedSink}s.
- */
- protected static final class XmlWriteOperation<T> extends WriteOperation<T> {
+ /** {@link WriteOperation} for XML {@link FileBasedSink}s. */
+ protected static final class XmlWriteOperation<T> extends WriteOperation<Void, T> {
public XmlWriteOperation(XmlSink<T> sink) {
super(sink);
}
@@ -112,10 +101,8 @@
}
}
- /**
- * A {@link Writer} that can write objects as XML elements.
- */
- protected static final class XmlWriter<T> extends Writer<T> {
+ /** A {@link Writer} that can write objects as XML elements. */
+ protected static final class XmlWriter<T> extends Writer<Void, T> {
final Marshaller marshaller;
private OutputStream os = null;
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
index 7aa42c5..921cd7a 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
@@ -42,8 +42,7 @@
import org.apache.beam.sdk.io.FileBasedSource;
import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.options.ValueProvider;
import org.codehaus.stax2.XMLInputFactory2;
/** Implementation of {@link XmlIO#read}. */
@@ -51,21 +50,29 @@
private static final String XML_VERSION = "1.1";
- private final XmlIO.Read<T> spec;
+ private final XmlIO.MappingConfiguration<T> configuration;
- XmlSource(XmlIO.Read<T> spec) {
- super(StaticValueProvider.of(spec.getFileOrPatternSpec()), spec.getMinBundleSize());
- this.spec = spec;
+ XmlSource(
+ ValueProvider<String> spec,
+ XmlIO.MappingConfiguration<T> configuration,
+ long minBundleSizeBytes) {
+ super(spec, minBundleSizeBytes);
+ this.configuration = configuration;
}
- private XmlSource(XmlIO.Read<T> spec, Metadata metadata, long startOffset, long endOffset) {
- super(metadata, spec.getMinBundleSize(), startOffset, endOffset);
- this.spec = spec;
+ private XmlSource(
+ XmlIO.MappingConfiguration<T> configuration,
+ long minBundleSizeBytes,
+ Metadata metadata,
+ long startOffset,
+ long endOffset) {
+ super(metadata, minBundleSizeBytes, startOffset, endOffset);
+ this.configuration = configuration;
}
@Override
protected FileBasedSource<T> createForSubrangeOfFile(Metadata metadata, long start, long end) {
- return new XmlSource<T>(spec.from(metadata.toString()), metadata, start, end);
+ return new XmlSource<T>(configuration, getMinBundleSize(), metadata, start, end);
}
@Override
@@ -74,19 +81,8 @@
}
@Override
- public void validate() {
- super.validate();
- spec.validate(null);
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- spec.populateDisplayData(builder);
- }
-
- @Override
- public Coder<T> getDefaultOutputCoder() {
- return JAXBCoder.of(spec.getRecordClass());
+ public Coder<T> getOutputCoder() {
+ return JAXBCoder.of(configuration.getRecordClass());
}
/**
@@ -137,10 +133,12 @@
// Set up a JAXB Unmarshaller that can be used to unmarshall record objects.
try {
- JAXBContext jaxbContext = JAXBContext.newInstance(getCurrentSource().spec.getRecordClass());
+ JAXBContext jaxbContext =
+ JAXBContext.newInstance(getCurrentSource().configuration.getRecordClass());
jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- if (getCurrentSource().spec.getValidationEventHandler() != null) {
- jaxbUnmarshaller.setEventHandler(getCurrentSource().spec.getValidationEventHandler());
+ if (getCurrentSource().configuration.getValidationEventHandler() != null) {
+ jaxbUnmarshaller.setEventHandler(
+ getCurrentSource().configuration.getValidationEventHandler());
}
} catch (JAXBException e) {
throw new RuntimeException(e);
@@ -179,10 +177,10 @@
byte[] dummyStartDocumentBytes =
(String.format(
"<?xml version=\"%s\" encoding=\""
- + getCurrentSource().spec.getCharset()
+ + getCurrentSource().configuration.getCharset()
+ "\"?><%s>",
- XML_VERSION, getCurrentSource().spec.getRootElement()))
- .getBytes(getCurrentSource().spec.getCharset());
+ XML_VERSION, getCurrentSource().configuration.getRootElement()))
+ .getBytes(getCurrentSource().configuration.getCharset());
preambleByteBuffer.write(dummyStartDocumentBytes);
// Gets the byte offset (in the input file) of the first record in ReadableByteChannel. This
// method returns the offset and stores any bytes that should be used when creating the XML
@@ -230,7 +228,8 @@
ByteBuffer buf = ByteBuffer.allocate(BUF_SIZE);
byte[] recordStartBytes =
- ("<" + getCurrentSource().spec.getRecordElement()).getBytes(StandardCharsets.UTF_8);
+ ("<" + getCurrentSource().configuration.getRecordElement())
+ .getBytes(StandardCharsets.UTF_8);
outer: while (channel.read(buf) > 0) {
buf.flip();
@@ -334,14 +333,14 @@
this.parser = xmlInputFactory.createXMLStreamReader(
new SequenceInputStream(
new ByteArrayInputStream(lookAhead), Channels.newInputStream(channel)),
- getCurrentSource().spec.getCharset());
+ getCurrentSource().configuration.getCharset());
// Current offset should be the offset before reading the record element.
while (true) {
int event = parser.next();
if (event == XMLStreamConstants.START_ELEMENT) {
String localName = parser.getLocalName();
- if (localName.equals(getCurrentSource().spec.getRecordElement())) {
+ if (localName.equals(getCurrentSource().configuration.getRecordElement())) {
break;
}
}
@@ -369,7 +368,7 @@
}
}
JAXBElement<T> jb =
- jaxbUnmarshaller.unmarshal(parser, getCurrentSource().spec.getRecordClass());
+ jaxbUnmarshaller.unmarshal(parser, getCurrentSource().configuration.getRecordClass());
currentRecord = jb.getValue();
return true;
} catch (JAXBException | XMLStreamException e) {
diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSinkTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSinkTest.java
index aa0c1c3..3834abd 100644
--- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSinkTest.java
+++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSinkTest.java
@@ -127,25 +127,25 @@
/** Validation ensures no fields are missing. */
@Test
public void testValidateXmlSinkMissingRecordClass() {
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
XmlIO.<Bird>write()
.to(testFilePrefix)
.withRootElement(testRootElement)
- .validate(null);
+ .expand(null);
}
@Test
public void testValidateXmlSinkMissingRootElement() {
- thrown.expect(NullPointerException.class);
+ thrown.expect(IllegalArgumentException.class);
XmlIO.<Bird>write().withRecordClass(Bird.class)
.to(testFilePrefix)
- .validate(null);
+ .expand(null);
}
@Test
public void testValidateXmlSinkMissingOutputDirectory() {
- thrown.expect(NullPointerException.class);
- XmlIO.<Bird>write().withRecordClass(Bird.class).withRootElement(testRootElement).validate(null);
+ thrown.expect(IllegalArgumentException.class);
+ XmlIO.<Bird>write().withRecordClass(Bird.class).withRootElement(testRootElement).expand(null);
}
/**
@@ -197,8 +197,8 @@
.withRecordClass(Integer.class);
DisplayData displayData = DisplayData.from(write);
-
- assertThat(displayData, hasDisplayItem("filenamePattern", "file-SSSSS-of-NNNNN.xml"));
+ assertThat(
+ displayData, hasDisplayItem("filenamePattern", "/path/to/file-SSSSS-of-NNNNN" + ".xml"));
assertThat(displayData, hasDisplayItem("rootElement", "bird"));
assertThat(displayData, hasDisplayItem("recordClass", Integer.class));
}
diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java
index abddcf9..a6adac6 100644
--- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java
+++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java
@@ -41,6 +41,7 @@
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.FileIO;
import org.apache.beam.sdk.io.Source.Reader;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -459,60 +460,6 @@
}
@Test
- public void testReadXMLNoRootElement() throws IOException {
- File file = tempFolder.newFile("trainXMLSmall");
- Files.write(file.toPath(), trainXML.getBytes(StandardCharsets.UTF_8));
-
- BoundedSource<Train> source =
- XmlIO.<Train>read()
- .from(file.toPath().toString())
- .withRecordElement("train")
- .withRecordClass(Train.class)
- .createSource();
-
- exception.expect(NullPointerException.class);
- exception.expectMessage(
- "rootElement is null. Use builder method withRootElement() to set this.");
- readEverythingFromReader(source.createReader(null));
- }
-
- @Test
- public void testReadXMLNoRecordElement() throws IOException {
- File file = tempFolder.newFile("trainXMLSmall");
- Files.write(file.toPath(), trainXML.getBytes(StandardCharsets.UTF_8));
-
- BoundedSource<Train> source =
- XmlIO.<Train>read()
- .from(file.toPath().toString())
- .withRootElement("trains")
- .withRecordClass(Train.class)
- .createSource();
-
- exception.expect(NullPointerException.class);
- exception.expectMessage(
- "recordElement is null. Use builder method withRecordElement() to set this.");
- readEverythingFromReader(source.createReader(null));
- }
-
- @Test
- public void testReadXMLNoRecordClass() throws IOException {
- File file = tempFolder.newFile("trainXMLSmall");
- Files.write(file.toPath(), trainXML.getBytes(StandardCharsets.UTF_8));
-
- BoundedSource<Train> source =
- XmlIO.<Train>read()
- .from(file.toPath().toString())
- .withRootElement("trains")
- .withRecordElement("train")
- .createSource();
-
- exception.expect(NullPointerException.class);
- exception.expectMessage(
- "recordClass is null. Use builder method withRecordClass() to set this.");
- readEverythingFromReader(source.createReader(null));
- }
-
- @Test
public void testReadXMLIncorrectRootElement() throws IOException {
File file = tempFolder.newFile("trainXMLSmall");
Files.write(file.toPath(), trainXML.getBytes(StandardCharsets.UTF_8));
@@ -938,7 +885,7 @@
@Test
@Category(NeedsRunner.class)
- public void testReadXMLFilePattern() throws IOException {
+ public void testReadXMLFilePatternUsingReadAndReadFiles() throws IOException {
List<Train> trains1 = generateRandomTrainList(20);
File file = createRandomTrainXML("temp1.xml", trains1);
List<Train> trains2 = generateRandomTrainList(10);
@@ -948,9 +895,9 @@
generateRandomTrainList(8);
createRandomTrainXML("otherfile.xml", trains1);
- PCollection<Train> output =
+ PCollection<Train> read =
p.apply(
- "ReadFileData",
+ "Read",
XmlIO.<Train>read()
.from(file.getParent() + "/" + "temp*.xml")
.withRootElement("trains")
@@ -958,12 +905,23 @@
.withRecordClass(Train.class)
.withMinBundleSize(1024));
+ PCollection<Train> readFiles =
+ p.apply(FileIO.match().filepattern(file.getParent() + "/" + "temp*.xml"))
+ .apply(FileIO.readMatches())
+ .apply(
+ "ReadFiles",
+ XmlIO.<Train>readFiles()
+ .withRootElement("trains")
+ .withRecordElement("train")
+ .withRecordClass(Train.class));
+
List<Train> expectedResults = new ArrayList<>();
expectedResults.addAll(trains1);
expectedResults.addAll(trains2);
expectedResults.addAll(trains3);
- PAssert.that(output).containsInAnyOrder(expectedResults);
+ PAssert.that(read).containsInAnyOrder(expectedResults);
+ PAssert.that(readFiles).containsInAnyOrder(expectedResults);
p.run();
}
diff --git a/sdks/java/java8tests/pom.xml b/sdks/java/java8tests/pom.xml
index b90a757..1fc84ed 100644
--- a/sdks/java/java8tests/pom.xml
+++ b/sdks/java/java8tests/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/javadoc/ant.xml b/sdks/java/javadoc/ant.xml
index 9a1880a..152b9dd 100644
--- a/sdks/java/javadoc/ant.xml
+++ b/sdks/java/javadoc/ant.xml
@@ -30,6 +30,7 @@
<fileset dir="..">
<include name="**/src/main/java/**/*.java"/>
<exclude name="**/maven-archetypes/**"/>
+ <exclude name="**/nexmark/**"/>
</fileset>
<!-- For each pathname, turn X/src/main/java/Y to Y. This
results in one Java source tree. -->
diff --git a/sdks/java/javadoc/pom.xml b/sdks/java/javadoc/pom.xml
index 54dae3a..85440ff 100644
--- a/sdks/java/javadoc/pom.xml
+++ b/sdks/java/javadoc/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
@@ -79,6 +79,11 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
@@ -89,6 +94,11 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-sketching</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-sorter</artifactId>
</dependency>
@@ -99,6 +109,26 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-amqp</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-cassandra</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-2</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-elasticsearch-tests-5</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-elasticsearch</artifactId>
</dependency>
@@ -124,6 +154,11 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-hcatalog</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-jdbc</artifactId>
</dependency>
@@ -152,6 +187,11 @@
<artifactId>beam-sdks-java-io-mqtt</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-solr</artifactId>
+ </dependency>
+
<!-- provided and optional dependencies.-->
<dependency>
<groupId>com.google.auto.service</groupId>
@@ -196,13 +236,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
</dependency>
</dependencies>
diff --git a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
index 58c5f22..d8117ce 100755
--- a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
@@ -22,7 +22,7 @@
# Usage: Invoke with no arguments from any working directory.
# The directory of this script. Assumes root of the maven-archetypes module.
-HERE="$(dirname $0)"
+HERE="$( dirname "$0" )"
# The directory of the examples-java and examples-java8 modules
EXAMPLES_ROOT="${HERE}/../../../../examples/java"
diff --git a/sdks/java/maven-archetypes/examples-java8/pom.xml b/sdks/java/maven-archetypes/examples-java8/pom.xml
index b57644d..fbab9ff 100644
--- a/sdks/java/maven-archetypes/examples-java8/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 5f34689..ffdb066 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -28,7 +28,23 @@
<properties>
<beam.version>@project.version@</beam.version>
- <surefire-plugin.version>2.20</surefire-plugin.version>
+
+ <bigquery.version>@bigquery.version@</bigquery.version>
+ <google-clients.version>@google-clients.version@</google-clients.version>
+ <guava.version>@guava.version@</guava.version>
+ <hamcrest.version>@hamcrest.version@</hamcrest.version>
+ <jackson.version>@jackson.version@</jackson.version>
+ <joda.version>@joda.version@</joda.version>
+ <junit.version>@junit.version@</junit.version>
+ <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+ <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+ <maven-jar-plugin.version>@maven-jar-plugin.version@</maven-jar-plugin.version>
+ <maven-shade-plugin.version>@maven-shade-plugin.version@</maven-shade-plugin.version>
+ <mockito.version>@mockito.version@</mockito.version>
+ <pubsub.version>@pubsub.version@</pubsub.version>
+ <slf4j.version>@slf4j.version@</slf4j.version>
+ <spark.version>@spark.version@</spark.version>
+ <surefire-plugin.version>@surefire-plugin.version@</surefire-plugin.version>
</properties>
<repositories>
@@ -50,7 +66,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.5.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
@@ -80,6 +96,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
</plugin>
<!--
@@ -89,7 +106,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -97,6 +114,7 @@
<goal>shade</goal>
</goals>
<configuration>
+ <finalName>${project.artifactId}-bundled-${project.version}</finalName>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -122,7 +140,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.4.0</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
</configuration>
@@ -205,6 +223,19 @@
</profile>
<profile>
+ <id>gearpump-runner</id>
+ <!-- Makes the GearpumpRunner available when running a pipeline. -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <version>${beam.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
<id>spark-runner</id>
<!-- Makes the SparkRunner available when running a pipeline. Additionally,
overrides some Spark dependencies to Beam-compatible versions. -->
@@ -224,7 +255,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>1.6.2</version>
+ <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
@@ -236,7 +267,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_2.10</artifactId>
- <version>@jackson.version@</version>
+ <version>${jackson.version}</version>
<scope>runtime</scope>
</dependency>
</dependencies>
@@ -261,7 +292,7 @@
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
- <version>1.22.0</version>
+ <version>${google-clients.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -276,7 +307,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
- <version>v2-rev295-1.22.0</version>
+ <version>${bigquery.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -290,7 +321,7 @@
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
- <version>1.22.0</version>
+ <version>${google-clients.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -304,7 +335,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-pubsub</artifactId>
- <version>v1-rev10-1.22.0</version>
+ <version>${pubsub.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -318,26 +349,26 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <version>2.4</version>
+ <version>${joda.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>20.0</version>
+ <version>${guava.version}</version>
</dependency>
<!-- Add slf4j API frontend binding with JUL backend -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
<!-- When loaded at runtime this will wire up slf4j to the JUL backend -->
<scope>runtime</scope>
</dependency>
@@ -347,19 +378,19 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
+ <version>${hamcrest.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.12</version>
+ <version>${junit.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
+ <version>${mockito.version}</version>
<scope>test</scope>
</dependency>
diff --git a/sdks/java/maven-archetypes/examples/generate-sources.sh b/sdks/java/maven-archetypes/examples/generate-sources.sh
index d9109ac..62af772 100755
--- a/sdks/java/maven-archetypes/examples/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples/generate-sources.sh
@@ -21,7 +21,7 @@
# Usage: Invoke with no arguments from any working directory.
# The directory of this script. Assumes root of the maven-archetypes module.
-HERE="$(dirname $0)"
+HERE="$( dirname "$0" )"
# The directory of the examples-java module
EXAMPLES_ROOT="${HERE}/../../../../examples/java"
diff --git a/sdks/java/maven-archetypes/examples/pom.xml b/sdks/java/maven-archetypes/examples/pom.xml
index c1378cb..e658c3b 100644
--- a/sdks/java/maven-archetypes/examples/pom.xml
+++ b/sdks/java/maven-archetypes/examples/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index a3d7b8f..b8b9c9f 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -28,7 +28,22 @@
<properties>
<beam.version>@project.version@</beam.version>
- <surefire-plugin.version>2.20</surefire-plugin.version>
+
+ <bigquery.version>@bigquery.version@</bigquery.version>
+ <google-clients.version>@google-clients.version@</google-clients.version>
+ <guava.version>@guava.version@</guava.version>
+ <hamcrest.version>@hamcrest.version@</hamcrest.version>
+ <jackson.version>@jackson.version@</jackson.version>
+ <joda.version>@joda.version@</joda.version>
+ <junit.version>@junit.version@</junit.version>
+ <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+ <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+ <maven-jar-plugin.version>@maven-jar-plugin.version@</maven-jar-plugin.version>
+ <maven-shade-plugin.version>@maven-shade-plugin.version@</maven-shade-plugin.version>
+ <pubsub.version>@pubsub.version@</pubsub.version>
+ <slf4j.version>@slf4j.version@</slf4j.version>
+ <spark.version>@spark.version@</spark.version>
+ <surefire-plugin.version>@surefire-plugin.version@</surefire-plugin.version>
</properties>
<repositories>
@@ -50,7 +65,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.5.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>${targetPlatform}</source>
<target>${targetPlatform}</target>
@@ -80,6 +95,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
</plugin>
<!--
@@ -89,7 +105,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -97,6 +113,7 @@
<goal>shade</goal>
</goals>
<configuration>
+ <finalName>${project.artifactId}-bundled-${project.version}</finalName>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -122,7 +139,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.4.0</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
</configuration>
@@ -224,7 +241,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
- <version>1.6.2</version>
+ <version>${spark.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
@@ -236,7 +253,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_2.10</artifactId>
- <version>@jackson.version@</version>
+ <version>${jackson.version}</version>
<scope>runtime</scope>
</dependency>
</dependencies>
@@ -262,7 +279,7 @@
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
- <version>1.22.0</version>
+ <version>${google-clients.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -276,7 +293,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
- <version>v2-rev295-1.22.0</version>
+ <version>${bigquery.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -290,7 +307,7 @@
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
- <version>1.22.0</version>
+ <version>${google-clients.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -304,7 +321,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-pubsub</artifactId>
- <version>v1-rev10-1.22.0</version>
+ <version>${pubsub.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
@@ -318,26 +335,26 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <version>2.4</version>
+ <version>${joda.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>20.0</version>
+ <version>${guava.version}</version>
</dependency>
<!-- Add slf4j API frontend binding with JUL backend -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
<!-- When loaded at runtime this will wire up slf4j to the JUL backend -->
<scope>runtime</scope>
</dependency>
@@ -347,13 +364,13 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
+ <version>${hamcrest.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.12</version>
+ <version>${junit.version}</version>
</dependency>
<!-- The DirectRunner is needed for unit tests. -->
diff --git a/sdks/java/maven-archetypes/pom.xml b/sdks/java/maven-archetypes/pom.xml
index b7fe274..2d0cef0 100644
--- a/sdks/java/maven-archetypes/pom.xml
+++ b/sdks/java/maven-archetypes/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/maven-archetypes/starter/pom.xml b/sdks/java/maven-archetypes/starter/pom.xml
index 06b41c8..9891409 100644
--- a/sdks/java/maven-archetypes/starter/pom.xml
+++ b/sdks/java/maven-archetypes/starter/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
index dddd5ab..f4fb9f8 100644
--- a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
@@ -26,6 +26,10 @@
<properties>
<beam.version>@project.version@</beam.version>
+
+ <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+ <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+ <slf4j.version>@slf4j.version@</slf4j.version>
</properties>
<repositories>
@@ -47,7 +51,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.5.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>${targetPlatform}</source>
<target>${targetPlatform}</target>
@@ -60,7 +64,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.4.0</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
</configuration>
@@ -94,12 +98,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index 39fefd6..91da6eb 100644
--- a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -26,6 +26,10 @@
<properties>
<beam.version>@project.version@</beam.version>
+
+ <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+ <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+ <slf4j.version>@slf4j.version@</slf4j.version>
</properties>
<repositories>
@@ -47,7 +51,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.5.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
@@ -60,7 +64,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.4.0</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
</configuration>
@@ -94,12 +98,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
- <version>1.7.14</version>
+ <version>${slf4j.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/sdks/java/nexmark/pom.xml b/sdks/java/nexmark/pom.xml
new file mode 100644
index 0000000..8210ddc
--- /dev/null
+++ b/sdks/java/nexmark/pom.xml
@@ -0,0 +1,265 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-nexmark</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: Nexmark</name>
+ <packaging>jar</packaging>
+
+ <profiles>
+
+ <!--
+ The direct runner is available by default.
+ You can also include it on the classpath explicitly using the profile below
+ -->
+ <profile>
+ <id>direct-runner</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>apex-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-apex</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>flink-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-flink_2.10</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>spark-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-spark</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>dataflow-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-bundled-${project.version}</finalName>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Avro plugin for automatic code generation -->
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>schemas</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ </goals>
+ <configuration>
+ <sourceDirectory>${project.basedir}/src/main/</sourceDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/java</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-bigquery</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
new file mode 100644
index 0000000..ab2284c
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * An implementation of the 'NEXMark queries' for Beam.
+ * These are multiple queries over a three table schema representing an online auction system:
+ * <ul>
+ * <li>{@link Person} represents a person submitting an item for auction and/or making a bid
+ * on an auction.
+ * <li>{@link Auction} represents an item under auction.
+ * <li>{@link Bid} represents a bid for an item under auction.
+ * </ul>
+ * The queries exercise many aspects of the Beam model.
+ *
+ * <p>We synthesize the creation of people, auctions and bids in real-time. The data is not
+ * particularly sensible.
+ *
+ * <p>See <a href="http://datalab.cs.pdx.edu/niagaraST/NEXMark/">
+ * http://datalab.cs.pdx.edu/niagaraST/NEXMark/</a>
+ */
+public class Main<OptionT extends NexmarkOptions> {
+
+ /**
+ * Entry point.
+ */
+ void runAll(OptionT options, NexmarkLauncher nexmarkLauncher) {
+ Instant start = Instant.now();
+ Map<NexmarkConfiguration, NexmarkPerf> baseline = loadBaseline(options.getBaselineFilename());
+ Map<NexmarkConfiguration, NexmarkPerf> actual = new LinkedHashMap<>();
+ Iterable<NexmarkConfiguration> configurations = options.getSuite().getConfigurations(options);
+
+ boolean successful = true;
+ try {
+ // Run all the configurations.
+ for (NexmarkConfiguration configuration : configurations) {
+ NexmarkPerf perf = nexmarkLauncher.run(configuration);
+ if (perf != null) {
+ if (perf.errors == null || perf.errors.size() > 0) {
+ successful = false;
+ }
+ appendPerf(options.getPerfFilename(), configuration, perf);
+ actual.put(configuration, perf);
+ // Summarize what we've run so far.
+ saveSummary(null, configurations, actual, baseline, start);
+ }
+ }
+ } finally {
+ if (options.getMonitorJobs()) {
+ // Report overall performance.
+ saveSummary(options.getSummaryFilename(), configurations, actual, baseline, start);
+ saveJavascript(options.getJavascriptFilename(), configurations, actual, baseline, start);
+ }
+ }
+
+ if (!successful) {
+ throw new RuntimeException("Execution was not successful");
+ }
+ }
+
+ /**
+ * Append the pair of {@code configuration} and {@code perf} to perf file.
+ */
+ private void appendPerf(
+ @Nullable String perfFilename, NexmarkConfiguration configuration,
+ NexmarkPerf perf) {
+ if (perfFilename == null) {
+ return;
+ }
+ List<String> lines = new ArrayList<>();
+ lines.add("");
+ lines.add(String.format("# %s", Instant.now()));
+ lines.add(String.format("# %s", configuration.toShortString()));
+ lines.add(configuration.toString());
+ lines.add(perf.toString());
+ try {
+ Files.write(Paths.get(perfFilename), lines, StandardCharsets.UTF_8, StandardOpenOption.CREATE,
+ StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to write perf file: ", e);
+ }
+ NexmarkUtils.console("appended results to perf file %s.", perfFilename);
+ }
+
+ /**
+ * Load the baseline perf.
+ */
+ @Nullable
+ private static Map<NexmarkConfiguration, NexmarkPerf> loadBaseline(
+ @Nullable String baselineFilename) {
+ if (baselineFilename == null) {
+ return null;
+ }
+ Map<NexmarkConfiguration, NexmarkPerf> baseline = new LinkedHashMap<>();
+ List<String> lines;
+ try {
+ lines = Files.readAllLines(Paths.get(baselineFilename), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to read baseline perf file: ", e);
+ }
+ for (int i = 0; i < lines.size(); i++) {
+ if (lines.get(i).startsWith("#") || lines.get(i).trim().isEmpty()) {
+ continue;
+ }
+ NexmarkConfiguration configuration = NexmarkConfiguration.fromString(lines.get(i++));
+ NexmarkPerf perf = NexmarkPerf.fromString(lines.get(i));
+ baseline.put(configuration, perf);
+ }
+ NexmarkUtils.console("loaded %d entries from baseline file %s.", baseline.size(),
+ baselineFilename);
+ return baseline;
+ }
+
+ private static final String LINE =
+ "==========================================================================================";
+
+ /**
+ * Print summary of {@code actual} vs (if non-null) {@code baseline}.
+ */
+ private static void saveSummary(
+ @Nullable String summaryFilename,
+ Iterable<NexmarkConfiguration> configurations, Map<NexmarkConfiguration, NexmarkPerf> actual,
+ @Nullable Map<NexmarkConfiguration, NexmarkPerf> baseline, Instant start) {
+ List<String> lines = new ArrayList<>();
+
+ lines.add("");
+ lines.add(LINE);
+
+ lines.add(
+ String.format("Run started %s and ran for %s", start, new Duration(start, Instant.now())));
+ lines.add("");
+
+ lines.add("Default configuration:");
+ lines.add(NexmarkConfiguration.DEFAULT.toString());
+ lines.add("");
+
+ lines.add("Configurations:");
+ lines.add(" Conf Description");
+ int conf = 0;
+ for (NexmarkConfiguration configuration : configurations) {
+ lines.add(String.format(" %04d %s", conf++, configuration.toShortString()));
+ NexmarkPerf actualPerf = actual.get(configuration);
+ if (actualPerf != null && actualPerf.jobId != null) {
+ lines.add(String.format(" %4s [Ran as job %s]", "", actualPerf.jobId));
+ }
+ }
+
+ lines.add("");
+ lines.add("Performance:");
+ lines.add(String.format(" %4s %12s %12s %12s %12s %12s %12s", "Conf", "Runtime(sec)",
+ "(Baseline)", "Events(/sec)", "(Baseline)", "Results", "(Baseline)"));
+ conf = 0;
+ for (NexmarkConfiguration configuration : configurations) {
+ String line = String.format(" %04d ", conf++);
+ NexmarkPerf actualPerf = actual.get(configuration);
+ if (actualPerf == null) {
+ line += "*** not run ***";
+ } else {
+ NexmarkPerf baselinePerf = baseline == null ? null : baseline.get(configuration);
+ double runtimeSec = actualPerf.runtimeSec;
+ line += String.format("%12.1f ", runtimeSec);
+ if (baselinePerf == null) {
+ line += String.format("%12s ", "");
+ } else {
+ double baselineRuntimeSec = baselinePerf.runtimeSec;
+ double diff = ((runtimeSec - baselineRuntimeSec) / baselineRuntimeSec) * 100.0;
+ line += String.format("%+11.2f%% ", diff);
+ }
+
+ double eventsPerSec = actualPerf.eventsPerSec;
+ line += String.format("%12.1f ", eventsPerSec);
+ if (baselinePerf == null) {
+ line += String.format("%12s ", "");
+ } else {
+ double baselineEventsPerSec = baselinePerf.eventsPerSec;
+ double diff = ((eventsPerSec - baselineEventsPerSec) / baselineEventsPerSec) * 100.0;
+ line += String.format("%+11.2f%% ", diff);
+ }
+
+ long numResults = actualPerf.numResults;
+ line += String.format("%12d ", numResults);
+ if (baselinePerf == null) {
+ line += String.format("%12s", "");
+ } else {
+ long baselineNumResults = baselinePerf.numResults;
+ long diff = numResults - baselineNumResults;
+ line += String.format("%+12d", diff);
+ }
+ }
+ lines.add(line);
+
+ if (actualPerf != null) {
+ List<String> errors = actualPerf.errors;
+ if (errors == null) {
+ errors = new ArrayList<>();
+ errors.add("NexmarkGoogleRunner returned null errors list");
+ }
+ for (String error : errors) {
+ lines.add(String.format(" %4s *** %s ***", "", error));
+ }
+ }
+ }
+
+ lines.add(LINE);
+ lines.add("");
+
+ for (String line : lines) {
+ System.out.println(line);
+ }
+
+ if (summaryFilename != null) {
+ try {
+ Files.write(Paths.get(summaryFilename), lines, StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to save summary file: ", e);
+ }
+ NexmarkUtils.console("appended summary to summary file %s.", summaryFilename);
+ }
+ }
+
+ /**
+ * Write all perf data and any baselines to a javascript file which can be used by
+ * graphing page etc.
+ */
+ private static void saveJavascript(
+ @Nullable String javascriptFilename,
+ Iterable<NexmarkConfiguration> configurations, Map<NexmarkConfiguration, NexmarkPerf> actual,
+ @Nullable Map<NexmarkConfiguration, NexmarkPerf> baseline, Instant start) {
+ if (javascriptFilename == null) {
+ return;
+ }
+
+ List<String> lines = new ArrayList<>();
+ lines.add(String.format(
+ "// Run started %s and ran for %s", start, new Duration(start, Instant.now())));
+ lines.add("var all = [");
+
+ for (NexmarkConfiguration configuration : configurations) {
+ lines.add(" {");
+ lines.add(String.format(" config: %s", configuration));
+ NexmarkPerf actualPerf = actual.get(configuration);
+ if (actualPerf != null) {
+ lines.add(String.format(" ,perf: %s", actualPerf));
+ }
+ NexmarkPerf baselinePerf = baseline == null ? null : baseline.get(configuration);
+ if (baselinePerf != null) {
+ lines.add(String.format(" ,baseline: %s", baselinePerf));
+ }
+ lines.add(" },");
+ }
+
+ lines.add("];");
+
+ try {
+ Files.write(Paths.get(javascriptFilename), lines, StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to save javascript file: ", e);
+ }
+ NexmarkUtils.console("saved javascript to file %s.", javascriptFilename);
+ }
+
+ public static void main(String[] args) {
+ NexmarkOptions options = PipelineOptionsFactory.fromArgs(args)
+ .withValidation()
+ .as(NexmarkOptions.class);
+ NexmarkLauncher<NexmarkOptions> nexmarkLauncher = new NexmarkLauncher<>(options);
+ new Main<>().runAll(options, nexmarkLauncher);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Monitor.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Monitor.java
new file mode 100644
index 0000000..f45c387
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Monitor.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Distribution;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * A monitor of elements with support for later retrieving their metrics.
+ *
+ * @param <T> Type of element we are monitoring.
+ */
+public class Monitor<T extends KnownSize> implements Serializable {
+ private class MonitorDoFn extends DoFn<T, T> {
+ final Counter elementCounter =
+ Metrics.counter(name , prefix + ".elements");
+ final Counter bytesCounter =
+ Metrics.counter(name , prefix + ".bytes");
+ final Distribution startTime =
+ Metrics.distribution(name , prefix + ".startTime");
+ final Distribution endTime =
+ Metrics.distribution(name , prefix + ".endTime");
+ final Distribution startTimestamp =
+ Metrics.distribution(name , prefix + ".startTimestamp");
+ final Distribution endTimestamp =
+ Metrics.distribution(name , prefix + ".endTimestamp");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ elementCounter.inc();
+ bytesCounter.inc(c.element().sizeInBytes());
+ long now = System.currentTimeMillis();
+ startTime.update(now);
+ endTime.update(now);
+ startTimestamp.update(c.timestamp().getMillis());
+ endTimestamp.update(c.timestamp().getMillis());
+ c.output(c.element());
+ }
+ }
+
+ public final String name;
+ public final String prefix;
+ private final MonitorDoFn doFn;
+ private final PTransform<PCollection<? extends T>, PCollection<T>> transform;
+
+ public Monitor(String name, String prefix) {
+ this.name = name;
+ this.prefix = prefix;
+ doFn = new MonitorDoFn();
+ transform = ParDo.of(doFn);
+ }
+
+ public PTransform<PCollection<? extends T>, PCollection<T>> getTransform() {
+ return transform;
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkConfiguration.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkConfiguration.java
new file mode 100644
index 0000000..904fcd5
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkConfiguration.java
@@ -0,0 +1,721 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Configuration controlling how a query is run. May be supplied by command line or
+ * programmatically. We only capture properties which may influence the resulting
+ * pipeline performance, as captured by {@link NexmarkPerf}.
+ */
+public class NexmarkConfiguration implements Serializable {
+ public static final NexmarkConfiguration DEFAULT = new NexmarkConfiguration();
+
+ /** If {@literal true}, include additional debugging and monitoring stats. */
+ @JsonProperty
+ public boolean debug = true;
+
+ /** Which query to run, in [0,9]. */
+ @JsonProperty
+ public int query = 0;
+
+ /** Where events come from. */
+ @JsonProperty
+ public NexmarkUtils.SourceType sourceType = NexmarkUtils.SourceType.DIRECT;
+
+ /** Where results go to. */
+ @JsonProperty
+ public NexmarkUtils.SinkType sinkType = NexmarkUtils.SinkType.DEVNULL;
+
+ /**
+ * Control whether pub/sub publishing is done in a stand-alone pipeline or is integrated
+ * into the overall query pipeline.
+ */
+ @JsonProperty
+ public NexmarkUtils.PubSubMode pubSubMode = NexmarkUtils.PubSubMode.COMBINED;
+
+ /**
+ * Number of events to generate. If zero, generate as many as possible without overflowing
+ * internal counters etc.
+ */
+ @JsonProperty
+ public long numEvents = 100000;
+
+ /**
+ * Number of event generators to use. Each generates events in its own timeline.
+ */
+ @JsonProperty
+ public int numEventGenerators = 100;
+
+ /**
+ * Shape of event rate curve.
+ */
+ @JsonProperty
+ public NexmarkUtils.RateShape rateShape = NexmarkUtils.RateShape.SINE;
+
+ /**
+ * Initial overall event rate (in {@link #rateUnit}).
+ */
+ @JsonProperty
+ public int firstEventRate = 10000;
+
+ /**
+ * Next overall event rate (in {@link #rateUnit}).
+ */
+ @JsonProperty
+ public int nextEventRate = 10000;
+
+ /**
+ * Unit for rates.
+ */
+ @JsonProperty
+ public NexmarkUtils.RateUnit rateUnit = NexmarkUtils.RateUnit.PER_SECOND;
+
+ /**
+ * Overall period of rate shape, in seconds.
+ */
+ @JsonProperty
+ public int ratePeriodSec = 600;
+
+ /**
+ * Time in seconds to preload the subscription with data, at the initial input rate of the
+ * pipeline.
+ */
+ @JsonProperty
+ public int preloadSeconds = 0;
+
+ /**
+ * Timeout for stream pipelines to stop in seconds.
+ */
+ @JsonProperty
+ public int streamTimeout = 240;
+
+ /**
+ * If true, and in streaming mode, generate events only when they are due according to their
+ * timestamp.
+ */
+ @JsonProperty
+ public boolean isRateLimited = false;
+
+ /**
+ * If true, use wallclock time as event time. Otherwise, use a deterministic
+ * time in the past so that multiple runs will see exactly the same event streams
+ * and should thus have exactly the same results.
+ */
+ @JsonProperty
+ public boolean useWallclockEventTime = false;
+
+ /** Average idealized size of a 'new person' event, in bytes. */
+ @JsonProperty
+ public int avgPersonByteSize = 200;
+
+ /** Average idealized size of a 'new auction' event, in bytes. */
+ @JsonProperty
+ public int avgAuctionByteSize = 500;
+
+ /** Average idealized size of a 'bid' event, in bytes. */
+ @JsonProperty
+ public int avgBidByteSize = 100;
+
+ /** Ratio of bids to 'hot' auctions compared to all other auctions. */
+ @JsonProperty
+ public int hotAuctionRatio = 2;
+
+ /** Ratio of auctions for 'hot' sellers compared to all other people. */
+ @JsonProperty
+ public int hotSellersRatio = 4;
+
+ /** Ratio of bids for 'hot' bidders compared to all other people. */
+ @JsonProperty
+ public int hotBiddersRatio = 4;
+
+ /** Window size, in seconds, for queries 3, 5, 7 and 8. */
+ @JsonProperty
+ public long windowSizeSec = 10;
+
+ /** Sliding window period, in seconds, for query 5. */
+ @JsonProperty
+ public long windowPeriodSec = 5;
+
+ /** Number of seconds to hold back events according to their reported timestamp. */
+ @JsonProperty
+ public long watermarkHoldbackSec = 0;
+
+ /** Average number of auction which should be inflight at any time, per generator. */
+ @JsonProperty
+ public int numInFlightAuctions = 100;
+
+ /** Maximum number of people to consider as active for placing auctions or bids. */
+ @JsonProperty
+ public int numActivePeople = 1000;
+
+ /** Coder strategy to follow. */
+ @JsonProperty
+ public NexmarkUtils.CoderStrategy coderStrategy = NexmarkUtils.CoderStrategy.HAND;
+
+ /**
+ * Delay, in milliseconds, for each event. This will peg one core for this number
+ * of milliseconds to simulate CPU-bound computation.
+ */
+ @JsonProperty
+ public long cpuDelayMs = 0;
+
+ /**
+ * Extra data, in bytes, to save to persistent state for each event. This will force
+ * i/o all the way to durable storage to simulate an I/O-bound computation.
+ */
+ @JsonProperty
+ public long diskBusyBytes = 0;
+
+ /**
+ * Skip factor for query 2. We select bids for every {@code auctionSkip}'th auction.
+ */
+ @JsonProperty
+ public int auctionSkip = 123;
+
+ /**
+ * Fanout for queries 4 (groups by category id), 5 and 7 (find a global maximum).
+ */
+ @JsonProperty
+ public int fanout = 5;
+
+ /**
+ * Maximum waiting time to clean personState in query3
+ * (ie maximum waiting of the auctions related to person in state in seconds in event time).
+ */
+ @JsonProperty
+ public int maxAuctionsWaitingTime = 600;
+
+ /**
+ * Length of occasional delay to impose on events (in seconds).
+ */
+ @JsonProperty
+ public long occasionalDelaySec = 3;
+
+ /**
+ * Probability that an event will be delayed by delayS.
+ */
+ @JsonProperty
+ public double probDelayedEvent = 0.1;
+
+ /**
+ * Maximum size of each log file (in events). For Query10 only.
+ */
+ @JsonProperty
+ public int maxLogEvents = 100_000;
+
+ /**
+ * If true, use pub/sub publish time instead of event time.
+ */
+ @JsonProperty
+ public boolean usePubsubPublishTime = false;
+
+ /**
+ * Number of events in out-of-order groups. 1 implies no out-of-order events. 1000 implies
+ * every 1000 events per generator are emitted in pseudo-random order.
+ */
+ @JsonProperty
+ public long outOfOrderGroupSize = 1;
+
+ /**
+ * Replace any properties of this configuration which have been supplied by the command line.
+ */
+ public void overrideFromOptions(NexmarkOptions options) {
+ if (options.getDebug() != null) {
+ debug = options.getDebug();
+ }
+ if (options.getQuery() != null) {
+ query = options.getQuery();
+ }
+ if (options.getSourceType() != null) {
+ sourceType = options.getSourceType();
+ }
+ if (options.getSinkType() != null) {
+ sinkType = options.getSinkType();
+ }
+ if (options.getPubSubMode() != null) {
+ pubSubMode = options.getPubSubMode();
+ }
+ if (options.getNumEvents() != null) {
+ numEvents = options.getNumEvents();
+ }
+ if (options.getNumEventGenerators() != null) {
+ numEventGenerators = options.getNumEventGenerators();
+ }
+ if (options.getRateShape() != null) {
+ rateShape = options.getRateShape();
+ }
+ if (options.getFirstEventRate() != null) {
+ firstEventRate = options.getFirstEventRate();
+ }
+ if (options.getNextEventRate() != null) {
+ nextEventRate = options.getNextEventRate();
+ }
+ if (options.getRateUnit() != null) {
+ rateUnit = options.getRateUnit();
+ }
+ if (options.getRatePeriodSec() != null) {
+ ratePeriodSec = options.getRatePeriodSec();
+ }
+ if (options.getPreloadSeconds() != null) {
+ preloadSeconds = options.getPreloadSeconds();
+ }
+ if (options.getStreamTimeout() != null) {
+ streamTimeout = options.getStreamTimeout();
+ }
+ if (options.getIsRateLimited() != null) {
+ isRateLimited = options.getIsRateLimited();
+ }
+ if (options.getUseWallclockEventTime() != null) {
+ useWallclockEventTime = options.getUseWallclockEventTime();
+ }
+ if (options.getAvgPersonByteSize() != null) {
+ avgPersonByteSize = options.getAvgPersonByteSize();
+ }
+ if (options.getAvgAuctionByteSize() != null) {
+ avgAuctionByteSize = options.getAvgAuctionByteSize();
+ }
+ if (options.getAvgBidByteSize() != null) {
+ avgBidByteSize = options.getAvgBidByteSize();
+ }
+ if (options.getHotAuctionRatio() != null) {
+ hotAuctionRatio = options.getHotAuctionRatio();
+ }
+ if (options.getHotSellersRatio() != null) {
+ hotSellersRatio = options.getHotSellersRatio();
+ }
+ if (options.getHotBiddersRatio() != null) {
+ hotBiddersRatio = options.getHotBiddersRatio();
+ }
+ if (options.getWindowSizeSec() != null) {
+ windowSizeSec = options.getWindowSizeSec();
+ }
+ if (options.getWindowPeriodSec() != null) {
+ windowPeriodSec = options.getWindowPeriodSec();
+ }
+ if (options.getWatermarkHoldbackSec() != null) {
+ watermarkHoldbackSec = options.getWatermarkHoldbackSec();
+ }
+ if (options.getNumInFlightAuctions() != null) {
+ numInFlightAuctions = options.getNumInFlightAuctions();
+ }
+ if (options.getNumActivePeople() != null) {
+ numActivePeople = options.getNumActivePeople();
+ }
+ if (options.getCoderStrategy() != null) {
+ coderStrategy = options.getCoderStrategy();
+ }
+ if (options.getCpuDelayMs() != null) {
+ cpuDelayMs = options.getCpuDelayMs();
+ }
+ if (options.getDiskBusyBytes() != null) {
+ diskBusyBytes = options.getDiskBusyBytes();
+ }
+ if (options.getAuctionSkip() != null) {
+ auctionSkip = options.getAuctionSkip();
+ }
+ if (options.getFanout() != null) {
+ fanout = options.getFanout();
+ }
+ if (options.getMaxAuctionsWaitingTime() != null) {
+ fanout = options.getMaxAuctionsWaitingTime();
+ }
+ if (options.getOccasionalDelaySec() != null) {
+ occasionalDelaySec = options.getOccasionalDelaySec();
+ }
+ if (options.getProbDelayedEvent() != null) {
+ probDelayedEvent = options.getProbDelayedEvent();
+ }
+ if (options.getMaxLogEvents() != null) {
+ maxLogEvents = options.getMaxLogEvents();
+ }
+ if (options.getUsePubsubPublishTime() != null) {
+ usePubsubPublishTime = options.getUsePubsubPublishTime();
+ }
+ if (options.getOutOfOrderGroupSize() != null) {
+ outOfOrderGroupSize = options.getOutOfOrderGroupSize();
+ }
+ }
+
+ /**
+ * Return copy of configuration with given label.
+ */
+ public NexmarkConfiguration copy() {
+ NexmarkConfiguration result;
+ result = new NexmarkConfiguration();
+ result.debug = debug;
+ result.query = query;
+ result.sourceType = sourceType;
+ result.sinkType = sinkType;
+ result.pubSubMode = pubSubMode;
+ result.numEvents = numEvents;
+ result.numEventGenerators = numEventGenerators;
+ result.rateShape = rateShape;
+ result.firstEventRate = firstEventRate;
+ result.nextEventRate = nextEventRate;
+ result.rateUnit = rateUnit;
+ result.ratePeriodSec = ratePeriodSec;
+ result.preloadSeconds = preloadSeconds;
+ result.streamTimeout = streamTimeout;
+ result.isRateLimited = isRateLimited;
+ result.useWallclockEventTime = useWallclockEventTime;
+ result.avgPersonByteSize = avgPersonByteSize;
+ result.avgAuctionByteSize = avgAuctionByteSize;
+ result.avgBidByteSize = avgBidByteSize;
+ result.hotAuctionRatio = hotAuctionRatio;
+ result.hotSellersRatio = hotSellersRatio;
+ result.hotBiddersRatio = hotBiddersRatio;
+ result.windowSizeSec = windowSizeSec;
+ result.windowPeriodSec = windowPeriodSec;
+ result.watermarkHoldbackSec = watermarkHoldbackSec;
+ result.numInFlightAuctions = numInFlightAuctions;
+ result.numActivePeople = numActivePeople;
+ result.coderStrategy = coderStrategy;
+ result.cpuDelayMs = cpuDelayMs;
+ result.diskBusyBytes = diskBusyBytes;
+ result.auctionSkip = auctionSkip;
+ result.fanout = fanout;
+ result.maxAuctionsWaitingTime = maxAuctionsWaitingTime;
+ result.occasionalDelaySec = occasionalDelaySec;
+ result.probDelayedEvent = probDelayedEvent;
+ result.maxLogEvents = maxLogEvents;
+ result.usePubsubPublishTime = usePubsubPublishTime;
+ result.outOfOrderGroupSize = outOfOrderGroupSize;
+ return result;
+ }
+
+ /**
+ * Return short description of configuration (suitable for use in logging). We only render
+ * the core fields plus those which do not have default values.
+ */
+ public String toShortString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(String.format("query:%d", query));
+ if (debug != DEFAULT.debug) {
+ sb.append(String.format("; debug:%s", debug));
+ }
+ if (sourceType != DEFAULT.sourceType) {
+ sb.append(String.format("; sourceType:%s", sourceType));
+ }
+ if (sinkType != DEFAULT.sinkType) {
+ sb.append(String.format("; sinkType:%s", sinkType));
+ }
+ if (pubSubMode != DEFAULT.pubSubMode) {
+ sb.append(String.format("; pubSubMode:%s", pubSubMode));
+ }
+ if (numEvents != DEFAULT.numEvents) {
+ sb.append(String.format("; numEvents:%d", numEvents));
+ }
+ if (numEventGenerators != DEFAULT.numEventGenerators) {
+ sb.append(String.format("; numEventGenerators:%d", numEventGenerators));
+ }
+ if (rateShape != DEFAULT.rateShape) {
+ sb.append(String.format("; rateShape:%s", rateShape));
+ }
+ if (firstEventRate != DEFAULT.firstEventRate || nextEventRate != DEFAULT.nextEventRate) {
+ sb.append(String.format("; firstEventRate:%d", firstEventRate));
+ sb.append(String.format("; nextEventRate:%d", nextEventRate));
+ }
+ if (rateUnit != DEFAULT.rateUnit) {
+ sb.append(String.format("; rateUnit:%s", rateUnit));
+ }
+ if (ratePeriodSec != DEFAULT.ratePeriodSec) {
+ sb.append(String.format("; ratePeriodSec:%d", ratePeriodSec));
+ }
+ if (preloadSeconds != DEFAULT.preloadSeconds) {
+ sb.append(String.format("; preloadSeconds:%d", preloadSeconds));
+ }
+ if (streamTimeout != DEFAULT.streamTimeout) {
+ sb.append(String.format("; streamTimeout:%d", streamTimeout));
+ }
+ if (isRateLimited != DEFAULT.isRateLimited) {
+ sb.append(String.format("; isRateLimited:%s", isRateLimited));
+ }
+ if (useWallclockEventTime != DEFAULT.useWallclockEventTime) {
+ sb.append(String.format("; useWallclockEventTime:%s", useWallclockEventTime));
+ }
+ if (avgPersonByteSize != DEFAULT.avgPersonByteSize) {
+ sb.append(String.format("; avgPersonByteSize:%d", avgPersonByteSize));
+ }
+ if (avgAuctionByteSize != DEFAULT.avgAuctionByteSize) {
+ sb.append(String.format("; avgAuctionByteSize:%d", avgAuctionByteSize));
+ }
+ if (avgBidByteSize != DEFAULT.avgBidByteSize) {
+ sb.append(String.format("; avgBidByteSize:%d", avgBidByteSize));
+ }
+ if (hotAuctionRatio != DEFAULT.hotAuctionRatio) {
+ sb.append(String.format("; hotAuctionRatio:%d", hotAuctionRatio));
+ }
+ if (hotSellersRatio != DEFAULT.hotSellersRatio) {
+ sb.append(String.format("; hotSellersRatio:%d", hotSellersRatio));
+ }
+ if (hotBiddersRatio != DEFAULT.hotBiddersRatio) {
+ sb.append(String.format("; hotBiddersRatio:%d", hotBiddersRatio));
+ }
+ if (windowSizeSec != DEFAULT.windowSizeSec) {
+ sb.append(String.format("; windowSizeSec:%d", windowSizeSec));
+ }
+ if (windowPeriodSec != DEFAULT.windowPeriodSec) {
+ sb.append(String.format("; windowPeriodSec:%d", windowPeriodSec));
+ }
+ if (watermarkHoldbackSec != DEFAULT.watermarkHoldbackSec) {
+ sb.append(String.format("; watermarkHoldbackSec:%d", watermarkHoldbackSec));
+ }
+ if (numInFlightAuctions != DEFAULT.numInFlightAuctions) {
+ sb.append(String.format("; numInFlightAuctions:%d", numInFlightAuctions));
+ }
+ if (numActivePeople != DEFAULT.numActivePeople) {
+ sb.append(String.format("; numActivePeople:%d", numActivePeople));
+ }
+ if (coderStrategy != DEFAULT.coderStrategy) {
+ sb.append(String.format("; coderStrategy:%s", coderStrategy));
+ }
+ if (cpuDelayMs != DEFAULT.cpuDelayMs) {
+ sb.append(String.format("; cpuSlowdownMs:%d", cpuDelayMs));
+ }
+ if (diskBusyBytes != DEFAULT.diskBusyBytes) {
+ sb.append(String.format("; diskBuysBytes:%d", diskBusyBytes));
+ }
+ if (auctionSkip != DEFAULT.auctionSkip) {
+ sb.append(String.format("; auctionSkip:%d", auctionSkip));
+ }
+ if (fanout != DEFAULT.fanout) {
+ sb.append(String.format("; fanout:%d", fanout));
+ }
+ if (maxAuctionsWaitingTime != DEFAULT.maxAuctionsWaitingTime) {
+ sb.append(String.format("; maxAuctionsWaitingTime:%d", fanout));
+ }
+ if (occasionalDelaySec != DEFAULT.occasionalDelaySec) {
+ sb.append(String.format("; occasionalDelaySec:%d", occasionalDelaySec));
+ }
+ if (probDelayedEvent != DEFAULT.probDelayedEvent) {
+ sb.append(String.format("; probDelayedEvent:%f", probDelayedEvent));
+ }
+ if (maxLogEvents != DEFAULT.maxLogEvents) {
+ sb.append(String.format("; maxLogEvents:%d", maxLogEvents));
+ }
+ if (usePubsubPublishTime != DEFAULT.usePubsubPublishTime) {
+ sb.append(String.format("; usePubsubPublishTime:%s", usePubsubPublishTime));
+ }
+ if (outOfOrderGroupSize != DEFAULT.outOfOrderGroupSize) {
+ sb.append(String.format("; outOfOrderGroupSize:%d", outOfOrderGroupSize));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Return full description as a string.
+ */
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Parse an object from {@code string}.
+ */
+ public static NexmarkConfiguration fromString(String string) {
+ try {
+ return NexmarkUtils.MAPPER.readValue(string, NexmarkConfiguration.class);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to parse nexmark configuration: ", e);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ debug,
+ query,
+ sourceType,
+ sinkType,
+ pubSubMode,
+ numEvents,
+ numEventGenerators,
+ rateShape,
+ firstEventRate,
+ nextEventRate,
+ rateUnit,
+ ratePeriodSec,
+ preloadSeconds,
+ streamTimeout,
+ isRateLimited,
+ useWallclockEventTime,
+ avgPersonByteSize,
+ avgAuctionByteSize,
+ avgBidByteSize,
+ hotAuctionRatio,
+ hotSellersRatio,
+ hotBiddersRatio,
+ windowSizeSec,
+ windowPeriodSec,
+ watermarkHoldbackSec,
+ numInFlightAuctions,
+ numActivePeople,
+ coderStrategy,
+ cpuDelayMs,
+ diskBusyBytes,
+ auctionSkip,
+ fanout,
+ maxAuctionsWaitingTime,
+ occasionalDelaySec,
+ probDelayedEvent,
+ maxLogEvents,
+ usePubsubPublishTime,
+ outOfOrderGroupSize);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ NexmarkConfiguration other = (NexmarkConfiguration) obj;
+ if (debug != other.debug) {
+ return false;
+ }
+ if (auctionSkip != other.auctionSkip) {
+ return false;
+ }
+ if (avgAuctionByteSize != other.avgAuctionByteSize) {
+ return false;
+ }
+ if (avgBidByteSize != other.avgBidByteSize) {
+ return false;
+ }
+ if (avgPersonByteSize != other.avgPersonByteSize) {
+ return false;
+ }
+ if (coderStrategy != other.coderStrategy) {
+ return false;
+ }
+ if (cpuDelayMs != other.cpuDelayMs) {
+ return false;
+ }
+ if (diskBusyBytes != other.diskBusyBytes) {
+ return false;
+ }
+ if (fanout != other.fanout) {
+ return false;
+ }
+ if (maxAuctionsWaitingTime != other.maxAuctionsWaitingTime) {
+ return false;
+ }
+ if (firstEventRate != other.firstEventRate) {
+ return false;
+ }
+ if (hotAuctionRatio != other.hotAuctionRatio) {
+ return false;
+ }
+ if (hotBiddersRatio != other.hotBiddersRatio) {
+ return false;
+ }
+ if (hotSellersRatio != other.hotSellersRatio) {
+ return false;
+ }
+ if (isRateLimited != other.isRateLimited) {
+ return false;
+ }
+ if (maxLogEvents != other.maxLogEvents) {
+ return false;
+ }
+ if (nextEventRate != other.nextEventRate) {
+ return false;
+ }
+ if (rateUnit != other.rateUnit) {
+ return false;
+ }
+ if (numEventGenerators != other.numEventGenerators) {
+ return false;
+ }
+ if (numEvents != other.numEvents) {
+ return false;
+ }
+ if (numInFlightAuctions != other.numInFlightAuctions) {
+ return false;
+ }
+ if (numActivePeople != other.numActivePeople) {
+ return false;
+ }
+ if (occasionalDelaySec != other.occasionalDelaySec) {
+ return false;
+ }
+ if (preloadSeconds != other.preloadSeconds) {
+ return false;
+ }
+ if (streamTimeout != other.streamTimeout) {
+ return false;
+ }
+ if (Double.doubleToLongBits(probDelayedEvent)
+ != Double.doubleToLongBits(other.probDelayedEvent)) {
+ return false;
+ }
+ if (pubSubMode != other.pubSubMode) {
+ return false;
+ }
+ if (ratePeriodSec != other.ratePeriodSec) {
+ return false;
+ }
+ if (rateShape != other.rateShape) {
+ return false;
+ }
+ if (query != other.query) {
+ return false;
+ }
+ if (sinkType != other.sinkType) {
+ return false;
+ }
+ if (sourceType != other.sourceType) {
+ return false;
+ }
+ if (useWallclockEventTime != other.useWallclockEventTime) {
+ return false;
+ }
+ if (watermarkHoldbackSec != other.watermarkHoldbackSec) {
+ return false;
+ }
+ if (windowPeriodSec != other.windowPeriodSec) {
+ return false;
+ }
+ if (windowSizeSec != other.windowSizeSec) {
+ return false;
+ }
+ if (usePubsubPublishTime != other.usePubsubPublishTime) {
+ return false;
+ }
+ if (outOfOrderGroupSize != other.outOfOrderGroupSize) {
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
new file mode 100644
index 0000000..550fbd2
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
@@ -0,0 +1,1157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
+import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
+import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
+import org.apache.beam.sdk.metrics.DistributionResult;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricQueryResults;
+import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricsFilter;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.nexmark.queries.NexmarkQuery;
+import org.apache.beam.sdk.nexmark.queries.NexmarkQueryModel;
+import org.apache.beam.sdk.nexmark.queries.Query0;
+import org.apache.beam.sdk.nexmark.queries.Query0Model;
+import org.apache.beam.sdk.nexmark.queries.Query1;
+import org.apache.beam.sdk.nexmark.queries.Query10;
+import org.apache.beam.sdk.nexmark.queries.Query11;
+import org.apache.beam.sdk.nexmark.queries.Query12;
+import org.apache.beam.sdk.nexmark.queries.Query1Model;
+import org.apache.beam.sdk.nexmark.queries.Query2;
+import org.apache.beam.sdk.nexmark.queries.Query2Model;
+import org.apache.beam.sdk.nexmark.queries.Query3;
+import org.apache.beam.sdk.nexmark.queries.Query3Model;
+import org.apache.beam.sdk.nexmark.queries.Query4;
+import org.apache.beam.sdk.nexmark.queries.Query4Model;
+import org.apache.beam.sdk.nexmark.queries.Query5;
+import org.apache.beam.sdk.nexmark.queries.Query5Model;
+import org.apache.beam.sdk.nexmark.queries.Query6;
+import org.apache.beam.sdk.nexmark.queries.Query6Model;
+import org.apache.beam.sdk.nexmark.queries.Query7;
+import org.apache.beam.sdk.nexmark.queries.Query7Model;
+import org.apache.beam.sdk.nexmark.queries.Query8;
+import org.apache.beam.sdk.nexmark.queries.Query8Model;
+import org.apache.beam.sdk.nexmark.queries.Query9;
+import org.apache.beam.sdk.nexmark.queries.Query9Model;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.Duration;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Run a single Nexmark query using a given configuration.
+ */
+public class NexmarkLauncher<OptionT extends NexmarkOptions> {
+ private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(NexmarkLauncher.class);
+ /**
+ * Minimum number of samples needed for 'stead-state' rate calculation.
+ */
+ private static final int MIN_SAMPLES = 9;
+ /**
+ * Minimum length of time over which to consider samples for 'steady-state' rate calculation.
+ */
+ private static final Duration MIN_WINDOW = Duration.standardMinutes(2);
+ /**
+ * Delay between perf samples.
+ */
+ private static final Duration PERF_DELAY = Duration.standardSeconds(15);
+ /**
+ * How long to let streaming pipeline run after all events have been generated and we've
+ * seen no activity.
+ */
+ private static final Duration DONE_DELAY = Duration.standardMinutes(1);
+ /**
+ * How long to allow no activity without warning.
+ */
+ private static final Duration STUCK_WARNING_DELAY = Duration.standardMinutes(10);
+ /**
+ * How long to let streaming pipeline run after we've
+ * seen no activity, even if all events have not been generated.
+ */
+ private static final Duration STUCK_TERMINATE_DELAY = Duration.standardDays(3);
+ /**
+ * NexmarkOptions shared by all runs.
+ */
+ private final OptionT options;
+
+ /**
+ * Which configuration we are running.
+ */
+ @Nullable
+ private NexmarkConfiguration configuration;
+
+ /**
+ * If in --pubsubMode=COMBINED, the event monitor for the publisher pipeline. Otherwise null.
+ */
+ @Nullable
+ private Monitor<Event> publisherMonitor;
+
+ /**
+ * If in --pubsubMode=COMBINED, the pipeline result for the publisher pipeline. Otherwise null.
+ */
+ @Nullable
+ private PipelineResult publisherResult;
+
+ /**
+ * Result for the main pipeline.
+ */
+ @Nullable
+ private PipelineResult mainResult;
+
+ /**
+ * Query name we are running.
+ */
+ @Nullable
+ private String queryName;
+
+ public NexmarkLauncher(OptionT options) {
+ this.options = options;
+ }
+
+
+ /**
+ * Is this query running in streaming mode?
+ */
+ private boolean isStreaming() {
+ return options.isStreaming();
+ }
+
+ /**
+ * Return maximum number of workers.
+ */
+ private int maxNumWorkers() {
+ return 5;
+ }
+
+ /**
+ * Return the current value for a long counter, or a default value if can't be retrieved.
+ * Note this uses only attempted metrics because some runners don't support committed metrics.
+ */
+ private long getCounterMetric(PipelineResult result, String namespace, String name,
+ long defaultValue) {
+ MetricQueryResults metrics = result.metrics().queryMetrics(
+ MetricsFilter.builder().addNameFilter(MetricNameFilter.named(namespace, name)).build());
+ Iterable<MetricResult<Long>> counters = metrics.counters();
+ try {
+ MetricResult<Long> metricResult = counters.iterator().next();
+ return metricResult.attempted();
+ } catch (NoSuchElementException e) {
+ LOG.error("Failed to get metric {}, from namespace {}", name, namespace);
+ }
+ return defaultValue;
+ }
+
+ /**
+ * Return the current value for a long counter, or a default value if can't be retrieved.
+ * Note this uses only attempted metrics because some runners don't support committed metrics.
+ */
+ private long getDistributionMetric(PipelineResult result, String namespace, String name,
+ DistributionType distType, long defaultValue) {
+ MetricQueryResults metrics = result.metrics().queryMetrics(
+ MetricsFilter.builder().addNameFilter(MetricNameFilter.named(namespace, name)).build());
+ Iterable<MetricResult<DistributionResult>> distributions = metrics.distributions();
+ try {
+ MetricResult<DistributionResult> distributionResult = distributions.iterator().next();
+ switch (distType) {
+ case MIN:
+ return distributionResult.attempted().min();
+ case MAX:
+ return distributionResult.attempted().max();
+ default:
+ return defaultValue;
+ }
+ } catch (NoSuchElementException e) {
+ LOG.error(
+ "Failed to get distribution metric {} for namespace {}",
+ name,
+ namespace);
+ }
+ return defaultValue;
+ }
+
+ private enum DistributionType {MIN, MAX}
+
+ /**
+ * Return the current value for a time counter, or -1 if can't be retrieved.
+ */
+ private long getTimestampMetric(long now, long value) {
+ // timestamp metrics are used to monitor time of execution of transforms.
+ // If result timestamp metric is too far from now, consider that metric is erroneous
+
+ if (Math.abs(value - now) > Duration.standardDays(10000).getMillis()) {
+ return -1;
+ }
+ return value;
+ }
+
+ /**
+ * Find a 'steady state' events/sec from {@code snapshots} and
+ * store it in {@code perf} if found.
+ */
+ private void captureSteadyState(NexmarkPerf perf, List<NexmarkPerf.ProgressSnapshot> snapshots) {
+ if (!options.isStreaming()) {
+ return;
+ }
+
+ // Find the first sample with actual event and result counts.
+ int dataStart = 0;
+ for (; dataStart < snapshots.size(); dataStart++) {
+ if (snapshots.get(dataStart).numEvents >= 0 && snapshots.get(dataStart).numResults >= 0) {
+ break;
+ }
+ }
+
+ // Find the last sample which demonstrated progress.
+ int dataEnd = snapshots.size() - 1;
+ for (; dataEnd > dataStart; dataEnd--) {
+ if (snapshots.get(dataEnd).anyActivity(snapshots.get(dataEnd - 1))) {
+ break;
+ }
+ }
+
+ int numSamples = dataEnd - dataStart + 1;
+ if (numSamples < MIN_SAMPLES) {
+ // Not enough samples.
+ NexmarkUtils.console("%d samples not enough to calculate steady-state event rate",
+ numSamples);
+ return;
+ }
+
+ // We'll look at only the middle third samples.
+ int sampleStart = dataStart + numSamples / 3;
+ int sampleEnd = dataEnd - numSamples / 3;
+
+ double sampleSec =
+ snapshots.get(sampleEnd).secSinceStart - snapshots.get(sampleStart).secSinceStart;
+ if (sampleSec < MIN_WINDOW.getStandardSeconds()) {
+ // Not sampled over enough time.
+ NexmarkUtils.console(
+ "sample of %.1f sec not long enough to calculate steady-state event rate",
+ sampleSec);
+ return;
+ }
+
+ // Find rate with least squares error.
+ double sumxx = 0.0;
+ double sumxy = 0.0;
+ long prevNumEvents = -1;
+ for (int i = sampleStart; i <= sampleEnd; i++) {
+ if (prevNumEvents == snapshots.get(i).numEvents) {
+ // Skip samples with no change in number of events since they contribute no data.
+ continue;
+ }
+ // Use the effective runtime instead of wallclock time so we can
+ // insulate ourselves from delays and stutters in the query manager.
+ double x = snapshots.get(i).runtimeSec;
+ prevNumEvents = snapshots.get(i).numEvents;
+ double y = prevNumEvents;
+ sumxx += x * x;
+ sumxy += x * y;
+ }
+ double eventsPerSec = sumxy / sumxx;
+ NexmarkUtils.console("revising events/sec from %.1f to %.1f", perf.eventsPerSec, eventsPerSec);
+ perf.eventsPerSec = eventsPerSec;
+ }
+
+ /**
+ * Return the current performance given {@code eventMonitor} and {@code resultMonitor}.
+ */
+ private NexmarkPerf currentPerf(
+ long startMsSinceEpoch, long now, PipelineResult result,
+ List<NexmarkPerf.ProgressSnapshot> snapshots, Monitor<?> eventMonitor,
+ Monitor<?> resultMonitor) {
+ NexmarkPerf perf = new NexmarkPerf();
+
+ long numEvents =
+ getCounterMetric(result, eventMonitor.name, eventMonitor.prefix + ".elements", -1);
+ long numEventBytes =
+ getCounterMetric(result, eventMonitor.name, eventMonitor.prefix + ".bytes", -1);
+ long eventStart =
+ getTimestampMetric(now,
+ getDistributionMetric(result, eventMonitor.name, eventMonitor.prefix + ".startTime",
+ DistributionType.MIN, -1));
+ long eventEnd =
+ getTimestampMetric(now,
+ getDistributionMetric(result, eventMonitor.name, eventMonitor.prefix + ".endTime",
+ DistributionType.MAX, -1));
+
+ long numResults =
+ getCounterMetric(result, resultMonitor.name, resultMonitor.prefix + ".elements", -1);
+ long numResultBytes =
+ getCounterMetric(result, resultMonitor.name, resultMonitor.prefix + ".bytes", -1);
+ long resultStart =
+ getTimestampMetric(now,
+ getDistributionMetric(result, resultMonitor.name, resultMonitor.prefix + ".startTime",
+ DistributionType.MIN, -1));
+ long resultEnd =
+ getTimestampMetric(now,
+ getDistributionMetric(result, resultMonitor.name, resultMonitor.prefix + ".endTime",
+ DistributionType.MAX, -1));
+ long timestampStart =
+ getTimestampMetric(now,
+ getDistributionMetric(result,
+ resultMonitor.name, resultMonitor.prefix + ".startTimestamp",
+ DistributionType.MIN, -1));
+ long timestampEnd =
+ getTimestampMetric(now,
+ getDistributionMetric(result,
+ resultMonitor.name, resultMonitor.prefix + ".endTimestamp",
+ DistributionType.MAX, -1));
+
+ long effectiveEnd = -1;
+ if (eventEnd >= 0 && resultEnd >= 0) {
+ // It is possible for events to be generated after the last result was emitted.
+ // (Eg Query 2, which only yields results for a small prefix of the event stream.)
+ // So use the max of last event and last result times.
+ effectiveEnd = Math.max(eventEnd, resultEnd);
+ } else if (resultEnd >= 0) {
+ effectiveEnd = resultEnd;
+ } else if (eventEnd >= 0) {
+ // During startup we may have no result yet, but we would still like to track how
+ // long the pipeline has been running.
+ effectiveEnd = eventEnd;
+ }
+
+ if (effectiveEnd >= 0 && eventStart >= 0 && effectiveEnd >= eventStart) {
+ perf.runtimeSec = (effectiveEnd - eventStart) / 1000.0;
+ }
+
+ if (numEvents >= 0) {
+ perf.numEvents = numEvents;
+ }
+
+ if (numEvents >= 0 && perf.runtimeSec > 0.0) {
+ // For streaming we may later replace this with a 'steady-state' value calculated
+ // from the progress snapshots.
+ perf.eventsPerSec = numEvents / perf.runtimeSec;
+ }
+
+ if (numEventBytes >= 0 && perf.runtimeSec > 0.0) {
+ perf.eventBytesPerSec = numEventBytes / perf.runtimeSec;
+ }
+
+ if (numResults >= 0) {
+ perf.numResults = numResults;
+ }
+
+ if (numResults >= 0 && perf.runtimeSec > 0.0) {
+ perf.resultsPerSec = numResults / perf.runtimeSec;
+ }
+
+ if (numResultBytes >= 0 && perf.runtimeSec > 0.0) {
+ perf.resultBytesPerSec = numResultBytes / perf.runtimeSec;
+ }
+
+ if (eventStart >= 0) {
+ perf.startupDelaySec = (eventStart - startMsSinceEpoch) / 1000.0;
+ }
+
+ if (resultStart >= 0 && eventStart >= 0 && resultStart >= eventStart) {
+ perf.processingDelaySec = (resultStart - eventStart) / 1000.0;
+ }
+
+ if (timestampStart >= 0 && timestampEnd >= 0 && perf.runtimeSec > 0.0) {
+ double eventRuntimeSec = (timestampEnd - timestampStart) / 1000.0;
+ perf.timeDilation = eventRuntimeSec / perf.runtimeSec;
+ }
+
+ if (resultEnd >= 0) {
+ // Fill in the shutdown delay assuming the job has now finished.
+ perf.shutdownDelaySec = (now - resultEnd) / 1000.0;
+ }
+
+ // As soon as available, try to capture cumulative cost at this point too.
+
+ NexmarkPerf.ProgressSnapshot snapshot = new NexmarkPerf.ProgressSnapshot();
+ snapshot.secSinceStart = (now - startMsSinceEpoch) / 1000.0;
+ snapshot.runtimeSec = perf.runtimeSec;
+ snapshot.numEvents = numEvents;
+ snapshot.numResults = numResults;
+ snapshots.add(snapshot);
+
+ captureSteadyState(perf, snapshots);
+
+ return perf;
+ }
+
+ /**
+ * Build and run a pipeline using specified options.
+ */
+ interface PipelineBuilder<OptionT extends NexmarkOptions> {
+ void build(OptionT publishOnlyOptions);
+ }
+
+ /**
+ * Invoke the builder with options suitable for running a publish-only child pipeline.
+ */
+ private void invokeBuilderForPublishOnlyPipeline(PipelineBuilder<NexmarkOptions> builder) {
+ builder.build(options);
+ }
+
+ /**
+ * Monitor the performance and progress of a running job. Return final performance if
+ * it was measured.
+ */
+ @Nullable
+ private NexmarkPerf monitor(NexmarkQuery query) {
+ if (!options.getMonitorJobs()) {
+ return null;
+ }
+
+ if (configuration.debug) {
+ NexmarkUtils.console("Waiting for main pipeline to 'finish'");
+ } else {
+ NexmarkUtils.console("--debug=false, so job will not self-cancel");
+ }
+
+ PipelineResult job = mainResult;
+ PipelineResult publisherJob = publisherResult;
+ List<NexmarkPerf.ProgressSnapshot> snapshots = new ArrayList<>();
+ long startMsSinceEpoch = System.currentTimeMillis();
+ long endMsSinceEpoch = -1;
+ if (options.getRunningTimeMinutes() != null) {
+ endMsSinceEpoch = startMsSinceEpoch
+ + Duration.standardMinutes(options.getRunningTimeMinutes()).getMillis()
+ - Duration.standardSeconds(configuration.preloadSeconds).getMillis();
+ }
+ long lastActivityMsSinceEpoch = -1;
+ NexmarkPerf perf = null;
+ boolean waitingForShutdown = false;
+ boolean publisherCancelled = false;
+ List<String> errors = new ArrayList<>();
+
+ while (true) {
+ long now = System.currentTimeMillis();
+ if (endMsSinceEpoch >= 0 && now > endMsSinceEpoch && !waitingForShutdown) {
+ NexmarkUtils.console("Reached end of test, cancelling job");
+ try {
+ job.cancel();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to cancel main job: ", e);
+ }
+ if (publisherResult != null) {
+ try {
+ publisherJob.cancel();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to cancel publisher job: ", e);
+ }
+ publisherCancelled = true;
+ }
+ waitingForShutdown = true;
+ }
+
+ PipelineResult.State state = job.getState();
+ NexmarkUtils.console("%s %s%s", state, queryName,
+ waitingForShutdown ? " (waiting for shutdown)" : "");
+
+ NexmarkPerf currPerf;
+ if (configuration.debug) {
+ currPerf = currentPerf(startMsSinceEpoch, now, job, snapshots,
+ query.eventMonitor, query.resultMonitor);
+ } else {
+ currPerf = null;
+ }
+
+ if (perf == null || perf.anyActivity(currPerf)) {
+ lastActivityMsSinceEpoch = now;
+ }
+
+ if (options.isStreaming() && !waitingForShutdown) {
+ Duration quietFor = new Duration(lastActivityMsSinceEpoch, now);
+ long fatalCount = getCounterMetric(job, query.getName(), "fatal", 0);
+ if (fatalCount > 0) {
+ NexmarkUtils.console("job has fatal errors, cancelling.");
+ errors.add(String.format("Pipeline reported %s fatal errors", fatalCount));
+ waitingForShutdown = true;
+ } else if (configuration.debug && configuration.numEvents > 0
+ && currPerf.numEvents == configuration.numEvents
+ && currPerf.numResults >= 0 && quietFor.isLongerThan(DONE_DELAY)) {
+ NexmarkUtils.console("streaming query appears to have finished, cancelling job.");
+ waitingForShutdown = true;
+ } else if (quietFor.isLongerThan(STUCK_TERMINATE_DELAY)) {
+ NexmarkUtils.console("streaming query appears to have gotten stuck, cancelling job.");
+ errors.add("Streaming job was cancelled since appeared stuck");
+ waitingForShutdown = true;
+ } else if (quietFor.isLongerThan(STUCK_WARNING_DELAY)) {
+ NexmarkUtils.console("WARNING: streaming query appears to have been stuck for %d min.",
+ quietFor.getStandardMinutes());
+ errors.add(
+ String.format("Streaming query was stuck for %d min", quietFor.getStandardMinutes()));
+ }
+
+ if (waitingForShutdown) {
+ try {
+ job.cancel();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to cancel main job: ", e);
+ }
+ }
+ }
+
+ perf = currPerf;
+
+ boolean running = true;
+ switch (state) {
+ case UNKNOWN:
+ case STOPPED:
+ case RUNNING:
+ // Keep going.
+ break;
+ case DONE:
+ // All done.
+ running = false;
+ break;
+ case CANCELLED:
+ running = false;
+ if (!waitingForShutdown) {
+ errors.add("Job was unexpectedly cancelled");
+ }
+ break;
+ case FAILED:
+ case UPDATED:
+ // Abnormal termination.
+ running = false;
+ errors.add("Job was unexpectedly updated");
+ break;
+ }
+
+ if (!running) {
+ break;
+ }
+
+ if (lastActivityMsSinceEpoch == now) {
+ NexmarkUtils.console("new perf %s", perf);
+ } else {
+ NexmarkUtils.console("no activity");
+ }
+
+ try {
+ Thread.sleep(PERF_DELAY.getMillis());
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ NexmarkUtils.console("Interrupted: pipeline is still running");
+ }
+ }
+
+ perf.errors = errors;
+ perf.snapshots = snapshots;
+
+ if (publisherResult != null) {
+ NexmarkUtils.console("Shutting down publisher pipeline.");
+ try {
+ if (!publisherCancelled) {
+ publisherJob.cancel();
+ }
+ publisherJob.waitUntilFinish(Duration.standardMinutes(5));
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to cancel publisher job: ", e);
+ }
+ }
+
+ return perf;
+ }
+
+ // ================================================================================
+ // Basic sources and sinks
+ // ================================================================================
+
+ /**
+ * Return a topic name.
+ */
+ private String shortTopic(long now) {
+ String baseTopic = options.getPubsubTopic();
+ if (Strings.isNullOrEmpty(baseTopic)) {
+ throw new RuntimeException("Missing --pubsubTopic");
+ }
+ switch (options.getResourceNameMode()) {
+ case VERBATIM:
+ return baseTopic;
+ case QUERY:
+ return String.format("%s_%s_source", baseTopic, queryName);
+ case QUERY_AND_SALT:
+ return String.format("%s_%s_%d_source", baseTopic, queryName, now);
+ }
+ throw new RuntimeException("Unrecognized enum " + options.getResourceNameMode());
+ }
+
+ /**
+ * Return a subscription name.
+ */
+ private String shortSubscription(long now) {
+ String baseSubscription = options.getPubsubSubscription();
+ if (Strings.isNullOrEmpty(baseSubscription)) {
+ throw new RuntimeException("Missing --pubsubSubscription");
+ }
+ switch (options.getResourceNameMode()) {
+ case VERBATIM:
+ return baseSubscription;
+ case QUERY:
+ return String.format("%s_%s_source", baseSubscription, queryName);
+ case QUERY_AND_SALT:
+ return String.format("%s_%s_%d_source", baseSubscription, queryName, now);
+ }
+ throw new RuntimeException("Unrecognized enum " + options.getResourceNameMode());
+ }
+
+ /**
+ * Return a file name for plain text.
+ */
+ private String textFilename(long now) {
+ String baseFilename = options.getOutputPath();
+ if (Strings.isNullOrEmpty(baseFilename)) {
+ throw new RuntimeException("Missing --outputPath");
+ }
+ switch (options.getResourceNameMode()) {
+ case VERBATIM:
+ return baseFilename;
+ case QUERY:
+ return String.format("%s/nexmark_%s.txt", baseFilename, queryName);
+ case QUERY_AND_SALT:
+ return String.format("%s/nexmark_%s_%d.txt", baseFilename, queryName, now);
+ }
+ throw new RuntimeException("Unrecognized enum " + options.getResourceNameMode());
+ }
+
+ /**
+ * Return a BigQuery table spec.
+ */
+ private String tableSpec(long now, String version) {
+ String baseTableName = options.getBigQueryTable();
+ if (Strings.isNullOrEmpty(baseTableName)) {
+ throw new RuntimeException("Missing --bigQueryTable");
+ }
+ switch (options.getResourceNameMode()) {
+ case VERBATIM:
+ return String.format("%s:nexmark.%s_%s",
+ options.getProject(), baseTableName, version);
+ case QUERY:
+ return String.format("%s:nexmark.%s_%s_%s",
+ options.getProject(), baseTableName, queryName, version);
+ case QUERY_AND_SALT:
+ return String.format("%s:nexmark.%s_%s_%s_%d",
+ options.getProject(), baseTableName, queryName, version, now);
+ }
+ throw new RuntimeException("Unrecognized enum " + options.getResourceNameMode());
+ }
+
+ /**
+ * Return a directory for logs.
+ */
+ private String logsDir(long now) {
+ String baseFilename = options.getOutputPath();
+ if (Strings.isNullOrEmpty(baseFilename)) {
+ throw new RuntimeException("Missing --outputPath");
+ }
+ switch (options.getResourceNameMode()) {
+ case VERBATIM:
+ return baseFilename;
+ case QUERY:
+ return String.format("%s/logs_%s", baseFilename, queryName);
+ case QUERY_AND_SALT:
+ return String.format("%s/logs_%s_%d", baseFilename, queryName, now);
+ }
+ throw new RuntimeException("Unrecognized enum " + options.getResourceNameMode());
+ }
+
+ /**
+ * Return a source of synthetic events.
+ */
+ private PCollection<Event> sourceEventsFromSynthetic(Pipeline p) {
+ if (isStreaming()) {
+ NexmarkUtils.console("Generating %d events in streaming mode", configuration.numEvents);
+ return p.apply(queryName + ".ReadUnbounded", NexmarkUtils.streamEventsSource(configuration));
+ } else {
+ NexmarkUtils.console("Generating %d events in batch mode", configuration.numEvents);
+ return p.apply(queryName + ".ReadBounded", NexmarkUtils.batchEventsSource(configuration));
+ }
+ }
+
+ /**
+ * Return source of events from Pubsub.
+ */
+ private PCollection<Event> sourceEventsFromPubsub(Pipeline p, long now) {
+ String shortSubscription = shortSubscription(now);
+ NexmarkUtils.console("Reading events from Pubsub %s", shortSubscription);
+
+ PubsubIO.Read<PubsubMessage> io =
+ PubsubIO.readMessagesWithAttributes().fromSubscription(shortSubscription)
+ .withIdAttribute(NexmarkUtils.PUBSUB_ID);
+ if (!configuration.usePubsubPublishTime) {
+ io = io.withTimestampAttribute(NexmarkUtils.PUBSUB_TIMESTAMP);
+ }
+
+ return p
+ .apply(queryName + ".ReadPubsubEvents", io)
+ .apply(queryName + ".PubsubMessageToEvent", ParDo.of(new DoFn<PubsubMessage, Event>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ byte[] payload = c.element().getPayload();
+ try {
+ Event event = CoderUtils.decodeFromByteArray(Event.CODER, payload);
+ c.output(event);
+ } catch (CoderException e) {
+ LOG.error("Error while decoding Event from pusbSub message: serialization error");
+ }
+ }
+ }));
+ }
+
+ /**
+ * Return Avro source of events from {@code options.getInputFilePrefix}.
+ */
+ private PCollection<Event> sourceEventsFromAvro(Pipeline p) {
+ String filename = options.getInputPath();
+ if (Strings.isNullOrEmpty(filename)) {
+ throw new RuntimeException("Missing --inputPath");
+ }
+ NexmarkUtils.console("Reading events from Avro files at %s", filename);
+ return p
+ .apply(queryName + ".ReadAvroEvents", AvroIO.read(Event.class)
+ .from(filename + "*.avro"))
+ .apply("OutputWithTimestamp", NexmarkQuery.EVENT_TIMESTAMP_FROM_DATA);
+ }
+
+ /**
+ * Send {@code events} to Pubsub.
+ */
+ private void sinkEventsToPubsub(PCollection<Event> events, long now) {
+ String shortTopic = shortTopic(now);
+ NexmarkUtils.console("Writing events to Pubsub %s", shortTopic);
+
+ PubsubIO.Write<PubsubMessage> io =
+ PubsubIO.writeMessages().to(shortTopic)
+ .withIdAttribute(NexmarkUtils.PUBSUB_ID);
+ if (!configuration.usePubsubPublishTime) {
+ io = io.withTimestampAttribute(NexmarkUtils.PUBSUB_TIMESTAMP);
+ }
+
+ events.apply(queryName + ".EventToPubsubMessage",
+ ParDo.of(new DoFn<Event, PubsubMessage>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ try {
+ byte[] payload = CoderUtils.encodeToByteArray(Event.CODER, c.element());
+ c.output(new PubsubMessage(payload, new HashMap<String, String>()));
+ } catch (CoderException e1) {
+ LOG.error("Error while sending Event {} to pusbSub: serialization error",
+ c.element().toString());
+ }
+ }
+ })
+ )
+ .apply(queryName + ".WritePubsubEvents", io);
+ }
+
+ /**
+ * Send {@code formattedResults} to Pubsub.
+ */
+ private void sinkResultsToPubsub(PCollection<String> formattedResults, long now) {
+ String shortTopic = shortTopic(now);
+ NexmarkUtils.console("Writing results to Pubsub %s", shortTopic);
+ PubsubIO.Write<String> io =
+ PubsubIO.writeStrings().to(shortTopic)
+ .withIdAttribute(NexmarkUtils.PUBSUB_ID);
+ if (!configuration.usePubsubPublishTime) {
+ io = io.withTimestampAttribute(NexmarkUtils.PUBSUB_TIMESTAMP);
+ }
+ formattedResults.apply(queryName + ".WritePubsubResults", io);
+ }
+
+ /**
+ * Sink all raw Events in {@code source} to {@code options.getOutputPath}.
+ * This will configure the job to write the following files:
+ * <ul>
+ * <li>{@code $outputPath/event*.avro} All Event entities.
+ * <li>{@code $outputPath/auction*.avro} Auction entities.
+ * <li>{@code $outputPath/bid*.avro} Bid entities.
+ * <li>{@code $outputPath/person*.avro} Person entities.
+ * </ul>
+ *
+ * @param source A PCollection of events.
+ */
+ private void sinkEventsToAvro(PCollection<Event> source) {
+ String filename = options.getOutputPath();
+ if (Strings.isNullOrEmpty(filename)) {
+ throw new RuntimeException("Missing --outputPath");
+ }
+ NexmarkUtils.console("Writing events to Avro files at %s", filename);
+ source.apply(queryName + ".WriteAvroEvents",
+ AvroIO.write(Event.class).to(filename + "/event").withSuffix(".avro"));
+ source.apply(NexmarkQuery.JUST_BIDS)
+ .apply(queryName + ".WriteAvroBids",
+ AvroIO.write(Bid.class).to(filename + "/bid").withSuffix(".avro"));
+ source.apply(NexmarkQuery.JUST_NEW_AUCTIONS)
+ .apply(queryName + ".WriteAvroAuctions",
+ AvroIO.write(Auction.class).to(filename + "/auction").withSuffix(".avro"));
+ source.apply(NexmarkQuery.JUST_NEW_PERSONS)
+ .apply(queryName + ".WriteAvroPeople",
+ AvroIO.write(Person.class).to(filename + "/person").withSuffix(".avro"));
+ }
+
+ /**
+ * Send {@code formattedResults} to text files.
+ */
+ private void sinkResultsToText(PCollection<String> formattedResults, long now) {
+ String filename = textFilename(now);
+ NexmarkUtils.console("Writing results to text files at %s", filename);
+ formattedResults.apply(queryName + ".WriteTextResults",
+ TextIO.write().to(filename));
+ }
+
+ private static class StringToTableRow extends DoFn<String, TableRow> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ int n = ThreadLocalRandom.current().nextInt(10);
+ List<TableRow> records = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ records.add(new TableRow().set("index", i).set("value", Integer.toString(i)));
+ }
+ c.output(new TableRow().set("result", c.element()).set("records", records));
+ }
+ }
+
+ /**
+ * Send {@code formattedResults} to BigQuery.
+ */
+ private void sinkResultsToBigQuery(
+ PCollection<String> formattedResults, long now,
+ String version) {
+ String tableSpec = tableSpec(now, version);
+ TableSchema tableSchema =
+ new TableSchema().setFields(ImmutableList.of(
+ new TableFieldSchema().setName("result").setType("STRING"),
+ new TableFieldSchema().setName("records").setMode("REPEATED").setType("RECORD")
+ .setFields(ImmutableList.of(
+ new TableFieldSchema().setName("index").setType("INTEGER"),
+ new TableFieldSchema().setName("value").setType("STRING")))));
+ NexmarkUtils.console("Writing results to BigQuery table %s", tableSpec);
+ BigQueryIO.Write io =
+ BigQueryIO.write().to(tableSpec)
+ .withSchema(tableSchema)
+ .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
+ .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND);
+ formattedResults
+ .apply(queryName + ".StringToTableRow", ParDo.of(new StringToTableRow()))
+ .apply(queryName + ".WriteBigQueryResults", io);
+ }
+
+ // ================================================================================
+ // Construct overall pipeline
+ // ================================================================================
+
+ /**
+ * Return source of events for this run, or null if we are simply publishing events
+ * to Pubsub.
+ */
+ private PCollection<Event> createSource(Pipeline p, final long now) {
+ PCollection<Event> source = null;
+ switch (configuration.sourceType) {
+ case DIRECT:
+ source = sourceEventsFromSynthetic(p);
+ break;
+ case AVRO:
+ source = sourceEventsFromAvro(p);
+ break;
+ case PUBSUB:
+ // Setup the sink for the publisher.
+ switch (configuration.pubSubMode) {
+ case SUBSCRIBE_ONLY:
+ // Nothing to publish.
+ break;
+ case PUBLISH_ONLY:
+ // Send synthesized events to Pubsub in this job.
+ sinkEventsToPubsub(sourceEventsFromSynthetic(p).apply(queryName + ".Snoop",
+ NexmarkUtils.snoop(queryName)), now);
+ break;
+ case COMBINED:
+ // Send synthesized events to Pubsub in separate publisher job.
+ // We won't start the main pipeline until the publisher has sent the pre-load events.
+ // We'll shutdown the publisher job when we notice the main job has finished.
+ invokeBuilderForPublishOnlyPipeline(new PipelineBuilder<NexmarkOptions>() {
+ @Override
+ public void build(NexmarkOptions publishOnlyOptions) {
+ Pipeline sp = Pipeline.create(options);
+ NexmarkUtils.setupPipeline(configuration.coderStrategy, sp);
+ publisherMonitor = new Monitor<>(queryName, "publisher");
+ sinkEventsToPubsub(
+ sourceEventsFromSynthetic(sp)
+ .apply(queryName + ".Monitor", publisherMonitor.getTransform()),
+ now);
+ publisherResult = sp.run();
+ }
+ });
+ break;
+ }
+
+ // Setup the source for the consumer.
+ switch (configuration.pubSubMode) {
+ case PUBLISH_ONLY:
+ // Nothing to consume. Leave source null.
+ break;
+ case SUBSCRIBE_ONLY:
+ case COMBINED:
+ // Read events from pubsub.
+ source = sourceEventsFromPubsub(p, now);
+ break;
+ }
+ break;
+ }
+ return source;
+ }
+
+ private static final TupleTag<String> MAIN = new TupleTag<String>(){};
+ private static final TupleTag<String> SIDE = new TupleTag<String>(){};
+
+ private static class PartitionDoFn extends DoFn<String, String> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (c.element().hashCode() % 2 == 0) {
+ c.output(c.element());
+ } else {
+ c.output(SIDE, c.element());
+ }
+ }
+ }
+
+ /**
+ * Consume {@code results}.
+ */
+ private void sink(PCollection<TimestampedValue<KnownSize>> results, long now) {
+ if (configuration.sinkType == NexmarkUtils.SinkType.COUNT_ONLY) {
+ // Avoid the cost of formatting the results.
+ results.apply(queryName + ".DevNull", NexmarkUtils.devNull(queryName));
+ return;
+ }
+
+ PCollection<String> formattedResults =
+ results.apply(queryName + ".Format", NexmarkUtils.format(queryName));
+ if (options.getLogResults()) {
+ formattedResults = formattedResults.apply(queryName + ".Results.Log",
+ NexmarkUtils.<String>log(queryName + ".Results"));
+ }
+
+ switch (configuration.sinkType) {
+ case DEVNULL:
+ // Discard all results
+ formattedResults.apply(queryName + ".DevNull", NexmarkUtils.devNull(queryName));
+ break;
+ case PUBSUB:
+ sinkResultsToPubsub(formattedResults, now);
+ break;
+ case TEXT:
+ sinkResultsToText(formattedResults, now);
+ break;
+ case AVRO:
+ NexmarkUtils.console(
+ "WARNING: with --sinkType=AVRO, actual query results will be discarded.");
+ break;
+ case BIGQUERY:
+ // Multiple BigQuery backends to mimic what most customers do.
+ PCollectionTuple res = formattedResults.apply(queryName + ".Partition",
+ ParDo.of(new PartitionDoFn()).withOutputTags(MAIN, TupleTagList.of(SIDE)));
+ sinkResultsToBigQuery(res.get(MAIN), now, "main");
+ sinkResultsToBigQuery(res.get(SIDE), now, "side");
+ sinkResultsToBigQuery(formattedResults, now, "copy");
+ break;
+ case COUNT_ONLY:
+ // Short-circuited above.
+ throw new RuntimeException();
+ }
+ }
+
+ // ================================================================================
+ // Entry point
+ // ================================================================================
+
+ /**
+ * Calculate the distribution of the expected rate of results per minute (in event time, not
+ * wallclock time).
+ */
+ private void modelResultRates(NexmarkQueryModel model) {
+ List<Long> counts = Lists.newArrayList(model.simulator().resultsPerWindow());
+ Collections.sort(counts);
+ int n = counts.size();
+ if (n < 5) {
+ NexmarkUtils.console("Query%d: only %d samples", model.configuration.query, n);
+ } else {
+ NexmarkUtils.console("Query%d: N:%d; min:%d; 1st%%:%d; mean:%d; 3rd%%:%d; max:%d",
+ model.configuration.query, n, counts.get(0), counts.get(n / 4),
+ counts.get(n / 2),
+ counts.get(n - 1 - n / 4), counts.get(n - 1));
+ }
+ }
+
+ /**
+ * Run {@code configuration} and return its performance if possible.
+ */
+ @Nullable
+ public NexmarkPerf run(NexmarkConfiguration runConfiguration) {
+ if (options.getManageResources() && !options.getMonitorJobs()) {
+ throw new RuntimeException("If using --manageResources then must also use --monitorJobs.");
+ }
+
+ //
+ // Setup per-run state.
+ //
+ checkState(configuration == null);
+ checkState(queryName == null);
+ configuration = runConfiguration;
+
+ try {
+ NexmarkUtils.console("Running %s", configuration.toShortString());
+
+ if (configuration.numEvents < 0) {
+ NexmarkUtils.console("skipping since configuration is disabled");
+ return null;
+ }
+
+ List<NexmarkQuery> queries = Arrays.asList(new Query0(configuration),
+ new Query1(configuration),
+ new Query2(configuration),
+ new Query3(configuration),
+ new Query4(configuration),
+ new Query5(configuration),
+ new Query6(configuration),
+ new Query7(configuration),
+ new Query8(configuration),
+ new Query9(configuration),
+ new Query10(configuration),
+ new Query11(configuration),
+ new Query12(configuration));
+ NexmarkQuery query = queries.get(configuration.query);
+ queryName = query.getName();
+
+ List<NexmarkQueryModel> models = Arrays.asList(
+ new Query0Model(configuration),
+ new Query1Model(configuration),
+ new Query2Model(configuration),
+ new Query3Model(configuration),
+ new Query4Model(configuration),
+ new Query5Model(configuration),
+ new Query6Model(configuration),
+ new Query7Model(configuration),
+ new Query8Model(configuration),
+ new Query9Model(configuration),
+ null,
+ null,
+ null);
+ NexmarkQueryModel model = models.get(configuration.query);
+
+ if (options.getJustModelResultRate()) {
+ if (model == null) {
+ throw new RuntimeException(String.format("No model for %s", queryName));
+ }
+ modelResultRates(model);
+ return null;
+ }
+
+ long now = System.currentTimeMillis();
+ Pipeline p = Pipeline.create(options);
+ NexmarkUtils.setupPipeline(configuration.coderStrategy, p);
+
+ // Generate events.
+ PCollection<Event> source = createSource(p, now);
+
+ if (options.getLogEvents()) {
+ source = source.apply(queryName + ".Events.Log",
+ NexmarkUtils.<Event>log(queryName + ".Events"));
+ }
+
+ // Source will be null if source type is PUBSUB and mode is PUBLISH_ONLY.
+ // In that case there's nothing more to add to pipeline.
+ if (source != null) {
+ // Optionally sink events in Avro format.
+ // (Query results are ignored).
+ if (configuration.sinkType == NexmarkUtils.SinkType.AVRO) {
+ sinkEventsToAvro(source);
+ }
+
+ // Query 10 logs all events to Google Cloud storage files. It could generate a lot of logs,
+ // so, set parallelism. Also set the output path where to write log files.
+ if (configuration.query == 10) {
+ String path = null;
+ if (options.getOutputPath() != null && !options.getOutputPath().isEmpty()) {
+ path = logsDir(now);
+ }
+ ((Query10) query).setOutputPath(path);
+ ((Query10) query).setMaxNumWorkers(maxNumWorkers());
+ }
+
+ // Apply query.
+ PCollection<TimestampedValue<KnownSize>> results = source.apply(query);
+
+ if (options.getAssertCorrectness()) {
+ if (model == null) {
+ throw new RuntimeException(String.format("No model for %s", queryName));
+ }
+ // We know all our streams have a finite number of elements.
+ results.setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
+ // If we have a finite number of events then assert our pipeline's
+ // results match those of a model using the same sequence of events.
+ PAssert.that(results).satisfies(model.assertionFor());
+ }
+
+ // Output results.
+ sink(results, now);
+ }
+
+ mainResult = p.run();
+ mainResult.waitUntilFinish(Duration.standardSeconds(configuration.streamTimeout));
+ return monitor(query);
+ } finally {
+ configuration = null;
+ queryName = null;
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
new file mode 100644
index 0000000..2a2a5a7
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.StreamingOptions;
+
+/**
+ * Command line flags.
+ */
+public interface NexmarkOptions
+ extends ApplicationNameOptions, GcpOptions, PipelineOptions, StreamingOptions {
+ @Description("Which suite to run. Default is to use command line arguments for one job.")
+ @Default.Enum("DEFAULT")
+ NexmarkSuite getSuite();
+
+ void setSuite(NexmarkSuite suite);
+
+ @Description("If true, monitor the jobs as they run.")
+ @Default.Boolean(false)
+ boolean getMonitorJobs();
+
+ void setMonitorJobs(boolean monitorJobs);
+
+ @Description("Where the events come from.")
+ @Nullable
+ NexmarkUtils.SourceType getSourceType();
+
+ void setSourceType(NexmarkUtils.SourceType sourceType);
+
+ @Description("Prefix for input files if using avro input")
+ @Nullable
+ String getInputPath();
+
+ void setInputPath(String inputPath);
+
+ @Description("Where results go.")
+ @Nullable
+ NexmarkUtils.SinkType getSinkType();
+
+ void setSinkType(NexmarkUtils.SinkType sinkType);
+
+ @Description("Which mode to run in when source is PUBSUB.")
+ @Nullable
+ NexmarkUtils.PubSubMode getPubSubMode();
+
+ void setPubSubMode(NexmarkUtils.PubSubMode pubSubMode);
+
+ @Description("Which query to run.")
+ @Nullable
+ Integer getQuery();
+
+ void setQuery(Integer query);
+
+ @Description("Prefix for output files if using text output for results or running Query 10.")
+ @Nullable
+ String getOutputPath();
+
+ void setOutputPath(String outputPath);
+
+ @Description("Base name of pubsub topic to publish to in streaming mode.")
+ @Nullable
+ @Default.String("nexmark")
+ String getPubsubTopic();
+
+ void setPubsubTopic(String pubsubTopic);
+
+ @Description("Base name of pubsub subscription to read from in streaming mode.")
+ @Nullable
+ @Default.String("nexmark")
+ String getPubsubSubscription();
+
+ void setPubsubSubscription(String pubsubSubscription);
+
+ @Description("Base name of BigQuery table name if using BigQuery output.")
+ @Nullable
+ @Default.String("nexmark")
+ String getBigQueryTable();
+
+ void setBigQueryTable(String bigQueryTable);
+
+ @Description("Approximate number of events to generate. "
+ + "Zero for effectively unlimited in streaming mode.")
+ @Nullable
+ Long getNumEvents();
+
+ void setNumEvents(Long numEvents);
+
+ @Description("Time in seconds to preload the subscription with data, at the initial input rate "
+ + "of the pipeline.")
+ @Nullable
+ Integer getPreloadSeconds();
+
+ void setPreloadSeconds(Integer preloadSeconds);
+
+ @Description(
+ "Time in seconds to wait in pipelineResult.waitUntilFinish(), useful in streaming mode")
+ @Nullable
+ Integer getStreamTimeout();
+
+ void setStreamTimeout(Integer streamTimeout);
+
+ @Description("Number of unbounded sources to create events.")
+ @Nullable
+ Integer getNumEventGenerators();
+
+ void setNumEventGenerators(Integer numEventGenerators);
+
+ @Description("Shape of event rate curve.")
+ @Nullable
+ NexmarkUtils.RateShape getRateShape();
+
+ void setRateShape(NexmarkUtils.RateShape rateShape);
+
+ @Description("Initial overall event rate (in --rateUnit).")
+ @Nullable
+ Integer getFirstEventRate();
+
+ void setFirstEventRate(Integer firstEventRate);
+
+ @Description("Next overall event rate (in --rateUnit).")
+ @Nullable
+ Integer getNextEventRate();
+
+ void setNextEventRate(Integer nextEventRate);
+
+ @Description("Unit for rates.")
+ @Nullable
+ NexmarkUtils.RateUnit getRateUnit();
+
+ void setRateUnit(NexmarkUtils.RateUnit rateUnit);
+
+ @Description("Overall period of rate shape, in seconds.")
+ @Nullable
+ Integer getRatePeriodSec();
+
+ void setRatePeriodSec(Integer ratePeriodSec);
+
+ @Description("If true, relay events in real time in streaming mode.")
+ @Nullable
+ Boolean getIsRateLimited();
+
+ void setIsRateLimited(Boolean isRateLimited);
+
+ @Description("If true, use wallclock time as event time. Otherwise, use a deterministic"
+ + " time in the past so that multiple runs will see exactly the same event streams"
+ + " and should thus have exactly the same results.")
+ @Nullable
+ Boolean getUseWallclockEventTime();
+
+ void setUseWallclockEventTime(Boolean useWallclockEventTime);
+
+ @Description("Assert pipeline results match model results.")
+ @Nullable
+ boolean getAssertCorrectness();
+
+ void setAssertCorrectness(boolean assertCorrectness);
+
+ @Description("Log all input events.")
+ @Nullable
+ boolean getLogEvents();
+
+ void setLogEvents(boolean logEvents);
+
+ @Description("Log all query results.")
+ @Nullable
+ boolean getLogResults();
+
+ void setLogResults(boolean logResults);
+
+ @Description("Average size in bytes for a person record.")
+ @Nullable
+ Integer getAvgPersonByteSize();
+
+ void setAvgPersonByteSize(Integer avgPersonByteSize);
+
+ @Description("Average size in bytes for an auction record.")
+ @Nullable
+ Integer getAvgAuctionByteSize();
+
+ void setAvgAuctionByteSize(Integer avgAuctionByteSize);
+
+ @Description("Average size in bytes for a bid record.")
+ @Nullable
+ Integer getAvgBidByteSize();
+
+ void setAvgBidByteSize(Integer avgBidByteSize);
+
+ @Description("Ratio of bids for 'hot' auctions above the background.")
+ @Nullable
+ Integer getHotAuctionRatio();
+
+ void setHotAuctionRatio(Integer hotAuctionRatio);
+
+ @Description("Ratio of auctions for 'hot' sellers above the background.")
+ @Nullable
+ Integer getHotSellersRatio();
+
+ void setHotSellersRatio(Integer hotSellersRatio);
+
+ @Description("Ratio of auctions for 'hot' bidders above the background.")
+ @Nullable
+ Integer getHotBiddersRatio();
+
+ void setHotBiddersRatio(Integer hotBiddersRatio);
+
+ @Description("Window size in seconds.")
+ @Nullable
+ Long getWindowSizeSec();
+
+ void setWindowSizeSec(Long windowSizeSec);
+
+ @Description("Window period in seconds.")
+ @Nullable
+ Long getWindowPeriodSec();
+
+ void setWindowPeriodSec(Long windowPeriodSec);
+
+ @Description("If in streaming mode, the holdback for watermark in seconds.")
+ @Nullable
+ Long getWatermarkHoldbackSec();
+
+ void setWatermarkHoldbackSec(Long watermarkHoldbackSec);
+
+ @Description("Roughly how many auctions should be in flight for each generator.")
+ @Nullable
+ Integer getNumInFlightAuctions();
+
+ void setNumInFlightAuctions(Integer numInFlightAuctions);
+
+
+ @Description("Maximum number of people to consider as active for placing auctions or bids.")
+ @Nullable
+ Integer getNumActivePeople();
+
+ void setNumActivePeople(Integer numActivePeople);
+
+ @Description("Filename of perf data to append to.")
+ @Nullable
+ String getPerfFilename();
+
+ void setPerfFilename(String perfFilename);
+
+ @Description("Filename of baseline perf data to read from.")
+ @Nullable
+ String getBaselineFilename();
+
+ void setBaselineFilename(String baselineFilename);
+
+ @Description("Filename of summary perf data to append to.")
+ @Nullable
+ String getSummaryFilename();
+
+ void setSummaryFilename(String summaryFilename);
+
+ @Description("Filename for javascript capturing all perf data and any baselines.")
+ @Nullable
+ String getJavascriptFilename();
+
+ void setJavascriptFilename(String javascriptFilename);
+
+ @Description("If true, don't run the actual query. Instead, calculate the distribution "
+ + "of number of query results per (event time) minute according to the query model.")
+ @Nullable
+ boolean getJustModelResultRate();
+
+ void setJustModelResultRate(boolean justModelResultRate);
+
+ @Description("Coder strategy to use.")
+ @Nullable
+ NexmarkUtils.CoderStrategy getCoderStrategy();
+
+ void setCoderStrategy(NexmarkUtils.CoderStrategy coderStrategy);
+
+ @Description("Delay, in milliseconds, for each event. We will peg one core for this "
+ + "number of milliseconds to simulate CPU-bound computation.")
+ @Nullable
+ Long getCpuDelayMs();
+
+ void setCpuDelayMs(Long cpuDelayMs);
+
+ @Description("Extra data, in bytes, to save to persistent state for each event. "
+ + "This will force I/O all the way to durable storage to simulate an "
+ + "I/O-bound computation.")
+ @Nullable
+ Long getDiskBusyBytes();
+
+ void setDiskBusyBytes(Long diskBusyBytes);
+
+ @Description("Skip factor for query 2. We select bids for every {@code auctionSkip}'th auction")
+ @Nullable
+ Integer getAuctionSkip();
+
+ void setAuctionSkip(Integer auctionSkip);
+
+ @Description("Fanout for queries 4 (groups by category id) and 7 (finds a global maximum).")
+ @Nullable
+ Integer getFanout();
+
+ void setFanout(Integer fanout);
+
+ @Description("Maximum waiting time to clean personState in query3 "
+ + "(ie maximum waiting of the auctions related to person in state in seconds in event time).")
+ @Nullable
+ Integer getMaxAuctionsWaitingTime();
+
+ void setMaxAuctionsWaitingTime(Integer fanout);
+
+ @Description("Length of occasional delay to impose on events (in seconds).")
+ @Nullable
+ Long getOccasionalDelaySec();
+
+ void setOccasionalDelaySec(Long occasionalDelaySec);
+
+ @Description("Probability that an event will be delayed by delayS.")
+ @Nullable
+ Double getProbDelayedEvent();
+
+ void setProbDelayedEvent(Double probDelayedEvent);
+
+ @Description("Maximum size of each log file (in events). For Query10 only.")
+ @Nullable
+ Integer getMaxLogEvents();
+
+ void setMaxLogEvents(Integer maxLogEvents);
+
+ @Description("How to derive names of resources.")
+ @Default.Enum("QUERY_AND_SALT")
+ NexmarkUtils.ResourceNameMode getResourceNameMode();
+
+ void setResourceNameMode(NexmarkUtils.ResourceNameMode mode);
+
+ @Description("If true, manage the creation and cleanup of topics, subscriptions and gcs files.")
+ @Default.Boolean(true)
+ boolean getManageResources();
+
+ void setManageResources(boolean manageResources);
+
+ @Description("If true, use pub/sub publish time instead of event time.")
+ @Nullable
+ Boolean getUsePubsubPublishTime();
+
+ void setUsePubsubPublishTime(Boolean usePubsubPublishTime);
+
+ @Description("Number of events in out-of-order groups. 1 implies no out-of-order events. "
+ + "1000 implies every 1000 events per generator are emitted in pseudo-random order.")
+ @Nullable
+ Long getOutOfOrderGroupSize();
+
+ void setOutOfOrderGroupSize(Long outOfOrderGroupSize);
+
+ @Description("If false, do not add the Monitor and Snoop transforms.")
+ @Nullable
+ Boolean getDebug();
+
+ void setDebug(Boolean value);
+
+ @Description("If set, cancel running pipelines after this long")
+ @Nullable
+ Long getRunningTimeMinutes();
+
+ void setRunningTimeMinutes(Long value);
+
+ @Description("If set and --monitorJobs is true, check that the system watermark is never more "
+ + "than this far behind real time")
+ @Nullable
+ Long getMaxSystemLagSeconds();
+
+ void setMaxSystemLagSeconds(Long value);
+
+ @Description("If set and --monitorJobs is true, check that the data watermark is never more "
+ + "than this far behind real time")
+ @Nullable
+ Long getMaxDataLagSeconds();
+
+ void setMaxDataLagSeconds(Long value);
+
+ @Description("Only start validating watermarks after this many seconds")
+ @Nullable
+ Long getWatermarkValidationDelaySeconds();
+
+ void setWatermarkValidationDelaySeconds(Long value);
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkPerf.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkPerf.java
new file mode 100644
index 0000000..2edf4e8
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkPerf.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Summary of performance for a particular run of a configuration.
+ */
+public class NexmarkPerf {
+ /**
+ * A sample of the number of events and number of results (if known) generated at
+ * a particular time.
+ */
+ public static class ProgressSnapshot {
+ /** Seconds since job was started (in wallclock time). */
+ @JsonProperty
+ double secSinceStart;
+
+ /** Job runtime in seconds (time from first event to last generated event or output result). */
+ @JsonProperty
+ double runtimeSec;
+
+ /** Cumulative number of events generated. -1 if not known. */
+ @JsonProperty
+ long numEvents;
+
+ /** Cumulative number of results emitted. -1 if not known. */
+ @JsonProperty
+ long numResults;
+
+ /**
+ * Return true if there looks to be activity between {@code this} and {@code that}
+ * snapshots.
+ */
+ public boolean anyActivity(ProgressSnapshot that) {
+ if (runtimeSec != that.runtimeSec) {
+ // An event or result end timestamp looks to have changed.
+ return true;
+ }
+ if (numEvents != that.numEvents) {
+ // Some more events were generated.
+ return true;
+ }
+ if (numResults != that.numResults) {
+ // Some more results were emitted.
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Progess snapshots. Null if not yet calculated.
+ */
+ @JsonProperty
+ @Nullable
+ public List<ProgressSnapshot> snapshots = null;
+
+ /**
+ * Effective runtime, in seconds. Measured from timestamp of first generated event to latest of
+ * timestamp of last generated event and last emitted result. -1 if not known.
+ */
+ @JsonProperty
+ public double runtimeSec = -1.0;
+
+ /**
+ * Number of events generated. -1 if not known.
+ */
+ @JsonProperty
+ public long numEvents = -1;
+
+ /**
+ * Number of events generated per second of runtime. For batch this is number of events
+ * over the above runtime. For streaming this is the 'steady-state' event generation rate sampled
+ * over the lifetime of the job. -1 if not known.
+ */
+ @JsonProperty
+ public double eventsPerSec = -1.0;
+
+ /**
+ * Number of event bytes generated per second of runtime. -1 if not known.
+ */
+ @JsonProperty
+ public double eventBytesPerSec = -1.0;
+
+ /**
+ * Number of results emitted. -1 if not known.
+ */
+ @JsonProperty
+ public long numResults = -1;
+
+ /**
+ * Number of results generated per second of runtime. -1 if not known.
+ */
+ @JsonProperty
+ public double resultsPerSec = -1.0;
+
+ /**
+ * Number of result bytes generated per second of runtime. -1 if not known.
+ */
+ @JsonProperty
+ public double resultBytesPerSec = -1.0;
+
+ /**
+ * Delay between start of job and first event in second. -1 if not known.
+ */
+ @JsonProperty
+ public double startupDelaySec = -1.0;
+
+ /**
+ * Delay between first event and first result in seconds. -1 if not known.
+ */
+ @JsonProperty
+ public double processingDelaySec = -1.0;
+
+ /**
+ * Delay between last result and job completion in seconds. -1 if not known.
+ */
+ @JsonProperty
+ public double shutdownDelaySec = -1.0;
+
+ /**
+ * Time-dilation factor. Calculate as event time advancement rate relative to real time.
+ * Greater than one implies we processed events faster than they would have been generated
+ * in real time. Less than one implies we could not keep up with events in real time.
+ * -1 if not known.
+ */
+ @JsonProperty
+ double timeDilation = -1.0;
+
+ /**
+ * List of errors encountered during job execution.
+ */
+ @JsonProperty
+ @Nullable
+ public List<String> errors = null;
+
+ /**
+ * The job id this perf was drawn from. Null if not known.
+ */
+ @JsonProperty
+ @Nullable
+ public String jobId = null;
+
+ /**
+ * Return a JSON representation of performance.
+ */
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Parse a {@link NexmarkPerf} object from JSON {@code string}.
+ */
+ public static NexmarkPerf fromString(String string) {
+ try {
+ return NexmarkUtils.MAPPER.readValue(string, NexmarkPerf.class);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to parse nexmark perf: ", e);
+ }
+ }
+
+ /**
+ * Return true if there looks to be activity between {@code this} and {@code that}
+ * perf values.
+ */
+ public boolean anyActivity(NexmarkPerf that) {
+ if (runtimeSec != that.runtimeSec) {
+ // An event or result end timestamp looks to have changed.
+ return true;
+ }
+ if (numEvents != that.numEvents) {
+ // Some more events were generated.
+ return true;
+ }
+ if (numResults != that.numResults) {
+ // Some more results were emitted.
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkSuite.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkSuite.java
new file mode 100644
index 0000000..d38cb7b
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkSuite.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A set of {@link NexmarkConfiguration}s.
+ */
+public enum NexmarkSuite {
+ /**
+ * The default.
+ */
+ DEFAULT(defaultConf()),
+
+ /**
+ * Sweep through all queries using the default configuration.
+ * 100k/10k events (depending on query).
+ */
+ SMOKE(smoke()),
+
+ /**
+ * As for SMOKE, but with 10m/1m events.
+ */
+ STRESS(stress()),
+
+ /**
+ * As for SMOKE, but with 1b/100m events.
+ */
+ FULL_THROTTLE(fullThrottle());
+
+ private static List<NexmarkConfiguration> defaultConf() {
+ List<NexmarkConfiguration> configurations = new ArrayList<>();
+ NexmarkConfiguration configuration = new NexmarkConfiguration();
+ configurations.add(configuration);
+ return configurations;
+ }
+
+ private static List<NexmarkConfiguration> smoke() {
+ List<NexmarkConfiguration> configurations = new ArrayList<>();
+ for (int query = 0; query <= 12; query++) {
+ NexmarkConfiguration configuration = NexmarkConfiguration.DEFAULT.copy();
+ configuration.query = query;
+ configuration.numEvents = 100_000;
+ if (query == 4 || query == 6 || query == 9) {
+ // Scale back so overall runtimes are reasonably close across all queries.
+ configuration.numEvents /= 10;
+ }
+ configurations.add(configuration);
+ }
+ return configurations;
+ }
+
+ private static List<NexmarkConfiguration> stress() {
+ List<NexmarkConfiguration> configurations = smoke();
+ for (NexmarkConfiguration configuration : configurations) {
+ if (configuration.numEvents >= 0) {
+ configuration.numEvents *= 1000;
+ }
+ }
+ return configurations;
+ }
+
+ private static List<NexmarkConfiguration> fullThrottle() {
+ List<NexmarkConfiguration> configurations = smoke();
+ for (NexmarkConfiguration configuration : configurations) {
+ if (configuration.numEvents >= 0) {
+ configuration.numEvents *= 1000;
+ }
+ }
+ return configurations;
+ }
+
+ private final List<NexmarkConfiguration> configurations;
+
+ NexmarkSuite(List<NexmarkConfiguration> configurations) {
+ this.configurations = configurations;
+ }
+
+ /**
+ * Return the configurations corresponding to this suite. We'll override each configuration
+ * with any set command line flags, except for --isStreaming which is only respected for
+ * the {@link #DEFAULT} suite.
+ */
+ public Iterable<NexmarkConfiguration> getConfigurations(NexmarkOptions options) {
+ Set<NexmarkConfiguration> results = new LinkedHashSet<>();
+ for (NexmarkConfiguration configuration : configurations) {
+ NexmarkConfiguration result = configuration.copy();
+ result.overrideFromOptions(options);
+ results.add(result);
+ }
+ return results;
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
new file mode 100644
index 0000000..fa1ef16
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
@@ -0,0 +1,674 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.hash.Hashing;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AvroCoder;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.AuctionCount;
+import org.apache.beam.sdk.nexmark.model.AuctionPrice;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.BidsPerSession;
+import org.apache.beam.sdk.nexmark.model.CategoryPrice;
+import org.apache.beam.sdk.nexmark.model.Done;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.IdNameReserve;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.NameCityStateId;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.nexmark.model.SellerPrice;
+import org.apache.beam.sdk.nexmark.sources.BoundedEventSource;
+import org.apache.beam.sdk.nexmark.sources.Generator;
+import org.apache.beam.sdk.nexmark.sources.GeneratorConfig;
+import org.apache.beam.sdk.nexmark.sources.UnboundedEventSource;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Odd's 'n Ends used throughout queries and driver.
+ */
+public class NexmarkUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(NexmarkUtils.class);
+
+ /**
+ * Mapper for (de)serializing JSON.
+ */
+ public static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /**
+ * Possible sources for events.
+ */
+ public enum SourceType {
+ /**
+ * Produce events directly.
+ */
+ DIRECT,
+ /**
+ * Read events from an Avro file.
+ */
+ AVRO,
+ /**
+ * Read from a PubSub topic. It will be fed the same synthetic events by this pipeline.
+ */
+ PUBSUB
+ }
+
+ /**
+ * Possible sinks for query results.
+ */
+ public enum SinkType {
+ /**
+ * Discard all results.
+ */
+ COUNT_ONLY,
+ /**
+ * Discard all results after converting them to strings.
+ */
+ DEVNULL,
+ /**
+ * Write to a PubSub topic. It will be drained by this pipeline.
+ */
+ PUBSUB,
+ /**
+ * Write to a text file. Only works in batch mode.
+ */
+ TEXT,
+ /**
+ * Write raw Events to Avro. Only works in batch mode.
+ */
+ AVRO,
+ /**
+ * Write raw Events to BigQuery.
+ */
+ BIGQUERY,
+ }
+
+ /**
+ * Pub/sub mode to run in.
+ */
+ public enum PubSubMode {
+ /**
+ * Publish events to pub/sub, but don't run the query.
+ */
+ PUBLISH_ONLY,
+ /**
+ * Consume events from pub/sub and run the query, but don't publish.
+ */
+ SUBSCRIBE_ONLY,
+ /**
+ * Both publish and consume, but as separate jobs.
+ */
+ COMBINED
+ }
+
+ /**
+ * Coder strategies.
+ */
+ public enum CoderStrategy {
+ /**
+ * Hand-written.
+ */
+ HAND,
+ /**
+ * Avro.
+ */
+ AVRO,
+ /**
+ * Java serialization.
+ */
+ JAVA
+ }
+
+ /**
+ * How to determine resource names.
+ */
+ public enum ResourceNameMode {
+ /** Names are used as provided. */
+ VERBATIM,
+ /** Names are suffixed with the query being run. */
+ QUERY,
+ /** Names are suffixed with the query being run and a random number. */
+ QUERY_AND_SALT
+ }
+
+ /**
+ * Units for rates.
+ */
+ public enum RateUnit {
+ PER_SECOND(1_000_000L),
+ PER_MINUTE(60_000_000L);
+
+ RateUnit(long usPerUnit) {
+ this.usPerUnit = usPerUnit;
+ }
+
+ /**
+ * Number of microseconds per unit.
+ */
+ private final long usPerUnit;
+
+ /**
+ * Number of microseconds between events at given rate.
+ */
+ public long rateToPeriodUs(long rate) {
+ return (usPerUnit + rate / 2) / rate;
+ }
+ }
+
+ /**
+ * Shape of event rate.
+ */
+ public enum RateShape {
+ SQUARE,
+ SINE;
+
+ /**
+ * Number of steps used to approximate sine wave.
+ */
+ private static final int N = 10;
+
+ /**
+ * Return inter-event delay, in microseconds, for each generator
+ * to follow in order to achieve {@code rate} at {@code unit} using {@code numGenerators}.
+ */
+ public long interEventDelayUs(int rate, RateUnit unit, int numGenerators) {
+ return unit.rateToPeriodUs(rate) * numGenerators;
+ }
+
+ /**
+ * Return array of successive inter-event delays, in microseconds, for each generator
+ * to follow in order to achieve this shape with {@code firstRate/nextRate} at
+ * {@code unit} using {@code numGenerators}.
+ */
+ public long[] interEventDelayUs(
+ int firstRate, int nextRate, RateUnit unit, int numGenerators) {
+ if (firstRate == nextRate) {
+ long[] interEventDelayUs = new long[1];
+ interEventDelayUs[0] = unit.rateToPeriodUs(firstRate) * numGenerators;
+ return interEventDelayUs;
+ }
+
+ switch (this) {
+ case SQUARE: {
+ long[] interEventDelayUs = new long[2];
+ interEventDelayUs[0] = unit.rateToPeriodUs(firstRate) * numGenerators;
+ interEventDelayUs[1] = unit.rateToPeriodUs(nextRate) * numGenerators;
+ return interEventDelayUs;
+ }
+ case SINE: {
+ double mid = (firstRate + nextRate) / 2.0;
+ double amp = (firstRate - nextRate) / 2.0; // may be -ve
+ long[] interEventDelayUs = new long[N];
+ for (int i = 0; i < N; i++) {
+ double r = (2.0 * Math.PI * i) / N;
+ double rate = mid + amp * Math.cos(r);
+ interEventDelayUs[i] = unit.rateToPeriodUs(Math.round(rate)) * numGenerators;
+ }
+ return interEventDelayUs;
+ }
+ }
+ throw new RuntimeException(); // switch should be exhaustive
+ }
+
+ /**
+ * Return delay between steps, in seconds, for result of {@link #interEventDelayUs}, so
+ * as to cycle through the entire sequence every {@code ratePeriodSec}.
+ */
+ public int stepLengthSec(int ratePeriodSec) {
+ int n = 0;
+ switch (this) {
+ case SQUARE:
+ n = 2;
+ break;
+ case SINE:
+ n = N;
+ break;
+ }
+ return (ratePeriodSec + n - 1) / n;
+ }
+ }
+
+ /**
+ * Set to true to capture all info messages. The logging level flags don't currently work.
+ */
+ private static final boolean LOG_INFO = false;
+
+ /**
+ * Set to true to capture all error messages. The logging level flags don't currently work.
+ */
+ private static final boolean LOG_ERROR = true;
+
+ /**
+ * Set to true to log directly to stdout. If run using Google Dataflow, you can watch the results
+ * in real-time with: tail -f /var/log/dataflow/streaming-harness/harness-stdout.log
+ */
+ private static final boolean LOG_TO_CONSOLE = false;
+
+ /**
+ * Log info message.
+ */
+ public static void info(String format, Object... args) {
+ if (LOG_INFO) {
+ LOG.info(String.format(format, args));
+ if (LOG_TO_CONSOLE) {
+ System.out.println(String.format(format, args));
+ }
+ }
+ }
+
+ /**
+ * Log message to console. For client side only.
+ */
+ public static void console(String format, Object... args) {
+ System.out.printf("%s %s%n", Instant.now(), String.format(format, args));
+ }
+
+ /**
+ * Label to use for timestamps on pub/sub messages.
+ */
+ public static final String PUBSUB_TIMESTAMP = "timestamp";
+
+ /**
+ * Label to use for windmill ids on pub/sub messages.
+ */
+ public static final String PUBSUB_ID = "id";
+
+ /**
+ * All events will be given a timestamp relative to this time (ms since epoch).
+ */
+ private static final long BASE_TIME = Instant.parse("2015-07-15T00:00:00.000Z").getMillis();
+
+ /**
+ * Instants guaranteed to be strictly before and after all event timestamps, and which won't
+ * be subject to underflow/overflow.
+ */
+ public static final Instant BEGINNING_OF_TIME = new Instant(0).plus(Duration.standardDays(365));
+ public static final Instant END_OF_TIME =
+ BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(365));
+
+ /**
+ * Setup pipeline with codes and some other options.
+ */
+ public static void setupPipeline(CoderStrategy coderStrategy, Pipeline p) {
+ CoderRegistry registry = p.getCoderRegistry();
+ switch (coderStrategy) {
+ case HAND:
+ registry.registerCoderForClass(Auction.class, Auction.CODER);
+ registry.registerCoderForClass(AuctionBid.class, AuctionBid.CODER);
+ registry.registerCoderForClass(AuctionCount.class, AuctionCount.CODER);
+ registry.registerCoderForClass(AuctionPrice.class, AuctionPrice.CODER);
+ registry.registerCoderForClass(Bid.class, Bid.CODER);
+ registry.registerCoderForClass(CategoryPrice.class, CategoryPrice.CODER);
+ registry.registerCoderForClass(Event.class, Event.CODER);
+ registry.registerCoderForClass(IdNameReserve.class, IdNameReserve.CODER);
+ registry.registerCoderForClass(NameCityStateId.class, NameCityStateId.CODER);
+ registry.registerCoderForClass(Person.class, Person.CODER);
+ registry.registerCoderForClass(SellerPrice.class, SellerPrice.CODER);
+ registry.registerCoderForClass(Done.class, Done.CODER);
+ registry.registerCoderForClass(BidsPerSession.class, BidsPerSession.CODER);
+ break;
+ case AVRO:
+ registry.registerCoderProvider(AvroCoder.getCoderProvider());
+ break;
+ case JAVA:
+ registry.registerCoderProvider(SerializableCoder.getCoderProvider());
+ break;
+ }
+ }
+
+ /**
+ * Return a generator config to match the given {@code options}.
+ */
+ private static GeneratorConfig standardGeneratorConfig(NexmarkConfiguration configuration) {
+ return new GeneratorConfig(configuration,
+ configuration.useWallclockEventTime ? System.currentTimeMillis()
+ : BASE_TIME, 0,
+ configuration.numEvents, 0);
+ }
+
+ /**
+ * Return an iterator of events using the 'standard' generator config.
+ */
+ public static Iterator<TimestampedValue<Event>> standardEventIterator(
+ NexmarkConfiguration configuration) {
+ return new Generator(standardGeneratorConfig(configuration));
+ }
+
+ /**
+ * Return a transform which yields a finite number of synthesized events generated
+ * as a batch.
+ */
+ public static PTransform<PBegin, PCollection<Event>> batchEventsSource(
+ NexmarkConfiguration configuration) {
+ return Read.from(new BoundedEventSource(standardGeneratorConfig(configuration),
+ configuration.numEventGenerators));
+ }
+
+ /**
+ * Return a transform which yields a finite number of synthesized events generated
+ * on-the-fly in real time.
+ */
+ public static PTransform<PBegin, PCollection<Event>> streamEventsSource(
+ NexmarkConfiguration configuration) {
+ return Read.from(new UnboundedEventSource(NexmarkUtils.standardGeneratorConfig(configuration),
+ configuration.numEventGenerators,
+ configuration.watermarkHoldbackSec,
+ configuration.isRateLimited));
+ }
+
+ /**
+ * Return a transform to pass-through events, but count them as they go by.
+ */
+ public static ParDo.SingleOutput<Event, Event> snoop(final String name) {
+ return ParDo.of(new DoFn<Event, Event>() {
+ final Counter eventCounter = Metrics.counter(name, "events");
+ final Counter newPersonCounter = Metrics.counter(name, "newPersons");
+ final Counter newAuctionCounter = Metrics.counter(name, "newAuctions");
+ final Counter bidCounter = Metrics.counter(name, "bids");
+ final Counter endOfStreamCounter = Metrics.counter(name, "endOfStream");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ eventCounter.inc();
+ if (c.element().newPerson != null) {
+ newPersonCounter.inc();
+ } else if (c.element().newAuction != null) {
+ newAuctionCounter.inc();
+ } else if (c.element().bid != null) {
+ bidCounter.inc();
+ } else {
+ endOfStreamCounter.inc();
+ }
+ info("%s snooping element %s", name, c.element());
+ c.output(c.element());
+ }
+ });
+ }
+
+ /**
+ * Return a transform to count and discard each element.
+ */
+ public static <T> ParDo.SingleOutput<T, Void> devNull(final String name) {
+ return ParDo.of(new DoFn<T, Void>() {
+ final Counter discardedCounterMetric = Metrics.counter(name, "discarded");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ discardedCounterMetric.inc();
+ }
+ });
+ }
+
+ /**
+ * Return a transform to log each element, passing it through unchanged.
+ */
+ public static <T> ParDo.SingleOutput<T, T> log(final String name) {
+ return ParDo.of(new DoFn<T, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ LOG.info("%s: %s", name, c.element());
+ c.output(c.element());
+ }
+ });
+ }
+
+ /**
+ * Return a transform to format each element as a string.
+ */
+ public static <T> ParDo.SingleOutput<T, String> format(final String name) {
+ return ParDo.of(new DoFn<T, String>() {
+ final Counter recordCounterMetric = Metrics.counter(name, "records");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ recordCounterMetric.inc();
+ c.output(c.element().toString());
+ }
+ });
+ }
+
+ /**
+ * Return a transform to make explicit the timestamp of each element.
+ */
+ public static <T> ParDo.SingleOutput<T, TimestampedValue<T>> stamp(String name) {
+ return ParDo.of(new DoFn<T, TimestampedValue<T>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(TimestampedValue.of(c.element(), c.timestamp()));
+ }
+ });
+ }
+
+ /**
+ * Return a transform to reduce a stream to a single, order-invariant long hash.
+ */
+ public static <T> PTransform<PCollection<T>, PCollection<Long>> hash(
+ final long numEvents, String name) {
+ return new PTransform<PCollection<T>, PCollection<Long>>(name) {
+ @Override
+ public PCollection<Long> expand(PCollection<T> input) {
+ return input.apply(Window.<T>into(new GlobalWindows())
+ .triggering(AfterPane.elementCountAtLeast((int) numEvents))
+ .withAllowedLateness(Duration.standardDays(1))
+ .discardingFiredPanes())
+
+ .apply(name + ".Hash", ParDo.of(new DoFn<T, Long>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ long hash =
+ Hashing.murmur3_128()
+ .newHasher()
+ .putLong(c.timestamp().getMillis())
+ .putString(c.element().toString(), StandardCharsets.UTF_8)
+ .hash()
+ .asLong();
+ c.output(hash);
+ }
+ }))
+
+ .apply(Combine.globally(new Combine.BinaryCombineFn<Long>() {
+ @Override
+ public Long apply(Long left, Long right) {
+ return left ^ right;
+ }
+ }));
+ }
+ };
+ }
+
+ private static final long MASK = (1L << 16) - 1L;
+ private static final long HASH = 0x243F6A8885A308D3L;
+ private static final long INIT_PLAINTEXT = 50000L;
+
+ /**
+ * Return a transform to keep the CPU busy for given milliseconds on every record.
+ */
+ public static <T> ParDo.SingleOutput<T, T> cpuDelay(String name, final long delayMs) {
+ return ParDo.of(new DoFn<T, T>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ long now = System.currentTimeMillis();
+ long end = now + delayMs;
+ while (now < end) {
+ // Find plaintext which hashes to HASH in lowest MASK bits.
+ // Values chosen to roughly take 1ms on typical workstation.
+ long p = INIT_PLAINTEXT;
+ while (true) {
+ long t = Hashing.murmur3_128().hashLong(p).asLong();
+ if ((t & MASK) == (HASH & MASK)) {
+ break;
+ }
+ p++;
+ }
+ now = System.currentTimeMillis();
+ }
+ c.output(c.element());
+ }
+ });
+ }
+
+ private static final int MAX_BUFFER_SIZE = 1 << 24;
+
+ private static class DiskBusyTransform<T> extends PTransform<PCollection<T>, PCollection<T>>{
+
+ private long bytes;
+
+ private DiskBusyTransform(long bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override public PCollection<T> expand(PCollection<T> input) {
+ // Add dummy key to be able to use State API
+ PCollection<KV<Integer, T>> kvCollection = input
+ .apply("diskBusy.keyElements", ParDo.of(new DoFn<T, KV<Integer, T>>() {
+
+ @ProcessElement public void processElement(ProcessContext context) {
+ context.output(KV.of(0, context.element()));
+ }
+ }));
+ // Apply actual transform that generates disk IO using state API
+ PCollection<T> output = kvCollection
+ .apply("diskBusy.generateIO", ParDo.of(new DoFn<KV<Integer, T>, T>() {
+
+ private static final String DISK_BUSY = "diskBusy";
+
+ @StateId(DISK_BUSY) private final StateSpec<ValueState<byte[]>> spec = StateSpecs
+ .value(ByteArrayCoder.of());
+
+ @ProcessElement public void processElement(ProcessContext c,
+ @StateId(DISK_BUSY) ValueState<byte[]> state) {
+ long remain = bytes;
+ long now = System.currentTimeMillis();
+ while (remain > 0) {
+ long thisBytes = Math.min(remain, MAX_BUFFER_SIZE);
+ remain -= thisBytes;
+ byte[] arr = new byte[(int) thisBytes];
+ for (int i = 0; i < thisBytes; i++) {
+ arr[i] = (byte) now;
+ }
+ state.write(arr);
+ now = System.currentTimeMillis();
+ }
+ c.output(c.element().getValue());
+ }
+ }));
+ return output;
+ }
+ }
+
+
+ /**
+ * Return a transform to write given number of bytes to durable store on every record.
+ */
+ public static <T> PTransform<PCollection<T>, PCollection<T>> diskBusy(final long bytes) {
+ return new DiskBusyTransform<>(bytes);
+ }
+
+ /**
+ * Return a transform to cast each element to {@link KnownSize}.
+ */
+ private static <T extends KnownSize> ParDo.SingleOutput<T, KnownSize> castToKnownSize() {
+ return ParDo.of(new DoFn<T, KnownSize>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element());
+ }
+ });
+ }
+
+ /**
+ * A coder for instances of {@code T} cast up to {@link KnownSize}.
+ *
+ * @param <T> True type of object.
+ */
+ private static class CastingCoder<T extends KnownSize> extends CustomCoder<KnownSize> {
+ private final Coder<T> trueCoder;
+
+ public CastingCoder(Coder<T> trueCoder) {
+ this.trueCoder = trueCoder;
+ }
+
+ @Override
+ public void encode(KnownSize value, OutputStream outStream)
+ throws CoderException, IOException {
+ @SuppressWarnings("unchecked")
+ T typedValue = (T) value;
+ trueCoder.encode(typedValue, outStream);
+ }
+
+ @Override
+ public KnownSize decode(InputStream inStream)
+ throws CoderException, IOException {
+ return trueCoder.decode(inStream);
+ }
+ }
+
+ /**
+ * Return a coder for {@code KnownSize} that are known to be exactly of type {@code T}.
+ */
+ private static <T extends KnownSize> Coder<KnownSize> makeCastingCoder(Coder<T> trueCoder) {
+ return new CastingCoder<>(trueCoder);
+ }
+
+ /**
+ * Return {@code elements} as {@code KnownSize}s.
+ */
+ public static <T extends KnownSize> PCollection<KnownSize> castToKnownSize(
+ final String name, PCollection<T> elements) {
+ return elements.apply(name + ".Forget", castToKnownSize())
+ .setCoder(makeCastingCoder(elements.getCoder()));
+ }
+
+ // Do not instantiate.
+ private NexmarkUtils() {
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java
new file mode 100644
index 0000000..6a37ade
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * An auction submitted by a person.
+ */
+public class Auction implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+
+ public static final Coder<Auction> CODER = new CustomCoder<Auction>() {
+ @Override
+ public void encode(Auction value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.id, outStream);
+ STRING_CODER.encode(value.itemName, outStream);
+ STRING_CODER.encode(value.description, outStream);
+ LONG_CODER.encode(value.initialBid, outStream);
+ LONG_CODER.encode(value.reserve, outStream);
+ LONG_CODER.encode(value.dateTime, outStream);
+ LONG_CODER.encode(value.expires, outStream);
+ LONG_CODER.encode(value.seller, outStream);
+ LONG_CODER.encode(value.category, outStream);
+ STRING_CODER.encode(value.extra, outStream);
+ }
+
+ @Override
+ public Auction decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long id = LONG_CODER.decode(inStream);
+ String itemName = STRING_CODER.decode(inStream);
+ String description = STRING_CODER.decode(inStream);
+ long initialBid = LONG_CODER.decode(inStream);
+ long reserve = LONG_CODER.decode(inStream);
+ long dateTime = LONG_CODER.decode(inStream);
+ long expires = LONG_CODER.decode(inStream);
+ long seller = LONG_CODER.decode(inStream);
+ long category = LONG_CODER.decode(inStream);
+ String extra = STRING_CODER.decode(inStream);
+ return new Auction(
+ id, itemName, description, initialBid, reserve, dateTime, expires, seller, category,
+ extra);
+ }
+ };
+
+
+ /** Id of auction. */
+ @JsonProperty
+ public final long id; // primary key
+
+ /** Extra auction properties. */
+ @JsonProperty
+ private final String itemName;
+
+ @JsonProperty
+ private final String description;
+
+ /** Initial bid price, in cents. */
+ @JsonProperty
+ private final long initialBid;
+
+ /** Reserve price, in cents. */
+ @JsonProperty
+ public final long reserve;
+
+ @JsonProperty
+ public final long dateTime;
+
+ /** When does auction expire? (ms since epoch). Bids at or after this time are ignored. */
+ @JsonProperty
+ public final long expires;
+
+ /** Id of person who instigated auction. */
+ @JsonProperty
+ public final long seller; // foreign key: Person.id
+
+ /** Id of category auction is listed under. */
+ @JsonProperty
+ public final long category; // foreign key: Category.id
+
+ /** Additional arbitrary payload for performance testing. */
+ @JsonProperty
+ private final String extra;
+
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private Auction() {
+ id = 0;
+ itemName = null;
+ description = null;
+ initialBid = 0;
+ reserve = 0;
+ dateTime = 0;
+ expires = 0;
+ seller = 0;
+ category = 0;
+ extra = null;
+ }
+
+ public Auction(long id, String itemName, String description, long initialBid, long reserve,
+ long dateTime, long expires, long seller, long category, String extra) {
+ this.id = id;
+ this.itemName = itemName;
+ this.description = description;
+ this.initialBid = initialBid;
+ this.reserve = reserve;
+ this.dateTime = dateTime;
+ this.expires = expires;
+ this.seller = seller;
+ this.category = category;
+ this.extra = extra;
+ }
+
+ /**
+ * Return a copy of auction which capture the given annotation.
+ * (Used for debugging).
+ */
+ public Auction withAnnotation(String annotation) {
+ return new Auction(id, itemName, description, initialBid, reserve, dateTime, expires, seller,
+ category, annotation + ": " + extra);
+ }
+
+ /**
+ * Does auction have {@code annotation}? (Used for debugging.)
+ */
+ public boolean hasAnnotation(String annotation) {
+ return extra.startsWith(annotation + ": ");
+ }
+
+ /**
+ * Remove {@code annotation} from auction. (Used for debugging.)
+ */
+ public Auction withoutAnnotation(String annotation) {
+ if (hasAnnotation(annotation)) {
+ return new Auction(id, itemName, description, initialBid, reserve, dateTime, expires, seller,
+ category, extra.substring(annotation.length() + 2));
+ } else {
+ return this;
+ }
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + itemName.length() + 1 + description.length() + 1 + 8 + 8 + 8 + 8 + 8 + 8
+ + extra.length() + 1;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java
new file mode 100644
index 0000000..cb1aac5
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.queries.WinningBids;
+
+/**
+ * Result of {@link WinningBids} transform.
+ */
+public class AuctionBid implements KnownSize, Serializable {
+ public static final Coder<AuctionBid> CODER = new CustomCoder<AuctionBid>() {
+ @Override
+ public void encode(AuctionBid value, OutputStream outStream)
+ throws CoderException, IOException {
+ Auction.CODER.encode(value.auction, outStream);
+ Bid.CODER.encode(value.bid, outStream);
+ }
+
+ @Override
+ public AuctionBid decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ Auction auction = Auction.CODER.decode(inStream);
+ Bid bid = Bid.CODER.decode(inStream);
+ return new AuctionBid(auction, bid);
+ }
+ };
+
+ @JsonProperty
+ public final Auction auction;
+
+ @JsonProperty
+ public final Bid bid;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private AuctionBid() {
+ auction = null;
+ bid = null;
+ }
+
+ public AuctionBid(Auction auction, Bid bid) {
+ this.auction = auction;
+ this.bid = bid;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return auction.sizeInBytes() + bid.sizeInBytes();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionCount.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionCount.java
new file mode 100644
index 0000000..4d15d25
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionCount.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of Query5.
+ */
+public class AuctionCount implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+
+ public static final Coder<AuctionCount> CODER = new CustomCoder<AuctionCount>() {
+ @Override
+ public void encode(AuctionCount value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.auction, outStream);
+ LONG_CODER.encode(value.count, outStream);
+ }
+
+ @Override
+ public AuctionCount decode(InputStream inStream)
+ throws CoderException, IOException {
+ long auction = LONG_CODER.decode(inStream);
+ long count = LONG_CODER.decode(inStream);
+ return new AuctionCount(auction, count);
+ }
+ };
+
+ @JsonProperty private final long auction;
+
+ @JsonProperty private final long count;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private AuctionCount() {
+ auction = 0;
+ count = 0;
+ }
+
+ public AuctionCount(long auction, long count) {
+ this.auction = auction;
+ this.count = count;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionPrice.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionPrice.java
new file mode 100644
index 0000000..f4fe881
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionPrice.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of Query2.
+ */
+public class AuctionPrice implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+
+ public static final Coder<AuctionPrice> CODER = new CustomCoder<AuctionPrice>() {
+ @Override
+ public void encode(AuctionPrice value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.auction, outStream);
+ LONG_CODER.encode(value.price, outStream);
+ }
+
+ @Override
+ public AuctionPrice decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long auction = LONG_CODER.decode(inStream);
+ long price = LONG_CODER.decode(inStream);
+ return new AuctionPrice(auction, price);
+ }
+ };
+
+ @JsonProperty
+ private final long auction;
+
+ /** Price in cents. */
+ @JsonProperty
+ private final long price;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private AuctionPrice() {
+ auction = 0;
+ price = 0;
+ }
+
+ public AuctionPrice(long auction, long price) {
+ this.auction = auction;
+ this.price = price;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Bid.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Bid.java
new file mode 100644
index 0000000..b465e62
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Bid.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * A bid for an item on auction.
+ */
+public class Bid implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+
+ public static final Coder<Bid> CODER = new CustomCoder<Bid>() {
+ @Override
+ public void encode(Bid value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.auction, outStream);
+ LONG_CODER.encode(value.bidder, outStream);
+ LONG_CODER.encode(value.price, outStream);
+ LONG_CODER.encode(value.dateTime, outStream);
+ STRING_CODER.encode(value.extra, outStream);
+ }
+
+ @Override
+ public Bid decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long auction = LONG_CODER.decode(inStream);
+ long bidder = LONG_CODER.decode(inStream);
+ long price = LONG_CODER.decode(inStream);
+ long dateTime = LONG_CODER.decode(inStream);
+ String extra = STRING_CODER.decode(inStream);
+ return new Bid(auction, bidder, price, dateTime, extra);
+ }
+
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ /**
+ * Comparator to order bids by ascending price then descending time
+ * (for finding winning bids).
+ */
+ public static final Comparator<Bid> PRICE_THEN_DESCENDING_TIME = new Comparator<Bid>() {
+ @Override
+ public int compare(Bid left, Bid right) {
+ int i = Double.compare(left.price, right.price);
+ if (i != 0) {
+ return i;
+ }
+ return Long.compare(right.dateTime, left.dateTime);
+ }
+ };
+
+ /**
+ * Comparator to order bids by ascending time then ascending price.
+ * (for finding most recent bids).
+ */
+ public static final Comparator<Bid> ASCENDING_TIME_THEN_PRICE = new Comparator<Bid>() {
+ @Override
+ public int compare(Bid left, Bid right) {
+ int i = Long.compare(left.dateTime, right.dateTime);
+ if (i != 0) {
+ return i;
+ }
+ return Double.compare(left.price, right.price);
+ }
+ };
+
+ /** Id of auction this bid is for. */
+ @JsonProperty
+ public final long auction; // foreign key: Auction.id
+
+ /** Id of person bidding in auction. */
+ @JsonProperty
+ public final long bidder; // foreign key: Person.id
+
+ /** Price of bid, in cents. */
+ @JsonProperty
+ public final long price;
+
+ /**
+ * Instant at which bid was made (ms since epoch).
+ * NOTE: This may be earlier than the system's event time.
+ */
+ @JsonProperty
+ public final long dateTime;
+
+ /** Additional arbitrary payload for performance testing. */
+ @JsonProperty
+ public final String extra;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private Bid() {
+ auction = 0;
+ bidder = 0;
+ price = 0;
+ dateTime = 0;
+ extra = null;
+ }
+
+ public Bid(long auction, long bidder, long price, long dateTime, String extra) {
+ this.auction = auction;
+ this.bidder = bidder;
+ this.price = price;
+ this.dateTime = dateTime;
+ this.extra = extra;
+ }
+
+ /**
+ * Return a copy of bid which capture the given annotation.
+ * (Used for debugging).
+ */
+ public Bid withAnnotation(String annotation) {
+ return new Bid(auction, bidder, price, dateTime, annotation + ": " + extra);
+ }
+
+ /**
+ * Does bid have {@code annotation}? (Used for debugging.)
+ */
+ public boolean hasAnnotation(String annotation) {
+ return extra.startsWith(annotation + ": ");
+ }
+
+ /**
+ * Remove {@code annotation} from bid. (Used for debugging.)
+ */
+ public Bid withoutAnnotation(String annotation) {
+ if (hasAnnotation(annotation)) {
+ return new Bid(auction, bidder, price, dateTime, extra.substring(annotation.length() + 2));
+ } else {
+ return this;
+ }
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + 8 + 8 + 8 + extra.length() + 1;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java
new file mode 100644
index 0000000..84e23e7
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of query 11.
+ */
+public class BidsPerSession implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+
+ public static final Coder<BidsPerSession> CODER = new CustomCoder<BidsPerSession>() {
+ @Override
+ public void encode(BidsPerSession value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.personId, outStream);
+ LONG_CODER.encode(value.bidsPerSession, outStream);
+ }
+
+ @Override
+ public BidsPerSession decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long personId = LONG_CODER.decode(inStream);
+ long bidsPerSession = LONG_CODER.decode(inStream);
+ return new BidsPerSession(personId, bidsPerSession);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ private final long personId;
+
+ @JsonProperty
+ private final long bidsPerSession;
+
+ public BidsPerSession() {
+ personId = 0;
+ bidsPerSession = 0;
+ }
+
+ public BidsPerSession(long personId, long bidsPerSession) {
+ this.personId = personId;
+ this.bidsPerSession = bidsPerSession;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ // Two longs.
+ return 8 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java
new file mode 100644
index 0000000..3b33635
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of Query4.
+ */
+public class CategoryPrice implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<Integer> INT_CODER = VarIntCoder.of();
+
+ public static final Coder<CategoryPrice> CODER = new CustomCoder<CategoryPrice>() {
+ @Override
+ public void encode(CategoryPrice value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.category, outStream);
+ LONG_CODER.encode(value.price, outStream);
+ INT_CODER.encode(value.isLast ? 1 : 0, outStream);
+ }
+
+ @Override
+ public CategoryPrice decode(InputStream inStream)
+ throws CoderException, IOException {
+ long category = LONG_CODER.decode(inStream);
+ long price = LONG_CODER.decode(inStream);
+ boolean isLast = INT_CODER.decode(inStream) != 0;
+ return new CategoryPrice(category, price, isLast);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ public final long category;
+
+ /** Price in cents. */
+ @JsonProperty
+ public final long price;
+
+ @JsonProperty
+ public final boolean isLast;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private CategoryPrice() {
+ category = 0;
+ price = 0;
+ isLast = false;
+ }
+
+ public CategoryPrice(long category, long price, boolean isLast) {
+ this.category = category;
+ this.price = price;
+ this.isLast = isLast;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + 8 + 1;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java
new file mode 100644
index 0000000..e285041
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of query 10.
+ */
+public class Done implements KnownSize, Serializable {
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+
+ public static final Coder<Done> CODER = new CustomCoder<Done>() {
+ @Override
+ public void encode(Done value, OutputStream outStream)
+ throws CoderException, IOException {
+ STRING_CODER.encode(value.message, outStream);
+ }
+
+ @Override
+ public Done decode(InputStream inStream)
+ throws CoderException, IOException {
+ String message = STRING_CODER.decode(inStream);
+ return new Done(message);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ private final String message;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ public Done() {
+ message = null;
+ }
+
+ public Done(String message) {
+ this.message = message;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return message.length();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java
new file mode 100644
index 0000000..880cfe4
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+
+/**
+ * An event in the auction system, either a (new) {@link Person}, a (new) {@link Auction}, or a
+ * {@link Bid}.
+ */
+public class Event implements KnownSize, Serializable {
+ private enum Tag {
+ PERSON(0),
+ AUCTION(1),
+ BID(2);
+
+ private int value = -1;
+
+ Tag(int value){
+ this.value = value;
+ }
+ }
+ private static final Coder<Integer> INT_CODER = VarIntCoder.of();
+
+ public static final Coder<Event> CODER =
+ new CustomCoder<Event>() {
+ @Override
+ public void encode(Event value, OutputStream outStream) throws IOException {
+ if (value.newPerson != null) {
+ INT_CODER.encode(Tag.PERSON.value, outStream);
+ Person.CODER.encode(value.newPerson, outStream);
+ } else if (value.newAuction != null) {
+ INT_CODER.encode(Tag.AUCTION.value, outStream);
+ Auction.CODER.encode(value.newAuction, outStream);
+ } else if (value.bid != null) {
+ INT_CODER.encode(Tag.BID.value, outStream);
+ Bid.CODER.encode(value.bid, outStream);
+ } else {
+ throw new RuntimeException("invalid event");
+ }
+ }
+
+ @Override
+ public Event decode(InputStream inStream) throws IOException {
+ int tag = INT_CODER.decode(inStream);
+ if (tag == Tag.PERSON.value) {
+ Person person = Person.CODER.decode(inStream);
+ return new Event(person);
+ } else if (tag == Tag.AUCTION.value) {
+ Auction auction = Auction.CODER.decode(inStream);
+ return new Event(auction);
+ } else if (tag == Tag.BID.value) {
+ Bid bid = Bid.CODER.decode(inStream);
+ return new Event(bid);
+ } else {
+ throw new RuntimeException("invalid event encoding");
+ }
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @Nullable
+ @org.apache.avro.reflect.Nullable
+ public final Person newPerson;
+
+ @Nullable
+ @org.apache.avro.reflect.Nullable
+ public final Auction newAuction;
+
+ @Nullable
+ @org.apache.avro.reflect.Nullable
+ public final Bid bid;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private Event() {
+ newPerson = null;
+ newAuction = null;
+ bid = null;
+ }
+
+ public Event(Person newPerson) {
+ this.newPerson = newPerson;
+ newAuction = null;
+ bid = null;
+ }
+
+ public Event(Auction newAuction) {
+ newPerson = null;
+ this.newAuction = newAuction;
+ bid = null;
+ }
+
+ public Event(Bid bid) {
+ newPerson = null;
+ newAuction = null;
+ this.bid = bid;
+ }
+
+ /** Return a copy of event which captures {@code annotation}. (Used for debugging). */
+ public Event withAnnotation(String annotation) {
+ if (newPerson != null) {
+ return new Event(newPerson.withAnnotation(annotation));
+ } else if (newAuction != null) {
+ return new Event(newAuction.withAnnotation(annotation));
+ } else {
+ return new Event(bid.withAnnotation(annotation));
+ }
+ }
+
+ /** Does event have {@code annotation}? (Used for debugging.) */
+ public boolean hasAnnotation(String annotation) {
+ if (newPerson != null) {
+ return newPerson.hasAnnotation(annotation);
+ } else if (newAuction != null) {
+ return newAuction.hasAnnotation(annotation);
+ } else {
+ return bid.hasAnnotation(annotation);
+ }
+ }
+
+ @Override
+ public long sizeInBytes() {
+ if (newPerson != null) {
+ return 1 + newPerson.sizeInBytes();
+ } else if (newAuction != null) {
+ return 1 + newAuction.sizeInBytes();
+ } else if (bid != null) {
+ return 1 + bid.sizeInBytes();
+ } else {
+ throw new RuntimeException("invalid event");
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (newPerson != null) {
+ return newPerson.toString();
+ } else if (newAuction != null) {
+ return newAuction.toString();
+ } else if (bid != null) {
+ return bid.toString();
+ } else {
+ throw new RuntimeException("invalid event");
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java
new file mode 100644
index 0000000..0519f5d
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result type of Query8.
+ */
+public class IdNameReserve implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+
+ public static final Coder<IdNameReserve> CODER = new CustomCoder<IdNameReserve>() {
+ @Override
+ public void encode(IdNameReserve value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.id, outStream);
+ STRING_CODER.encode(value.name, outStream);
+ LONG_CODER.encode(value.reserve, outStream);
+ }
+
+ @Override
+ public IdNameReserve decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long id = LONG_CODER.decode(inStream);
+ String name = STRING_CODER.decode(inStream);
+ long reserve = LONG_CODER.decode(inStream);
+ return new IdNameReserve(id, name, reserve);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ private final long id;
+
+ @JsonProperty
+ private final String name;
+
+ /** Reserve price in cents. */
+ @JsonProperty
+ private final long reserve;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private IdNameReserve() {
+ id = 0;
+ name = null;
+ reserve = 0;
+ }
+
+ public IdNameReserve(long id, String name, long reserve) {
+ this.id = id;
+ this.name = name;
+ this.reserve = reserve;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + name.length() + 1 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/KnownSize.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/KnownSize.java
new file mode 100644
index 0000000..45af3fc
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/KnownSize.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+/**
+ * Interface for elements which can quickly estimate their encoded byte size.
+ */
+public interface KnownSize {
+ long sizeInBytes();
+}
+
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/NameCityStateId.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/NameCityStateId.java
new file mode 100644
index 0000000..55fca62
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/NameCityStateId.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of Query3.
+ */
+public class NameCityStateId implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+
+ public static final Coder<NameCityStateId> CODER = new CustomCoder<NameCityStateId>() {
+ @Override
+ public void encode(NameCityStateId value, OutputStream outStream)
+ throws CoderException, IOException {
+ STRING_CODER.encode(value.name, outStream);
+ STRING_CODER.encode(value.city, outStream);
+ STRING_CODER.encode(value.state, outStream);
+ LONG_CODER.encode(value.id, outStream);
+ }
+
+ @Override
+ public NameCityStateId decode(InputStream inStream)
+ throws CoderException, IOException {
+ String name = STRING_CODER.decode(inStream);
+ String city = STRING_CODER.decode(inStream);
+ String state = STRING_CODER.decode(inStream);
+ long id = LONG_CODER.decode(inStream);
+ return new NameCityStateId(name, city, state, id);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ private final String name;
+
+ @JsonProperty
+ private final String city;
+
+ @JsonProperty
+ private final String state;
+
+ @JsonProperty
+ private final long id;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private NameCityStateId() {
+ name = null;
+ city = null;
+ state = null;
+ id = 0;
+ }
+
+ public NameCityStateId(String name, String city, String state, long id) {
+ this.name = name;
+ this.city = city;
+ this.state = state;
+ this.id = id;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return name.length() + 1 + city.length() + 1 + state.length() + 1 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java
new file mode 100644
index 0000000..800f937
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * A person either creating an auction or making a bid.
+ */
+public class Person implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+ private static final Coder<String> STRING_CODER = StringUtf8Coder.of();
+ public static final Coder<Person> CODER = new CustomCoder<Person>() {
+ @Override
+ public void encode(Person value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.id, outStream);
+ STRING_CODER.encode(value.name, outStream);
+ STRING_CODER.encode(value.emailAddress, outStream);
+ STRING_CODER.encode(value.creditCard, outStream);
+ STRING_CODER.encode(value.city, outStream);
+ STRING_CODER.encode(value.state, outStream);
+ LONG_CODER.encode(value.dateTime, outStream);
+ STRING_CODER.encode(value.extra, outStream);
+ }
+
+ @Override
+ public Person decode(InputStream inStream)
+ throws CoderException, IOException {
+ long id = LONG_CODER.decode(inStream);
+ String name = STRING_CODER.decode(inStream);
+ String emailAddress = STRING_CODER.decode(inStream);
+ String creditCard = STRING_CODER.decode(inStream);
+ String city = STRING_CODER.decode(inStream);
+ String state = STRING_CODER.decode(inStream);
+ long dateTime = LONG_CODER.decode(inStream);
+ String extra = STRING_CODER.decode(inStream);
+ return new Person(id, name, emailAddress, creditCard, city, state, dateTime, extra);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ /** Id of person. */
+ @JsonProperty
+ public final long id; // primary key
+
+ /** Extra person properties. */
+ @JsonProperty
+ public final String name;
+
+ @JsonProperty
+ private final String emailAddress;
+
+ @JsonProperty
+ private final String creditCard;
+
+ @JsonProperty
+ public final String city;
+
+ @JsonProperty
+ public final String state;
+
+ @JsonProperty
+ public final long dateTime;
+
+ /** Additional arbitrary payload for performance testing. */
+ @JsonProperty
+ private final String extra;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private Person() {
+ id = 0;
+ name = null;
+ emailAddress = null;
+ creditCard = null;
+ city = null;
+ state = null;
+ dateTime = 0;
+ extra = null;
+ }
+
+ public Person(long id, String name, String emailAddress, String creditCard, String city,
+ String state, long dateTime, String extra) {
+ this.id = id;
+ this.name = name;
+ this.emailAddress = emailAddress;
+ this.creditCard = creditCard;
+ this.city = city;
+ this.state = state;
+ this.dateTime = dateTime;
+ this.extra = extra;
+ }
+
+ /**
+ * Return a copy of person which capture the given annotation.
+ * (Used for debugging).
+ */
+ public Person withAnnotation(String annotation) {
+ return new Person(id, name, emailAddress, creditCard, city, state, dateTime,
+ annotation + ": " + extra);
+ }
+
+ /**
+ * Does person have {@code annotation}? (Used for debugging.)
+ */
+ public boolean hasAnnotation(String annotation) {
+ return extra.startsWith(annotation + ": ");
+ }
+
+ /**
+ * Remove {@code annotation} from person. (Used for debugging.)
+ */
+ public Person withoutAnnotation(String annotation) {
+ if (hasAnnotation(annotation)) {
+ return new Person(id, name, emailAddress, creditCard, city, state, dateTime,
+ extra.substring(annotation.length() + 2));
+ } else {
+ return this;
+ }
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + name.length() + 1 + emailAddress.length() + 1 + creditCard.length() + 1
+ + city.length() + 1 + state.length() + 8 + 1 + extra.length() + 1;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java
new file mode 100644
index 0000000..82b551c
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+
+/**
+ * Result of Query6.
+ */
+public class SellerPrice implements KnownSize, Serializable {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+
+ public static final Coder<SellerPrice> CODER = new CustomCoder<SellerPrice>() {
+ @Override
+ public void encode(SellerPrice value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.seller, outStream);
+ LONG_CODER.encode(value.price, outStream);
+ }
+
+ @Override
+ public SellerPrice decode(
+ InputStream inStream)
+ throws CoderException, IOException {
+ long seller = LONG_CODER.decode(inStream);
+ long price = LONG_CODER.decode(inStream);
+ return new SellerPrice(seller, price);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ @JsonProperty
+ public final long seller;
+
+ /** Price in cents. */
+ @JsonProperty
+ private final long price;
+
+ // For Avro only.
+ @SuppressWarnings("unused")
+ private SellerPrice() {
+ seller = 0;
+ price = 0;
+ }
+
+ public SellerPrice(long seller, long price) {
+ this.seller = seller;
+ this.price = price;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return 8 + 8;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return NexmarkUtils.MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/package-info.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/package-info.java
new file mode 100644
index 0000000..3b4bb63
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Nexmark Benchmark Model.
+ */
+package org.apache.beam.sdk.nexmark.model;
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
new file mode 100644
index 0000000..62218a4
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Nexmark test suite.
+ */
+package org.apache.beam.sdk.nexmark;
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/AbstractSimulator.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/AbstractSimulator.java
new file mode 100644
index 0000000..6f4ad56
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/AbstractSimulator.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * Abstract base class for simulator of a query.
+ *
+ * @param <InputT> Type of input elements.
+ * @param <OutputT> Type of output elements.
+ */
+public abstract class AbstractSimulator<InputT, OutputT> {
+ /** Window size for action bucket sampling. */
+ private static final Duration WINDOW_SIZE = Duration.standardMinutes(1);
+
+ /** Input event stream we should draw from. */
+ private final Iterator<TimestampedValue<InputT>> input;
+
+ /** Set to true when no more results. */
+ private boolean isDone;
+
+ /**
+ * Results which have not yet been returned by the {@link #results} iterator.
+ */
+ private final List<TimestampedValue<OutputT>> pendingResults;
+
+ /**
+ * Current window timestamp (ms since epoch).
+ */
+ private long currentWindow;
+
+ /**
+ * Number of (possibly intermediate) results for the current window.
+ */
+ private long currentCount;
+
+ /**
+ * Result counts per window which have not yet been returned by the {@link #resultsPerWindow}
+ * iterator.
+ */
+ private final List<Long> pendingCounts;
+
+ public AbstractSimulator(Iterator<TimestampedValue<InputT>> input) {
+ this.input = input;
+ isDone = false;
+ pendingResults = new ArrayList<>();
+ currentWindow = BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis();
+ currentCount = 0;
+ pendingCounts = new ArrayList<>();
+ }
+
+ /** Called by implementors of {@link #run}: Fetch the next input element. */
+ @Nullable
+ TimestampedValue<InputT> nextInput() {
+ if (!input.hasNext()) {
+ return null;
+ }
+ TimestampedValue<InputT> timestampedInput = input.next();
+ NexmarkUtils.info("input: %s", timestampedInput);
+ return timestampedInput;
+ }
+
+ /**
+ * Called by implementors of {@link #run}: Capture an intermediate result, for the purpose of
+ * recording the expected activity of the query over time.
+ */
+ void addIntermediateResult(TimestampedValue<OutputT> result) {
+ NexmarkUtils.info("intermediate result: %s", result);
+ updateCounts(result.getTimestamp());
+ }
+
+ /**
+ * Called by implementors of {@link #run}: Capture a final result, for the purpose of checking
+ * semantic correctness.
+ */
+ void addResult(TimestampedValue<OutputT> result) {
+ NexmarkUtils.info("result: %s", result);
+ pendingResults.add(result);
+ updateCounts(result.getTimestamp());
+ }
+
+ /**
+ * Update window and counts.
+ */
+ private void updateCounts(Instant timestamp) {
+ long window = timestamp.getMillis() - timestamp.getMillis() % WINDOW_SIZE.getMillis();
+ if (window > currentWindow) {
+ if (currentWindow > BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()) {
+ pendingCounts.add(currentCount);
+ }
+ currentCount = 0;
+ currentWindow = window;
+ }
+ currentCount++;
+ }
+
+ /** Called by implementors of {@link #run}: Record that no more results will be emitted. */
+ void allDone() {
+ isDone = true;
+ }
+
+ /**
+ * Overridden by derived classes to do the next increment of work. Each call should
+ * call one or more of {@link #nextInput}, {@link #addIntermediateResult}, {@link #addResult}
+ * or {@link #allDone}. It is ok for a single call to emit more than one result via
+ * {@link #addResult}. It is ok for a single call to run the entire simulation, though
+ * this will prevent the {@link #results} and {@link #resultsPerWindow} iterators to
+ * stall.
+ */
+ protected abstract void run();
+
+ /**
+ * Return iterator over all expected timestamped results. The underlying simulator state is
+ * changed. Only one of {@link #results} or {@link #resultsPerWindow} can be called.
+ */
+ public Iterator<TimestampedValue<OutputT>> results() {
+ return new Iterator<TimestampedValue<OutputT>>() {
+ @Override
+ public boolean hasNext() {
+ while (true) {
+ if (!pendingResults.isEmpty()) {
+ return true;
+ }
+ if (isDone) {
+ return false;
+ }
+ run();
+ }
+ }
+
+ @Override
+ public TimestampedValue<OutputT> next() {
+ TimestampedValue<OutputT> result = pendingResults.get(0);
+ pendingResults.remove(0);
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ /**
+ * Return an iterator over the number of results per {@link #WINDOW_SIZE} period. The underlying
+ * simulator state is changed. Only one of {@link #results} or {@link #resultsPerWindow} can be
+ * called.
+ */
+ public Iterator<Long> resultsPerWindow() {
+ return new Iterator<Long>() {
+ @Override
+ public boolean hasNext() {
+ while (true) {
+ if (!pendingCounts.isEmpty()) {
+ return true;
+ }
+ if (isDone) {
+ if (currentCount > 0) {
+ pendingCounts.add(currentCount);
+ currentCount = 0;
+ currentWindow = BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis();
+ return true;
+ } else {
+ return false;
+ }
+ }
+ run();
+ }
+ }
+
+ @Override
+ public Long next() {
+ Long result = pendingCounts.get(0);
+ pendingCounts.remove(0);
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQuery.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQuery.java
new file mode 100644
index 0000000..d070058
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQuery.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.Monitor;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.Instant;
+
+/**
+ * Base class for the eight 'NEXMark' queries. Supplies some fragments common to
+ * multiple queries.
+ */
+public abstract class NexmarkQuery
+ extends PTransform<PCollection<Event>, PCollection<TimestampedValue<KnownSize>>> {
+ public static final TupleTag<Auction> AUCTION_TAG = new TupleTag<>("auctions");
+ public static final TupleTag<Bid> BID_TAG = new TupleTag<>("bids");
+ static final TupleTag<Person> PERSON_TAG = new TupleTag<>("person");
+
+ /** Predicate to detect a new person event. */
+ private static final SerializableFunction<Event, Boolean> IS_NEW_PERSON =
+ new SerializableFunction<Event, Boolean>() {
+ @Override
+ public Boolean apply(Event event) {
+ return event.newPerson != null;
+ }
+ };
+
+ /** DoFn to convert a new person event to a person. */
+ private static final DoFn<Event, Person> AS_PERSON = new DoFn<Event, Person>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().newPerson);
+ }
+ };
+
+ /** Predicate to detect a new auction event. */
+ private static final SerializableFunction<Event, Boolean> IS_NEW_AUCTION =
+ new SerializableFunction<Event, Boolean>() {
+ @Override
+ public Boolean apply(Event event) {
+ return event.newAuction != null;
+ }
+ };
+
+ /** DoFn to convert a new auction event to an auction. */
+ private static final DoFn<Event, Auction> AS_AUCTION = new DoFn<Event, Auction>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().newAuction);
+ }
+ };
+
+ /** Predicate to detect a new bid event. */
+ private static final SerializableFunction<Event, Boolean> IS_BID =
+ new SerializableFunction<Event, Boolean>() {
+ @Override
+ public Boolean apply(Event event) {
+ return event.bid != null;
+ }
+ };
+
+ /** DoFn to convert a bid event to a bid. */
+ private static final DoFn<Event, Bid> AS_BID = new DoFn<Event, Bid>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().bid);
+ }
+ };
+
+ /** Transform to key each person by their id. */
+ static final ParDo.SingleOutput<Person, KV<Long, Person>> PERSON_BY_ID =
+ ParDo.of(new DoFn<Person, KV<Long, Person>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(KV.of(c.element().id, c.element()));
+ }
+ });
+
+ /** Transform to key each auction by its id. */
+ static final ParDo.SingleOutput<Auction, KV<Long, Auction>> AUCTION_BY_ID =
+ ParDo.of(new DoFn<Auction, KV<Long, Auction>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(KV.of(c.element().id, c.element()));
+ }
+ });
+
+ /** Transform to key each auction by its seller id. */
+ static final ParDo.SingleOutput<Auction, KV<Long, Auction>> AUCTION_BY_SELLER =
+ ParDo.of(new DoFn<Auction, KV<Long, Auction>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(KV.of(c.element().seller, c.element()));
+ }
+ });
+
+ /** Transform to key each bid by it's auction id. */
+ static final ParDo.SingleOutput<Bid, KV<Long, Bid>> BID_BY_AUCTION =
+ ParDo.of(new DoFn<Bid, KV<Long, Bid>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(KV.of(c.element().auction, c.element()));
+ }
+ });
+
+ /** Transform to project the auction id from each bid. */
+ static final ParDo.SingleOutput<Bid, Long> BID_TO_AUCTION =
+ ParDo.of(new DoFn<Bid, Long>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().auction);
+ }
+ });
+
+ /** Transform to project the price from each bid. */
+ static final ParDo.SingleOutput<Bid, Long> BID_TO_PRICE =
+ ParDo.of(new DoFn<Bid, Long>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().price);
+ }
+ });
+
+ /** Transform to emit each event with the timestamp embedded within it. */
+ public static final ParDo.SingleOutput<Event, Event> EVENT_TIMESTAMP_FROM_DATA =
+ ParDo.of(new DoFn<Event, Event>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Event e = c.element();
+ if (e.bid != null) {
+ c.outputWithTimestamp(e, new Instant(e.bid.dateTime));
+ } else if (e.newPerson != null) {
+ c.outputWithTimestamp(e, new Instant(e.newPerson.dateTime));
+ } else if (e.newAuction != null) {
+ c.outputWithTimestamp(e, new Instant(e.newAuction.dateTime));
+ }
+ }
+ });
+
+ /**
+ * Transform to filter for just the new auction events.
+ */
+ public static final PTransform<PCollection<Event>, PCollection<Auction>> JUST_NEW_AUCTIONS =
+ new PTransform<PCollection<Event>, PCollection<Auction>>("justNewAuctions") {
+ @Override
+ public PCollection<Auction> expand(PCollection<Event> input) {
+ return input.apply("IsNewAuction", Filter.by(IS_NEW_AUCTION))
+ .apply("AsAuction", ParDo.of(AS_AUCTION));
+ }
+ };
+
+ /**
+ * Transform to filter for just the new person events.
+ */
+ public static final PTransform<PCollection<Event>, PCollection<Person>> JUST_NEW_PERSONS =
+ new PTransform<PCollection<Event>, PCollection<Person>>("justNewPersons") {
+ @Override
+ public PCollection<Person> expand(PCollection<Event> input) {
+ return input.apply("IsNewPerson", Filter.by(IS_NEW_PERSON))
+ .apply("AsPerson", ParDo.of(AS_PERSON));
+ }
+ };
+
+ /**
+ * Transform to filter for just the bid events.
+ */
+ public static final PTransform<PCollection<Event>, PCollection<Bid>> JUST_BIDS =
+ new PTransform<PCollection<Event>, PCollection<Bid>>("justBids") {
+ @Override
+ public PCollection<Bid> expand(PCollection<Event> input) {
+ return input.apply("IsBid", Filter.by(IS_BID))
+ .apply("AsBid", ParDo.of(AS_BID));
+ }
+ };
+
+ final NexmarkConfiguration configuration;
+ public final Monitor<Event> eventMonitor;
+ public final Monitor<KnownSize> resultMonitor;
+ private final Monitor<Event> endOfStreamMonitor;
+ private final Counter fatalCounter;
+
+ NexmarkQuery(NexmarkConfiguration configuration, String name) {
+ super(name);
+ this.configuration = configuration;
+ if (configuration.debug) {
+ eventMonitor = new Monitor<>(name + ".Events", "event");
+ resultMonitor = new Monitor<>(name + ".Results", "result");
+ endOfStreamMonitor = new Monitor<>(name + ".EndOfStream", "end");
+ fatalCounter = Metrics.counter(name , "fatal");
+ } else {
+ eventMonitor = null;
+ resultMonitor = null;
+ endOfStreamMonitor = null;
+ fatalCounter = null;
+ }
+ }
+
+ /**
+ * Implement the actual query. All we know about the result is it has a known encoded size.
+ */
+ protected abstract PCollection<KnownSize> applyPrim(PCollection<Event> events);
+
+ @Override
+ public PCollection<TimestampedValue<KnownSize>> expand(PCollection<Event> events) {
+
+ if (configuration.debug) {
+ events =
+ events
+ // Monitor events as they go by.
+ .apply(name + ".Monitor", eventMonitor.getTransform())
+ // Count each type of event.
+ .apply(name + ".Snoop", NexmarkUtils.snoop(name));
+ }
+
+ if (configuration.cpuDelayMs > 0) {
+ // Slow down by pegging one core at 100%.
+ events = events.apply(name + ".CpuDelay",
+ NexmarkUtils.<Event>cpuDelay(name, configuration.cpuDelayMs));
+ }
+
+ if (configuration.diskBusyBytes > 0) {
+ // Slow down by forcing bytes to durable store.
+ events = events.apply(name + ".DiskBusy",
+ NexmarkUtils.<Event>diskBusy(configuration.diskBusyBytes));
+ }
+
+ // Run the query.
+ PCollection<KnownSize> queryResults = applyPrim(events);
+
+ if (configuration.debug) {
+ // Monitor results as they go by.
+ queryResults = queryResults.apply(name + ".Debug", resultMonitor.getTransform());
+ }
+
+ // Timestamp the query results.
+ return queryResults.apply(name + ".Stamp", NexmarkUtils.<KnownSize>stamp(name));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQueryModel.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQueryModel.java
new file mode 100644
index 0000000..2efab3e
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQueryModel.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.hamcrest.core.IsEqual;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Assert;
+
+/**
+ * Base class for models of the eight NEXMark queries. Provides an assertion function which can be
+ * applied against the actual query results to check their consistency with the model.
+ */
+public abstract class NexmarkQueryModel implements Serializable {
+ public final NexmarkConfiguration configuration;
+
+ NexmarkQueryModel(NexmarkConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Return the start of the most recent window of {@code size} and {@code period} which ends
+ * strictly before {@code timestamp}.
+ */
+ static Instant windowStart(Duration size, Duration period, Instant timestamp) {
+ long ts = timestamp.getMillis();
+ long p = period.getMillis();
+ long lim = ts - ts % p;
+ long s = size.getMillis();
+ return new Instant(lim - s);
+ }
+
+ /** Convert {@code itr} to strings capturing values, timestamps and order. */
+ static <T> List<String> toValueTimestampOrder(Iterator<TimestampedValue<T>> itr) {
+ List<String> strings = new ArrayList<>();
+ while (itr.hasNext()) {
+ strings.add(itr.next().toString());
+ }
+ return strings;
+ }
+
+ /** Convert {@code itr} to strings capturing values and order. */
+ static <T> List<String> toValueOrder(Iterator<TimestampedValue<T>> itr) {
+ List<String> strings = new ArrayList<>();
+ while (itr.hasNext()) {
+ strings.add(itr.next().getValue().toString());
+ }
+ return strings;
+ }
+
+ /** Convert {@code itr} to strings capturing values only. */
+ static <T> Set<String> toValue(Iterator<TimestampedValue<T>> itr) {
+ Set<String> strings = new HashSet<>();
+ while (itr.hasNext()) {
+ strings.add(itr.next().getValue().toString());
+ }
+ return strings;
+ }
+
+ /** Return simulator for query. */
+ public abstract AbstractSimulator<?, ?> simulator();
+
+ /** Return sub-sequence of results which are significant for model. */
+ Iterable<TimestampedValue<KnownSize>> relevantResults(
+ Iterable<TimestampedValue<KnownSize>> results) {
+ return results;
+ }
+
+ /**
+ * Convert iterator of elements to collection of strings to use when testing coherence of model
+ * against actual query results.
+ */
+ protected abstract <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr);
+
+ /** Return assertion to use on results of pipeline for this query. */
+ public SerializableFunction<Iterable<TimestampedValue<KnownSize>>, Void> assertionFor() {
+ final Collection<String> expectedStrings = toCollection(simulator().results());
+
+ return new SerializableFunction<Iterable<TimestampedValue<KnownSize>>, Void>() {
+ @Override
+ @Nullable
+ public Void apply(Iterable<TimestampedValue<KnownSize>> actual) {
+ Collection<String> actualStrings = toCollection(relevantResults(actual).iterator());
+ Assert.assertThat("wrong pipeline output", actualStrings,
+ IsEqual.equalTo(expectedStrings));
+ return null;
+ }
+ };
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0.java
new file mode 100644
index 0000000..68bf78e
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Query 0: Pass events through unchanged. However, force them to do a round trip through
+ * serialization so that we measure the impact of the choice of coders.
+ */
+public class Query0 extends NexmarkQuery {
+ public Query0(NexmarkConfiguration configuration) {
+ super(configuration, "Query0");
+ }
+
+ private PCollection<Event> applyTyped(PCollection<Event> events) {
+ final Coder<Event> coder = events.getCoder();
+ return events
+ // Force round trip through coder.
+ .apply(name + ".Serialize",
+ ParDo.of(new DoFn<Event, Event>() {
+ private final Counter bytesMetric =
+ Metrics.counter(name , "bytes");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws CoderException, IOException {
+ ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ coder.encode(c.element(), outStream, Coder.Context.OUTER);
+ byte[] byteArray = outStream.toByteArray();
+ bytesMetric.inc((long) byteArray.length);
+ ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
+ Event event = coder.decode(inStream, Coder.Context.OUTER);
+ c.output(event);
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0Model.java
new file mode 100644
index 0000000..0e73a21
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0Model.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+/**
+ * A direct implementation of {@link Query0}.
+ */
+public class Query0Model extends NexmarkQueryModel {
+ /**
+ * Simulator for query 0.
+ */
+ private static class Simulator extends AbstractSimulator<Event, Event> {
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ allDone();
+ return;
+ }
+ addResult(timestampedEvent);
+ }
+ }
+
+ public Query0Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValueTimestampOrder(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1.java
new file mode 100644
index 0000000..810cd87
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Query 1, 'Currency Conversion'. Convert each bid value from dollars to euros.
+ * In CQL syntax:
+ *
+ * <pre>
+ * SELECT Istream(auction, DOLTOEUR(price), bidder, datetime)
+ * FROM bid [ROWS UNBOUNDED];
+ * </pre>
+ *
+ * <p>To make things more interesting, allow the 'currency conversion' to be arbitrarily
+ * slowed down.
+ */
+public class Query1 extends NexmarkQuery {
+ public Query1(NexmarkConfiguration configuration) {
+ super(configuration, "Query1");
+ }
+
+ private PCollection<Bid> applyTyped(PCollection<Event> events) {
+ return events
+ // Only want the bid events.
+ .apply(JUST_BIDS)
+
+ // Map the conversion function over all bids.
+ .apply(name + ".ToEuros",
+ ParDo.of(new DoFn<Bid, Bid>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Bid bid = c.element();
+ c.output(new Bid(
+ bid.auction, bid.bidder, (bid.price * 89) / 100, bid.dateTime, bid.extra));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
new file mode 100644
index 0000000..1c4e443
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Done;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.AfterEach;
+import org.apache.beam.sdk.transforms.windowing.AfterFirst;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Query "10", 'Log to sharded files' (Not in original suite.)
+ *
+ * <p>Every windowSizeSec, save all events from the last period into 2*maxWorkers log files.
+ */
+public class Query10 extends NexmarkQuery {
+ private static final Logger LOG = LoggerFactory.getLogger(Query10.class);
+ private static final int CHANNEL_BUFFER = 8 << 20; // 8MB
+ private static final int NUM_SHARDS_PER_WORKER = 5;
+ private static final Duration LATE_BATCHING_PERIOD = Duration.standardSeconds(10);
+
+ /**
+ * Capture everything we need to know about the records in a single output file.
+ */
+ private static class OutputFile implements Serializable {
+ /** Maximum possible timestamp of records in file. */
+ private final Instant maxTimestamp;
+ /** Shard within window. */
+ private final String shard;
+ /** Index of file in all files in shard. */
+ private final long index;
+ /** Timing of records in this file. */
+ private final PaneInfo.Timing timing;
+ /** Path to file containing records, or {@literal null} if no output required. */
+ @Nullable
+ private final String filename;
+
+ public OutputFile(
+ Instant maxTimestamp,
+ String shard,
+ long index,
+ PaneInfo.Timing timing,
+ @Nullable String filename) {
+ this.maxTimestamp = maxTimestamp;
+ this.shard = shard;
+ this.index = index;
+ this.timing = timing;
+ this.filename = filename;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s %s %d %s %s%n", maxTimestamp, shard, index, timing, filename);
+ }
+ }
+
+ /**
+ * GCS uri prefix for all log and 'finished' files. If null they won't be written.
+ */
+ @Nullable
+ private String outputPath;
+
+ /**
+ * Maximum number of workers, used to determine log sharding factor.
+ */
+ private int maxNumWorkers;
+
+ public Query10(NexmarkConfiguration configuration) {
+ super(configuration, "Query10");
+ }
+
+ public void setOutputPath(@Nullable String outputPath) {
+ this.outputPath = outputPath;
+ }
+
+ public void setMaxNumWorkers(int maxNumWorkers) {
+ this.maxNumWorkers = maxNumWorkers;
+ }
+
+ /**
+ * Return channel for writing bytes to GCS.
+ */
+ private WritableByteChannel openWritableGcsFile(GcsOptions options, String filename)
+ throws IOException {
+ //TODO
+ // Fix after PR: right now this is a specific Google added use case
+ // Discuss it on ML: shall we keep GCS or use HDFS or use a generic beam filesystem way.
+ throw new UnsupportedOperationException("Disabled after removal of GcsIOChannelFactory");
+ }
+
+ /** Return a short string to describe {@code timing}. */
+ private String timingToString(PaneInfo.Timing timing) {
+ switch (timing) {
+ case EARLY:
+ return "E";
+ case ON_TIME:
+ return "O";
+ case LATE:
+ return "L";
+ }
+ throw new RuntimeException(); // cases are exhaustive
+ }
+
+ /** Construct an {@link OutputFile} for {@code pane} in {@code window} for {@code shard}. */
+ private OutputFile outputFileFor(BoundedWindow window, String shard, PaneInfo pane) {
+ @Nullable String filename =
+ outputPath == null
+ ? null
+ : String.format("%s/LOG-%s-%s-%03d-%s-%x",
+ outputPath, window.maxTimestamp(), shard, pane.getIndex(),
+ timingToString(pane.getTiming()),
+ ThreadLocalRandom.current().nextLong());
+ return new OutputFile(window.maxTimestamp(), shard, pane.getIndex(),
+ pane.getTiming(), filename);
+ }
+
+ /**
+ * Return path to which we should write the index for {@code window}, or {@literal null}
+ * if no output required.
+ */
+ @Nullable
+ private String indexPathFor(BoundedWindow window) {
+ if (outputPath == null) {
+ return null;
+ }
+ return String.format("%s/INDEX-%s", outputPath, window.maxTimestamp());
+ }
+
+ private PCollection<Done> applyTyped(PCollection<Event> events) {
+ final int numLogShards = maxNumWorkers * NUM_SHARDS_PER_WORKER;
+
+ return events
+ .apply(name + ".ShardEvents",
+ ParDo.of(new DoFn<Event, KV<String, Event>>() {
+ private final Counter lateCounter = Metrics.counter(name , "actuallyLateEvent");
+ private final Counter onTimeCounter = Metrics.counter(name , "onTimeCounter");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (c.element().hasAnnotation("LATE")) {
+ lateCounter.inc();
+ LOG.info("Observed late: %s", c.element());
+ } else {
+ onTimeCounter.inc();
+ }
+ int shardNum = (int) Math.abs((long) c.element().hashCode() % numLogShards);
+ String shard = String.format("shard-%05d-of-%05d", shardNum, numLogShards);
+ c.output(KV.of(shard, c.element()));
+ }
+ }))
+ .apply(name + ".WindowEvents",
+ Window.<KV<String, Event>>into(
+ FixedWindows.of(Duration.standardSeconds(configuration.windowSizeSec)))
+ .triggering(AfterEach.inOrder(
+ Repeatedly
+ .forever(AfterPane.elementCountAtLeast(configuration.maxLogEvents))
+ .orFinally(AfterWatermark.pastEndOfWindow()),
+ Repeatedly.forever(
+ AfterFirst.of(AfterPane.elementCountAtLeast(configuration.maxLogEvents),
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(LATE_BATCHING_PERIOD)))))
+ .discardingFiredPanes()
+ // Use a 1 day allowed lateness so that any forgotten hold will stall the
+ // pipeline for that period and be very noticeable.
+ .withAllowedLateness(Duration.standardDays(1)))
+ .apply(name + ".GroupByKey", GroupByKey.<String, Event>create())
+ .apply(name + ".CheckForLateEvents",
+ ParDo.of(new DoFn<KV<String, Iterable<Event>>,
+ KV<String, Iterable<Event>>>() {
+ private final Counter earlyCounter = Metrics.counter(name , "earlyShard");
+ private final Counter onTimeCounter = Metrics.counter(name , "onTimeShard");
+ private final Counter lateCounter = Metrics.counter(name , "lateShard");
+ private final Counter unexpectedLatePaneCounter =
+ Metrics.counter(name , "ERROR_unexpectedLatePane");
+ private final Counter unexpectedOnTimeElementCounter =
+ Metrics.counter(name , "ERROR_unexpectedOnTimeElement");
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window) {
+ int numLate = 0;
+ int numOnTime = 0;
+ for (Event event : c.element().getValue()) {
+ if (event.hasAnnotation("LATE")) {
+ numLate++;
+ } else {
+ numOnTime++;
+ }
+ }
+ String shard = c.element().getKey();
+ LOG.info(String.format(
+ "%s with timestamp %s has %d actually late and %d on-time "
+ + "elements in pane %s for window %s",
+ shard, c.timestamp(), numLate, numOnTime, c.pane(),
+ window.maxTimestamp()));
+ if (c.pane().getTiming() == PaneInfo.Timing.LATE) {
+ if (numLate == 0) {
+ LOG.error(
+ "ERROR! No late events in late pane for %s", shard);
+ unexpectedLatePaneCounter.inc();
+ }
+ if (numOnTime > 0) {
+ LOG.error(
+ "ERROR! Have %d on-time events in late pane for %s",
+ numOnTime, shard);
+ unexpectedOnTimeElementCounter.inc();
+ }
+ lateCounter.inc();
+ } else if (c.pane().getTiming() == PaneInfo.Timing.EARLY) {
+ if (numOnTime + numLate < configuration.maxLogEvents) {
+ LOG.error(
+ "ERROR! Only have %d events in early pane for %s",
+ numOnTime + numLate, shard);
+ }
+ earlyCounter.inc();
+ } else {
+ onTimeCounter.inc();
+ }
+ c.output(c.element());
+ }
+ }))
+ .apply(name + ".UploadEvents",
+ ParDo.of(new DoFn<KV<String, Iterable<Event>>,
+ KV<Void, OutputFile>>() {
+ private final Counter savedFileCounter = Metrics.counter(name , "savedFile");
+ private final Counter writtenRecordsCounter = Metrics.counter(name , "writtenRecords");
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window)
+ throws IOException {
+ String shard = c.element().getKey();
+ GcsOptions options = c.getPipelineOptions().as(GcsOptions.class);
+ OutputFile outputFile = outputFileFor(window, shard, c.pane());
+ LOG.info(String.format(
+ "Writing %s with record timestamp %s, window timestamp %s, pane %s",
+ shard, c.timestamp(), window.maxTimestamp(), c.pane()));
+ if (outputFile.filename != null) {
+ LOG.info("Beginning write to '%s'", outputFile.filename);
+ int n = 0;
+ try (OutputStream output =
+ Channels.newOutputStream(openWritableGcsFile(options, outputFile
+ .filename))) {
+ for (Event event : c.element().getValue()) {
+ Event.CODER.encode(event, output, Coder.Context.OUTER);
+ writtenRecordsCounter.inc();
+ if (++n % 10000 == 0) {
+ LOG.info("So far written %d records to '%s'", n,
+ outputFile.filename);
+ }
+ }
+ }
+ LOG.info("Written all %d records to '%s'", n, outputFile.filename);
+ }
+ savedFileCounter.inc();
+ c.output(KV.<Void, OutputFile>of(null, outputFile));
+ }
+ }))
+ // Clear fancy triggering from above.
+ .apply(name + ".WindowLogFiles", Window.<KV<Void, OutputFile>>into(
+ FixedWindows.of(Duration.standardSeconds(configuration.windowSizeSec)))
+ .triggering(AfterWatermark.pastEndOfWindow())
+ // We expect no late data here, but we'll assume the worst so we can detect any.
+ .withAllowedLateness(Duration.standardDays(1))
+ .discardingFiredPanes())
+ // this GroupByKey allows to have one file per window
+ .apply(name + ".GroupByKey2", GroupByKey.<Void, OutputFile>create())
+ .apply(name + ".Index",
+ ParDo.of(new DoFn<KV<Void, Iterable<OutputFile>>, Done>() {
+ private final Counter unexpectedLateCounter =
+ Metrics.counter(name , "ERROR_unexpectedLate");
+ private final Counter unexpectedEarlyCounter =
+ Metrics.counter(name , "ERROR_unexpectedEarly");
+ private final Counter unexpectedIndexCounter =
+ Metrics.counter(name , "ERROR_unexpectedIndex");
+ private final Counter finalizedCounter = Metrics.counter(name , "indexed");
+
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window)
+ throws IOException {
+ if (c.pane().getTiming() == Timing.LATE) {
+ unexpectedLateCounter.inc();
+ LOG.error("ERROR! Unexpected LATE pane: %s", c.pane());
+ } else if (c.pane().getTiming() == Timing.EARLY) {
+ unexpectedEarlyCounter.inc();
+ LOG.error("ERROR! Unexpected EARLY pane: %s", c.pane());
+ } else if (c.pane().getTiming() == Timing.ON_TIME
+ && c.pane().getIndex() != 0) {
+ unexpectedIndexCounter.inc();
+ LOG.error("ERROR! Unexpected ON_TIME pane index: %s", c.pane());
+ } else {
+ GcsOptions options = c.getPipelineOptions().as(GcsOptions.class);
+ LOG.info(
+ "Index with record timestamp %s, window timestamp %s, pane %s",
+ c.timestamp(), window.maxTimestamp(), c.pane());
+
+ @Nullable String filename = indexPathFor(window);
+ if (filename != null) {
+ LOG.info("Beginning write to '%s'", filename);
+ int n = 0;
+ try (OutputStream output =
+ Channels.newOutputStream(
+ openWritableGcsFile(options, filename))) {
+ for (OutputFile outputFile : c.element().getValue()) {
+ output.write(outputFile.toString().getBytes("UTF-8"));
+ n++;
+ }
+ }
+ LOG.info("Written all %d lines to '%s'", n, filename);
+ }
+ c.output(
+ new Done("written for timestamp " + window.maxTimestamp()));
+ finalizedCounter.inc();
+ }
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query11.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query11.java
new file mode 100644
index 0000000..47e7c00
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query11.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.BidsPerSession;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query "11", 'User sessions' (Not in original suite.)
+ *
+ * <p>Group bids by the same user into sessions with {@code windowSizeSec} max gap.
+ * However limit the session to at most {@code maxLogEvents}. Emit the number of
+ * bids per session.
+ */
+public class Query11 extends NexmarkQuery {
+ public Query11(NexmarkConfiguration configuration) {
+ super(configuration, "Query11");
+ }
+
+ private PCollection<BidsPerSession> applyTyped(PCollection<Event> events) {
+ PCollection<Long> bidders = events.apply(JUST_BIDS).apply(name + ".Rekey",
+ ParDo.of(new DoFn<Bid, Long>() {
+
+ @ProcessElement public void processElement(ProcessContext c) {
+ Bid bid = c.element();
+ c.output(bid.bidder);
+ }
+ }));
+
+ PCollection<Long> biddersWindowed = bidders.apply(
+ Window.<Long>into(
+ Sessions.withGapDuration(Duration.standardSeconds(configuration.windowSizeSec)))
+ .triggering(
+ Repeatedly.forever(AfterPane.elementCountAtLeast(configuration.maxLogEvents)))
+ .discardingFiredPanes()
+ .withAllowedLateness(Duration.standardSeconds(configuration.occasionalDelaySec / 2)));
+ return biddersWindowed.apply(Count.<Long>perElement())
+ .apply(name + ".ToResult", ParDo.of(new DoFn<KV<Long, Long>, BidsPerSession>() {
+
+ @ProcessElement public void processElement(ProcessContext c) {
+ c.output(new BidsPerSession(c.element().getKey(), c.element().getValue()));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query12.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query12.java
new file mode 100644
index 0000000..0f4b232
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query12.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.BidsPerSession;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query "12", 'Processing time windows' (Not in original suite.)
+ *
+ * <p>Group bids by the same user into processing time windows of windowSize. Emit the count
+ * of bids per window.
+ */
+public class Query12 extends NexmarkQuery {
+ public Query12(NexmarkConfiguration configuration) {
+ super(configuration, "Query12");
+ }
+
+ private PCollection<BidsPerSession> applyTyped(PCollection<Event> events) {
+ return events
+ .apply(JUST_BIDS)
+ .apply(ParDo.of(new DoFn<Bid, Long>() {
+ @ProcessElement
+ public void processElement(ProcessContext c){
+ c.output(c.element().bidder);
+ }
+ }))
+ .apply(Window.<Long>into(new GlobalWindows())
+ .triggering(
+ Repeatedly.forever(
+ AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(
+ Duration.standardSeconds(configuration.windowSizeSec))))
+ .discardingFiredPanes()
+ .withAllowedLateness(Duration.ZERO))
+ .apply(Count.<Long>perElement())
+ .apply(name + ".ToResult",
+ ParDo.of(new DoFn<KV<Long, Long>, BidsPerSession>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(
+ new BidsPerSession(c.element().getKey(), c.element().getValue()));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1Model.java
new file mode 100644
index 0000000..76c182a
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1Model.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+/**
+ * A direct implementation of {@link Query1}.
+ */
+public class Query1Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 1.
+ */
+ private static class Simulator extends AbstractSimulator<Event, Bid> {
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ allDone();
+ return;
+ }
+ Event event = timestampedEvent.getValue();
+ if (event.bid == null) {
+ // Ignore non-bid events.
+ return;
+ }
+ Bid bid = event.bid;
+ Bid resultBid =
+ new Bid(bid.auction, bid.bidder, bid.price * 89 / 100, bid.dateTime, bid.extra);
+ TimestampedValue<Bid> result =
+ TimestampedValue.of(resultBid, timestampedEvent.getTimestamp());
+ addResult(result);
+ }
+ }
+
+ public Query1Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValueTimestampOrder(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2.java
new file mode 100644
index 0000000..c5ab992
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.AuctionPrice;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Query 2, 'Filtering. Find bids with specific auction ids and show their bid price.
+ * In CQL syntax:
+ *
+ * <pre>
+ * SELECT Rstream(auction, price)
+ * FROM Bid [NOW]
+ * WHERE auction = 1007 OR auction = 1020 OR auction = 2001 OR auction = 2019 OR auction = 2087;
+ * </pre>
+ *
+ * <p>As written that query will only yield a few hundred results over event streams of
+ * arbitrary size. To make it more interesting we instead choose bids for every
+ * {@code auctionSkip}'th auction.
+ */
+public class Query2 extends NexmarkQuery {
+ public Query2(NexmarkConfiguration configuration) {
+ super(configuration, "Query2");
+ }
+
+ private PCollection<AuctionPrice> applyTyped(PCollection<Event> events) {
+ return events
+ // Only want the bid events.
+ .apply(JUST_BIDS)
+
+ // Select just the bids for the auctions we care about.
+ .apply(Filter.by(new SerializableFunction<Bid, Boolean>() {
+ @Override
+ public Boolean apply(Bid bid) {
+ return bid.auction % configuration.auctionSkip == 0;
+ }
+ }))
+
+ // Project just auction id and price.
+ .apply(name + ".Project",
+ ParDo.of(new DoFn<Bid, AuctionPrice>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Bid bid = c.element();
+ c.output(new AuctionPrice(bid.auction, bid.price));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2Model.java
new file mode 100644
index 0000000..33a1f8d
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2Model.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.AuctionPrice;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+/**
+ * A direct implementation of {@link Query2}.
+ */
+public class Query2Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 2.
+ */
+ private class Simulator extends AbstractSimulator<Event, AuctionPrice> {
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ allDone();
+ return;
+ }
+ Event event = timestampedEvent.getValue();
+ if (event.bid == null) {
+ // Ignore non bid events.
+ return;
+ }
+ Bid bid = event.bid;
+ if (bid.auction % configuration.auctionSkip != 0) {
+ // Ignore bids for auctions we don't care about.
+ return;
+ }
+ AuctionPrice auctionPrice = new AuctionPrice(bid.auction, bid.price);
+ TimestampedValue<AuctionPrice> result =
+ TimestampedValue.of(auctionPrice, timestampedEvent.getTimestamp());
+ addResult(result);
+ }
+ }
+
+ public Query2Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValueTimestampOrder(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java
new file mode 100644
index 0000000..6f8d72d
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.NameCityStateId;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Query 3, 'Local Item Suggestion'. Who is selling in OR, ID or CA in category 10, and for what
+ * auction ids? In CQL syntax:
+ *
+ * <pre>
+ * SELECT Istream(P.name, P.city, P.state, A.id)
+ * FROM Auction A [ROWS UNBOUNDED], Person P [ROWS UNBOUNDED]
+ * WHERE A.seller = P.id AND (P.state = `OR' OR P.state = `ID' OR P.state = `CA') AND A.category
+ * = 10;
+ * </pre>
+ *
+ * <p>We'll implement this query to allow 'new auction' events to come before the 'new person'
+ * events for the auction seller. Those auctions will be stored until the matching person is seen.
+ * Then all subsequent auctions for a person will use the stored person record.
+ *
+ * <p>A real system would use an external system to maintain the id-to-person association.
+ */
+public class Query3 extends NexmarkQuery {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Query3.class);
+ private final JoinDoFn joinDoFn;
+
+ public Query3(NexmarkConfiguration configuration) {
+ super(configuration, "Query3");
+ joinDoFn = new JoinDoFn(name, configuration.maxAuctionsWaitingTime);
+ }
+
+ private PCollection<NameCityStateId> applyTyped(PCollection<Event> events) {
+ int numEventsInPane = 30;
+
+ PCollection<Event> eventsWindowed =
+ events.apply(
+ Window.<Event>into(new GlobalWindows())
+ .triggering(Repeatedly.forever((AfterPane.elementCountAtLeast(numEventsInPane))))
+ .discardingFiredPanes()
+ .withAllowedLateness(Duration.ZERO));
+ PCollection<KV<Long, Auction>> auctionsBySellerId =
+ eventsWindowed
+ // Only want the new auction events.
+ .apply(JUST_NEW_AUCTIONS)
+
+ // We only want auctions in category 10.
+ .apply(
+ name + ".InCategory",
+ Filter.by(
+ new SerializableFunction<Auction, Boolean>() {
+
+ @Override
+ public Boolean apply(Auction auction) {
+ return auction.category == 10;
+ }
+ }))
+
+ // Key auctions by their seller id.
+ .apply("AuctionBySeller", AUCTION_BY_SELLER);
+
+ PCollection<KV<Long, Person>> personsById =
+ eventsWindowed
+ // Only want the new people events.
+ .apply(JUST_NEW_PERSONS)
+
+ // We only want people in OR, ID, CA.
+ .apply(
+ name + ".InState",
+ Filter.by(
+ new SerializableFunction<Person, Boolean>() {
+
+ @Override
+ public Boolean apply(Person person) {
+ return person.state.equals("OR")
+ || person.state.equals("ID")
+ || person.state.equals("CA");
+ }
+ }))
+
+ // Key people by their id.
+ .apply("PersonById", PERSON_BY_ID);
+
+ return
+ // Join auctions and people.
+ // concatenate KeyedPCollections
+ KeyedPCollectionTuple.of(AUCTION_TAG, auctionsBySellerId)
+ .and(PERSON_TAG, personsById)
+ // group auctions and persons by personId
+ .apply(CoGroupByKey.<Long>create())
+ .apply(name + ".Join", ParDo.of(joinDoFn))
+
+ // Project what we want.
+ .apply(
+ name + ".Project",
+ ParDo.of(
+ new DoFn<KV<Auction, Person>, NameCityStateId>() {
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Auction auction = c.element().getKey();
+ Person person = c.element().getValue();
+ c.output(
+ new NameCityStateId(person.name, person.city, person.state, auction.id));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+
+ /**
+ * Join {@code auctions} and {@code people} by person id and emit their cross-product one pair at
+ * a time.
+ *
+ * <p>We know a person may submit any number of auctions. Thus new person event must have the
+ * person record stored in persistent state in order to match future auctions by that person.
+ *
+ * <p>However we know that each auction is associated with at most one person, so only need to
+ * store auction records in persistent state until we have seen the corresponding person record.
+ * And of course may have already seen that record.
+ */
+ private static class JoinDoFn extends DoFn<KV<Long, CoGbkResult>, KV<Auction, Person>> {
+
+ private final int maxAuctionsWaitingTime;
+ private static final String AUCTIONS = "auctions";
+ private static final String PERSON = "person";
+
+ @StateId(PERSON)
+ private static final StateSpec<ValueState<Person>> personSpec =
+ StateSpecs.value(Person.CODER);
+
+ private static final String PERSON_STATE_EXPIRING = "personStateExpiring";
+
+ @StateId(AUCTIONS)
+ private final StateSpec<ValueState<List<Auction>>> auctionsSpec =
+ StateSpecs.value(ListCoder.of(Auction.CODER));
+
+ @TimerId(PERSON_STATE_EXPIRING)
+ private final TimerSpec timerSpec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ // Used to refer the metrics namespace
+ private final String name;
+
+ private final Counter newAuctionCounter;
+ private final Counter newPersonCounter;
+ private final Counter newNewOutputCounter;
+ private final Counter newOldOutputCounter;
+ private final Counter oldNewOutputCounter;
+ private final Counter fatalCounter;
+
+ private JoinDoFn(String name, int maxAuctionsWaitingTime) {
+ this.name = name;
+ this.maxAuctionsWaitingTime = maxAuctionsWaitingTime;
+ newAuctionCounter = Metrics.counter(name, "newAuction");
+ newPersonCounter = Metrics.counter(name, "newPerson");
+ newNewOutputCounter = Metrics.counter(name, "newNewOutput");
+ newOldOutputCounter = Metrics.counter(name, "newOldOutput");
+ oldNewOutputCounter = Metrics.counter(name, "oldNewOutput");
+ fatalCounter = Metrics.counter(name , "fatal");
+ }
+
+ @ProcessElement
+ public void processElement(
+ ProcessContext c,
+ @TimerId(PERSON_STATE_EXPIRING) Timer timer,
+ @StateId(PERSON) ValueState<Person> personState,
+ @StateId(AUCTIONS) ValueState<List<Auction>> auctionsState) {
+ // We would *almost* implement this by rewindowing into the global window and
+ // running a combiner over the result. The combiner's accumulator would be the
+ // state we use below. However, combiners cannot emit intermediate results, thus
+ // we need to wait for the pending ReduceFn API.
+
+ Person existingPerson = personState.read();
+ if (existingPerson != null) {
+ // We've already seen the new person event for this person id.
+ // We can join with any new auctions on-the-fly without needing any
+ // additional persistent state.
+ for (Auction newAuction : c.element().getValue().getAll(AUCTION_TAG)) {
+ newAuctionCounter.inc();
+ newOldOutputCounter.inc();
+ c.output(KV.of(newAuction, existingPerson));
+ }
+ return;
+ }
+
+ Person theNewPerson = null;
+ for (Person newPerson : c.element().getValue().getAll(PERSON_TAG)) {
+ if (theNewPerson == null) {
+ theNewPerson = newPerson;
+ } else {
+ if (theNewPerson.equals(newPerson)) {
+ LOG.error("Duplicate person {}", theNewPerson);
+ } else {
+ LOG.error("Conflicting persons {} and {}", theNewPerson, newPerson);
+ }
+ fatalCounter.inc();
+ continue;
+ }
+ newPersonCounter.inc();
+ // We've now seen the person for this person id so can flush any
+ // pending auctions for the same seller id (an auction is done by only one seller).
+ List<Auction> pendingAuctions = auctionsState.read();
+ if (pendingAuctions != null) {
+ for (Auction pendingAuction : pendingAuctions) {
+ oldNewOutputCounter.inc();
+ c.output(KV.of(pendingAuction, newPerson));
+ }
+ auctionsState.clear();
+ }
+ // Also deal with any new auctions.
+ for (Auction newAuction : c.element().getValue().getAll(AUCTION_TAG)) {
+ newAuctionCounter.inc();
+ newNewOutputCounter.inc();
+ c.output(KV.of(newAuction, newPerson));
+ }
+ // Remember this person for any future auctions.
+ personState.write(newPerson);
+ //set a time out to clear this state
+ Instant firingTime = new Instant(newPerson.dateTime)
+ .plus(Duration.standardSeconds(maxAuctionsWaitingTime));
+ timer.set(firingTime);
+ }
+ if (theNewPerson != null) {
+ return;
+ }
+
+ // We'll need to remember the auctions until we see the corresponding
+ // new person event.
+ List<Auction> pendingAuctions = auctionsState.read();
+ if (pendingAuctions == null) {
+ pendingAuctions = new ArrayList<>();
+ }
+ for (Auction newAuction : c.element().getValue().getAll(AUCTION_TAG)) {
+ newAuctionCounter.inc();
+ pendingAuctions.add(newAuction);
+ }
+ auctionsState.write(pendingAuctions);
+ }
+
+ @OnTimer(PERSON_STATE_EXPIRING)
+ public void onTimerCallback(
+ OnTimerContext context,
+ @StateId(PERSON) ValueState<Person> personState) {
+ personState.clear();
+ }
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java
new file mode 100644
index 0000000..94f24cb
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.NameCityStateId;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+
+/**
+ * A direct implementation of {@link Query3}.
+ */
+public class Query3Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 3.
+ */
+ private static class Simulator extends AbstractSimulator<Event, NameCityStateId> {
+ /** Auctions, indexed by seller id. */
+ private final Multimap<Long, Auction> newAuctions;
+
+ /** Persons, indexed by id. */
+ private final Map<Long, Person> newPersons;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ newPersons = new HashMap<>();
+ newAuctions = ArrayListMultimap.create();
+ }
+
+ /**
+ * Capture new result.
+ */
+ private void addResult(Auction auction, Person person, Instant timestamp) {
+ TimestampedValue<NameCityStateId> result = TimestampedValue.of(
+ new NameCityStateId(person.name, person.city, person.state, auction.id), timestamp);
+ addResult(result);
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ allDone();
+ return;
+ }
+ Event event = timestampedEvent.getValue();
+ if (event.bid != null) {
+ // Ignore bid events.
+ return;
+ }
+
+ Instant timestamp = timestampedEvent.getTimestamp();
+
+ if (event.newAuction != null) {
+ // Only want auctions in category 10.
+ if (event.newAuction.category == 10) {
+ // Join new auction with existing person, if any.
+ Person person = newPersons.get(event.newAuction.seller);
+ if (person != null) {
+ addResult(event.newAuction, person, timestamp);
+ } else {
+ // Remember auction for future new person event.
+ newAuctions.put(event.newAuction.seller, event.newAuction);
+ }
+ }
+ } else {
+ // Only want people in OR, ID or CA.
+ if (event.newPerson.state.equals("OR") || event.newPerson.state.equals("ID")
+ || event.newPerson.state.equals("CA")) {
+ // Join new person with existing auctions.
+ for (Auction auction : newAuctions.get(event.newPerson.id)) {
+ addResult(auction, event.newPerson, timestamp);
+ }
+ // We'll never need these auctions again.
+ newAuctions.removeAll(event.newPerson.id);
+ // Remember person for future auctions.
+ newPersons.put(event.newPerson.id, event.newPerson);
+ }
+ }
+ }
+ }
+
+ public Query3Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4.java
new file mode 100644
index 0000000..3c1cf3b
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.Monitor;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.CategoryPrice;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Mean;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query 4, 'Average Price for a Category'. Select the average of the wining bid prices for all
+ * closed auctions in each category. In CQL syntax:
+ *
+ * <pre>{@code
+ * SELECT Istream(AVG(Q.final))
+ * FROM Category C, (SELECT Rstream(MAX(B.price) AS final, A.category)
+ * FROM Auction A [ROWS UNBOUNDED], Bid B [ROWS UNBOUNDED]
+ * WHERE A.id=B.auction AND B.datetime < A.expires AND A.expires < CURRENT_TIME
+ * GROUP BY A.id, A.category) Q
+ * WHERE Q.category = C.id
+ * GROUP BY C.id;
+ * }</pre>
+ *
+ * <p>For extra spiciness our implementation differs slightly from the above:
+ * <ul>
+ * <li>We select both the average winning price and the category.
+ * <li>We don't bother joining with a static category table, since it's contents are never used.
+ * <li>We only consider bids which are above the auction's reserve price.
+ * <li>We accept the highest-price, earliest valid bid as the winner.
+ * <li>We calculate the averages oven a sliding window of size {@code windowSizeSec} and
+ * period {@code windowPeriodSec}.
+ * </ul>
+ */
+public class Query4 extends NexmarkQuery {
+ private final Monitor<AuctionBid> winningBidsMonitor;
+
+ public Query4(NexmarkConfiguration configuration) {
+ super(configuration, "Query4");
+ winningBidsMonitor = new Monitor<>(name + ".WinningBids", "winning");
+ }
+
+ private PCollection<CategoryPrice> applyTyped(PCollection<Event> events) {
+ PCollection<AuctionBid> winningBids =
+ events
+ // Find the winning bid for each closed auction.
+ .apply(new WinningBids(name + ".WinningBids", configuration));
+
+ // Monitor winning bids
+ winningBids = winningBids.apply(name + ".WinningBidsMonitor",
+ winningBidsMonitor.getTransform());
+
+ return winningBids
+ // Key the winning bid price by the auction category.
+ .apply(name + ".Rekey",
+ ParDo.of(new DoFn<AuctionBid, KV<Long, Long>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Auction auction = c.element().auction;
+ Bid bid = c.element().bid;
+ c.output(KV.of(auction.category, bid.price));
+ }
+ }))
+
+ // Re-window so we can calculate a sliding average
+ .apply(Window.<KV<Long, Long>>into(
+ SlidingWindows.of(Duration.standardSeconds(configuration.windowSizeSec))
+ .every(Duration.standardSeconds(configuration.windowPeriodSec))))
+
+ // Find the average of the winning bids for each category.
+ // Make sure we share the work for each category between workers.
+ .apply(Mean.<Long, Long>perKey().withHotKeyFanout(configuration.fanout))
+
+ // For testing against Query4Model, capture which results are 'final'.
+ .apply(name + ".Project",
+ ParDo.of(new DoFn<KV<Long, Double>, CategoryPrice>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(new CategoryPrice(c.element().getKey(),
+ Math.round(c.element().getValue()), c.pane().isLast()));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4Model.java
new file mode 100644
index 0000000..84274a8
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4Model.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.CategoryPrice;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Assert;
+
+/**
+ * A direct implementation of {@link Query4}.
+ */
+public class Query4Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 4.
+ */
+ private class Simulator extends AbstractSimulator<AuctionBid, CategoryPrice> {
+ /** The prices and categories for all winning bids in the last window size. */
+ private final List<TimestampedValue<CategoryPrice>> winningPricesByCategory;
+
+ /** Timestamp of last result (ms since epoch). */
+ private Instant lastTimestamp;
+
+ /** When oldest active window starts. */
+ private Instant windowStart;
+
+ /** The last seen result for each category. */
+ private final Map<Long, TimestampedValue<CategoryPrice>> lastSeenResults;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(new WinningBidsSimulator(configuration).results());
+ winningPricesByCategory = new ArrayList<>();
+ lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ windowStart = NexmarkUtils.BEGINNING_OF_TIME;
+ lastSeenResults = new TreeMap<>();
+ }
+
+ /**
+ * Calculate the average bid price for each category for all winning bids
+ * which are strictly before {@code end}.
+ */
+ private void averages(Instant end) {
+ Map<Long, Long> counts = new TreeMap<>();
+ Map<Long, Long> totals = new TreeMap<>();
+ for (TimestampedValue<CategoryPrice> value : winningPricesByCategory) {
+ if (!value.getTimestamp().isBefore(end)) {
+ continue;
+ }
+ long category = value.getValue().category;
+ long price = value.getValue().price;
+ Long count = counts.get(category);
+ if (count == null) {
+ count = 1L;
+ } else {
+ count += 1;
+ }
+ counts.put(category, count);
+ Long total = totals.get(category);
+ if (total == null) {
+ total = price;
+ } else {
+ total += price;
+ }
+ totals.put(category, total);
+ }
+ for (Map.Entry<Long, Long> entry : counts.entrySet()) {
+ long category = entry.getKey();
+ long count = entry.getValue();
+ long total = totals.get(category);
+ TimestampedValue<CategoryPrice> result = TimestampedValue.of(
+ new CategoryPrice(category, Math.round((double) total / count), true), lastTimestamp);
+ addIntermediateResult(result);
+ lastSeenResults.put(category, result);
+ }
+ }
+
+ /**
+ * Calculate averages for any windows which can now be retired. Also prune entries
+ * which can no longer contribute to any future window.
+ */
+ private void prune(Instant newWindowStart) {
+ while (!newWindowStart.equals(windowStart)) {
+ averages(windowStart.plus(Duration.standardSeconds(configuration.windowSizeSec)));
+ windowStart = windowStart.plus(Duration.standardSeconds(configuration.windowPeriodSec));
+ Iterator<TimestampedValue<CategoryPrice>> itr = winningPricesByCategory.iterator();
+ while (itr.hasNext()) {
+ if (itr.next().getTimestamp().isBefore(windowStart)) {
+ itr.remove();
+ }
+ }
+ if (winningPricesByCategory.isEmpty()) {
+ windowStart = newWindowStart;
+ }
+ }
+ }
+
+ /**
+ * Capture the winning bid.
+ */
+ private void captureWinningBid(Auction auction, Bid bid, Instant timestamp) {
+ winningPricesByCategory.add(
+ TimestampedValue.of(new CategoryPrice(auction.category, bid.price, false), timestamp));
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<AuctionBid> timestampedWinningBid = nextInput();
+ if (timestampedWinningBid == null) {
+ prune(NexmarkUtils.END_OF_TIME);
+ for (TimestampedValue<CategoryPrice> result : lastSeenResults.values()) {
+ addResult(result);
+ }
+ allDone();
+ return;
+ }
+ lastTimestamp = timestampedWinningBid.getTimestamp();
+ Instant newWindowStart = windowStart(Duration.standardSeconds(configuration.windowSizeSec),
+ Duration.standardSeconds(configuration.windowPeriodSec), lastTimestamp);
+ prune(newWindowStart);
+ captureWinningBid(timestampedWinningBid.getValue().auction,
+ timestampedWinningBid.getValue().bid, lastTimestamp);
+ }
+ }
+
+ public Query4Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected Iterable<TimestampedValue<KnownSize>> relevantResults(
+ Iterable<TimestampedValue<KnownSize>> results) {
+ // Find the last (in processing time) reported average price for each category.
+ Map<Long, TimestampedValue<KnownSize>> finalAverages = new TreeMap<>();
+ for (TimestampedValue<KnownSize> obj : results) {
+ Assert.assertTrue("have CategoryPrice", obj.getValue() instanceof CategoryPrice);
+ CategoryPrice categoryPrice = (CategoryPrice) obj.getValue();
+ if (categoryPrice.isLast) {
+ finalAverages.put(
+ categoryPrice.category,
+ TimestampedValue.of((KnownSize) categoryPrice, obj.getTimestamp()));
+ }
+ }
+
+ return finalAverages.values();
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java
new file mode 100644
index 0000000..d027cb3
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.AuctionCount;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query 5, 'Hot Items'. Which auctions have seen the most bids in the last hour (updated every
+ * minute). In CQL syntax:
+ *
+ * <pre>{@code
+ * SELECT Rstream(auction)
+ * FROM (SELECT B1.auction, count(*) AS num
+ * FROM Bid [RANGE 60 MINUTE SLIDE 1 MINUTE] B1
+ * GROUP BY B1.auction)
+ * WHERE num >= ALL (SELECT count(*)
+ * FROM Bid [RANGE 60 MINUTE SLIDE 1 MINUTE] B2
+ * GROUP BY B2.auction);
+ * }</pre>
+ *
+ * <p>To make things a bit more dynamic and easier to test we use much shorter windows, and
+ * we'll also preserve the bid counts.
+ */
+public class Query5 extends NexmarkQuery {
+ public Query5(NexmarkConfiguration configuration) {
+ super(configuration, "Query5");
+ }
+
+ private PCollection<AuctionCount> applyTyped(PCollection<Event> events) {
+ return events
+ // Only want the bid events.
+ .apply(JUST_BIDS)
+ // Window the bids into sliding windows.
+ .apply(
+ Window.<Bid>into(
+ SlidingWindows.of(Duration.standardSeconds(configuration.windowSizeSec))
+ .every(Duration.standardSeconds(configuration.windowPeriodSec))))
+ // Project just the auction id.
+ .apply("BidToAuction", BID_TO_AUCTION)
+
+ // Count the number of bids per auction id.
+ .apply(Count.<Long>perElement())
+
+ // We'll want to keep all auctions with the maximal number of bids.
+ // Start by lifting each into a singleton list.
+ // need to do so because bellow combine returns a list of auctions in the key in case of
+ // equal number of bids. Combine needs to have same input type and return type.
+ .apply(
+ name + ".ToSingletons",
+ ParDo.of(
+ new DoFn<KV<Long, Long>, KV<List<Long>, Long>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(
+ KV.of(
+ Collections.singletonList(c.element().getKey()),
+ c.element().getValue()));
+ }
+ }))
+
+ // Keep only the auction ids with the most bids.
+ .apply(
+ Combine.globally(
+ new Combine.BinaryCombineFn<KV<List<Long>, Long>>() {
+ @Override
+ public KV<List<Long>, Long> apply(
+ KV<List<Long>, Long> left, KV<List<Long>, Long> right) {
+ List<Long> leftBestAuctions = left.getKey();
+ long leftCount = left.getValue();
+ List<Long> rightBestAuctions = right.getKey();
+ long rightCount = right.getValue();
+ if (leftCount > rightCount) {
+ return left;
+ } else if (leftCount < rightCount) {
+ return right;
+ } else {
+ List<Long> newBestAuctions = new ArrayList<>();
+ newBestAuctions.addAll(leftBestAuctions);
+ newBestAuctions.addAll(rightBestAuctions);
+ return KV.of(newBestAuctions, leftCount);
+ }
+ }
+ })
+ .withoutDefaults()
+ .withFanout(configuration.fanout))
+
+ // Project into result.
+ .apply(
+ name + ".Select",
+ ParDo.of(
+ new DoFn<KV<List<Long>, Long>, AuctionCount>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ long count = c.element().getValue();
+ for (long auction : c.element().getKey()) {
+ c.output(new AuctionCount(auction, count));
+ }
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5Model.java
new file mode 100644
index 0000000..7ed0709
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5Model.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.AuctionCount;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A direct implementation of {@link Query5}.
+ */
+public class Query5Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 5.
+ */
+ private class Simulator extends AbstractSimulator<Event, AuctionCount> {
+ /** Time of bids still contributing to open windows, indexed by their auction id. */
+ private final Map<Long, List<Instant>> bids;
+
+ /** When oldest active window starts. */
+ private Instant windowStart;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ bids = new TreeMap<>();
+ windowStart = NexmarkUtils.BEGINNING_OF_TIME;
+ }
+
+ /**
+ * Count bids per auction id for bids strictly before {@code end}. Add the auction ids with
+ * the maximum number of bids to results.
+ */
+ private void countBids(Instant end) {
+ Map<Long, Long> counts = new TreeMap<>();
+ long maxCount = 0L;
+ for (Map.Entry<Long, List<Instant>> entry : bids.entrySet()) {
+ long count = 0L;
+ long auction = entry.getKey();
+ for (Instant bid : entry.getValue()) {
+ if (bid.isBefore(end)) {
+ count++;
+ }
+ }
+ if (count > 0) {
+ counts.put(auction, count);
+ maxCount = Math.max(maxCount, count);
+ }
+ }
+ for (Map.Entry<Long, Long> entry : counts.entrySet()) {
+ long auction = entry.getKey();
+ long count = entry.getValue();
+ if (count == maxCount) {
+ AuctionCount result = new AuctionCount(auction, count);
+ addResult(TimestampedValue.of(result, end));
+ }
+ }
+ }
+
+ /**
+ * Retire bids which are strictly before {@code cutoff}. Return true if there are any bids
+ * remaining.
+ */
+ private boolean retireBids(Instant cutoff) {
+ boolean anyRemain = false;
+ for (Map.Entry<Long, List<Instant>> entry : bids.entrySet()) {
+ long auction = entry.getKey();
+ Iterator<Instant> itr = entry.getValue().iterator();
+ while (itr.hasNext()) {
+ Instant bid = itr.next();
+ if (bid.isBefore(cutoff)) {
+ NexmarkUtils.info("retire: %s for %s", bid, auction);
+ itr.remove();
+ } else {
+ anyRemain = true;
+ }
+ }
+ }
+ return anyRemain;
+ }
+
+ /**
+ * Retire active windows until we've reached {@code newWindowStart}.
+ */
+ private void retireWindows(Instant newWindowStart) {
+ while (!newWindowStart.equals(windowStart)) {
+ NexmarkUtils.info("retiring window %s, aiming for %s", windowStart, newWindowStart);
+ // Count bids in the window (windowStart, windowStart + size].
+ countBids(windowStart.plus(Duration.standardSeconds(configuration.windowSizeSec)));
+ // Advance the window.
+ windowStart = windowStart.plus(Duration.standardSeconds(configuration.windowPeriodSec));
+ // Retire bids which will never contribute to a future window.
+ if (!retireBids(windowStart)) {
+ // Can fast forward to latest window since no more outstanding bids.
+ windowStart = newWindowStart;
+ }
+ }
+ }
+
+ /**
+ * Add bid to state.
+ */
+ private void captureBid(Bid bid, Instant timestamp) {
+ List<Instant> existing = bids.get(bid.auction);
+ if (existing == null) {
+ existing = new ArrayList<>();
+ bids.put(bid.auction, existing);
+ }
+ existing.add(timestamp);
+ }
+
+ @Override
+ public void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ // Drain the remaining windows.
+ retireWindows(NexmarkUtils.END_OF_TIME);
+ allDone();
+ return;
+ }
+
+ Event event = timestampedEvent.getValue();
+ if (event.bid == null) {
+ // Ignore non-bid events.
+ return;
+ }
+ Instant timestamp = timestampedEvent.getTimestamp();
+ Instant newWindowStart = windowStart(Duration.standardSeconds(configuration.windowSizeSec),
+ Duration.standardSeconds(configuration.windowPeriodSec), timestamp);
+ // Capture results from any windows we can now retire.
+ retireWindows(newWindowStart);
+ // Capture current bid.
+ captureBid(event.bid, timestamp);
+ }
+ }
+
+ public Query5Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java
new file mode 100644
index 0000000..bc6b12c
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.SellerPrice;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query 6, 'Average Selling Price by Seller'. Select the average selling price over the
+ * last 10 closed auctions by the same seller. In CQL syntax:
+ *
+ * <pre>{@code
+ * SELECT Istream(AVG(Q.final), Q.seller)
+ * FROM (SELECT Rstream(MAX(B.price) AS final, A.seller)
+ * FROM Auction A [ROWS UNBOUNDED], Bid B [ROWS UNBOUNDED]
+ * WHERE A.id=B.auction AND B.datetime < A.expires AND A.expires < CURRENT_TIME
+ * GROUP BY A.id, A.seller) [PARTITION BY A.seller ROWS 10] Q
+ * GROUP BY Q.seller;
+ * }</pre>
+ *
+ * <p>We are a little more exact with selecting winning bids: see {@link WinningBids}.
+ */
+public class Query6 extends NexmarkQuery {
+ /**
+ * Combiner to keep track of up to {@code maxNumBids} of the most recent wining bids and calculate
+ * their average selling price.
+ */
+ private static class MovingMeanSellingPrice extends Combine.CombineFn<Bid, List<Bid>, Long> {
+ private final int maxNumBids;
+
+ public MovingMeanSellingPrice(int maxNumBids) {
+ this.maxNumBids = maxNumBids;
+ }
+
+ @Override
+ public List<Bid> createAccumulator() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<Bid> addInput(List<Bid> accumulator, Bid input) {
+ accumulator.add(input);
+ Collections.sort(accumulator, Bid.ASCENDING_TIME_THEN_PRICE);
+ if (accumulator.size() > maxNumBids) {
+ accumulator.remove(0);
+ }
+ return accumulator;
+ }
+
+ @Override
+ public List<Bid> mergeAccumulators(Iterable<List<Bid>> accumulators) {
+ List<Bid> result = new ArrayList<>();
+ for (List<Bid> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ Collections.sort(result, Bid.ASCENDING_TIME_THEN_PRICE);
+ if (result.size() > maxNumBids) {
+ result = Lists.newArrayList(result.listIterator(result.size() - maxNumBids));
+ }
+ return result;
+ }
+
+ @Override
+ public Long extractOutput(List<Bid> accumulator) {
+ if (accumulator.isEmpty()) {
+ return 0L;
+ }
+ long sumOfPrice = 0;
+ for (Bid bid : accumulator) {
+ sumOfPrice += bid.price;
+ }
+ return Math.round((double) sumOfPrice / accumulator.size());
+ }
+ }
+
+ public Query6(NexmarkConfiguration configuration) {
+ super(configuration, "Query6");
+ }
+
+ private PCollection<SellerPrice> applyTyped(PCollection<Event> events) {
+ return events
+ // Find the winning bid for each closed auction.
+ .apply(new WinningBids(name + ".WinningBids", configuration))
+
+ // Key the winning bid by the seller id.
+ .apply(name + ".Rekey",
+ ParDo.of(new DoFn<AuctionBid, KV<Long, Bid>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ Auction auction = c.element().auction;
+ Bid bid = c.element().bid;
+ c.output(KV.of(auction.seller, bid));
+ }
+ }))
+
+ // Re-window to update on every wining bid.
+ .apply(
+ Window.<KV<Long, Bid>>into(new GlobalWindows())
+ .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.ZERO))
+
+ // Find the average of last 10 winning bids for each seller.
+ .apply(Combine.<Long, Bid, Long>perKey(new MovingMeanSellingPrice(10)))
+
+ // Project into our datatype.
+ .apply(name + ".Select",
+ ParDo.of(new DoFn<KV<Long, Long>, SellerPrice>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(new SellerPrice(c.element().getKey(), c.element().getValue()));
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6Model.java
new file mode 100644
index 0000000..b5152d8
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6Model.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.SellerPrice;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+import org.junit.Assert;
+
+/**
+ * A direct implementation of {@link Query6}.
+ */
+public class Query6Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 6.
+ */
+ private static class Simulator extends AbstractSimulator<AuctionBid, SellerPrice> {
+ /** The cumulative count of winning bids, indexed by seller id. */
+ private final Map<Long, Long> numWinningBidsPerSeller;
+
+ /** The cumulative total of winning bid prices, indexed by seller id. */
+ private final Map<Long, Long> totalWinningBidPricesPerSeller;
+
+ private Instant lastTimestamp;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(new WinningBidsSimulator(configuration).results());
+ numWinningBidsPerSeller = new TreeMap<>();
+ totalWinningBidPricesPerSeller = new TreeMap<>();
+ lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ }
+
+ /**
+ * Update the per-seller running counts/sums.
+ */
+ private void captureWinningBid(Auction auction, Bid bid, Instant timestamp) {
+ NexmarkUtils.info("winning auction, bid: %s, %s", auction, bid);
+ Long count = numWinningBidsPerSeller.get(auction.seller);
+ if (count == null) {
+ count = 1L;
+ } else {
+ count += 1;
+ }
+ numWinningBidsPerSeller.put(auction.seller, count);
+ Long total = totalWinningBidPricesPerSeller.get(auction.seller);
+ if (total == null) {
+ total = bid.price;
+ } else {
+ total += bid.price;
+ }
+ totalWinningBidPricesPerSeller.put(auction.seller, total);
+ TimestampedValue<SellerPrice> intermediateResult = TimestampedValue.of(
+ new SellerPrice(auction.seller, Math.round((double) total / count)), timestamp);
+ addIntermediateResult(intermediateResult);
+ }
+
+
+ @Override
+ protected void run() {
+ TimestampedValue<AuctionBid> timestampedWinningBid = nextInput();
+ if (timestampedWinningBid == null) {
+ for (Map.Entry<Long, Long> entry : numWinningBidsPerSeller.entrySet()) {
+ long seller = entry.getKey();
+ long count = entry.getValue();
+ long total = totalWinningBidPricesPerSeller.get(seller);
+ addResult(TimestampedValue.of(
+ new SellerPrice(seller, Math.round((double) total / count)), lastTimestamp));
+ }
+ allDone();
+ return;
+ }
+
+ lastTimestamp = timestampedWinningBid.getTimestamp();
+ captureWinningBid(timestampedWinningBid.getValue().auction,
+ timestampedWinningBid.getValue().bid, lastTimestamp);
+ }
+ }
+
+ public Query6Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected Iterable<TimestampedValue<KnownSize>> relevantResults(
+ Iterable<TimestampedValue<KnownSize>> results) {
+ // Find the last (in processing time) reported average price for each seller.
+ Map<Long, TimestampedValue<KnownSize>> finalAverages = new TreeMap<>();
+ for (TimestampedValue<KnownSize> obj : results) {
+ Assert.assertTrue("have SellerPrice", obj.getValue() instanceof SellerPrice);
+ SellerPrice sellerPrice = (SellerPrice) obj.getValue();
+ finalAverages.put(
+ sellerPrice.seller, TimestampedValue.of((KnownSize) sellerPrice, obj.getTimestamp()));
+ }
+ return finalAverages.values();
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7.java
new file mode 100644
index 0000000..71b75c3
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Max;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import org.joda.time.Duration;
+
+/**
+ * Query 7, 'Highest Bid'. Select the bids with the highest bid
+ * price in the last minute. In CQL syntax:
+ *
+ * <pre>
+ * SELECT Rstream(B.auction, B.price, B.bidder)
+ * FROM Bid [RANGE 1 MINUTE SLIDE 1 MINUTE] B
+ * WHERE B.price = (SELECT MAX(B1.price)
+ * FROM BID [RANGE 1 MINUTE SLIDE 1 MINUTE] B1);
+ * </pre>
+ *
+ * <p>We will use a shorter window to help make testing easier. We'll also implement this using
+ * a side-input in order to exercise that functionality. (A combiner, as used in Query 5, is
+ * a more efficient approach.).
+ */
+public class Query7 extends NexmarkQuery {
+ public Query7(NexmarkConfiguration configuration) {
+ super(configuration, "Query7");
+ }
+
+ private PCollection<Bid> applyTyped(PCollection<Event> events) {
+ // Window the bids.
+ PCollection<Bid> slidingBids = events.apply(JUST_BIDS).apply(
+ Window.<Bid>into(FixedWindows.of(Duration.standardSeconds(configuration.windowSizeSec))));
+
+ // Find the largest price in all bids.
+ // NOTE: It would be more efficient to write this query much as we did for Query5, using
+ // a binary combiner to accumulate the bids with maximal price. As written this query
+ // requires an additional scan per window, with the associated cost of snapshotted state and
+ // its I/O. We'll keep this implementation since it illustrates the use of side inputs.
+ final PCollectionView<Long> maxPriceView =
+ slidingBids
+ .apply("BidToPrice", BID_TO_PRICE)
+ .apply(Max.longsGlobally().withFanout(configuration.fanout).asSingletonView());
+
+ return slidingBids
+ // Select all bids which have that maximum price (there may be more than one).
+ .apply(name + ".Select", ParDo
+ .of(new DoFn<Bid, Bid>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ long maxPrice = c.sideInput(maxPriceView);
+ Bid bid = c.element();
+ if (bid.price == maxPrice) {
+ c.output(bid);
+ }
+ }
+ })
+ .withSideInputs(maxPriceView));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7Model.java
new file mode 100644
index 0000000..4011746
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7Model.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A direct implementation of {@link Query7}.
+ */
+public class Query7Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 7.
+ */
+ private class Simulator extends AbstractSimulator<Event, Bid> {
+ /** Bids with highest bid price seen in the current window. */
+ private final List<Bid> highestBids;
+
+ /** When current window started. */
+ private Instant windowStart;
+
+ private Instant lastTimestamp;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ highestBids = new ArrayList<>();
+ windowStart = NexmarkUtils.BEGINNING_OF_TIME;
+ lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ }
+
+ /**
+ * Transfer the currently winning bids into results and retire them.
+ */
+ private void retireWindow(Instant timestamp) {
+ for (Bid bid : highestBids) {
+ addResult(TimestampedValue.of(bid, timestamp));
+ }
+ highestBids.clear();
+ }
+
+ /**
+ * Keep just the highest price bid.
+ */
+ private void captureBid(Bid bid) {
+ Iterator<Bid> itr = highestBids.iterator();
+ boolean isWinning = true;
+ while (itr.hasNext()) {
+ Bid existingBid = itr.next();
+ if (existingBid.price > bid.price) {
+ isWinning = false;
+ break;
+ }
+ NexmarkUtils.info("smaller price: %s", existingBid);
+ itr.remove();
+ }
+ if (isWinning) {
+ NexmarkUtils.info("larger price: %s", bid);
+ highestBids.add(bid);
+ }
+ }
+
+ @Override
+ protected void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ // Capture all remaining bids in results.
+ retireWindow(lastTimestamp);
+ allDone();
+ return;
+ }
+
+ Event event = timestampedEvent.getValue();
+ if (event.bid == null) {
+ // Ignore non-bid events.
+ return;
+ }
+ lastTimestamp = timestampedEvent.getTimestamp();
+ Instant newWindowStart = windowStart(Duration.standardSeconds(configuration.windowSizeSec),
+ Duration.standardSeconds(configuration.windowSizeSec), lastTimestamp);
+ if (!newWindowStart.equals(windowStart)) {
+ // Capture highest priced bids in current window and retire it.
+ retireWindow(lastTimestamp);
+ windowStart = newWindowStart;
+ }
+ // Keep only the highest bids.
+ captureBid(event.bid);
+ }
+ }
+
+ public Query7Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValueOrder(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8.java
new file mode 100644
index 0000000..def7cb3
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.IdNameReserve;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+
+/**
+ * Query 8, 'Monitor New Users'. Select people who have entered the system and created auctions
+ * in the last 12 hours, updated every 12 hours. In CQL syntax:
+ *
+ * <pre>
+ * SELECT Rstream(P.id, P.name, A.reserve)
+ * FROM Person [RANGE 12 HOUR] P, Auction [RANGE 12 HOUR] A
+ * WHERE P.id = A.seller;
+ * </pre>
+ *
+ * <p>To make things a bit more dynamic and easier to test we'll use a much shorter window.
+ */
+public class Query8 extends NexmarkQuery {
+ public Query8(NexmarkConfiguration configuration) {
+ super(configuration, "Query8");
+ }
+
+ private PCollection<IdNameReserve> applyTyped(PCollection<Event> events) {
+ // Window and key new people by their id.
+ PCollection<KV<Long, Person>> personsById =
+ events
+ .apply(JUST_NEW_PERSONS)
+ .apply("Query8.WindowPersons",
+ Window.<Person>into(
+ FixedWindows.of(Duration.standardSeconds(configuration.windowSizeSec))))
+ .apply("PersonById", PERSON_BY_ID);
+
+ // Window and key new auctions by their id.
+ PCollection<KV<Long, Auction>> auctionsBySeller =
+ events.apply(JUST_NEW_AUCTIONS)
+ .apply("Query8.WindowAuctions",
+ Window.<Auction>into(
+ FixedWindows.of(Duration.standardSeconds(configuration.windowSizeSec))))
+ .apply("AuctionBySeller", AUCTION_BY_SELLER);
+
+ // Join people and auctions and project the person id, name and auction reserve price.
+ return KeyedPCollectionTuple.of(PERSON_TAG, personsById)
+ .and(AUCTION_TAG, auctionsBySeller)
+ .apply(CoGroupByKey.<Long>create())
+ .apply(name + ".Select",
+ ParDo.of(new DoFn<KV<Long, CoGbkResult>, IdNameReserve>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ @Nullable Person person = c.element().getValue().getOnly(PERSON_TAG, null);
+ if (person == null) {
+ // Person was not created in last window period.
+ return;
+ }
+ for (Auction auction : c.element().getValue().getAll(AUCTION_TAG)) {
+ c.output(new IdNameReserve(person.id, person.name, auction.reserve));
+ }
+ }
+ }));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java
new file mode 100644
index 0000000..351cef7
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.IdNameReserve;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A direct implementation of {@link Query8}.
+ */
+public class Query8Model extends NexmarkQueryModel implements Serializable {
+ /**
+ * Simulator for query 8.
+ */
+ private class Simulator extends AbstractSimulator<Event, IdNameReserve> {
+ /** New persons seen in the current window, indexed by id. */
+ private final Map<Long, Person> newPersons;
+
+ /** New auctions seen in the current window, indexed by seller id. */
+ private final Multimap<Long, Auction> newAuctions;
+
+ /** When did the current window start. */
+ private Instant windowStart;
+
+ public Simulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ newPersons = new HashMap<>();
+ newAuctions = ArrayListMultimap.create();
+ windowStart = NexmarkUtils.BEGINNING_OF_TIME;
+ }
+
+ /**
+ * Retire all persons added in last window.
+ */
+ private void retirePersons() {
+ for (Map.Entry<Long, Person> entry : newPersons.entrySet()) {
+ NexmarkUtils.info("retire: %s", entry.getValue());
+ }
+ newPersons.clear();
+ }
+
+ /**
+ * Retire all auctions added in last window.
+ */
+ private void retireAuctions() {
+ for (Map.Entry<Long, Auction> entry : newAuctions.entries()) {
+ NexmarkUtils.info("retire: %s", entry.getValue());
+ }
+ newAuctions.clear();
+ }
+
+ /**
+ * Capture new result.
+ */
+ private void addResult(Auction auction, Person person, Instant timestamp) {
+ addResult(TimestampedValue.of(
+ new IdNameReserve(person.id, person.name, auction.reserve), timestamp));
+ }
+
+ @Override
+ public void run() {
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ allDone();
+ return;
+ }
+
+ Event event = timestampedEvent.getValue();
+ if (event.bid != null) {
+ // Ignore bid events.
+ // Keep looking for next events.
+ return;
+ }
+ Instant timestamp = timestampedEvent.getTimestamp();
+ Instant newWindowStart = windowStart(Duration.standardSeconds(configuration.windowSizeSec),
+ Duration.standardSeconds(configuration.windowSizeSec), timestamp);
+ if (!newWindowStart.equals(windowStart)) {
+ // Retire this window.
+ retirePersons();
+ retireAuctions();
+ windowStart = newWindowStart;
+ }
+
+ if (event.newAuction != null) {
+ // Join new auction with existing person, if any.
+ Person person = newPersons.get(event.newAuction.seller);
+ if (person != null) {
+ addResult(event.newAuction, person, timestamp);
+ } else {
+ // Remember auction for future new people.
+ newAuctions.put(event.newAuction.seller, event.newAuction);
+ }
+ } else { // event is not an auction, nor a bid, so it is a person
+ // Join new person with existing auctions.
+ for (Auction auction : newAuctions.get(event.newPerson.id)) {
+ addResult(auction, event.newPerson, timestamp);
+ }
+ // We'll never need these auctions again.
+ newAuctions.removeAll(event.newPerson.id);
+ // Remember person for future auctions.
+ newPersons.put(event.newPerson.id, event.newPerson);
+ }
+ }
+ }
+
+ public Query8Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new Simulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9.java
new file mode 100644
index 0000000..5f11e4e
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Query "9", 'Winning bids'. Select just the winning bids. Not in original NEXMark suite, but
+ * handy for testing. See {@link WinningBids} for the details.
+ */
+public class Query9 extends NexmarkQuery {
+ public Query9(NexmarkConfiguration configuration) {
+ super(configuration, "Query9");
+ }
+
+ private PCollection<AuctionBid> applyTyped(PCollection<Event> events) {
+ return events.apply(new WinningBids(name, configuration));
+ }
+
+ @Override
+ protected PCollection<KnownSize> applyPrim(PCollection<Event> events) {
+ return NexmarkUtils.castToKnownSize(name, applyTyped(events));
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9Model.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9Model.java
new file mode 100644
index 0000000..48d792e
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9Model.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+/**
+ * A direct implementation of {@link Query9}.
+ */
+public class Query9Model extends NexmarkQueryModel implements Serializable {
+ public Query9Model(NexmarkConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ public AbstractSimulator<?, ?> simulator() {
+ return new WinningBidsSimulator(configuration);
+ }
+
+ @Override
+ protected <T> Collection<String> toCollection(Iterator<TimestampedValue<T>> itr) {
+ return toValue(itr);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java
new file mode 100644
index 0000000..bc553c9
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java
@@ -0,0 +1,418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.sources.GeneratorConfig;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Instant;
+
+/**
+ * A transform to find the winning bid for each closed auction. In pseudo CQL syntax:
+ *
+ * <pre>{@code
+ * SELECT Rstream(A.*, B.auction, B.bidder, MAX(B.price), B.dateTime)
+ * FROM Auction A [ROWS UNBOUNDED], Bid B [ROWS UNBOUNDED]
+ * WHERE A.id = B.auction AND B.datetime < A.expires AND A.expires < CURRENT_TIME
+ * GROUP BY A.id
+ * }</pre>
+ *
+ * <p>We will also check that the winning bid is above the auction reserve. Note that
+ * we ignore the auction opening bid value since it has no impact on which bid eventually wins,
+ * if any.
+ *
+ * <p>Our implementation will use a custom windowing function in order to bring bids and
+ * auctions together without requiring global state.
+ */
+public class WinningBids extends PTransform<PCollection<Event>, PCollection<AuctionBid>> {
+ /** Windows for open auctions and bids. */
+ private static class AuctionOrBidWindow extends IntervalWindow {
+ /** Id of auction this window is for. */
+ public final long auction;
+
+ /**
+ * True if this window represents an actual auction, and thus has a start/end
+ * time matching that of the auction. False if this window represents a bid, and
+ * thus has an unbounded start/end time.
+ */
+ public final boolean isAuctionWindow;
+
+ /** For avro only. */
+ private AuctionOrBidWindow() {
+ super(TIMESTAMP_MIN_VALUE, TIMESTAMP_MAX_VALUE);
+ auction = 0;
+ isAuctionWindow = false;
+ }
+
+ private AuctionOrBidWindow(
+ Instant start, Instant end, long auctionId, boolean isAuctionWindow) {
+ super(start, end);
+ this.auction = auctionId;
+ this.isAuctionWindow = isAuctionWindow;
+ }
+
+ /** Return an auction window for {@code auction}. */
+ public static AuctionOrBidWindow forAuction(Instant timestamp, Auction auction) {
+ return new AuctionOrBidWindow(timestamp, new Instant(auction.expires), auction.id, true);
+ }
+
+ /**
+ * Return a bid window for {@code bid}. It should later be merged into
+ * the corresponding auction window. However, it is possible this bid is for an already
+ * expired auction, or for an auction which the system has not yet seen. So we
+ * give the bid a bit of wiggle room in its interval.
+ */
+ public static AuctionOrBidWindow forBid(
+ long expectedAuctionDurationMs, Instant timestamp, Bid bid) {
+ // At this point we don't know which auctions are still valid, and the bid may
+ // be for an auction which won't start until some unknown time in the future
+ // (due to Generator.AUCTION_ID_LEAD in Generator.nextBid).
+ // A real system would atomically reconcile bids and auctions by a separate mechanism.
+ // If we give bids an unbounded window it is possible a bid for an auction which
+ // has already expired would cause the system watermark to stall, since that window
+ // would never be retired.
+ // Instead, we will just give the bid a finite window which expires at
+ // the upper bound of auctions assuming the auction starts at the same time as the bid,
+ // and assuming the system is running at its lowest event rate (as per interEventDelayUs).
+ return new AuctionOrBidWindow(
+ timestamp, timestamp.plus(expectedAuctionDurationMs * 2), bid.auction, false);
+ }
+
+ /** Is this an auction window? */
+ public boolean isAuctionWindow() {
+ return isAuctionWindow;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("AuctionOrBidWindow{start:%s; end:%s; auction:%d; isAuctionWindow:%s}",
+ start(), end(), auction, isAuctionWindow);
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ AuctionOrBidWindow that = (AuctionOrBidWindow) o;
+ return (isAuctionWindow == that.isAuctionWindow) && (auction == that.auction);
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(super.hashCode(), isAuctionWindow, auction);
+ }
+ }
+
+ /**
+ * Encodes an {@link AuctionOrBidWindow} as an {@link IntervalWindow} and an auction id long.
+ */
+ private static class AuctionOrBidWindowCoder extends CustomCoder<AuctionOrBidWindow> {
+ private static final AuctionOrBidWindowCoder INSTANCE = new AuctionOrBidWindowCoder();
+ private static final Coder<IntervalWindow> SUPER_CODER = IntervalWindow.getCoder();
+ private static final Coder<Long> ID_CODER = VarLongCoder.of();
+ private static final Coder<Integer> INT_CODER = VarIntCoder.of();
+
+ @JsonCreator
+ public static AuctionOrBidWindowCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(AuctionOrBidWindow window, OutputStream outStream)
+ throws IOException, CoderException {
+ SUPER_CODER.encode(window, outStream);
+ ID_CODER.encode(window.auction, outStream);
+ INT_CODER.encode(window.isAuctionWindow ? 1 : 0, outStream);
+ }
+
+ @Override
+ public AuctionOrBidWindow decode(InputStream inStream)
+ throws IOException, CoderException {
+ IntervalWindow superWindow = SUPER_CODER.decode(inStream);
+ long auction = ID_CODER.decode(inStream);
+ boolean isAuctionWindow = INT_CODER.decode(inStream) != 0;
+ return new AuctionOrBidWindow(
+ superWindow.start(), superWindow.end(), auction, isAuctionWindow);
+ }
+
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+
+ @Override
+ public Object structuralValue(AuctionOrBidWindow value) {
+ return value;
+ }
+ }
+
+ /** Assign events to auction windows and merges them intelligently. */
+ private static class AuctionOrBidWindowFn extends WindowFn<Event, AuctionOrBidWindow> {
+ /** Expected duration of auctions in ms. */
+ private final long expectedAuctionDurationMs;
+
+ public AuctionOrBidWindowFn(long expectedAuctionDurationMs) {
+ this.expectedAuctionDurationMs = expectedAuctionDurationMs;
+ }
+
+ @Override
+ public Collection<AuctionOrBidWindow> assignWindows(AssignContext c) {
+ Event event = c.element();
+ if (event.newAuction != null) {
+ // Assign auctions to an auction window which expires at the auction's close.
+ return Collections
+ .singletonList(AuctionOrBidWindow.forAuction(c.timestamp(), event.newAuction));
+ } else if (event.bid != null) {
+ // Assign bids to a temporary bid window which will later be merged into the appropriate
+ // auction window.
+ return Collections.singletonList(
+ AuctionOrBidWindow.forBid(expectedAuctionDurationMs, c.timestamp(), event.bid));
+ } else {
+ // Don't assign people to any window. They will thus be dropped.
+ return Collections.emptyList();
+ }
+ }
+
+ @Override
+ public void mergeWindows(MergeContext c) throws Exception {
+ // Split and index the auction and bid windows by auction id.
+ Map<Long, AuctionOrBidWindow> idToTrueAuctionWindow = new TreeMap<>();
+ Map<Long, List<AuctionOrBidWindow>> idToBidAuctionWindows = new TreeMap<>();
+ for (AuctionOrBidWindow window : c.windows()) {
+ if (window.isAuctionWindow()) {
+ idToTrueAuctionWindow.put(window.auction, window);
+ } else {
+ List<AuctionOrBidWindow> bidWindows = idToBidAuctionWindows.get(window.auction);
+ if (bidWindows == null) {
+ bidWindows = new ArrayList<>();
+ idToBidAuctionWindows.put(window.auction, bidWindows);
+ }
+ bidWindows.add(window);
+ }
+ }
+
+ // Merge all 'bid' windows into their corresponding 'auction' window, provided the
+ // auction has not expired.
+ for (Map.Entry<Long, AuctionOrBidWindow> entry : idToTrueAuctionWindow.entrySet()) {
+ long auction = entry.getKey();
+ AuctionOrBidWindow auctionWindow = entry.getValue();
+ List<AuctionOrBidWindow> bidWindows = idToBidAuctionWindows.get(auction);
+ if (bidWindows != null) {
+ List<AuctionOrBidWindow> toBeMerged = new ArrayList<>();
+ for (AuctionOrBidWindow bidWindow : bidWindows) {
+ if (bidWindow.start().isBefore(auctionWindow.end())) {
+ toBeMerged.add(bidWindow);
+ }
+ // else: This bid window will remain until its expire time, at which point it
+ // will expire without ever contributing to an output.
+ }
+ if (!toBeMerged.isEmpty()) {
+ toBeMerged.add(auctionWindow);
+ c.merge(toBeMerged, auctionWindow);
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean isCompatible(WindowFn<?, ?> other) {
+ return other instanceof AuctionOrBidWindowFn;
+ }
+
+ @Override
+ public Coder<AuctionOrBidWindow> windowCoder() {
+ return AuctionOrBidWindowCoder.of();
+ }
+
+ @Override
+ public WindowMappingFn<AuctionOrBidWindow> getDefaultWindowMappingFn() {
+ throw new UnsupportedOperationException("AuctionWindowFn not supported for side inputs");
+ }
+
+ /**
+ * Below we will GBK auctions and bids on their auction ids. Then we will reduce those
+ * per id to emit {@code (auction, winning bid)} pairs for auctions which have expired with at
+ * least one valid bid. We would like those output pairs to have a timestamp of the auction's
+ * expiry (since that's the earliest we know for sure we have the correct winner). We would
+ * also like to make that winning results are available to following stages at the auction's
+ * expiry.
+ *
+ * <p>Each result of the GBK will have a timestamp of the min of the result of this object's
+ * assignOutputTime over all records which end up in one of its iterables. Thus we get the
+ * desired behavior if we ignore each record's timestamp and always return the auction window's
+ * 'maxTimestamp', which will correspond to the auction's expiry.
+ *
+ * <p>In contrast, if this object's assignOutputTime were to return 'inputTimestamp'
+ * (the usual implementation), then each GBK record will take as its timestamp the minimum of
+ * the timestamps of all bids and auctions within it, which will always be the auction's
+ * timestamp. An auction which expires well into the future would thus hold up the watermark
+ * of the GBK results until that auction expired. That in turn would hold up all winning pairs.
+ */
+ @Override
+ public Instant getOutputTime(
+ Instant inputTimestamp, AuctionOrBidWindow window) {
+ return window.maxTimestamp();
+ }
+ }
+
+ private final AuctionOrBidWindowFn auctionOrBidWindowFn;
+
+ public WinningBids(String name, NexmarkConfiguration configuration) {
+ super(name);
+ // What's the expected auction time (when the system is running at the lowest event rate).
+ long[] interEventDelayUs = configuration.rateShape.interEventDelayUs(
+ configuration.firstEventRate, configuration.nextEventRate,
+ configuration.rateUnit, configuration.numEventGenerators);
+ long longestDelayUs = 0;
+ for (long interEventDelayU : interEventDelayUs) {
+ longestDelayUs = Math.max(longestDelayUs, interEventDelayU);
+ }
+ // Adjust for proportion of auction events amongst all events.
+ longestDelayUs =
+ (longestDelayUs * GeneratorConfig.PROPORTION_DENOMINATOR)
+ / GeneratorConfig.AUCTION_PROPORTION;
+ // Adjust for number of in-flight auctions.
+ longestDelayUs = longestDelayUs * configuration.numInFlightAuctions;
+ long expectedAuctionDurationMs = (longestDelayUs + 999) / 1000;
+ NexmarkUtils.console("Expected auction duration is %d ms", expectedAuctionDurationMs);
+ auctionOrBidWindowFn = new AuctionOrBidWindowFn(expectedAuctionDurationMs);
+ }
+
+ @Override
+ public PCollection<AuctionBid> expand(PCollection<Event> events) {
+ // Window auctions and bids into custom auction windows. New people events will be discarded.
+ // This will allow us to bring bids and auctions together irrespective of how long
+ // each auction is open for.
+ events = events.apply("Window", Window.into(auctionOrBidWindowFn));
+
+ // Key auctions by their id.
+ PCollection<KV<Long, Auction>> auctionsById =
+ events.apply(NexmarkQuery.JUST_NEW_AUCTIONS)
+ .apply("AuctionById:", NexmarkQuery.AUCTION_BY_ID);
+
+ // Key bids by their auction id.
+ PCollection<KV<Long, Bid>> bidsByAuctionId =
+ events.apply(NexmarkQuery.JUST_BIDS).apply("BidByAuction", NexmarkQuery.BID_BY_AUCTION);
+
+ // Find the highest price valid bid for each closed auction.
+ return
+ // Join auctions and bids.
+ KeyedPCollectionTuple.of(NexmarkQuery.AUCTION_TAG, auctionsById)
+ .and(NexmarkQuery.BID_TAG, bidsByAuctionId)
+ .apply(CoGroupByKey.<Long>create())
+ // Filter and select.
+ .apply(name + ".Join",
+ ParDo.of(new DoFn<KV<Long, CoGbkResult>, AuctionBid>() {
+ private final Counter noAuctionCounter = Metrics.counter(name, "noAuction");
+ private final Counter underReserveCounter = Metrics.counter(name, "underReserve");
+ private final Counter noValidBidsCounter = Metrics.counter(name, "noValidBids");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ @Nullable Auction auction =
+ c.element().getValue().getOnly(NexmarkQuery.AUCTION_TAG, null);
+ if (auction == null) {
+ // We have bids without a matching auction. Give up.
+ noAuctionCounter.inc();
+ return;
+ }
+ // Find the current winning bid for auction.
+ // The earliest bid with the maximum price above the reserve wins.
+ Bid bestBid = null;
+ for (Bid bid : c.element().getValue().getAll(NexmarkQuery.BID_TAG)) {
+ // Bids too late for their auction will have been
+ // filtered out by the window merge function.
+ checkState(bid.dateTime < auction.expires);
+ if (bid.price < auction.reserve) {
+ // Bid price is below auction reserve.
+ underReserveCounter.inc();
+ continue;
+ }
+
+ if (bestBid == null
+ || Bid.PRICE_THEN_DESCENDING_TIME.compare(bid, bestBid) > 0) {
+ bestBid = bid;
+ }
+ }
+ if (bestBid == null) {
+ // We don't have any valid bids for auction.
+ noValidBidsCounter.inc();
+ return;
+ }
+ c.output(new AuctionBid(auction, bestBid));
+ }
+ }
+ ));
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(auctionOrBidWindowFn);
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ WinningBids that = (WinningBids) o;
+ return auctionOrBidWindowFn.equals(that.auctionOrBidWindowFn);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBidsSimulator.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBidsSimulator.java
new file mode 100644
index 0000000..69b64c0
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBidsSimulator.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.AuctionBid;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+
+/**
+ * A simulator of the {@code WinningBids} query.
+ */
+public class WinningBidsSimulator extends AbstractSimulator<Event, AuctionBid> {
+ /** Auctions currently still open, indexed by auction id. */
+ private final Map<Long, Auction> openAuctions;
+
+ /** The ids of auctions known to be closed. */
+ private final Set<Long> closedAuctions;
+
+ /** Current best valid bids for open auctions, indexed by auction id. */
+ private final Map<Long, Bid> bestBids;
+
+ /** Bids for auctions we havn't seen yet. */
+ private final List<Bid> bidsWithoutAuctions;
+
+ /**
+ * Timestamp of last new auction or bid event (ms since epoch).
+ */
+ private long lastTimestamp;
+
+ public WinningBidsSimulator(NexmarkConfiguration configuration) {
+ super(NexmarkUtils.standardEventIterator(configuration));
+ openAuctions = new TreeMap<>();
+ closedAuctions = new TreeSet<>();
+ bestBids = new TreeMap<>();
+ bidsWithoutAuctions = new ArrayList<>();
+ lastTimestamp = BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis();
+ }
+
+ /**
+ * Try to account for {@code bid} in state. Return true if bid has now been
+ * accounted for by {@code bestBids}.
+ */
+ private boolean captureBestBid(Bid bid, boolean shouldLog) {
+ if (closedAuctions.contains(bid.auction)) {
+ // Ignore bids for known, closed auctions.
+ if (shouldLog) {
+ NexmarkUtils.info("closed auction: %s", bid);
+ }
+ return true;
+ }
+ Auction auction = openAuctions.get(bid.auction);
+ if (auction == null) {
+ // We don't have an auction for this bid yet, so can't determine if it is
+ // winning or not.
+ if (shouldLog) {
+ NexmarkUtils.info("pending auction: %s", bid);
+ }
+ return false;
+ }
+ if (bid.price < auction.reserve) {
+ // Bid price is too low.
+ if (shouldLog) {
+ NexmarkUtils.info("below reserve: %s", bid);
+ }
+ return true;
+ }
+ Bid existingBid = bestBids.get(bid.auction);
+ if (existingBid == null || Bid.PRICE_THEN_DESCENDING_TIME.compare(existingBid, bid) < 0) {
+ // We've found a (new) best bid for a known auction.
+ bestBids.put(bid.auction, bid);
+ if (shouldLog) {
+ NexmarkUtils.info("new winning bid: %s", bid);
+ }
+ } else {
+ if (shouldLog) {
+ NexmarkUtils.info("ignoring low bid: %s", bid);
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Try to match bids without auctions to auctions.
+ */
+ private void flushBidsWithoutAuctions() {
+ Iterator<Bid> itr = bidsWithoutAuctions.iterator();
+ while (itr.hasNext()) {
+ Bid bid = itr.next();
+ if (captureBestBid(bid, false)) {
+ NexmarkUtils.info("bid now accounted for: %s", bid);
+ itr.remove();
+ }
+ }
+ }
+
+ /**
+ * Return the next winning bid for an expired auction relative to {@code timestamp}.
+ * Return null if no more winning bids, in which case all expired auctions will
+ * have been removed from our state. Retire auctions in order of expire time.
+ */
+ @Nullable
+ private TimestampedValue<AuctionBid> nextWinningBid(long timestamp) {
+ Map<Long, List<Long>> toBeRetired = new TreeMap<>();
+ for (Map.Entry<Long, Auction> entry : openAuctions.entrySet()) {
+ if (entry.getValue().expires <= timestamp) {
+ List<Long> idsAtTime = toBeRetired.get(entry.getValue().expires);
+ if (idsAtTime == null) {
+ idsAtTime = new ArrayList<>();
+ toBeRetired.put(entry.getValue().expires, idsAtTime);
+ }
+ idsAtTime.add(entry.getKey());
+ }
+ }
+ for (Map.Entry<Long, List<Long>> entry : toBeRetired.entrySet()) {
+ for (long id : entry.getValue()) {
+ Auction auction = openAuctions.get(id);
+ NexmarkUtils.info("retiring auction: %s", auction);
+ openAuctions.remove(id);
+ Bid bestBid = bestBids.get(id);
+ if (bestBid != null) {
+ TimestampedValue<AuctionBid> result =
+ TimestampedValue.of(new AuctionBid(auction, bestBid), new Instant(auction.expires));
+ NexmarkUtils.info("winning: %s", result);
+ return result;
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ protected void run() {
+ if (lastTimestamp > BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()) {
+ // We may have finally seen the auction a bid was intended for.
+ flushBidsWithoutAuctions();
+ TimestampedValue<AuctionBid> result = nextWinningBid(lastTimestamp);
+ if (result != null) {
+ addResult(result);
+ return;
+ }
+ }
+
+ TimestampedValue<Event> timestampedEvent = nextInput();
+ if (timestampedEvent == null) {
+ // No more events. Flush any still open auctions.
+ TimestampedValue<AuctionBid> result =
+ nextWinningBid(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis());
+ if (result == null) {
+ // We are done.
+ allDone();
+ return;
+ }
+ addResult(result);
+ return;
+ }
+
+ Event event = timestampedEvent.getValue();
+ if (event.newPerson != null) {
+ // Ignore new person events.
+ return;
+ }
+
+ lastTimestamp = timestampedEvent.getTimestamp().getMillis();
+ if (event.newAuction != null) {
+ // Add this new open auction to our state.
+ openAuctions.put(event.newAuction.id, event.newAuction);
+ } else {
+ if (!captureBestBid(event.bid, true)) {
+ // We don't know what to do with this bid yet.
+ NexmarkUtils.info("bid not yet accounted for: %s", event.bid);
+ bidsWithoutAuctions.add(event.bid);
+ }
+ }
+ // Keep looking for winning bids.
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/package-info.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/package-info.java
new file mode 100644
index 0000000..2ca5a1c
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Nexmark Queries.
+ */
+package org.apache.beam.sdk.nexmark.queries;
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSource.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSource.java
new file mode 100644
index 0000000..60124bb
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSource.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+
+/**
+ * A custom, bounded source of event records.
+ */
+public class BoundedEventSource extends BoundedSource<Event> {
+ /** Configuration we generate events against. */
+ private final GeneratorConfig config;
+
+ /** How many bounded sources to create. */
+ private final int numEventGenerators;
+
+ public BoundedEventSource(GeneratorConfig config, int numEventGenerators) {
+ this.config = config;
+ this.numEventGenerators = numEventGenerators;
+ }
+
+ /** A reader to pull events from the generator. */
+ private static class EventReader extends BoundedReader<Event> {
+ /**
+ * Event source we purporting to be reading from.
+ * (We can't use Java's capture-outer-class pointer since we must update
+ * this field on calls to splitAtFraction.)
+ */
+ private BoundedEventSource source;
+
+ /** Generator we are reading from. */
+ private final Generator generator;
+
+ private boolean reportedStop;
+
+ @Nullable
+ private TimestampedValue<Event> currentEvent;
+
+ public EventReader(BoundedEventSource source, GeneratorConfig config) {
+ this.source = source;
+ generator = new Generator(config);
+ reportedStop = false;
+ }
+
+ @Override
+ public synchronized boolean start() {
+ NexmarkUtils.info("starting bounded generator %s", generator);
+ return advance();
+ }
+
+ @Override
+ public synchronized boolean advance() {
+ if (!generator.hasNext()) {
+ // No more events.
+ if (!reportedStop) {
+ reportedStop = true;
+ NexmarkUtils.info("stopped bounded generator %s", generator);
+ }
+ return false;
+ }
+ currentEvent = generator.next();
+ return true;
+ }
+
+ @Override
+ public synchronized Event getCurrent() throws NoSuchElementException {
+ if (currentEvent == null) {
+ throw new NoSuchElementException();
+ }
+ return currentEvent.getValue();
+ }
+
+ @Override
+ public synchronized Instant getCurrentTimestamp() throws NoSuchElementException {
+ if (currentEvent == null) {
+ throw new NoSuchElementException();
+ }
+ return currentEvent.getTimestamp();
+ }
+
+ @Override
+ public void close() throws IOException {
+ // Nothing to close.
+ }
+
+ @Override
+ public synchronized Double getFractionConsumed() {
+ return generator.getFractionConsumed();
+ }
+
+ @Override
+ public synchronized BoundedSource<Event> getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ @Nullable
+ public synchronized BoundedEventSource splitAtFraction(double fraction) {
+ long startId = generator.getCurrentConfig().getStartEventId();
+ long stopId = generator.getCurrentConfig().getStopEventId();
+ long size = stopId - startId;
+ long splitEventId = startId + Math.min((int) (size * fraction), size);
+ if (splitEventId <= generator.getNextEventId() || splitEventId == stopId) {
+ // Already passed this position or split results in left or right being empty.
+ NexmarkUtils.info("split failed for bounded generator %s at %f", generator, fraction);
+ return null;
+ }
+
+ NexmarkUtils.info("about to split bounded generator %s at %d", generator, splitEventId);
+
+ // Scale back the event space of the current generator, and return a generator config
+ // representing the event space we just 'stole' from the current generator.
+ GeneratorConfig remainingConfig = generator.splitAtEventId(splitEventId);
+
+ NexmarkUtils.info("split bounded generator into %s and %s", generator, remainingConfig);
+
+ // At this point
+ // generator.events() ++ new Generator(remainingConfig).events()
+ // == originalGenerator.events()
+
+ // We need a new source to represent the now smaller key space for this reader, so
+ // that we can maintain the invariant that
+ // this.getCurrentSource().createReader(...)
+ // will yield the same output as this.
+ source = new BoundedEventSource(generator.getCurrentConfig(), source.numEventGenerators);
+
+ // Return a source from which we may read the 'stolen' event space.
+ return new BoundedEventSource(remainingConfig, source.numEventGenerators);
+ }
+ }
+
+ @Override
+ public List<BoundedEventSource> split(
+ long desiredBundleSizeBytes, PipelineOptions options) {
+ NexmarkUtils.info("slitting bounded source %s into %d sub-sources", config, numEventGenerators);
+ List<BoundedEventSource> results = new ArrayList<>();
+ // Ignore desiredBundleSizeBytes and use numEventGenerators instead.
+ for (GeneratorConfig subConfig : config.split(numEventGenerators)) {
+ results.add(new BoundedEventSource(subConfig, 1));
+ }
+ return results;
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions options) {
+ return config.getEstimatedSizeBytes();
+ }
+
+ @Override
+ public EventReader createReader(PipelineOptions options) {
+ NexmarkUtils.info("creating initial bounded reader for %s", config);
+ return new EventReader(this, config);
+ }
+
+ @Override
+ public void validate() {
+ // Nothing to validate.
+ }
+
+ @Override
+ public Coder<Event> getDefaultOutputCoder() {
+ return Event.CODER;
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/Generator.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/Generator.java
new file mode 100644
index 0000000..c368d72
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/Generator.java
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.nexmark.model.Auction;
+import org.apache.beam.sdk.nexmark.model.Bid;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.model.Person;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Instant;
+
+/**
+ * A generator for synthetic events. We try to make the data vaguely reasonable. We also ensure
+ * most primary key/foreign key relations are correct. Eg: a {@link Bid} event will usually have
+ * valid auction and bidder ids which can be joined to already-generated Auction and Person events.
+ *
+ * <p>To help with testing, we generate timestamps relative to a given {@code baseTime}. Each new
+ * event is given a timestamp advanced from the previous timestamp by {@code interEventDelayUs}
+ * (in microseconds). The event stream is thus fully deterministic and does not depend on
+ * wallclock time.
+ *
+ * <p>This class implements {@link org.apache.beam.sdk.io.UnboundedSource.CheckpointMark}
+ * so that we can resume generating events from a saved snapshot.
+ */
+public class Generator implements Iterator<TimestampedValue<Event>>, Serializable {
+ /**
+ * Keep the number of categories small so the example queries will find results even with
+ * a small batch of events.
+ */
+ private static final int NUM_CATEGORIES = 5;
+
+ /** Smallest random string size. */
+ private static final int MIN_STRING_LENGTH = 3;
+
+ /**
+ * Keep the number of states small so that the example queries will find results even with
+ * a small batch of events.
+ */
+ private static final List<String> US_STATES = Arrays.asList(("AZ,CA,ID,OR,WA,WY").split(","));
+
+ private static final List<String> US_CITIES =
+ Arrays.asList(
+ ("Phoenix,Los Angeles,San Francisco,Boise,Portland,Bend,Redmond,Seattle,Kent,Cheyenne")
+ .split(","));
+
+ private static final List<String> FIRST_NAMES =
+ Arrays.asList(("Peter,Paul,Luke,John,Saul,Vicky,Kate,Julie,Sarah,Deiter,Walter").split(","));
+
+ private static final List<String> LAST_NAMES =
+ Arrays.asList(("Shultz,Abrams,Spencer,White,Bartels,Walton,Smith,Jones,Noris").split(","));
+
+ /**
+ * Number of yet-to-be-created people and auction ids allowed.
+ */
+ private static final int PERSON_ID_LEAD = 10;
+ private static final int AUCTION_ID_LEAD = 10;
+
+ /**
+ * Fraction of people/auctions which may be 'hot' sellers/bidders/auctions are 1
+ * over these values.
+ */
+ private static final int HOT_AUCTION_RATIO = 100;
+ private static final int HOT_SELLER_RATIO = 100;
+ private static final int HOT_BIDDER_RATIO = 100;
+
+ /**
+ * Just enough state to be able to restore a generator back to where it was checkpointed.
+ */
+ public static class Checkpoint implements UnboundedSource.CheckpointMark {
+ private static final Coder<Long> LONG_CODER = VarLongCoder.of();
+
+ /** Coder for this class. */
+ public static final Coder<Checkpoint> CODER_INSTANCE =
+ new CustomCoder<Checkpoint>() {
+ @Override public void encode(Checkpoint value, OutputStream outStream)
+ throws CoderException, IOException {
+ LONG_CODER.encode(value.numEvents, outStream);
+ LONG_CODER.encode(value.wallclockBaseTime, outStream);
+ }
+
+ @Override
+ public Checkpoint decode(InputStream inStream)
+ throws CoderException, IOException {
+ long numEvents = LONG_CODER.decode(inStream);
+ long wallclockBaseTime = LONG_CODER.decode(inStream);
+ return new Checkpoint(numEvents, wallclockBaseTime);
+ }
+ @Override public void verifyDeterministic() throws NonDeterministicException {}
+ };
+
+ private final long numEvents;
+ private final long wallclockBaseTime;
+
+ private Checkpoint(long numEvents, long wallclockBaseTime) {
+ this.numEvents = numEvents;
+ this.wallclockBaseTime = wallclockBaseTime;
+ }
+
+ public Generator toGenerator(GeneratorConfig config) {
+ return new Generator(config, numEvents, wallclockBaseTime);
+ }
+
+ @Override
+ public void finalizeCheckpoint() throws IOException {
+ // Nothing to finalize.
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Generator.Checkpoint{numEvents:%d;wallclockBaseTime:%d}",
+ numEvents, wallclockBaseTime);
+ }
+ }
+
+ /**
+ * The next event and its various timestamps. Ordered by increasing wallclock timestamp, then
+ * (arbitrary but stable) event hash order.
+ */
+ public static class NextEvent implements Comparable<NextEvent> {
+ /** When, in wallclock time, should this event be emitted? */
+ public final long wallclockTimestamp;
+
+ /** When, in event time, should this event be considered to have occured? */
+ public final long eventTimestamp;
+
+ /** The event itself. */
+ public final Event event;
+
+ /** The minimum of this and all future event timestamps. */
+ public final long watermark;
+
+ public NextEvent(long wallclockTimestamp, long eventTimestamp, Event event, long watermark) {
+ this.wallclockTimestamp = wallclockTimestamp;
+ this.eventTimestamp = eventTimestamp;
+ this.event = event;
+ this.watermark = watermark;
+ }
+
+ /**
+ * Return a deep copy of next event with delay added to wallclock timestamp and
+ * event annotate as 'LATE'.
+ */
+ public NextEvent withDelay(long delayMs) {
+ return new NextEvent(
+ wallclockTimestamp + delayMs, eventTimestamp, event.withAnnotation("LATE"), watermark);
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ NextEvent nextEvent = (NextEvent) o;
+
+ return (wallclockTimestamp == nextEvent.wallclockTimestamp
+ && eventTimestamp == nextEvent.eventTimestamp
+ && watermark == nextEvent.watermark
+ && event.equals(nextEvent.event));
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(wallclockTimestamp, eventTimestamp, watermark, event);
+ }
+
+ @Override
+ public int compareTo(NextEvent other) {
+ int i = Long.compare(wallclockTimestamp, other.wallclockTimestamp);
+ if (i != 0) {
+ return i;
+ }
+ return Integer.compare(event.hashCode(), other.event.hashCode());
+ }
+ }
+
+ /**
+ * Configuration to generate events against. Note that it may be replaced by a call to
+ * {@link #splitAtEventId}.
+ */
+ private GeneratorConfig config;
+
+ /** Number of events generated by this generator. */
+ private long numEvents;
+
+ /**
+ * Wallclock time at which we emitted the first event (ms since epoch). Initially -1.
+ */
+ private long wallclockBaseTime;
+
+ private Generator(GeneratorConfig config, long numEvents, long wallclockBaseTime) {
+ checkNotNull(config);
+ this.config = config;
+ this.numEvents = numEvents;
+ this.wallclockBaseTime = wallclockBaseTime;
+ }
+
+ /**
+ * Create a fresh generator according to {@code config}.
+ */
+ public Generator(GeneratorConfig config) {
+ this(config, 0, -1);
+ }
+
+ /**
+ * Return a checkpoint for the current generator.
+ */
+ public Checkpoint toCheckpoint() {
+ return new Checkpoint(numEvents, wallclockBaseTime);
+ }
+
+ /**
+ * Return a deep copy of this generator.
+ */
+ public Generator copy() {
+ checkNotNull(config);
+ Generator result = new Generator(config, numEvents, wallclockBaseTime);
+ return result;
+ }
+
+ /**
+ * Return the current config for this generator. Note that configs may be replaced by {@link
+ * #splitAtEventId}.
+ */
+ public GeneratorConfig getCurrentConfig() {
+ return config;
+ }
+
+ /**
+ * Mutate this generator so that it will only generate events up to but not including
+ * {@code eventId}. Return a config to represent the events this generator will no longer yield.
+ * The generators will run in on a serial timeline.
+ */
+ public GeneratorConfig splitAtEventId(long eventId) {
+ long newMaxEvents = eventId - (config.firstEventId + config.firstEventNumber);
+ GeneratorConfig remainConfig = config.copyWith(config.firstEventId,
+ config.maxEvents - newMaxEvents, config.firstEventNumber + newMaxEvents);
+ config = config.copyWith(config.firstEventId, newMaxEvents, config.firstEventNumber);
+ return remainConfig;
+ }
+
+ /**
+ * Return the next 'event id'. Though events don't have ids we can simulate them to
+ * help with bookkeeping.
+ */
+ public long getNextEventId() {
+ return config.firstEventId + config.nextAdjustedEventNumber(numEvents);
+ }
+
+ /**
+ * Return the last valid person id (ignoring FIRST_PERSON_ID). Will be the current person id if
+ * due to generate a person.
+ */
+ private long lastBase0PersonId() {
+ long eventId = getNextEventId();
+ long epoch = eventId / GeneratorConfig.PROPORTION_DENOMINATOR;
+ long offset = eventId % GeneratorConfig.PROPORTION_DENOMINATOR;
+ if (offset >= GeneratorConfig.PERSON_PROPORTION) {
+ // About to generate an auction or bid.
+ // Go back to the last person generated in this epoch.
+ offset = GeneratorConfig.PERSON_PROPORTION - 1;
+ }
+ // About to generate a person.
+ return epoch * GeneratorConfig.PERSON_PROPORTION + offset;
+ }
+
+ /**
+ * Return the last valid auction id (ignoring FIRST_AUCTION_ID). Will be the current auction id if
+ * due to generate an auction.
+ */
+ private long lastBase0AuctionId() {
+ long eventId = getNextEventId();
+ long epoch = eventId / GeneratorConfig.PROPORTION_DENOMINATOR;
+ long offset = eventId % GeneratorConfig.PROPORTION_DENOMINATOR;
+ if (offset < GeneratorConfig.PERSON_PROPORTION) {
+ // About to generate a person.
+ // Go back to the last auction in the last epoch.
+ epoch--;
+ offset = GeneratorConfig.AUCTION_PROPORTION - 1;
+ } else if (offset >= GeneratorConfig.PERSON_PROPORTION + GeneratorConfig.AUCTION_PROPORTION) {
+ // About to generate a bid.
+ // Go back to the last auction generated in this epoch.
+ offset = GeneratorConfig.AUCTION_PROPORTION - 1;
+ } else {
+ // About to generate an auction.
+ offset -= GeneratorConfig.PERSON_PROPORTION;
+ }
+ return epoch * GeneratorConfig.AUCTION_PROPORTION + offset;
+ }
+
+ /** return a random US state. */
+ private static String nextUSState(Random random) {
+ return US_STATES.get(random.nextInt(US_STATES.size()));
+ }
+
+ /** Return a random US city. */
+ private static String nextUSCity(Random random) {
+ return US_CITIES.get(random.nextInt(US_CITIES.size()));
+ }
+
+ /** Return a random person name. */
+ private static String nextPersonName(Random random) {
+ return FIRST_NAMES.get(random.nextInt(FIRST_NAMES.size())) + " "
+ + LAST_NAMES.get(random.nextInt(LAST_NAMES.size()));
+ }
+
+ /** Return a random string of up to {@code maxLength}. */
+ private static String nextString(Random random, int maxLength) {
+ int len = MIN_STRING_LENGTH + random.nextInt(maxLength - MIN_STRING_LENGTH);
+ StringBuilder sb = new StringBuilder();
+ while (len-- > 0) {
+ if (random.nextInt(13) == 0) {
+ sb.append(' ');
+ } else {
+ sb.append((char) ('a' + random.nextInt(26)));
+ }
+ }
+ return sb.toString().trim();
+ }
+
+ /** Return a random string of exactly {@code length}. */
+ private static String nextExactString(Random random, int length) {
+ StringBuilder sb = new StringBuilder();
+ while (length-- > 0) {
+ sb.append((char) ('a' + random.nextInt(26)));
+ }
+ return sb.toString();
+ }
+
+ /** Return a random email address. */
+ private static String nextEmail(Random random) {
+ return nextString(random, 7) + "@" + nextString(random, 5) + ".com";
+ }
+
+ /** Return a random credit card number. */
+ private static String nextCreditCard(Random random) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 4; i++) {
+ if (i > 0) {
+ sb.append(' ');
+ }
+ sb.append(String.format("%04d", random.nextInt(10000)));
+ }
+ return sb.toString();
+ }
+
+ /** Return a random price. */
+ private static long nextPrice(Random random) {
+ return Math.round(Math.pow(10.0, random.nextDouble() * 6.0) * 100.0);
+ }
+
+ /** Return a random time delay, in milliseconds, for length of auctions. */
+ private long nextAuctionLengthMs(Random random, long timestamp) {
+ // What's our current event number?
+ long currentEventNumber = config.nextAdjustedEventNumber(numEvents);
+ // How many events till we've generated numInFlightAuctions?
+ long numEventsForAuctions =
+ (config.configuration.numInFlightAuctions * GeneratorConfig.PROPORTION_DENOMINATOR)
+ / GeneratorConfig.AUCTION_PROPORTION;
+ // When will the auction numInFlightAuctions beyond now be generated?
+ long futureAuction =
+ config.timestampAndInterEventDelayUsForEvent(currentEventNumber + numEventsForAuctions)
+ .getKey();
+ // System.out.printf("*** auction will be for %dms (%d events ahead) ***\n",
+ // futureAuction - timestamp, numEventsForAuctions);
+ // Choose a length with average horizonMs.
+ long horizonMs = futureAuction - timestamp;
+ return 1L + nextLong(random, Math.max(horizonMs * 2, 1L));
+ }
+
+ /**
+ * Return a random {@code string} such that {@code currentSize + string.length()} is on average
+ * {@code averageSize}.
+ */
+ private static String nextExtra(Random random, int currentSize, int desiredAverageSize) {
+ if (currentSize > desiredAverageSize) {
+ return "";
+ }
+ desiredAverageSize -= currentSize;
+ int delta = (int) Math.round(desiredAverageSize * 0.2);
+ int minSize = desiredAverageSize - delta;
+ int desiredSize = minSize + (delta == 0 ? 0 : random.nextInt(2 * delta));
+ return nextExactString(random, desiredSize);
+ }
+
+ /** Return a random long from {@code [0, n)}. */
+ private static long nextLong(Random random, long n) {
+ if (n < Integer.MAX_VALUE) {
+ return random.nextInt((int) n);
+ } else {
+ // WARNING: Very skewed distribution! Bad!
+ return Math.abs(random.nextLong() % n);
+ }
+ }
+
+ /**
+ * Generate and return a random person with next available id.
+ */
+ private Person nextPerson(Random random, long timestamp) {
+ long id = lastBase0PersonId() + GeneratorConfig.FIRST_PERSON_ID;
+ String name = nextPersonName(random);
+ String email = nextEmail(random);
+ String creditCard = nextCreditCard(random);
+ String city = nextUSCity(random);
+ String state = nextUSState(random);
+ int currentSize =
+ 8 + name.length() + email.length() + creditCard.length() + city.length() + state.length();
+ String extra = nextExtra(random, currentSize, config.configuration.avgPersonByteSize);
+ return new Person(id, name, email, creditCard, city, state, timestamp, extra);
+ }
+
+ /**
+ * Return a random person id (base 0).
+ */
+ private long nextBase0PersonId(Random random) {
+ // Choose a random person from any of the 'active' people, plus a few 'leads'.
+ // By limiting to 'active' we ensure the density of bids or auctions per person
+ // does not decrease over time for long running jobs.
+ // By choosing a person id ahead of the last valid person id we will make
+ // newPerson and newAuction events appear to have been swapped in time.
+ long numPeople = lastBase0PersonId() + 1;
+ long activePeople = Math.min(numPeople, config.configuration.numActivePeople);
+ long n = nextLong(random, activePeople + PERSON_ID_LEAD);
+ return numPeople - activePeople + n;
+ }
+
+ /**
+ * Return a random auction id (base 0).
+ */
+ private long nextBase0AuctionId(Random random) {
+ // Choose a random auction for any of those which are likely to still be in flight,
+ // plus a few 'leads'.
+ // Note that ideally we'd track non-expired auctions exactly, but that state
+ // is difficult to split.
+ long minAuction = Math.max(lastBase0AuctionId() - config.configuration.numInFlightAuctions, 0);
+ long maxAuction = lastBase0AuctionId();
+ return minAuction + nextLong(random, maxAuction - minAuction + 1 + AUCTION_ID_LEAD);
+ }
+
+ /**
+ * Generate and return a random auction with next available id.
+ */
+ private Auction nextAuction(Random random, long timestamp) {
+ long id = lastBase0AuctionId() + GeneratorConfig.FIRST_AUCTION_ID;
+
+ long seller;
+ // Here P(auction will be for a hot seller) = 1 - 1/hotSellersRatio.
+ if (random.nextInt(config.configuration.hotSellersRatio) > 0) {
+ // Choose the first person in the batch of last HOT_SELLER_RATIO people.
+ seller = (lastBase0PersonId() / HOT_SELLER_RATIO) * HOT_SELLER_RATIO;
+ } else {
+ seller = nextBase0PersonId(random);
+ }
+ seller += GeneratorConfig.FIRST_PERSON_ID;
+
+ long category = GeneratorConfig.FIRST_CATEGORY_ID + random.nextInt(NUM_CATEGORIES);
+ long initialBid = nextPrice(random);
+ long expires = timestamp + nextAuctionLengthMs(random, timestamp);
+ String name = nextString(random, 20);
+ String desc = nextString(random, 100);
+ long reserve = initialBid + nextPrice(random);
+ int currentSize = 8 + name.length() + desc.length() + 8 + 8 + 8 + 8 + 8;
+ String extra = nextExtra(random, currentSize, config.configuration.avgAuctionByteSize);
+ return new Auction(id, name, desc, initialBid, reserve, timestamp, expires, seller, category,
+ extra);
+ }
+
+ /**
+ * Generate and return a random bid with next available id.
+ */
+ private Bid nextBid(Random random, long timestamp) {
+ long auction;
+ // Here P(bid will be for a hot auction) = 1 - 1/hotAuctionRatio.
+ if (random.nextInt(config.configuration.hotAuctionRatio) > 0) {
+ // Choose the first auction in the batch of last HOT_AUCTION_RATIO auctions.
+ auction = (lastBase0AuctionId() / HOT_AUCTION_RATIO) * HOT_AUCTION_RATIO;
+ } else {
+ auction = nextBase0AuctionId(random);
+ }
+ auction += GeneratorConfig.FIRST_AUCTION_ID;
+
+ long bidder;
+ // Here P(bid will be by a hot bidder) = 1 - 1/hotBiddersRatio
+ if (random.nextInt(config.configuration.hotBiddersRatio) > 0) {
+ // Choose the second person (so hot bidders and hot sellers don't collide) in the batch of
+ // last HOT_BIDDER_RATIO people.
+ bidder = (lastBase0PersonId() / HOT_BIDDER_RATIO) * HOT_BIDDER_RATIO + 1;
+ } else {
+ bidder = nextBase0PersonId(random);
+ }
+ bidder += GeneratorConfig.FIRST_PERSON_ID;
+
+ long price = nextPrice(random);
+ int currentSize = 8 + 8 + 8 + 8;
+ String extra = nextExtra(random, currentSize, config.configuration.avgBidByteSize);
+ return new Bid(auction, bidder, price, timestamp, extra);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return numEvents < config.maxEvents;
+ }
+
+ /**
+ * Return the next event. The outer timestamp is in wallclock time and corresponds to
+ * when the event should fire. The inner timestamp is in event-time and represents the
+ * time the event is purported to have taken place in the simulation.
+ */
+ public NextEvent nextEvent() {
+ if (wallclockBaseTime < 0) {
+ wallclockBaseTime = System.currentTimeMillis();
+ }
+ // When, in event time, we should generate the event. Monotonic.
+ long eventTimestamp =
+ config.timestampAndInterEventDelayUsForEvent(config.nextEventNumber(numEvents)).getKey();
+ // When, in event time, the event should say it was generated. Depending on outOfOrderGroupSize
+ // may have local jitter.
+ long adjustedEventTimestamp =
+ config.timestampAndInterEventDelayUsForEvent(config.nextAdjustedEventNumber(numEvents))
+ .getKey();
+ // The minimum of this and all future adjusted event timestamps. Accounts for jitter in
+ // the event timestamp.
+ long watermark =
+ config.timestampAndInterEventDelayUsForEvent(config.nextEventNumberForWatermark(numEvents))
+ .getKey();
+ // When, in wallclock time, we should emit the event.
+ long wallclockTimestamp = wallclockBaseTime + (eventTimestamp - getCurrentConfig().baseTime);
+
+ // Seed the random number generator with the next 'event id'.
+ Random random = new Random(getNextEventId());
+ long rem = getNextEventId() % GeneratorConfig.PROPORTION_DENOMINATOR;
+
+ Event event;
+ if (rem < GeneratorConfig.PERSON_PROPORTION) {
+ event = new Event(nextPerson(random, adjustedEventTimestamp));
+ } else if (rem < GeneratorConfig.PERSON_PROPORTION + GeneratorConfig.AUCTION_PROPORTION) {
+ event = new Event(nextAuction(random, adjustedEventTimestamp));
+ } else {
+ event = new Event(nextBid(random, adjustedEventTimestamp));
+ }
+
+ numEvents++;
+ return new NextEvent(wallclockTimestamp, adjustedEventTimestamp, event, watermark);
+ }
+
+ @Override
+ public TimestampedValue<Event> next() {
+ NextEvent next = nextEvent();
+ return TimestampedValue.of(next.event, new Instant(next.eventTimestamp));
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Return how many microseconds till we emit the next event.
+ */
+ public long currentInterEventDelayUs() {
+ return config.timestampAndInterEventDelayUsForEvent(config.nextEventNumber(numEvents))
+ .getValue();
+ }
+
+ /**
+ * Return an estimate of fraction of output consumed.
+ */
+ public double getFractionConsumed() {
+ return (double) numEvents / config.maxEvents;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Generator{config:%s; numEvents:%d; wallclockBaseTime:%d}", config,
+ numEvents, wallclockBaseTime);
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/GeneratorConfig.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/GeneratorConfig.java
new file mode 100644
index 0000000..42183c6
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/GeneratorConfig.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.values.KV;
+
+/**
+ * Parameters controlling how {@link Generator} synthesizes {@link Event} elements.
+ */
+public class GeneratorConfig implements Serializable {
+
+ /**
+ * We start the ids at specific values to help ensure the queries find a match even on
+ * small synthesized dataset sizes.
+ */
+ public static final long FIRST_AUCTION_ID = 1000L;
+ public static final long FIRST_PERSON_ID = 1000L;
+ public static final long FIRST_CATEGORY_ID = 10L;
+
+ /**
+ * Proportions of people/auctions/bids to synthesize.
+ */
+ public static final int PERSON_PROPORTION = 1;
+ public static final int AUCTION_PROPORTION = 3;
+ private static final int BID_PROPORTION = 46;
+ public static final int PROPORTION_DENOMINATOR =
+ PERSON_PROPORTION + AUCTION_PROPORTION + BID_PROPORTION;
+
+ /**
+ * Environment options.
+ */
+ public final NexmarkConfiguration configuration;
+
+ /**
+ * Delay between events, in microseconds. If the array has more than one entry then
+ * the rate is changed every {@link #stepLengthSec}, and wraps around.
+ */
+ private final long[] interEventDelayUs;
+
+ /**
+ * Delay before changing the current inter-event delay.
+ */
+ private final long stepLengthSec;
+
+ /**
+ * Time for first event (ms since epoch).
+ */
+ public final long baseTime;
+
+ /**
+ * Event id of first event to be generated. Event ids are unique over all generators, and
+ * are used as a seed to generate each event's data.
+ */
+ public final long firstEventId;
+
+ /**
+ * Maximum number of events to generate.
+ */
+ public final long maxEvents;
+
+ /**
+ * First event number. Generators running in parallel time may share the same event number,
+ * and the event number is used to determine the event timestamp.
+ */
+ public final long firstEventNumber;
+
+ /**
+ * True period of epoch in milliseconds. Derived from above.
+ * (Ie time to run through cycle for all interEventDelayUs entries).
+ */
+ private final long epochPeriodMs;
+
+ /**
+ * Number of events per epoch. Derived from above.
+ * (Ie number of events to run through cycle for all interEventDelayUs entries).
+ */
+ private final long eventsPerEpoch;
+
+ public GeneratorConfig(
+ NexmarkConfiguration configuration, long baseTime, long firstEventId,
+ long maxEventsOrZero, long firstEventNumber) {
+ this.configuration = configuration;
+ this.interEventDelayUs = configuration.rateShape.interEventDelayUs(
+ configuration.firstEventRate, configuration.nextEventRate,
+ configuration.rateUnit, configuration.numEventGenerators);
+ this.stepLengthSec = configuration.rateShape.stepLengthSec(configuration.ratePeriodSec);
+ this.baseTime = baseTime;
+ this.firstEventId = firstEventId;
+ if (maxEventsOrZero == 0) {
+ // Scale maximum down to avoid overflow in getEstimatedSizeBytes.
+ this.maxEvents =
+ Long.MAX_VALUE / (PROPORTION_DENOMINATOR
+ * Math.max(
+ Math.max(configuration.avgPersonByteSize, configuration.avgAuctionByteSize),
+ configuration.avgBidByteSize));
+ } else {
+ this.maxEvents = maxEventsOrZero;
+ }
+ this.firstEventNumber = firstEventNumber;
+
+ long eventsPerEpoch = 0;
+ long epochPeriodMs = 0;
+ if (interEventDelayUs.length > 1) {
+ for (long interEventDelayU : interEventDelayUs) {
+ long numEventsForThisCycle = (stepLengthSec * 1_000_000L) / interEventDelayU;
+ eventsPerEpoch += numEventsForThisCycle;
+ epochPeriodMs += (numEventsForThisCycle * interEventDelayU) / 1000L;
+ }
+ }
+ this.eventsPerEpoch = eventsPerEpoch;
+ this.epochPeriodMs = epochPeriodMs;
+ }
+
+ /**
+ * Return a copy of this config.
+ */
+ public GeneratorConfig copy() {
+ GeneratorConfig result;
+ result = new GeneratorConfig(configuration, baseTime, firstEventId,
+ maxEvents, firstEventNumber);
+ return result;
+ }
+
+ /**
+ * Split this config into {@code n} sub-configs with roughly equal number of
+ * possible events, but distinct value spaces. The generators will run on parallel timelines.
+ * This config should no longer be used.
+ */
+ public List<GeneratorConfig> split(int n) {
+ List<GeneratorConfig> results = new ArrayList<>();
+ if (n == 1) {
+ // No split required.
+ results.add(this);
+ } else {
+ long subMaxEvents = maxEvents / n;
+ long subFirstEventId = firstEventId;
+ for (int i = 0; i < n; i++) {
+ if (i == n - 1) {
+ // Don't loose any events to round-down.
+ subMaxEvents = maxEvents - subMaxEvents * (n - 1);
+ }
+ results.add(copyWith(subFirstEventId, subMaxEvents, firstEventNumber));
+ subFirstEventId += subMaxEvents;
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Return copy of this config except with given parameters.
+ */
+ public GeneratorConfig copyWith(long firstEventId, long maxEvents, long firstEventNumber) {
+ return new GeneratorConfig(configuration, baseTime, firstEventId, maxEvents, firstEventNumber);
+ }
+
+ /**
+ * Return an estimate of the bytes needed by {@code numEvents}.
+ */
+ public long estimatedBytesForEvents(long numEvents) {
+ long numPersons =
+ (numEvents * GeneratorConfig.PERSON_PROPORTION) / GeneratorConfig.PROPORTION_DENOMINATOR;
+ long numAuctions = (numEvents * AUCTION_PROPORTION) / PROPORTION_DENOMINATOR;
+ long numBids = (numEvents * BID_PROPORTION) / PROPORTION_DENOMINATOR;
+ return numPersons * configuration.avgPersonByteSize
+ + numAuctions * configuration.avgAuctionByteSize
+ + numBids * configuration.avgBidByteSize;
+ }
+
+ /**
+ * Return an estimate of the byte-size of all events a generator for this config would yield.
+ */
+ public long getEstimatedSizeBytes() {
+ return estimatedBytesForEvents(maxEvents);
+ }
+
+ /**
+ * Return the first 'event id' which could be generated from this config. Though events don't
+ * have ids we can simulate them to help bookkeeping.
+ */
+ public long getStartEventId() {
+ return firstEventId + firstEventNumber;
+ }
+
+ /**
+ * Return one past the last 'event id' which could be generated from this config.
+ */
+ public long getStopEventId() {
+ return firstEventId + firstEventNumber + maxEvents;
+ }
+
+ /**
+ * Return the next event number for a generator which has so far emitted {@code numEvents}.
+ */
+ public long nextEventNumber(long numEvents) {
+ return firstEventNumber + numEvents;
+ }
+
+ /**
+ * Return the next event number for a generator which has so far emitted {@code numEvents},
+ * but adjusted to account for {@code outOfOrderGroupSize}.
+ */
+ public long nextAdjustedEventNumber(long numEvents) {
+ long n = configuration.outOfOrderGroupSize;
+ long eventNumber = nextEventNumber(numEvents);
+ long base = (eventNumber / n) * n;
+ long offset = (eventNumber * 953) % n;
+ return base + offset;
+ }
+
+ /**
+ * Return the event number who's event time will be a suitable watermark for
+ * a generator which has so far emitted {@code numEvents}.
+ */
+ public long nextEventNumberForWatermark(long numEvents) {
+ long n = configuration.outOfOrderGroupSize;
+ long eventNumber = nextEventNumber(numEvents);
+ return (eventNumber / n) * n;
+ }
+
+ /**
+ * What timestamp should the event with {@code eventNumber} have for this generator? And
+ * what inter-event delay (in microseconds) is current?
+ */
+ public KV<Long, Long> timestampAndInterEventDelayUsForEvent(long eventNumber) {
+ if (interEventDelayUs.length == 1) {
+ long timestamp = baseTime + (eventNumber * interEventDelayUs[0]) / 1000L;
+ return KV.of(timestamp, interEventDelayUs[0]);
+ }
+
+ long epoch = eventNumber / eventsPerEpoch;
+ long n = eventNumber % eventsPerEpoch;
+ long offsetInEpochMs = 0;
+ for (long interEventDelayU : interEventDelayUs) {
+ long numEventsForThisCycle = (stepLengthSec * 1_000_000L) / interEventDelayU;
+ if (n < numEventsForThisCycle) {
+ long offsetInCycleUs = n * interEventDelayU;
+ long timestamp =
+ baseTime + epoch * epochPeriodMs + offsetInEpochMs + (offsetInCycleUs / 1000L);
+ return KV.of(timestamp, interEventDelayU);
+ }
+ n -= numEventsForThisCycle;
+ offsetInEpochMs += (numEventsForThisCycle * interEventDelayU) / 1000L;
+ }
+ throw new RuntimeException("internal eventsPerEpoch incorrect"); // can't reach
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("GeneratorConfig");
+ sb.append("{configuration:");
+ sb.append(configuration.toString());
+ sb.append(";interEventDelayUs=[");
+ for (int i = 0; i < interEventDelayUs.length; i++) {
+ if (i > 0) {
+ sb.append(",");
+ }
+ sb.append(interEventDelayUs[i]);
+ }
+ sb.append("]");
+ sb.append(";stepLengthSec:");
+ sb.append(stepLengthSec);
+ sb.append(";baseTime:");
+ sb.append(baseTime);
+ sb.append(";firstEventId:");
+ sb.append(firstEventId);
+ sb.append(";maxEvents:");
+ sb.append(maxEvents);
+ sb.append(";firstEventNumber:");
+ sb.append(firstEventNumber);
+ sb.append(";epochPeriodMs:");
+ sb.append(epochPeriodMs);
+ sb.append(";eventsPerEpoch:");
+ sb.append(eventsPerEpoch);
+ sb.append("}");
+ return sb.toString();
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSource.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSource.java
new file mode 100644
index 0000000..8f5575c
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSource.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A custom, unbounded source of event records.
+ *
+ * <p>If {@code isRateLimited} is true, events become available for return from the reader such
+ * that the overall rate respect the {@code interEventDelayUs} period if possible. Otherwise,
+ * events are returned every time the system asks for one.
+ */
+public class UnboundedEventSource extends UnboundedSource<Event, Generator.Checkpoint> {
+ private static final Duration BACKLOG_PERIOD = Duration.standardSeconds(30);
+ private static final Logger LOG = LoggerFactory.getLogger(UnboundedEventSource.class);
+
+ /** Configuration for generator to use when reading synthetic events. May be split. */
+ private final GeneratorConfig config;
+
+ /** How many unbounded sources to create. */
+ private final int numEventGenerators;
+
+ /** How many seconds to hold back the watermark. */
+ private final long watermarkHoldbackSec;
+
+ /** Are we rate limiting the events? */
+ private final boolean isRateLimited;
+
+ public UnboundedEventSource(GeneratorConfig config, int numEventGenerators,
+ long watermarkHoldbackSec, boolean isRateLimited) {
+ this.config = config;
+ this.numEventGenerators = numEventGenerators;
+ this.watermarkHoldbackSec = watermarkHoldbackSec;
+ this.isRateLimited = isRateLimited;
+ }
+
+ /** A reader to pull events from the generator. */
+ private class EventReader extends UnboundedReader<Event> {
+ /** Generator we are reading from. */
+ private final Generator generator;
+
+ /**
+ * Current watermark (ms since epoch). Initially set to beginning of time.
+ * Then updated to be the time of the next generated event.
+ * Then, once all events have been generated, set to the end of time.
+ */
+ private long watermark;
+
+ /**
+ * Current backlog (ms), as delay between timestamp of last returned event and the timestamp
+ * we should be up to according to wall-clock time. Used only for logging.
+ */
+ private long backlogDurationMs;
+
+ /**
+ * Current backlog, as estimated number of event bytes we are behind, or null if
+ * unknown. Reported to callers.
+ */
+ @Nullable
+ private Long backlogBytes;
+
+ /**
+ * Wallclock time (ms since epoch) we last reported the backlog, or -1 if never reported.
+ */
+ private long lastReportedBacklogWallclock;
+
+ /**
+ * Event time (ms since epoch) of pending event at last reported backlog, or -1 if never
+ * calculated.
+ */
+ private long timestampAtLastReportedBacklogMs;
+
+ /** Next event to make 'current' when wallclock time has advanced sufficiently. */
+ @Nullable
+ private TimestampedValue<Event> pendingEvent;
+
+ /** Wallclock time when {@link #pendingEvent} is due, or -1 if no pending event. */
+ private long pendingEventWallclockTime;
+
+ /** Current event to return from getCurrent. */
+ @Nullable
+ private TimestampedValue<Event> currentEvent;
+
+ /** Events which have been held back so as to force them to be late. */
+ private final Queue<Generator.NextEvent> heldBackEvents = new PriorityQueue<>();
+
+ public EventReader(Generator generator) {
+ this.generator = generator;
+ watermark = NexmarkUtils.BEGINNING_OF_TIME.getMillis();
+ lastReportedBacklogWallclock = -1;
+ pendingEventWallclockTime = -1;
+ timestampAtLastReportedBacklogMs = -1;
+ }
+
+ public EventReader(GeneratorConfig config) {
+ this(new Generator(config));
+ }
+
+ @Override
+ public boolean start() {
+ LOG.trace("starting unbounded generator {}", generator);
+ return advance();
+ }
+
+
+ @Override
+ public boolean advance() {
+ long now = System.currentTimeMillis();
+
+ while (pendingEvent == null) {
+ if (!generator.hasNext() && heldBackEvents.isEmpty()) {
+ // No more events, EVER.
+ if (isRateLimited) {
+ updateBacklog(System.currentTimeMillis(), 0);
+ }
+ if (watermark < BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ watermark = BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis();
+ LOG.trace("stopped unbounded generator {}", generator);
+ }
+ return false;
+ }
+
+ Generator.NextEvent next = heldBackEvents.peek();
+ if (next != null && next.wallclockTimestamp <= now) {
+ // Time to use the held-back event.
+ heldBackEvents.poll();
+ LOG.debug("replaying held-back event {}ms behind watermark",
+ watermark - next.eventTimestamp);
+ } else if (generator.hasNext()) {
+ next = generator.nextEvent();
+ if (isRateLimited && config.configuration.probDelayedEvent > 0.0
+ && config.configuration.occasionalDelaySec > 0
+ && ThreadLocalRandom.current().nextDouble() < config.configuration.probDelayedEvent) {
+ // We'll hold back this event and go around again.
+ long delayMs =
+ ThreadLocalRandom.current().nextLong(config.configuration.occasionalDelaySec * 1000)
+ + 1L;
+ LOG.debug("delaying event by {}ms", delayMs);
+ heldBackEvents.add(next.withDelay(delayMs));
+ continue;
+ }
+ } else {
+ // Waiting for held-back event to fire.
+ if (isRateLimited) {
+ updateBacklog(now, 0);
+ }
+ return false;
+ }
+
+ pendingEventWallclockTime = next.wallclockTimestamp;
+ pendingEvent = TimestampedValue.of(next.event, new Instant(next.eventTimestamp));
+ long newWatermark =
+ next.watermark - Duration.standardSeconds(watermarkHoldbackSec).getMillis();
+ if (newWatermark > watermark) {
+ watermark = newWatermark;
+ }
+ }
+
+ if (isRateLimited) {
+ if (pendingEventWallclockTime > now) {
+ // We want this event to fire in the future. Try again later.
+ updateBacklog(now, 0);
+ return false;
+ }
+ updateBacklog(now, now - pendingEventWallclockTime);
+ }
+
+ // This event is ready to fire.
+ currentEvent = pendingEvent;
+ pendingEvent = null;
+ return true;
+ }
+
+ private void updateBacklog(long now, long newBacklogDurationMs) {
+ backlogDurationMs = newBacklogDurationMs;
+ long interEventDelayUs = generator.currentInterEventDelayUs();
+ if (interEventDelayUs != 0) {
+ long backlogEvents = (backlogDurationMs * 1000 + interEventDelayUs - 1) / interEventDelayUs;
+ backlogBytes = generator.getCurrentConfig().estimatedBytesForEvents(backlogEvents);
+ }
+ if (lastReportedBacklogWallclock < 0
+ || now - lastReportedBacklogWallclock > BACKLOG_PERIOD.getMillis()) {
+ double timeDialation = Double.NaN;
+ if (pendingEvent != null
+ && lastReportedBacklogWallclock >= 0
+ && timestampAtLastReportedBacklogMs >= 0) {
+ long wallclockProgressionMs = now - lastReportedBacklogWallclock;
+ long eventTimeProgressionMs =
+ pendingEvent.getTimestamp().getMillis() - timestampAtLastReportedBacklogMs;
+ timeDialation = (double) eventTimeProgressionMs / (double) wallclockProgressionMs;
+ }
+ LOG.debug(
+ "unbounded generator backlog now {}ms ({} bytes) at {}us interEventDelay "
+ + "with {} time dilation",
+ backlogDurationMs, backlogBytes, interEventDelayUs, timeDialation);
+ lastReportedBacklogWallclock = now;
+ if (pendingEvent != null) {
+ timestampAtLastReportedBacklogMs = pendingEvent.getTimestamp().getMillis();
+ }
+ }
+ }
+
+ @Override
+ public Event getCurrent() {
+ if (currentEvent == null) {
+ throw new NoSuchElementException();
+ }
+ return currentEvent.getValue();
+ }
+
+ @Override
+ public Instant getCurrentTimestamp() {
+ if (currentEvent == null) {
+ throw new NoSuchElementException();
+ }
+ return currentEvent.getTimestamp();
+ }
+
+ @Override
+ public void close() {
+ // Nothing to close.
+ }
+
+ @Override
+ public UnboundedEventSource getCurrentSource() {
+ return UnboundedEventSource.this;
+ }
+
+ @Override
+ public Instant getWatermark() {
+ return new Instant(watermark);
+ }
+
+ @Override
+ public Generator.Checkpoint getCheckpointMark() {
+ return generator.toCheckpoint();
+ }
+
+ @Override
+ public long getSplitBacklogBytes() {
+ return backlogBytes == null ? BACKLOG_UNKNOWN : backlogBytes;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("EventReader(%d, %d, %d)",
+ generator.getCurrentConfig().getStartEventId(), generator.getNextEventId(),
+ generator.getCurrentConfig().getStopEventId());
+ }
+ }
+
+ @Override
+ public Coder<Generator.Checkpoint> getCheckpointMarkCoder() {
+ return Generator.Checkpoint.CODER_INSTANCE;
+ }
+
+ @Override
+ public List<UnboundedEventSource> split(
+ int desiredNumSplits, PipelineOptions options) {
+ LOG.trace("splitting unbounded source into {} sub-sources", numEventGenerators);
+ List<UnboundedEventSource> results = new ArrayList<>();
+ // Ignore desiredNumSplits and use numEventGenerators instead.
+ for (GeneratorConfig subConfig : config.split(numEventGenerators)) {
+ results.add(new UnboundedEventSource(subConfig, 1, watermarkHoldbackSec, isRateLimited));
+ }
+ return results;
+ }
+
+ @Override
+ public EventReader createReader(
+ PipelineOptions options, @Nullable Generator.Checkpoint checkpoint) {
+ if (checkpoint == null) {
+ LOG.trace("creating initial unbounded reader for {}", config);
+ return new EventReader(config);
+ } else {
+ LOG.trace("resuming unbounded reader from {}", checkpoint);
+ return new EventReader(checkpoint.toGenerator(config));
+ }
+ }
+
+ @Override
+ public void validate() {
+ // Nothing to validate.
+ }
+
+ @Override
+ public Coder<Event> getDefaultOutputCoder() {
+ return Event.CODER;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "UnboundedEventSource(%d, %d)", config.getStartEventId(), config.getStopEventId());
+ }
+}
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/package-info.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/package-info.java
new file mode 100644
index 0000000..266af10
--- /dev/null
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Nexmark Synthetic Sources.
+ */
+package org.apache.beam.sdk.nexmark.sources;
diff --git a/sdks/java/nexmark/src/main/resources/log4j.properties b/sdks/java/nexmark/src/main/resources/log4j.properties
new file mode 100644
index 0000000..14f8acd
--- /dev/null
+++ b/sdks/java/nexmark/src/main/resources/log4j.properties
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the console
+log4j.rootCategory=DEBUG, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c: %m%n
+
+# General Beam loggers
+log4j.logger.org.apache.beam.runners.direct=WARN
+log4j.logger.org.apache.beam.sdk=WARN
+
+# Nexmark specific
+log4j.logger.org.apache.beam.sdk.nexmark=WARN
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.spark_project.jetty=WARN
+log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
+
+# Setting to quiet spark logs, Beam logs should standout
+log4j.logger.org.apache.beam.runners.spark=WARN
+log4j.logger.org.apache.spark=WARN
+log4j.logger.org.spark-project=WARN
+log4j.logger.io.netty=INFO
+
+# Settings to quiet flink logs
+log4j.logger.org.apache.flink=WARN
+
+# Settings to quiet apex logs
+log4j.logger.org.apache.beam.runners.apex=INFO
+log4j.logger.com.datatorrent=ERROR
+log4j.logger.org.apache.hadoop.metrics2=WARN
+log4j.logger.org.apache.commons=WARN
+log4j.logger.org.apache.hadoop.security=WARN
+log4j.logger.org.apache.hadoop.util=WARN
+
+# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
diff --git a/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/QueryTest.java b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/QueryTest.java
new file mode 100644
index 0000000..d8ac057
--- /dev/null
+++ b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/QueryTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.queries;
+
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkUtils;
+import org.apache.beam.sdk.nexmark.model.KnownSize;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesStatefulParDo;
+import org.apache.beam.sdk.testing.UsesTimersInParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test the various NEXMark queries yield results coherent with their models. */
+@RunWith(JUnit4.class)
+public class QueryTest {
+ private static final NexmarkConfiguration CONFIG = NexmarkConfiguration.DEFAULT.copy();
+
+ static {
+ // careful, results of tests are linked to numEventGenerators because of timestamp generation
+ CONFIG.numEventGenerators = 1;
+ CONFIG.numEvents = 1000;
+ }
+
+ @Rule public TestPipeline p = TestPipeline.create();
+
+ /** Test {@code query} matches {@code model}. */
+ private void queryMatchesModel(
+ String name, NexmarkQuery query, NexmarkQueryModel model, boolean streamingMode) {
+ NexmarkUtils.setupPipeline(NexmarkUtils.CoderStrategy.HAND, p);
+ PCollection<TimestampedValue<KnownSize>> results;
+ if (streamingMode) {
+ results =
+ p.apply(name + ".ReadUnBounded", NexmarkUtils.streamEventsSource(CONFIG)).apply(query);
+ } else {
+ results = p.apply(name + ".ReadBounded", NexmarkUtils.batchEventsSource(CONFIG)).apply(query);
+ }
+ PAssert.that(results).satisfies(model.assertionFor());
+ PipelineResult result = p.run();
+ result.waitUntilFinish();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query0MatchesModelBatch() {
+ queryMatchesModel("Query0TestBatch", new Query0(CONFIG), new Query0Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query0MatchesModelStreaming() {
+ queryMatchesModel("Query0TestStreaming", new Query0(CONFIG), new Query0Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query1MatchesModelBatch() {
+ queryMatchesModel("Query1TestBatch", new Query1(CONFIG), new Query1Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query1MatchesModelStreaming() {
+ queryMatchesModel("Query1TestStreaming", new Query1(CONFIG), new Query1Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query2MatchesModelBatch() {
+ queryMatchesModel("Query2TestBatch", new Query2(CONFIG), new Query2Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query2MatchesModelStreaming() {
+ queryMatchesModel("Query2TestStreaming", new Query2(CONFIG), new Query2Model(CONFIG), true);
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesStatefulParDo.class, UsesTimersInParDo.class})
+ public void query3MatchesModelBatch() {
+ queryMatchesModel("Query3TestBatch", new Query3(CONFIG), new Query3Model(CONFIG), false);
+ }
+
+ @Test
+ @Category({NeedsRunner.class, UsesStatefulParDo.class, UsesTimersInParDo.class})
+ public void query3MatchesModelStreaming() {
+ queryMatchesModel("Query3TestStreaming", new Query3(CONFIG), new Query3Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query4MatchesModelBatch() {
+ queryMatchesModel("Query4TestBatch", new Query4(CONFIG), new Query4Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query4MatchesModelStreaming() {
+ queryMatchesModel("Query4TestStreaming", new Query4(CONFIG), new Query4Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query5MatchesModelBatch() {
+ queryMatchesModel("Query5TestBatch", new Query5(CONFIG), new Query5Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query5MatchesModelStreaming() {
+ queryMatchesModel("Query5TestStreaming", new Query5(CONFIG), new Query5Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query6MatchesModelBatch() {
+ queryMatchesModel("Query6TestBatch", new Query6(CONFIG), new Query6Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query6MatchesModelStreaming() {
+ queryMatchesModel("Query6TestStreaming", new Query6(CONFIG), new Query6Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query7MatchesModelBatch() {
+ queryMatchesModel("Query7TestBatch", new Query7(CONFIG), new Query7Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query7MatchesModelStreaming() {
+ queryMatchesModel("Query7TestStreaming", new Query7(CONFIG), new Query7Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query8MatchesModelBatch() {
+ queryMatchesModel("Query8TestBatch", new Query8(CONFIG), new Query8Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query8MatchesModelStreaming() {
+ queryMatchesModel("Query8TestStreaming", new Query8(CONFIG), new Query8Model(CONFIG), true);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query9MatchesModelBatch() {
+ queryMatchesModel("Query9TestBatch", new Query9(CONFIG), new Query9Model(CONFIG), false);
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void query9MatchesModelStreaming() {
+ queryMatchesModel("Query9TestStreaming", new Query9(CONFIG), new Query9Model(CONFIG), true);
+ }
+}
diff --git a/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSourceTest.java b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSourceTest.java
new file mode 100644
index 0000000..3590d64
--- /dev/null
+++ b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSourceTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.NexmarkOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test {@link BoundedEventSource}.
+ */
+@RunWith(JUnit4.class)
+public class BoundedEventSourceTest {
+ private GeneratorConfig makeConfig(long n) {
+ return new GeneratorConfig(
+ NexmarkConfiguration.DEFAULT, System.currentTimeMillis(), 0, n, 0);
+ }
+
+ @Test
+ public void sourceAndReadersWork() throws Exception {
+ NexmarkOptions options = PipelineOptionsFactory.as(NexmarkOptions.class);
+ long n = 200L;
+ BoundedEventSource source = new BoundedEventSource(makeConfig(n), 1);
+
+ SourceTestUtils.assertUnstartedReaderReadsSameAsItsSource(
+ source.createReader(options), options);
+ }
+
+ @Test
+ public void splitAtFractionRespectsContract() throws Exception {
+ NexmarkOptions options = PipelineOptionsFactory.as(NexmarkOptions.class);
+ long n = 20L;
+ BoundedEventSource source = new BoundedEventSource(makeConfig(n), 1);
+
+ // Can't split if already consumed.
+ SourceTestUtils.assertSplitAtFractionFails(source, 10, 0.3, options);
+
+ SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent(source, 5, 0.3, options);
+
+ SourceTestUtils.assertSplitAtFractionExhaustive(source, options);
+ }
+
+ @Test
+ public void splitIntoBundlesRespectsContract() throws Exception {
+ NexmarkOptions options = PipelineOptionsFactory.as(NexmarkOptions.class);
+ long n = 200L;
+ BoundedEventSource source = new BoundedEventSource(makeConfig(n), 1);
+ SourceTestUtils.assertSourcesEqualReferenceSource(
+ source, source.split(10, options), options);
+ }
+}
diff --git a/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/GeneratorTest.java b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/GeneratorTest.java
new file mode 100644
index 0000000..9553d22
--- /dev/null
+++ b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/GeneratorTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test {@link Generator}.
+ */
+@RunWith(JUnit4.class)
+public class GeneratorTest {
+ private GeneratorConfig makeConfig(long n) {
+ return new GeneratorConfig(
+ NexmarkConfiguration.DEFAULT, System.currentTimeMillis(), 0, n, 0);
+ }
+
+ private <T> long consume(long n, Iterator<T> itr) {
+ for (long i = 0; i < n; i++) {
+ assertTrue(itr.hasNext());
+ itr.next();
+ }
+ return n;
+ }
+
+ private <T> long consume(Iterator<T> itr) {
+ long n = 0;
+ while (itr.hasNext()) {
+ itr.next();
+ n++;
+ }
+ return n;
+ }
+
+ @Test
+ public void splitAtFractionPreservesOverallEventCount() {
+ long n = 55729L;
+ GeneratorConfig initialConfig = makeConfig(n);
+ long expected = initialConfig.getStopEventId() - initialConfig.getStartEventId();
+
+ long actual = 0;
+
+ Generator initialGenerator = new Generator(initialConfig);
+
+ // Consume some events.
+ actual += consume(5000, initialGenerator);
+
+
+ // Split once.
+ GeneratorConfig remainConfig1 = initialGenerator.splitAtEventId(9000L);
+ Generator remainGenerator1 = new Generator(remainConfig1);
+
+ // Consume some more events.
+ actual += consume(2000, initialGenerator);
+ actual += consume(3000, remainGenerator1);
+
+ // Split again.
+ GeneratorConfig remainConfig2 = remainGenerator1.splitAtEventId(30000L);
+ Generator remainGenerator2 = new Generator(remainConfig2);
+
+ // Run to completion.
+ actual += consume(initialGenerator);
+ actual += consume(remainGenerator1);
+ actual += consume(remainGenerator2);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void splitPreservesOverallEventCount() {
+ long n = 51237L;
+ GeneratorConfig initialConfig = makeConfig(n);
+ long expected = initialConfig.getStopEventId() - initialConfig.getStartEventId();
+
+ List<Generator> generators = new ArrayList<>();
+ for (GeneratorConfig subConfig : initialConfig.split(20)) {
+ generators.add(new Generator(subConfig));
+ }
+
+ long actual = 0;
+ for (Generator generator : generators) {
+ actual += consume(generator);
+ }
+
+ assertEquals(expected, actual);
+ }
+}
diff --git a/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSourceTest.java b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSourceTest.java
new file mode 100644
index 0000000..3853ede
--- /dev/null
+++ b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSourceTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test UnboundedEventSource.
+ */
+@RunWith(JUnit4.class)
+public class UnboundedEventSourceTest {
+ private GeneratorConfig makeConfig(long n) {
+ return new GeneratorConfig(
+ NexmarkConfiguration.DEFAULT, System.currentTimeMillis(), 0, n, 0);
+ }
+
+ /**
+ * Helper for tracking which ids we've seen (so we can detect dups) and
+ * confirming reading events match the model events.
+ */
+ private static class EventIdChecker {
+ private final Set<Long> seenPersonIds = new HashSet<>();
+ private final Set<Long> seenAuctionIds = new HashSet<>();
+
+ public void add(Event event) {
+ if (event.newAuction != null) {
+ assertTrue(seenAuctionIds.add(event.newAuction.id));
+ } else if (event.newPerson != null) {
+ assertTrue(seenPersonIds.add(event.newPerson.id));
+ }
+ }
+
+ public void add(int n, UnboundedReader<Event> reader, Generator modelGenerator)
+ throws IOException {
+ for (int i = 0; i < n; i++) {
+ assertTrue(modelGenerator.hasNext());
+ Event modelEvent = modelGenerator.next().getValue();
+ assertTrue(reader.advance());
+ Event actualEvent = reader.getCurrent();
+ assertEquals(modelEvent.toString(), actualEvent.toString());
+ add(actualEvent);
+ }
+ }
+ }
+
+ /**
+ * Check aggressively checkpointing and resuming a reader gives us exactly the
+ * same event stream as reading directly.
+ */
+ @Test
+ public void resumeFromCheckpoint() throws IOException {
+ Random random = new Random(297);
+ int n = 47293;
+ GeneratorConfig config = makeConfig(n);
+ Generator modelGenerator = new Generator(config);
+
+ EventIdChecker checker = new EventIdChecker();
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ UnboundedEventSource source = new UnboundedEventSource(config, 1, 0, false);
+ UnboundedReader<Event> reader = source.createReader(options, null);
+
+ while (n > 0) {
+ int m = Math.min(459 + random.nextInt(455), n);
+ System.out.printf("reading %d...%n", m);
+ checker.add(m, reader, modelGenerator);
+ n -= m;
+ System.out.printf("splitting with %d remaining...%n", n);
+ CheckpointMark checkpointMark = reader.getCheckpointMark();
+ reader = source.createReader(options, (Generator.Checkpoint) checkpointMark);
+ }
+
+ assertFalse(reader.advance());
+ }
+}
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index 250c85a..c6ab234 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -40,9 +40,11 @@
<module>io</module>
<module>maven-archetypes</module>
<module>extensions</module>
+ <module>fn-execution</module>
<!-- javadoc runs directly from the root parent as the last module
in the build to be able to capture runner-specific javadoc.
<module>javadoc</module> -->
+ <module>nexmark</module>
</modules>
<profiles>
@@ -53,6 +55,7 @@
</activation>
<modules>
<module>harness</module>
+ <module>container</module>
<module>java8tests</module>
</modules>
</profile>
diff --git a/sdks/pom.xml b/sdks/pom.xml
index 32c329d..7c85489 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -33,9 +33,9 @@
<name>Apache Beam :: SDKs</name>
<modules>
- <module>common</module>
+ <module>go</module>
<module>java</module>
- <!--<module>python</module>-->
+ <module>python</module>
</modules>
<profiles>
diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in
index 41d80ef..c97e57a 100644
--- a/sdks/python/MANIFEST.in
+++ b/sdks/python/MANIFEST.in
@@ -15,6 +15,7 @@
# limitations under the License.
#
+include gen_protos.py
include README.md
include NOTICE
include LICENSE
diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py
index 8b772c9..791ebb7 100644
--- a/sdks/python/apache_beam/__init__.py
+++ b/sdks/python/apache_beam/__init__.py
@@ -15,11 +15,12 @@
# limitations under the License.
#
-"""Apache Beam SDK for Python.
+"""
+Apache Beam SDK for Python
+==========================
-Apache Beam <https://beam.apache.org/>
-provides a simple, powerful programming model for building both batch
-and streaming parallel data processing pipelines.
+`Apache Beam <https://beam.apache.org>`_ provides a simple, powerful programming
+model for building both batch and streaming parallel data processing pipelines.
The Apache Beam SDK for Python provides access to Apache Beam capabilities
from the Python programming language.
@@ -33,32 +34,40 @@
--------
The key concepts in this programming model are
-* PCollection: represents a collection of data, which could be
- bounded or unbounded in size.
-* PTransform: represents a computation that transforms input
- PCollections into output PCollections.
-* Pipeline: manages a directed acyclic graph of PTransforms and
- PCollections that is ready for execution.
-* Runner: specifies where and how the Pipeline should execute.
-* Reading and Writing Data: your pipeline can read from an external
- source and write to an external data sink.
+* :class:`~apache_beam.pvalue.PCollection`: represents a collection of data,
+ which could be bounded or unbounded in size.
+* :class:`~apache_beam.transforms.ptransform.PTransform`: represents a
+ computation that transforms input PCollections into output PCollections.
+* :class:`~apache_beam.pipeline.Pipeline`: manages a directed acyclic graph of
+ :class:`~apache_beam.transforms.ptransform.PTransform` s and
+ :class:`~apache_beam.pvalue.PCollection` s that is ready for execution.
+* :class:`~apache_beam.runners.runner.PipelineRunner`: specifies where and how
+ the pipeline should execute.
+* :class:`~apache_beam.io.iobase.Read`: read from an external source.
+* :class:`~apache_beam.io.iobase.Write`: write to an external data sink.
Typical usage
-------------
At the top of your source file::
- import apache_beam as beam
+ import apache_beam as beam
After this import statement
-* transform classes are available as beam.FlatMap, beam.GroupByKey, etc.
-* Pipeline class is available as beam.Pipeline
-* text read/write transforms are available as beam.io.ReadfromText,
- beam.io.WriteToText
+* Transform classes are available as
+ :class:`beam.FlatMap <apache_beam.transforms.core.FlatMap>`,
+ :class:`beam.GroupByKey <apache_beam.transforms.core.GroupByKey>`, etc.
+* Pipeline class is available as
+ :class:`beam.Pipeline <apache_beam.pipeline.Pipeline>`
+* Text read/write transforms are available as
+ :class:`beam.io.ReadFromText <apache_beam.io.textio.ReadFromText>`,
+ :class:`beam.io.WriteToText <apache_beam.io.textio.WriteToText>`.
Examples
--------
-The examples subdirectory has some examples.
+The `examples subdirectory
+<https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples>`_
+has some examples.
"""
diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py
index 10298bf..172ee74 100644
--- a/sdks/python/apache_beam/coders/coder_impl.py
+++ b/sdks/python/apache_beam/coders/coder_impl.py
@@ -26,28 +26,30 @@
For internal use only; no backwards-compatibility guarantees.
"""
+from __future__ import absolute_import
+
from types import NoneType
from apache_beam.coders import observable
-from apache_beam.utils.timestamp import Timestamp
+from apache_beam.utils import windowed_value
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
-from apache_beam.utils import windowed_value
+from apache_beam.utils.timestamp import Timestamp
# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
try:
- from stream import InputStream as create_InputStream
- from stream import OutputStream as create_OutputStream
- from stream import ByteCountingOutputStream
- from stream import get_varint_size
+ from .stream import InputStream as create_InputStream
+ from .stream import OutputStream as create_OutputStream
+ from .stream import ByteCountingOutputStream
+ from .stream import get_varint_size
globals()['create_InputStream'] = create_InputStream
globals()['create_OutputStream'] = create_OutputStream
globals()['ByteCountingOutputStream'] = ByteCountingOutputStream
except ImportError:
- from slow_stream import InputStream as create_InputStream
- from slow_stream import OutputStream as create_OutputStream
- from slow_stream import ByteCountingOutputStream
- from slow_stream import get_varint_size
+ from .slow_stream import InputStream as create_InputStream
+ from .slow_stream import OutputStream as create_OutputStream
+ from .slow_stream import ByteCountingOutputStream
+ from .slow_stream import get_varint_size
# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
@@ -710,6 +712,10 @@
timestamp = MAX_TIMESTAMP.micros
else:
timestamp *= 1000
+ if timestamp > MAX_TIMESTAMP.micros:
+ timestamp = MAX_TIMESTAMP.micros
+ if timestamp < MIN_TIMESTAMP.micros:
+ timestamp = MIN_TIMESTAMP.micros
windows = self._windows_coder.decode_from_stream(in_stream, True)
# Read PaneInfo encoded byte.
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index ce914dd..67d5adb 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -19,20 +19,23 @@
Only those coders listed in __all__ are part of the public API of this module.
"""
+from __future__ import absolute_import
import base64
import cPickle as pickle
+
import google.protobuf
from apache_beam.coders import coder_impl
-from apache_beam.utils import urns
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.utils import proto_utils
+from apache_beam.utils import urns
# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
try:
- from stream import get_varint_size
+ from .stream import get_varint_size
except ImportError:
- from slow_stream import get_varint_size
+ from .slow_stream import get_varint_size
# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
@@ -201,27 +204,79 @@
and self._dict_without_impl() == other._dict_without_impl())
# pylint: enable=protected-access
- def to_runner_api(self, context):
- """For internal use only; no backwards-compatibility guarantees.
+ _known_urns = {}
+
+ @classmethod
+ def register_urn(cls, urn, parameter_type, fn=None):
+ """Registers a urn with a constructor.
+
+ For example, if 'beam:fn:foo' had parameter type FooPayload, one could
+ write `RunnerApiFn.register_urn('bean:fn:foo', FooPayload, foo_from_proto)`
+ where foo_from_proto took as arguments a FooPayload and a PipelineContext.
+ This function can also be used as a decorator rather than passing the
+ callable in as the final parameter.
+
+ A corresponding to_runner_api_parameter method would be expected that
+ returns the tuple ('beam:fn:foo', FooPayload)
"""
- # TODO(BEAM-115): Use specialized URNs and components.
- from apache_beam.runners.api import beam_runner_api_pb2
+ def register(fn):
+ cls._known_urns[urn] = parameter_type, fn
+ return staticmethod(fn)
+ if fn:
+ # Used as a statement.
+ register(fn)
+ else:
+ # Used as a decorator.
+ return register
+
+ def to_runner_api(self, context):
+ urn, typed_param, components = self.to_runner_api_parameter(context)
return beam_runner_api_pb2.Coder(
spec=beam_runner_api_pb2.SdkFunctionSpec(
spec=beam_runner_api_pb2.FunctionSpec(
- urn=urns.PICKLED_CODER,
- parameter=proto_utils.pack_Any(
- google.protobuf.wrappers_pb2.BytesValue(
- value=serialize_coder(self))))))
+ urn=urn,
+ payload=typed_param.SerializeToString()
+ if typed_param is not None else None)),
+ component_coder_ids=[context.coders.get_id(c) for c in components])
+
+ @classmethod
+ def from_runner_api(cls, coder_proto, context):
+ """Converts from an SdkFunctionSpec to a Fn object.
+
+ Prefer registering a urn with its parameter type and constructor.
+ """
+ parameter_type, constructor = cls._known_urns[coder_proto.spec.spec.urn]
+ return constructor(
+ proto_utils.parse_Bytes(coder_proto.spec.spec.payload, parameter_type),
+ [context.coders.get_by_id(c) for c in coder_proto.component_coder_ids],
+ context)
+
+ def to_runner_api_parameter(self, context):
+ return (
+ urns.PICKLED_CODER,
+ google.protobuf.wrappers_pb2.BytesValue(value=serialize_coder(self)),
+ ())
@staticmethod
- def from_runner_api(proto, context):
- """For internal use only; no backwards-compatibility guarantees.
+ def register_structured_urn(urn, cls):
+ """Register a coder that's completely defined by its urn and its
+ component(s), if any, which are passed to construct the instance.
"""
- any_proto = proto.spec.spec.parameter
- bytes_proto = google.protobuf.wrappers_pb2.BytesValue()
- any_proto.Unpack(bytes_proto)
- return deserialize_coder(bytes_proto.value)
+ cls.to_runner_api_parameter = (
+ lambda self, unused_context: (urn, None, self._get_component_coders()))
+
+ # pylint: disable=unused-variable
+ @Coder.register_urn(urn, None)
+ def from_runner_api_parameter(unused_payload, components, unused_context):
+ if components:
+ return cls(*components)
+ else:
+ return cls()
+
+
+@Coder.register_urn(urns.PICKLED_CODER, google.protobuf.wrappers_pb2.BytesValue)
+def _pickle_from_runner_api_parameter(payload, components, context):
+ return deserialize_coder(payload.value)
class StrUtf8Coder(Coder):
@@ -286,6 +341,11 @@
def is_deterministic(self):
return True
+ def as_cloud_object(self):
+ return {
+ '@type': 'kind:bytes',
+ }
+
def __eq__(self, other):
return type(self) == type(other)
@@ -293,6 +353,9 @@
return hash(type(self))
+Coder.register_structured_urn(urns.BYTES_CODER, BytesCoder)
+
+
class VarIntCoder(FastCoder):
"""Variable-length integer coder."""
@@ -309,6 +372,9 @@
return hash(type(self))
+Coder.register_structured_urn(urns.VAR_INT_CODER, VarIntCoder)
+
+
class FloatCoder(FastCoder):
"""A coder used for floating-point values."""
@@ -365,7 +431,7 @@
# We need to use the dill pickler for objects of certain custom classes,
# including, for example, ones that contain lambdas.
try:
- return pickle.dumps(o)
+ return pickle.dumps(o, pickle.HIGHEST_PROTOCOL)
except Exception: # pylint: disable=broad-except
return dill.dumps(o)
@@ -426,7 +492,10 @@
"""Coder using Python's pickle functionality."""
def _create_impl(self):
- return coder_impl.CallbackCoderImpl(pickle.dumps, pickle.loads)
+ dumps = pickle.dumps
+ HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
+ return coder_impl.CallbackCoderImpl(
+ lambda x: dumps(x, HIGHEST_PROTOCOL), pickle.loads)
class DillCoder(_PickleCoderBase):
@@ -515,7 +584,7 @@
# than via a special Coder.
def encode(self, value):
- return base64.b64encode(pickle.dumps(value))
+ return base64.b64encode(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
def decode(self, encoded):
return pickle.loads(base64.b64decode(encoded))
@@ -639,6 +708,16 @@
def __hash__(self):
return hash(self._coders)
+ def to_runner_api_parameter(self, context):
+ if self.is_kv_coder():
+ return urns.KV_CODER, None, self.coders()
+ else:
+ return super(TupleCoder, self).to_runner_api_parameter(context)
+
+ @Coder.register_urn(urns.KV_CODER, None)
+ def from_runner_api_parameter(unused_payload, components, unused_context):
+ return TupleCoder(components)
+
class TupleSequenceCoder(FastCoder):
"""Coder of homogeneous tuple objects."""
@@ -710,6 +789,9 @@
return hash((type(self), self._elem_coder))
+Coder.register_structured_urn(urns.ITERABLE_CODER, IterableCoder)
+
+
class GlobalWindowCoder(SingletonCoder):
"""Coder for global windows."""
@@ -723,6 +805,9 @@
}
+Coder.register_structured_urn(urns.GLOBAL_WINDOW_CODER, GlobalWindowCoder)
+
+
class IntervalWindowCoder(FastCoder):
"""Coder for an window defined by a start timestamp and a duration."""
@@ -744,6 +829,9 @@
return hash(type(self))
+Coder.register_structured_urn(urns.INTERVAL_WINDOW_CODER, IntervalWindowCoder)
+
+
class WindowedValueCoder(FastCoder):
"""Coder for windowed values."""
@@ -800,6 +888,9 @@
(self.wrapped_value_coder, self.timestamp_coder, self.window_coder))
+Coder.register_structured_urn(urns.WINDOWED_VALUE_CODER, WindowedValueCoder)
+
+
class LengthPrefixCoder(FastCoder):
"""For internal use only; no backwards-compatibility guarantees.
@@ -839,3 +930,6 @@
def __hash__(self):
return hash((type(self), self._value_coder))
+
+
+Coder.register_structured_urn(urns.LENGTH_PREFIX_CODER, LengthPrefixCoder)
diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py
index c89e810..705de89 100644
--- a/sdks/python/apache_beam/coders/coders_test.py
+++ b/sdks/python/apache_beam/coders/coders_test.py
@@ -20,8 +20,8 @@
import logging
import unittest
-from apache_beam.coders import coders
from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
+from apache_beam.coders import coders
from apache_beam.coders.typecoders import registry as coders_registry
diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py
index c9b67b3..fc7279d 100644
--- a/sdks/python/apache_beam/coders/coders_test_common.py
+++ b/sdks/python/apache_beam/coders/coders_test_common.py
@@ -16,6 +16,7 @@
#
"""Tests common to all coder implementations."""
+from __future__ import absolute_import
import logging
import math
@@ -23,13 +24,16 @@
import dill
-import observable
+from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
+from apache_beam.coders import coders
+from apache_beam.runners import pipeline_context
from apache_beam.transforms import window
+from apache_beam.transforms.window import GlobalWindow
from apache_beam.utils import timestamp
from apache_beam.utils import windowed_value
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
-from apache_beam.coders import coders
-from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
+from . import observable
# Defined out of line for picklability.
@@ -88,7 +92,8 @@
self.assertEqual(coder.get_impl().get_estimated_size_and_observables(v),
(coder.get_impl().estimate_size(v), []))
copy1 = dill.loads(dill.dumps(coder))
- copy2 = dill.loads(dill.dumps(coder))
+ context = pipeline_context.PipelineContext()
+ copy2 = coders.Coder.from_runner_api(coder.to_runner_api(context), context)
for v in values:
self.assertEqual(v, copy1.decode(copy2.encode(v)))
if coder.is_deterministic():
@@ -116,7 +121,7 @@
(1, dict()), ('a', [dict()]))
def test_dill_coder(self):
- cell_value = (lambda x: lambda: x)(0).func_closure[0]
+ cell_value = (lambda x: lambda: x)(0).__closure__[0]
self.check_coder(coders.DillCoder(), 'a', 1, cell_value)
self.check_coder(
coders.TupleCoder((coders.VarIntCoder(), coders.DillCoder())),
@@ -287,6 +292,12 @@
# Test binary representation
self.assertEqual('\x7f\xdf;dZ\x1c\xac\t\x00\x00\x00\x01\x0f\x01',
coder.encode(window.GlobalWindows.windowed_value(1)))
+
+ # Test decoding large timestamp
+ self.assertEqual(
+ coder.decode('\x7f\xdf;dZ\x1c\xac\x08\x00\x00\x00\x01\x0f\x00'),
+ windowed_value.create(0, MIN_TIMESTAMP.micros, (GlobalWindow(),)))
+
# Test unnested
self.check_coder(
coders.WindowedValueCoder(coders.VarIntCoder()),
diff --git a/sdks/python/apache_beam/coders/observable_test.py b/sdks/python/apache_beam/coders/observable_test.py
index eaf1aec..09ca304 100644
--- a/sdks/python/apache_beam/coders/observable_test.py
+++ b/sdks/python/apache_beam/coders/observable_test.py
@@ -20,7 +20,6 @@
import logging
import unittest
-
from apache_beam.coders import observable
diff --git a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py
index 16b1b4d..433d33f 100644
--- a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py
+++ b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py
@@ -19,19 +19,19 @@
# source: sdks/java/core/src/main/proto/proto2_coder_test_messages.proto
import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
+
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
-
-
DESCRIPTOR = _descriptor.FileDescriptor(
name='apache_beam/coders/proto2_coder_test_messages.proto',
package='proto2_coder_test_messages',
diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py
index 5f98455..ca13b80 100644
--- a/sdks/python/apache_beam/coders/standard_coders_test.py
+++ b/sdks/python/apache_beam/coders/standard_coders_test.py
@@ -17,6 +17,7 @@
"""Unit tests for coders that must be consistent across all Beam SDKs.
"""
+from __future__ import print_function
import json
import logging
@@ -26,12 +27,12 @@
import yaml
-from apache_beam.coders import coders
from apache_beam.coders import coder_impl
+from apache_beam.coders import coders
+from apache_beam.transforms import window
+from apache_beam.transforms.window import IntervalWindow
from apache_beam.utils import windowed_value
from apache_beam.utils.timestamp import Timestamp
-from apache_beam.transforms.window import IntervalWindow
-from apache_beam.transforms import window
STANDARD_CODERS_YAML = os.path.join(
os.path.dirname(__file__), '..', 'testing', 'data', 'standard_coders.yaml')
@@ -125,14 +126,14 @@
@classmethod
def tearDownClass(cls):
if cls.fix and cls.to_fix:
- print "FIXING", len(cls.to_fix), "TESTS"
+ print("FIXING", len(cls.to_fix), "TESTS")
doc_sep = '\n---\n'
docs = open(STANDARD_CODERS_YAML).read().split(doc_sep)
def quote(s):
return json.dumps(s.decode('latin1')).replace(r'\u0000', r'\0')
for (doc_ix, expected_encoded), actual_encoded in cls.to_fix.items():
- print quote(expected_encoded), "->", quote(actual_encoded)
+ print(quote(expected_encoded), "->", quote(actual_encoded))
docs[doc_ix] = docs[doc_ix].replace(
quote(expected_encoded) + ':', quote(actual_encoded) + ':')
open(STANDARD_CODERS_YAML, 'w').write(doc_sep.join(docs))
diff --git a/sdks/python/apache_beam/coders/stream.pxd b/sdks/python/apache_beam/coders/stream.pxd
index 4e01a89..ade9b72 100644
--- a/sdks/python/apache_beam/coders/stream.pxd
+++ b/sdks/python/apache_beam/coders/stream.pxd
@@ -53,7 +53,7 @@
cdef bytes all
cdef char* allc
- cpdef size_t size(self) except? -1
+ cpdef ssize_t size(self) except? -1
cpdef bytes read(self, size_t len)
cpdef long read_byte(self) except? -1
cpdef libc.stdint.int64_t read_var_int64(self) except? -1
diff --git a/sdks/python/apache_beam/coders/stream.pyx b/sdks/python/apache_beam/coders/stream.pyx
index 8d97681..7c9521a 100644
--- a/sdks/python/apache_beam/coders/stream.pyx
+++ b/sdks/python/apache_beam/coders/stream.pyx
@@ -167,7 +167,7 @@
# unsigned char here.
return <long>(<unsigned char> self.allc[self.pos - 1])
- cpdef size_t size(self) except? -1:
+ cpdef ssize_t size(self) except? -1:
return len(self.all) - self.pos
cpdef bytes read_all(self, bint nested=False):
diff --git a/sdks/python/apache_beam/coders/stream_test.py b/sdks/python/apache_beam/coders/stream_test.py
index e6108b6..15bc5eb 100644
--- a/sdks/python/apache_beam/coders/stream_test.py
+++ b/sdks/python/apache_beam/coders/stream_test.py
@@ -21,7 +21,6 @@
import math
import unittest
-
from apache_beam.coders import slow_stream
diff --git a/sdks/python/apache_beam/coders/typecoders.py b/sdks/python/apache_beam/coders/typecoders.py
index 3894bb5..797aee5 100644
--- a/sdks/python/apache_beam/coders/typecoders.py
+++ b/sdks/python/apache_beam/coders/typecoders.py
@@ -70,7 +70,6 @@
from apache_beam.coders import coders
from apache_beam.typehints import typehints
-
__all__ = ['registry']
diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py
index f0acc3f..b556e65 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete.py
@@ -44,16 +44,17 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
+ def format_result(prefix_candidates):
+ (prefix, candidates) = prefix_candidates
+ return '%s: %s' % (prefix, candidates)
- (p # pylint: disable=expression-not-assigned
- | 'read' >> ReadFromText(known_args.input)
- | 'split' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- | 'TopPerPrefix' >> TopPerPrefix(5)
- | 'format' >> beam.Map(
- lambda (prefix, candidates): '%s: %s' % (prefix, candidates))
- | 'write' >> WriteToText(known_args.output))
- p.run()
+ (p # pylint: disable=expression-not-assigned
+ | 'read' >> ReadFromText(known_args.input)
+ | 'split' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+ | 'TopPerPrefix' >> TopPerPrefix(5)
+ | 'format' >> beam.Map(format_result)
+ | 'write' >> WriteToText(known_args.output))
class TopPerPrefix(beam.PTransform):
diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
index 378d222..888ce44 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
@@ -31,22 +31,21 @@
WORDS = ['this', 'this', 'that', 'to', 'to', 'to']
def test_top_prefixes(self):
- p = TestPipeline()
- words = p | beam.Create(self.WORDS)
- result = words | autocomplete.TopPerPrefix(5)
- # values must be hashable for now
- result = result | beam.Map(lambda (k, vs): (k, tuple(vs)))
- assert_that(result, equal_to(
- [
- ('t', ((3, 'to'), (2, 'this'), (1, 'that'))),
- ('to', ((3, 'to'), )),
- ('th', ((2, 'this'), (1, 'that'))),
- ('thi', ((2, 'this'), )),
- ('this', ((2, 'this'), )),
- ('tha', ((1, 'that'), )),
- ('that', ((1, 'that'), )),
- ]))
- p.run()
+ with TestPipeline() as p:
+ words = p | beam.Create(self.WORDS)
+ result = words | autocomplete.TopPerPrefix(5)
+ # values must be hashable for now
+ result = result | beam.Map(lambda k_vs: (k_vs[0], tuple(k_vs[1])))
+ assert_that(result, equal_to(
+ [
+ ('t', ((3, 'to'), (2, 'this'), (1, 'that'))),
+ ('to', ((3, 'to'), )),
+ ('th', ((2, 'this'), (1, 'that'))),
+ ('thi', ((2, 'this'), )),
+ ('this', ((2, 'this'), )),
+ ('tha', ((1, 'that'), )),
+ ('that', ((1, 'that'), )),
+ ]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py
index c709713..d0a5fb7 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py
@@ -31,14 +31,13 @@
import logging
import random
-
import apache_beam as beam
from apache_beam.io import WriteToText
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.typehints import Any
from apache_beam.typehints import Iterable
from apache_beam.typehints import Tuple
-from apache_beam.options.pipeline_options import PipelineOptions
-from apache_beam.options.pipeline_options import SetupOptions
@beam.typehints.with_output_types(Tuple[int, int, int])
@@ -113,14 +112,11 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- (p # pylint: disable=expression-not-assigned
- | EstimatePiTransform()
- | WriteToText(known_args.output, coder=JsonCoder()))
-
- # Actually run the pipeline (all operations above are deferred).
- p.run()
+ (p # pylint: disable=expression-not-assigned
+ | EstimatePiTransform()
+ | WriteToText(known_args.output, coder=JsonCoder()))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
index fd51309..3463313 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
@@ -22,8 +22,8 @@
from apache_beam.examples.complete import estimate_pi
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that
from apache_beam.testing.util import BeamAssertException
+from apache_beam.testing.util import assert_that
def in_between(lower, upper):
@@ -38,13 +38,13 @@
class EstimatePiTest(unittest.TestCase):
def test_basics(self):
- p = TestPipeline()
- result = p | 'Estimate' >> estimate_pi.EstimatePiTransform(5000)
+ with TestPipeline() as p:
+ result = p | 'Estimate' >> estimate_pi.EstimatePiTransform(5000)
- # Note: Probabilistically speaking this test can fail with a probability
- # that is very small (VERY) given that we run at least 500 thousand trials.
- assert_that(result, in_between(3.125, 3.155))
- p.run()
+ # Note: Probabilistically speaking this test can fail with a probability
+ # that is very small (VERY) given that we run at least 500 thousand
+ # trials.
+ assert_that(result, in_between(3.125, 3.155))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py
new file mode 100644
index 0000000..d8c60dd
--- /dev/null
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py
@@ -0,0 +1,393 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Fourth in a series of four pipelines that tell a story in a 'gaming' domain.
+
+New concepts: session windows and finding session duration; use of both
+singleton and non-singleton side inputs.
+
+This pipeline builds on the {@link LeaderBoard} functionality, and adds some
+"business intelligence" analysis: abuse detection and usage patterns. The
+pipeline derives the Mean user score sum for a window, and uses that information
+to identify likely spammers/robots. (The robots have a higher click rate than
+the human users). The 'robot' users are then filtered out when calculating the
+team scores.
+
+Additionally, user sessions are tracked: that is, we find bursts of user
+activity using session windows. Then, the mean session duration information is
+recorded in the context of subsequent fixed windowing. (This could be used to
+tell us what games are giving us greater user retention).
+
+Run injector.Injector to generate pubsub data for this pipeline. The Injector
+documentation provides more detail on how to do this. The injector is currently
+implemented in Java only, it can be used from the Java SDK.
+
+The PubSub topic you specify should be the same topic to which the Injector is
+publishing.
+
+To run the Java injector:
+<beam_root>/examples/java8$ mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
+ -Dexec.args="$PROJECT_ID $PUBSUB_TOPIC none"
+
+For a description of the usage and options, use -h or --help.
+
+To specify a different runner:
+ --runner YOUR_RUNNER
+
+NOTE: When specifying a different runner, additional runner-specific options
+ may have to be passed in as well
+
+EXAMPLES
+--------
+
+# DirectRunner
+python game_stats.py \
+ --project $PROJECT_ID \
+ --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
+ --dataset $BIGQUERY_DATASET
+
+# DataflowRunner
+python game_stats.py \
+ --project $PROJECT_ID \
+ --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
+ --dataset $BIGQUERY_DATASET \
+ --runner DataflowRunner \
+ --temp_location gs://$BUCKET/user_score/temp
+
+--------------------------------------------------------------------------------
+NOTE [BEAM-2354]: This example is not yet runnable by DataflowRunner.
+ The runner still needs support for:
+ * the --save_main_session flag when streaming is enabled
+ * combiners
+--------------------------------------------------------------------------------
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import csv
+import logging
+import sys
+import time
+from datetime import datetime
+
+import apache_beam as beam
+from apache_beam.metrics.metric import Metrics
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import StandardOptions
+
+
+def timestamp2str(t, fmt='%Y-%m-%d %H:%M:%S.000'):
+ """Converts a unix timestamp into a formatted string."""
+ return datetime.fromtimestamp(t).strftime(fmt)
+
+
+class ParseGameEventFn(beam.DoFn):
+ """Parses the raw game event info into a Python dictionary.
+
+ Each event line has the following format:
+ username,teamname,score,timestamp_in_ms,readable_time
+
+ e.g.:
+ user2_AsparagusPig,AsparagusPig,10,1445230923951,2015-11-02 09:09:28.224
+
+ The human-readable time string is not used here.
+ """
+ def __init__(self):
+ super(ParseGameEventFn, self).__init__()
+ self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
+
+ def process(self, elem):
+ try:
+ row = list(csv.reader([elem]))[0]
+ yield {
+ 'user': row[0],
+ 'team': row[1],
+ 'score': int(row[2]),
+ 'timestamp': int(row[3]) / 1000.0,
+ }
+ except: # pylint: disable=bare-except
+ # Log and count parse errors
+ self.num_parse_errors.inc()
+ logging.error('Parse error on "%s"', elem)
+
+
+class ExtractAndSumScore(beam.PTransform):
+ """A transform to extract key/score information and sum the scores.
+ The constructor argument `field` determines whether 'team' or 'user' info is
+ extracted.
+ """
+ def __init__(self, field):
+ super(ExtractAndSumScore, self).__init__()
+ self.field = field
+
+ def expand(self, pcoll):
+ return (pcoll
+ | beam.Map(lambda elem: (elem[self.field], elem['score']))
+ | beam.CombinePerKey(sum))
+
+
+class TeamScoresDict(beam.DoFn):
+ """Formats the data into a dictionary of BigQuery columns with their values
+
+ Receives a (team, score) pair, extracts the window start timestamp, and
+ formats everything together into a dictionary. The dictionary is in the format
+ {'bigquery_column': value}
+ """
+ def process(self, team_score, window=beam.DoFn.WindowParam):
+ team, score = team_score
+ start = timestamp2str(int(window.start))
+ yield {
+ 'team': team,
+ 'total_score': score,
+ 'window_start': start,
+ 'processing_time': timestamp2str(int(time.time()))
+ }
+
+
+class WriteToBigQuery(beam.PTransform):
+ """Generate, format, and write BigQuery table row information."""
+ def __init__(self, table_name, dataset, schema):
+ """Initializes the transform.
+ Args:
+ table_name: Name of the BigQuery table to use.
+ dataset: Name of the dataset to use.
+ schema: Dictionary in the format {'column_name': 'bigquery_type'}
+ """
+ super(WriteToBigQuery, self).__init__()
+ self.table_name = table_name
+ self.dataset = dataset
+ self.schema = schema
+
+ def get_schema(self):
+ """Build the output table schema."""
+ return ', '.join(
+ '%s:%s' % (col, self.schema[col]) for col in self.schema)
+
+ def get_table(self, pipeline):
+ """Utility to construct an output table reference."""
+ project = pipeline.options.view_as(GoogleCloudOptions).project
+ return '%s:%s.%s' % (project, self.dataset, self.table_name)
+
+ def expand(self, pcoll):
+ table = self.get_table(pcoll.pipeline)
+ return (
+ pcoll
+ | 'ConvertToRow' >> beam.Map(
+ lambda elem: {col: elem[col] for col in self.schema})
+ | beam.io.Write(beam.io.BigQuerySink(
+ table,
+ schema=self.get_schema(),
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)))
+
+
+# [START abuse_detect]
+class CalculateSpammyUsers(beam.PTransform):
+ """Filter out all but those users with a high clickrate, which we will
+ consider as 'spammy' uesrs.
+
+ We do this by finding the mean total score per user, then using that
+ information as a side input to filter out all but those user scores that are
+ larger than (mean * SCORE_WEIGHT).
+ """
+ SCORE_WEIGHT = 2.5
+
+ def expand(self, user_scores):
+ # Get the sum of scores for each user.
+ sum_scores = (
+ user_scores
+ | 'SumUsersScores' >> beam.CombinePerKey(sum))
+
+ # Extract the score from each element, and use it to find the global mean.
+ global_mean_score = (
+ sum_scores
+ | beam.Values()
+ | beam.CombineGlobally(beam.combiners.MeanCombineFn())\
+ .as_singleton_view())
+
+ # Filter the user sums using the global mean.
+ filtered = (
+ sum_scores
+ # Use the derived mean total score (global_mean_score) as a side input.
+ | 'ProcessAndFilter' >> beam.Filter(
+ lambda (_, score), global_mean:\
+ score > global_mean * self.SCORE_WEIGHT,
+ global_mean_score))
+ return filtered
+# [END abuse_detect]
+
+
+class UserSessionActivity(beam.DoFn):
+ """Calculate and output an element's session duration, in seconds."""
+ def process(self, elem, window=beam.DoFn.WindowParam):
+ yield (window.end.micros - window.start.micros) / 1000000
+
+
+def run(argv=None):
+ """Main entry point; defines and runs the hourly_team_score pipeline."""
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('--topic',
+ type=str,
+ required=True,
+ help='Pub/Sub topic to read from')
+ parser.add_argument('--dataset',
+ type=str,
+ required=True,
+ help='BigQuery Dataset to write tables to. '
+ 'Must already exist.')
+ parser.add_argument('--table_name',
+ type=str,
+ default='game_stats',
+ help='The BigQuery table name. Should not already exist.')
+ parser.add_argument('--fixed_window_duration',
+ type=int,
+ default=60,
+ help='Numeric value of fixed window duration for user '
+ 'analysis, in minutes')
+ parser.add_argument('--session_gap',
+ type=int,
+ default=5,
+ help='Numeric value of gap between user sessions, '
+ 'in minutes')
+ parser.add_argument('--user_activity_window_duration',
+ type=int,
+ default=30,
+ help='Numeric value of fixed window for finding mean of '
+ 'user session duration, in minutes')
+
+ args, pipeline_args = parser.parse_known_args(argv)
+
+ options = PipelineOptions(pipeline_args)
+
+ # We also require the --project option to access --dataset
+ if options.view_as(GoogleCloudOptions).project is None:
+ parser.print_usage()
+ print(sys.argv[0] + ': error: argument --project is required')
+ sys.exit(1)
+
+ fixed_window_duration = args.fixed_window_duration * 60
+ session_gap = args.session_gap * 60
+ user_activity_window_duration = args.user_activity_window_duration * 60
+
+ # We use the save_main_session option because one or more DoFn's in this
+ # workflow rely on global context (e.g., a module imported at module level).
+ options.view_as(SetupOptions).save_main_session = True
+
+ # Enforce that this pipeline is always run in streaming mode
+ options.view_as(StandardOptions).streaming = True
+
+ with beam.Pipeline(options=options) as p:
+ # Read events from Pub/Sub using custom timestamps
+ raw_events = (
+ p
+ | 'ReadPubSub' >> beam.io.gcp.pubsub.ReadStringsFromPubSub(args.topic)
+ | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
+ | 'AddEventTimestamps' >> beam.Map(
+ lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
+
+ # Extract username/score pairs from the event stream
+ user_events = (
+ raw_events
+ | 'ExtractUserScores' >> beam.Map(
+ lambda elem: (elem['user'], elem['score'])))
+
+ # Calculate the total score per user over fixed windows, and cumulative
+ # updates for late data
+ spammers_view = (
+ user_events
+ | 'UserFixedWindows' >> beam.WindowInto(
+ beam.window.FixedWindows(fixed_window_duration))
+
+ # Filter out everyone but those with (SCORE_WEIGHT * avg) clickrate.
+ # These might be robots/spammers.
+ | 'CalculateSpammyUsers' >> CalculateSpammyUsers()
+
+ # Derive a view from the collection of spammer users. It will be used as
+ # a side input in calculating the team score sums, below
+ | 'CreateSpammersView' >> beam.CombineGlobally(
+ beam.combiners.ToDictCombineFn()).as_singleton_view())
+
+ # [START filter_and_calc]
+ # Calculate the total score per team over fixed windows, and emit cumulative
+ # updates for late data. Uses the side input derived above --the set of
+ # suspected robots-- to filter out scores from those users from the sum.
+ # Write the results to BigQuery.
+ (raw_events # pylint: disable=expression-not-assigned
+ | 'WindowIntoFixedWindows' >> beam.WindowInto(
+ beam.window.FixedWindows(fixed_window_duration))
+
+ # Filter out the detected spammer users, using the side input derived above
+ | 'FilterOutSpammers' >> beam.Filter(
+ lambda elem, spammers: elem['user'] not in spammers,
+ spammers_view)
+ # Extract and sum teamname/score pairs from the event data.
+ | 'ExtractAndSumScore' >> ExtractAndSumScore('team')
+ # [END filter_and_calc]
+ | 'TeamScoresDict' >> beam.ParDo(TeamScoresDict())
+ | 'WriteTeamScoreSums' >> WriteToBigQuery(
+ args.table_name + '_teams', args.dataset, {
+ 'team': 'STRING',
+ 'total_score': 'INTEGER',
+ 'window_start': 'STRING',
+ 'processing_time': 'STRING',
+ }))
+
+ # [START session_calc]
+ # Detect user sessions-- that is, a burst of activity separated by a gap
+ # from further activity. Find and record the mean session lengths.
+ # This information could help the game designers track the changing user
+ # engagement as their set of game changes.
+ (user_events # pylint: disable=expression-not-assigned
+ | 'WindowIntoSessions' >> beam.WindowInto(
+ beam.window.Sessions(session_gap),
+ timestamp_combiner=beam.window.TimestampCombiner.OUTPUT_AT_EOW)
+
+ # For this use, we care only about the existence of the session, not any
+ # particular information aggregated over it, so we can just group by key
+ # and assign a "dummy value" of None.
+ | beam.CombinePerKey(lambda _: None)
+
+ # Get the duration of the session
+ | 'UserSessionActivity' >> beam.ParDo(UserSessionActivity())
+ # [END session_calc]
+
+ # [START rewindow]
+ # Re-window to process groups of session sums according to when the
+ # sessions complete
+ | 'WindowToExtractSessionMean' >> beam.WindowInto(
+ beam.window.FixedWindows(user_activity_window_duration))
+
+ # Find the mean session duration in each window
+ | beam.CombineGlobally(beam.combiners.MeanCombineFn()).without_defaults()
+ | 'FormatAvgSessionLength' >> beam.Map(
+ lambda elem: {'mean_duration': float(elem)})
+ | 'WriteAvgSessionLength' >> WriteToBigQuery(
+ args.table_name + '_sessions', args.dataset, {
+ 'mean_duration': 'FLOAT',
+ }))
+ # [END rewindow]
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ run()
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py
new file mode 100644
index 0000000..971f9dc
--- /dev/null
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py
@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Test for the game_stats example."""
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.examples.complete.game import game_stats
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+
+class GameStatsTest(unittest.TestCase):
+
+ SAMPLE_DATA = [
+ 'user1_team1,team1,18,1447686663000,2015-11-16 15:11:03.921',
+ 'user1_team1,team1,18,1447690263000,2015-11-16 16:11:03.921',
+ 'user2_team2,team2,2,1447690263000,2015-11-16 16:11:03.955',
+ 'user3_team3,team3,8,1447690263000,2015-11-16 16:11:03.955',
+ 'user4_team3,team3,5,1447690263000,2015-11-16 16:11:03.959',
+ 'user1_team1,team1,14,1447697463000,2015-11-16 18:11:03.955',
+ 'robot1_team1,team1,9000,1447697463000,2015-11-16 18:11:03.955',
+ 'robot2_team2,team2,1,1447697463000,2015-11-16 20:11:03.955',
+ 'robot2_team2,team2,9000,1447697463000,2015-11-16 21:11:03.955',
+ ]
+
+ def create_data(self, p):
+ return (p
+ | beam.Create(GameStatsTest.SAMPLE_DATA)
+ | beam.ParDo(game_stats.ParseGameEventFn())
+ | beam.Map(lambda elem:\
+ beam.window.TimestampedValue(elem, elem['timestamp'])))
+
+ def test_spammy_users(self):
+ with TestPipeline() as p:
+ result = (
+ self.create_data(p)
+ | beam.Map(lambda elem: (elem['user'], elem['score']))
+ | game_stats.CalculateSpammyUsers())
+ assert_that(result, equal_to([
+ ('robot1_team1', 9000), ('robot2_team2', 9001)]))
+
+ def test_game_stats_sessions(self):
+ session_gap = 5 * 60
+ user_activity_window_duration = 30 * 60
+ with TestPipeline() as p:
+ result = (
+ self.create_data(p)
+ | beam.Map(lambda elem: (elem['user'], elem['score']))
+ | 'WindowIntoSessions' >> beam.WindowInto(
+ beam.window.Sessions(session_gap),
+ timestamp_combiner=beam.window.TimestampCombiner.OUTPUT_AT_EOW)
+ | beam.CombinePerKey(lambda _: None)
+ | beam.ParDo(game_stats.UserSessionActivity())
+ | 'WindowToExtractSessionMean' >> beam.WindowInto(
+ beam.window.FixedWindows(user_activity_window_duration))
+ | beam.CombineGlobally(beam.combiners.MeanCombineFn())\
+ .without_defaults())
+ assert_that(result, equal_to([300.0, 300.0, 300.0]))
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
index e9d7188..b286a6a 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
@@ -18,7 +18,7 @@
"""Second in a series of four pipelines that tell a story in a 'gaming' domain.
In addition to the concepts introduced in `user_score`, new concepts include:
-windowing and element timestamps; use of `Filter`.
+windowing and element timestamps; use of `Filter`; using standalone DoFns.
This pipeline processes data collected from gaming events in batch, building on
`user_score` but using fixed windows. It calculates the sum of scores per team,
@@ -31,10 +31,6 @@
in that we don't get results from plays at the beginning of the batch's time
period until the batch is processed.
-To execute this pipeline using the static example input data, specify the
-`--dataset=YOUR-DATASET` flag along with other runner specific flags. (Note:
-BigQuery dataset you specify must already exist.)
-
Optionally include the `--input` argument to specify a batch input file. To
indicate a time after which the data should be filtered out, include the
`--stop_min` arg. E.g., `--stop_min=2015-10-18-23-59` indicates that any data
@@ -43,29 +39,62 @@
the `--start_min` arg. If you're using the default input
"gs://dataflow-samples/game/gaming_data*.csv", then
`--start_min=2015-11-16-16-10 --stop_min=2015-11-17-16-10` are good values.
+
+For a description of the usage and options, use -h or --help.
+
+To specify a different runner:
+ --runner YOUR_RUNNER
+
+NOTE: When specifying a different runner, additional runner-specific options
+ may have to be passed in as well
+
+EXAMPLES
+--------
+
+# DirectRunner
+python hourly_team_score.py \
+ --project $PROJECT_ID \
+ --dataset $BIGQUERY_DATASET
+
+# DataflowRunner
+python hourly_team_score.py \
+ --project $PROJECT_ID \
+ --dataset $BIGQUERY_DATASET \
+ --runner DataflowRunner \
+ --temp_location gs://$BUCKET/user_score/temp
"""
from __future__ import absolute_import
+from __future__ import print_function
import argparse
-import datetime
+import csv
import logging
+import sys
+import time
+from datetime import datetime
import apache_beam as beam
-from apache_beam import typehints
-from apache_beam.io import ReadFromText
-from apache_beam.metrics import Metrics
-from apache_beam.transforms.window import FixedWindows
-from apache_beam.transforms.window import TimestampedValue
-from apache_beam.typehints import with_input_types
-from apache_beam.typehints import with_output_types
+from apache_beam.metrics.metric import Metrics
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
-class ParseEventFn(beam.DoFn):
- """Parses the raw game event info into GameActionInfo tuples.
+def str2timestamp(s, fmt='%Y-%m-%d-%H-%M'):
+ """Converts a string into a unix timestamp."""
+ dt = datetime.strptime(s, fmt)
+ epoch = datetime.utcfromtimestamp(0)
+ return (dt - epoch).total_seconds()
+
+
+def timestamp2str(t, fmt='%Y-%m-%d %H:%M:%S.000'):
+ """Converts a unix timestamp into a formatted string."""
+ return datetime.fromtimestamp(t).strftime(fmt)
+
+
+class ParseGameEventFn(beam.DoFn):
+ """Parses the raw game event info into a Python dictionary.
Each event line has the following format:
username,teamname,score,timestamp_in_ms,readable_time
@@ -76,32 +105,26 @@
The human-readable time string is not used here.
"""
def __init__(self):
- super(ParseEventFn, self).__init__()
+ super(ParseGameEventFn, self).__init__()
self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
- def process(self, element):
- components = element.split(',')
+ def process(self, elem):
try:
- user = components[0].strip()
- team = components[1].strip()
- score = int(components[2].strip())
- timestamp = int(components[3].strip())
- yield {'user': user, 'team': team, 'score': score, 'timestamp': timestamp}
+ row = list(csv.reader([elem]))[0]
+ yield {
+ 'user': row[0],
+ 'team': row[1],
+ 'score': int(row[2]),
+ 'timestamp': int(row[3]) / 1000.0,
+ }
except: # pylint: disable=bare-except
- # Log and count parse errors.
+ # Log and count parse errors
self.num_parse_errors.inc()
- logging.info('Parse error on %s.', element)
-
-
-@with_input_types(ints=typehints.Iterable[int])
-@with_output_types(int)
-def sum_ints(ints):
- return sum(ints)
+ logging.error('Parse error on "%s"', elem)
class ExtractAndSumScore(beam.PTransform):
"""A transform to extract key/score information and sum the scores.
-
The constructor argument `field` determines whether 'team' or 'user' info is
extracted.
"""
@@ -111,75 +134,58 @@
def expand(self, pcoll):
return (pcoll
- | beam.Map(lambda info: (info[self.field], info['score']))
- | beam.CombinePerKey(sum_ints))
+ | beam.Map(lambda elem: (elem[self.field], elem['score']))
+ | beam.CombinePerKey(sum))
-def configure_bigquery_write():
+class TeamScoresDict(beam.DoFn):
+ """Formats the data into a dictionary of BigQuery columns with their values
- def window_start_format(element, window):
- dt = datetime.datetime.fromtimestamp(int(window.start))
- return dt.strftime('%Y-%m-%d %H:%M:%S')
-
- return [
- ('team', 'STRING', lambda e, w: e[0]),
- ('total_score', 'INTEGER', lambda e, w: e[1]),
- ('window_start', 'STRING', window_start_format),
- ]
-
-
-class WriteWindowedToBigQuery(beam.PTransform):
- """Generate, format, and write BigQuery table row information.
-
- This class may be used for writes that require access to the window
- information.
+ Receives a (team, score) pair, extracts the window start timestamp, and
+ formats everything together into a dictionary. The dictionary is in the format
+ {'bigquery_column': value}
"""
- def __init__(self, table_name, dataset, field_info):
- """Initializes the transform.
+ def process(self, team_score, window=beam.DoFn.WindowParam):
+ team, score = team_score
+ start = timestamp2str(int(window.start))
+ yield {
+ 'team': team,
+ 'total_score': score,
+ 'window_start': start,
+ 'processing_time': timestamp2str(int(time.time()))
+ }
+
+class WriteToBigQuery(beam.PTransform):
+ """Generate, format, and write BigQuery table row information."""
+ def __init__(self, table_name, dataset, schema):
+ """Initializes the transform.
Args:
table_name: Name of the BigQuery table to use.
dataset: Name of the dataset to use.
- field_info: List of tuples that holds information about output table field
- definitions. The tuples are in the
- (field_name, field_type, field_fn) format, where field_name is
- the name of the field, field_type is the BigQuery type of the
- field and field_fn is a lambda function to generate the field
- value from the element.
+ schema: Dictionary in the format {'column_name': 'bigquery_type'}
"""
- super(WriteWindowedToBigQuery, self).__init__()
+ super(WriteToBigQuery, self).__init__()
self.table_name = table_name
self.dataset = dataset
- self.field_info = field_info
+ self.schema = schema
def get_schema(self):
"""Build the output table schema."""
return ', '.join(
- '%s:%s' % (entry[0], entry[1]) for entry in self.field_info)
+ '%s:%s' % (col, self.schema[col]) for col in self.schema)
def get_table(self, pipeline):
"""Utility to construct an output table reference."""
project = pipeline.options.view_as(GoogleCloudOptions).project
return '%s:%s.%s' % (project, self.dataset, self.table_name)
- class BuildRowFn(beam.DoFn):
- """Convert each key/score pair into a BigQuery TableRow as specified."""
- def __init__(self, field_info):
- super(WriteWindowedToBigQuery.BuildRowFn, self).__init__()
- self.field_info = field_info
-
- def process(self, element, window=beam.DoFn.WindowParam):
- row = {}
- for entry in self.field_info:
- row[entry[0]] = entry[2](element, window)
- yield row
-
def expand(self, pcoll):
table = self.get_table(pcoll.pipeline)
return (
pcoll
- | 'ConvertToRow' >> beam.ParDo(
- WriteWindowedToBigQuery.BuildRowFn(self.field_info))
+ | 'ConvertToRow' >> beam.Map(
+ lambda elem: {col: elem[col] for col in self.schema})
| beam.io.Write(beam.io.BigQuerySink(
table,
schema=self.get_schema(),
@@ -187,26 +193,19 @@
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)))
-def string_to_timestamp(datetime_str):
- dt = datetime.datetime.strptime(datetime_str, '%Y-%m-%d-%H-%M')
- epoch = datetime.datetime.utcfromtimestamp(0)
- return (dt - epoch).total_seconds() * 1000.0
-
-
+# [START main]
class HourlyTeamScore(beam.PTransform):
def __init__(self, start_min, stop_min, window_duration):
super(HourlyTeamScore, self).__init__()
- self.start_min = start_min
- self.stop_min = stop_min
- self.window_duration = window_duration
+ self.start_timestamp = str2timestamp(start_min)
+ self.stop_timestamp = str2timestamp(stop_min)
+ self.window_duration_in_seconds = window_duration * 60
def expand(self, pcoll):
- start_min_filter = string_to_timestamp(self.start_min)
- end_min_filter = string_to_timestamp(self.stop_min)
-
return (
pcoll
- | 'ParseGameEvent' >> beam.ParDo(ParseEventFn())
+ | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
+
# Filter out data before and after the given times so that it is not
# included in the calculations. As we collect data in batches (say, by
# day), the batch for the day that we want to analyze could potentially
@@ -215,22 +214,24 @@
# (to scoop up late-arriving events from the day we're analyzing), we
# need to weed out events that fall after the time period we want to
# analyze.
+ # [START filter_by_time_range]
| 'FilterStartTime' >> beam.Filter(
- lambda element: element['timestamp'] > start_min_filter)
+ lambda elem: elem['timestamp'] > self.start_timestamp)
| 'FilterEndTime' >> beam.Filter(
- lambda element: element['timestamp'] < end_min_filter)
+ lambda elem: elem['timestamp'] < self.stop_timestamp)
+ # [END filter_by_time_range]
+
+ # [START add_timestamp_and_window]
# Add an element timestamp based on the event log, and apply fixed
# windowing.
- # Convert element['timestamp'] into seconds as expected by
- # TimestampedValue.
| 'AddEventTimestamps' >> beam.Map(
- lambda element: TimestampedValue(
- element, element['timestamp'] / 1000.0))
- # Convert window_duration into seconds as expected by FixedWindows.
- | 'FixedWindowsTeam' >> beam.WindowInto(FixedWindows(
- size=self.window_duration * 60))
+ lambda elem: beam.window.TimestampedValue(elem, elem['timestamp']))
+ | 'FixedWindowsTeam' >> beam.WindowInto(
+ beam.window.FixedWindows(self.window_duration_in_seconds))
+ # [END add_timestamp_and_window]
+
# Extract and sum teamname/score pairs from the event data.
- | 'ExtractTeamScore' >> ExtractAndSumScore('team'))
+ | 'ExtractAndSumScore' >> ExtractAndSumScore('team'))
def run(argv=None):
@@ -240,24 +241,23 @@
# The default maps to two large Google Cloud Storage files (each ~12GB)
# holding two subsequent day's worth (roughly) of data.
parser.add_argument('--input',
- dest='input',
- default='gs://dataflow-samples/game/gaming_data*.csv',
+ type=str,
+ default='gs://apache-beam-samples/game/gaming_data*.csv',
help='Path to the data file(s) containing game data.')
parser.add_argument('--dataset',
- dest='dataset',
+ type=str,
required=True,
help='BigQuery Dataset to write tables to. '
- 'Must already exist.')
+ 'Must already exist.')
parser.add_argument('--table_name',
- dest='table_name',
- default='hourly_team_score',
+ default='leader_board',
help='The BigQuery table name. Should not already exist.')
parser.add_argument('--window_duration',
type=int,
default=60,
help='Numeric value of fixed window duration, in minutes')
parser.add_argument('--start_min',
- dest='start_min',
+ type=str,
default='1970-01-01-00-00',
help='String representation of the first minute after '
'which to generate results in the format: '
@@ -265,7 +265,7 @@
'prior to that minute won\'t be included in the '
'sums.')
parser.add_argument('--stop_min',
- dest='stop_min',
+ type=str,
default='2100-01-01-00-00',
help='String representation of the first minute for '
'which to generate results in the format: '
@@ -273,21 +273,33 @@
'after to that minute won\'t be included in the '
'sums.')
- known_args, pipeline_args = parser.parse_known_args(argv)
+ args, pipeline_args = parser.parse_known_args(argv)
- pipeline_options = PipelineOptions(pipeline_args)
- p = beam.Pipeline(options=pipeline_options)
- pipeline_options.view_as(SetupOptions).save_main_session = True
+ options = PipelineOptions(pipeline_args)
- (p # pylint: disable=expression-not-assigned
- | ReadFromText(known_args.input)
- | HourlyTeamScore(
- known_args.start_min, known_args.stop_min, known_args.window_duration)
- | WriteWindowedToBigQuery(
- known_args.table_name, known_args.dataset, configure_bigquery_write()))
+ # We also require the --project option to access --dataset
+ if options.view_as(GoogleCloudOptions).project is None:
+ parser.print_usage()
+ print(sys.argv[0] + ': error: argument --project is required')
+ sys.exit(1)
- result = p.run()
- result.wait_until_finish()
+ # We use the save_main_session option because one or more DoFn's in this
+ # workflow rely on global context (e.g., a module imported at module level).
+ options.view_as(SetupOptions).save_main_session = True
+
+ with beam.Pipeline(options=options) as p:
+ (p # pylint: disable=expression-not-assigned
+ | 'ReadInputText' >> beam.io.ReadFromText(args.input)
+ | 'HourlyTeamScore' >> HourlyTeamScore(
+ args.start_min, args.stop_min, args.window_duration)
+ | 'TeamScoresDict' >> beam.ParDo(TeamScoresDict())
+ | 'WriteTeamScoreSums' >> WriteToBigQuery(
+ args.table_name, args.dataset, {
+ 'team': 'STRING',
+ 'total_score': 'INTEGER',
+ 'window_start': 'STRING',
+ }))
+# [END main]
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py
new file mode 100644
index 0000000..e207f26
--- /dev/null
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py
@@ -0,0 +1,349 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Third in a series of four pipelines that tell a story in a 'gaming' domain.
+
+Concepts include: processing unbounded data using fixed windows; use of custom
+timestamps and event-time processing; generation of early/speculative results;
+using AccumulationMode.ACCUMULATING to do cumulative processing of late-arriving
+data.
+
+This pipeline processes an unbounded stream of 'game events'. The calculation of
+the team scores uses fixed windowing based on event time (the time of the game
+play event), not processing time (the time that an event is processed by the
+pipeline). The pipeline calculates the sum of scores per team, for each window.
+By default, the team scores are calculated using one-hour windows.
+
+In contrast-- to demo another windowing option-- the user scores are calculated
+using a global window, which periodically (every ten minutes) emits cumulative
+user score sums.
+
+In contrast to the previous pipelines in the series, which used static, finite
+input data, here we're using an unbounded data source, which lets us provide
+speculative results, and allows handling of late data, at much lower latency.
+We can use the early/speculative results to keep a 'leaderboard' updated in
+near-realtime. Our handling of late data lets us generate correct results,
+e.g. for 'team prizes'. We're now outputting window results as they're
+calculated, giving us much lower latency than with the previous batch examples.
+
+Run injector.Injector to generate pubsub data for this pipeline. The Injector
+documentation provides more detail on how to do this. The injector is currently
+implemented in Java only, it can be used from the Java SDK.
+
+The PubSub topic you specify should be the same topic to which the Injector is
+publishing.
+
+To run the Java injector:
+<beam_root>/examples/java8$ mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
+ -Dexec.args="$PROJECT_ID $PUBSUB_TOPIC none"
+
+For a description of the usage and options, use -h or --help.
+
+To specify a different runner:
+ --runner YOUR_RUNNER
+
+NOTE: When specifying a different runner, additional runner-specific options
+ may have to be passed in as well
+
+EXAMPLES
+--------
+
+# DirectRunner
+python leader_board.py \
+ --project $PROJECT_ID \
+ --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
+ --dataset $BIGQUERY_DATASET
+
+# DataflowRunner
+python leader_board.py \
+ --project $PROJECT_ID \
+ --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
+ --dataset $BIGQUERY_DATASET \
+ --runner DataflowRunner \
+ --temp_location gs://$BUCKET/user_score/temp
+
+--------------------------------------------------------------------------------
+NOTE [BEAM-2354]: This example is not yet runnable by DataflowRunner.
+ The runner still needs support for:
+ * the --save_main_session flag when streaming is enabled
+--------------------------------------------------------------------------------
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import csv
+import logging
+import sys
+import time
+from datetime import datetime
+
+import apache_beam as beam
+from apache_beam.metrics.metric import Metrics
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.transforms import trigger
+
+
+def timestamp2str(t, fmt='%Y-%m-%d %H:%M:%S.000'):
+ """Converts a unix timestamp into a formatted string."""
+ return datetime.fromtimestamp(t).strftime(fmt)
+
+
+class ParseGameEventFn(beam.DoFn):
+ """Parses the raw game event info into a Python dictionary.
+
+ Each event line has the following format:
+ username,teamname,score,timestamp_in_ms,readable_time
+
+ e.g.:
+ user2_AsparagusPig,AsparagusPig,10,1445230923951,2015-11-02 09:09:28.224
+
+ The human-readable time string is not used here.
+ """
+ def __init__(self):
+ super(ParseGameEventFn, self).__init__()
+ self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
+
+ def process(self, elem):
+ try:
+ row = list(csv.reader([elem]))[0]
+ yield {
+ 'user': row[0],
+ 'team': row[1],
+ 'score': int(row[2]),
+ 'timestamp': int(row[3]) / 1000.0,
+ }
+ except: # pylint: disable=bare-except
+ # Log and count parse errors
+ self.num_parse_errors.inc()
+ logging.error('Parse error on "%s"', elem)
+
+
+class ExtractAndSumScore(beam.PTransform):
+ """A transform to extract key/score information and sum the scores.
+ The constructor argument `field` determines whether 'team' or 'user' info is
+ extracted.
+ """
+ def __init__(self, field):
+ super(ExtractAndSumScore, self).__init__()
+ self.field = field
+
+ def expand(self, pcoll):
+ return (pcoll
+ | beam.Map(lambda elem: (elem[self.field], elem['score']))
+ | beam.CombinePerKey(sum))
+
+
+class TeamScoresDict(beam.DoFn):
+ """Formats the data into a dictionary of BigQuery columns with their values
+
+ Receives a (team, score) pair, extracts the window start timestamp, and
+ formats everything together into a dictionary. The dictionary is in the format
+ {'bigquery_column': value}
+ """
+ def process(self, team_score, window=beam.DoFn.WindowParam):
+ team, score = team_score
+ start = timestamp2str(int(window.start))
+ yield {
+ 'team': team,
+ 'total_score': score,
+ 'window_start': start,
+ 'processing_time': timestamp2str(int(time.time()))
+ }
+
+
+class WriteToBigQuery(beam.PTransform):
+ """Generate, format, and write BigQuery table row information."""
+ def __init__(self, table_name, dataset, schema):
+ """Initializes the transform.
+ Args:
+ table_name: Name of the BigQuery table to use.
+ dataset: Name of the dataset to use.
+ schema: Dictionary in the format {'column_name': 'bigquery_type'}
+ """
+ super(WriteToBigQuery, self).__init__()
+ self.table_name = table_name
+ self.dataset = dataset
+ self.schema = schema
+
+ def get_schema(self):
+ """Build the output table schema."""
+ return ', '.join(
+ '%s:%s' % (col, self.schema[col]) for col in self.schema)
+
+ def get_table(self, pipeline):
+ """Utility to construct an output table reference."""
+ project = pipeline.options.view_as(GoogleCloudOptions).project
+ return '%s:%s.%s' % (project, self.dataset, self.table_name)
+
+ def expand(self, pcoll):
+ table = self.get_table(pcoll.pipeline)
+ return (
+ pcoll
+ | 'ConvertToRow' >> beam.Map(
+ lambda elem: {col: elem[col] for col in self.schema})
+ | beam.io.Write(beam.io.BigQuerySink(
+ table,
+ schema=self.get_schema(),
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)))
+
+
+# [START window_and_trigger]
+class CalculateTeamScores(beam.PTransform):
+ """Calculates scores for each team within the configured window duration.
+
+ Extract team/score pairs from the event stream, using hour-long windows by
+ default.
+ """
+ def __init__(self, team_window_duration, allowed_lateness):
+ super(CalculateTeamScores, self).__init__()
+ self.team_window_duration = team_window_duration * 60
+ self.allowed_lateness_seconds = allowed_lateness * 60
+
+ def expand(self, pcoll):
+ # NOTE: the behavior does not exactly match the Java example
+ # TODO: allowed_lateness not implemented yet in FixedWindows
+ # TODO: AfterProcessingTime not implemented yet, replace AfterCount
+ return (
+ pcoll
+ # We will get early (speculative) results as well as cumulative
+ # processing of late data.
+ | 'LeaderboardTeamFixedWindows' >> beam.WindowInto(
+ beam.window.FixedWindows(self.team_window_duration),
+ trigger=trigger.AfterWatermark(trigger.AfterCount(10),
+ trigger.AfterCount(20)),
+ accumulation_mode=trigger.AccumulationMode.ACCUMULATING)
+ # Extract and sum teamname/score pairs from the event data.
+ | 'ExtractAndSumScore' >> ExtractAndSumScore('team'))
+# [END window_and_trigger]
+
+
+# [START processing_time_trigger]
+class CalculateUserScores(beam.PTransform):
+ """Extract user/score pairs from the event stream using processing time, via
+ global windowing. Get periodic updates on all users' running scores.
+ """
+ def __init__(self, allowed_lateness):
+ super(CalculateUserScores, self).__init__()
+ self.allowed_lateness_seconds = allowed_lateness * 60
+
+ def expand(self, pcoll):
+ # NOTE: the behavior does not exactly match the Java example
+ # TODO: allowed_lateness not implemented yet in FixedWindows
+ # TODO: AfterProcessingTime not implemented yet, replace AfterCount
+ return (
+ pcoll
+ # Get periodic results every ten events.
+ | 'LeaderboardUserGlobalWindows' >> beam.WindowInto(
+ beam.window.GlobalWindows(),
+ trigger=trigger.Repeatedly(trigger.AfterCount(10)),
+ accumulation_mode=trigger.AccumulationMode.ACCUMULATING)
+ # Extract and sum username/score pairs from the event data.
+ | 'ExtractAndSumScore' >> ExtractAndSumScore('user'))
+# [END processing_time_trigger]
+
+
+def run(argv=None):
+ """Main entry point; defines and runs the hourly_team_score pipeline."""
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('--topic',
+ type=str,
+ required=True,
+ help='Pub/Sub topic to read from')
+ parser.add_argument('--dataset',
+ type=str,
+ required=True,
+ help='BigQuery Dataset to write tables to. '
+ 'Must already exist.')
+ parser.add_argument('--table_name',
+ default='leader_board',
+ help='The BigQuery table name. Should not already exist.')
+ parser.add_argument('--team_window_duration',
+ type=int,
+ default=60,
+ help='Numeric value of fixed window duration for team '
+ 'analysis, in minutes')
+ parser.add_argument('--allowed_lateness',
+ type=int,
+ default=120,
+ help='Numeric value of allowed data lateness, in minutes')
+
+ args, pipeline_args = parser.parse_known_args(argv)
+
+ options = PipelineOptions(pipeline_args)
+
+ # We also require the --project option to access --dataset
+ if options.view_as(GoogleCloudOptions).project is None:
+ parser.print_usage()
+ print(sys.argv[0] + ': error: argument --project is required')
+ sys.exit(1)
+
+ # We use the save_main_session option because one or more DoFn's in this
+ # workflow rely on global context (e.g., a module imported at module level).
+ options.view_as(SetupOptions).save_main_session = True
+
+ # Enforce that this pipeline is always run in streaming mode
+ options.view_as(StandardOptions).streaming = True
+
+ with beam.Pipeline(options=options) as p:
+ # Read game events from Pub/Sub using custom timestamps, which are extracted
+ # from the pubsub data elements, and parse the data.
+ events = (
+ p
+ | 'ReadPubSub' >> beam.io.gcp.pubsub.ReadStringsFromPubSub(args.topic)
+ | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
+ | 'AddEventTimestamps' >> beam.Map(
+ lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
+
+ # Get team scores and write the results to BigQuery
+ (events # pylint: disable=expression-not-assigned
+ | 'CalculateTeamScores' >> CalculateTeamScores(
+ args.team_window_duration, args.allowed_lateness)
+ | 'TeamScoresDict' >> beam.ParDo(TeamScoresDict())
+ | 'WriteTeamScoreSums' >> WriteToBigQuery(
+ args.table_name + '_teams', args.dataset, {
+ 'team': 'STRING',
+ 'total_score': 'INTEGER',
+ 'window_start': 'STRING',
+ 'processing_time': 'STRING',
+ }))
+
+ def format_user_score_sums(user_score):
+ (user, score) = user_score
+ return {'user': user, 'total_score': score}
+
+ # Get user scores and write the results to BigQuery
+ (events # pylint: disable=expression-not-assigned
+ | 'CalculateUserScores' >> CalculateUserScores(args.allowed_lateness)
+ | 'FormatUserScoreSums' >> beam.Map(format_user_score_sums)
+ | 'WriteUserScoreSums' >> WriteToBigQuery(
+ args.table_name + '_users', args.dataset, {
+ 'user': 'STRING',
+ 'total_score': 'INTEGER',
+ }))
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ run()
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py
new file mode 100644
index 0000000..aece264
--- /dev/null
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Test for the leader_board example."""
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.examples.complete.game import leader_board
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+
+class LeaderBoardTest(unittest.TestCase):
+
+ SAMPLE_DATA = [
+ 'user1_team1,team1,18,1447686663000,2015-11-16 15:11:03.921',
+ 'user1_team1,team1,18,1447690263000,2015-11-16 16:11:03.921',
+ 'user2_team2,team2,2,1447690263000,2015-11-16 16:11:03.955',
+ 'user3_team3,team3,8,1447690263000,2015-11-16 16:11:03.955',
+ 'user4_team3,team3,5,1447690263000,2015-11-16 16:11:03.959',
+ 'user1_team1,team1,14,1447697463000,2015-11-16 18:11:03.955',
+ ]
+
+ def create_data(self, p):
+ return (p
+ | beam.Create(LeaderBoardTest.SAMPLE_DATA)
+ | beam.ParDo(leader_board.ParseGameEventFn())
+ | beam.Map(lambda elem:\
+ beam.window.TimestampedValue(elem, elem['timestamp'])))
+
+ def test_leader_board_teams(self):
+ with TestPipeline() as p:
+ result = (
+ self.create_data(p)
+ | leader_board.CalculateTeamScores(
+ team_window_duration=60,
+ allowed_lateness=120))
+ assert_that(result, equal_to([
+ ('team1', 14), ('team1', 18), ('team1', 18), ('team2', 2),
+ ('team3', 13)]))
+
+ def test_leader_board_users(self):
+ with TestPipeline() as p:
+ result = (
+ self.create_data(p)
+ | leader_board.CalculateUserScores(allowed_lateness=120))
+ assert_that(result, equal_to([]))
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score.py b/sdks/python/apache_beam/examples/complete/game/user_score.py
index 389d2c6..3e3e547 100644
--- a/sdks/python/apache_beam/examples/complete/game/user_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/user_score.py
@@ -16,8 +16,9 @@
#
"""First in a series of four pipelines that tell a story in a 'gaming' domain.
-Concepts: batch processing; reading input from Google Cloud Storage and writing
-output to BigQuery; using standalone DoFns; use of the sum by key transform.
+Concepts: batch processing; reading input from Google Cloud Storage or a from a
+local text file, and writing output to a text file; using standalone DoFns; use
+of the CombinePerKey transform.
In this gaming scenario, many users play, as members of different teams, over
the course of a day, and their actions are logged for processing. Some of the
@@ -29,32 +30,41 @@
(collected, say, for each day). The batch processing will not include any late
data that arrives after the day's cutoff point.
-To execute this pipeline using the static example input data, specify the
-`--dataset=YOUR-DATASET` flag along with other runner specific flags. Note:
-The BigQuery dataset you specify must already exist. You can simply create a new
-empty BigQuery dataset if you don't have an existing one.
+For a description of the usage and options, use -h or --help.
-Optionally include the `--input` argument to specify a batch input file. See the
-`--input` default value for an example batch data file.
+To specify a different runner:
+ --runner YOUR_RUNNER
+
+NOTE: When specifying a different runner, additional runner-specific options
+ may have to be passed in as well
+
+EXAMPLES
+--------
+
+# DirectRunner
+python user_score.py \
+ --output /local/path/user_score/output
+
+# DataflowRunner
+python user_score.py \
+ --output gs://$BUCKET/user_score/output \
+ --runner DataflowRunner \
+ --project $PROJECT_ID \
+ --temp_location gs://$BUCKET/user_score/temp
"""
from __future__ import absolute_import
import argparse
+import csv
import logging
import apache_beam as beam
-from apache_beam import typehints
-from apache_beam.io import ReadFromText
-from apache_beam.metrics import Metrics
-from apache_beam.typehints import with_input_types
-from apache_beam.typehints import with_output_types
-from apache_beam.options.pipeline_options import GoogleCloudOptions
-from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.metrics.metric import Metrics
-class ParseEventFn(beam.DoFn):
- """Parses the raw game event info into GameActionInfo tuples.
+class ParseGameEventFn(beam.DoFn):
+ """Parses the raw game event info into a Python dictionary.
Each event line has the following format:
username,teamname,score,timestamp_in_ms,readable_time
@@ -65,32 +75,27 @@
The human-readable time string is not used here.
"""
def __init__(self):
- super(ParseEventFn, self).__init__()
+ super(ParseGameEventFn, self).__init__()
self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
- def process(self, element):
- components = element.split(',')
+ def process(self, elem):
try:
- user = components[0].strip()
- team = components[1].strip()
- score = int(components[2].strip())
- timestamp = int(components[3].strip())
- yield {'user': user, 'team': team, 'score': score, 'timestamp': timestamp}
+ row = list(csv.reader([elem]))[0]
+ yield {
+ 'user': row[0],
+ 'team': row[1],
+ 'score': int(row[2]),
+ 'timestamp': int(row[3]) / 1000.0,
+ }
except: # pylint: disable=bare-except
- # Log and count parse errors.
+ # Log and count parse errors
self.num_parse_errors.inc()
- logging.info('Parse error on %s.', element)
+ logging.error('Parse error on "%s"', elem)
-@with_input_types(ints=typehints.Iterable[int])
-@with_output_types(int)
-def sum_ints(ints):
- return sum(ints)
-
-
+# [START extract_and_sum_score]
class ExtractAndSumScore(beam.PTransform):
"""A transform to extract key/score information and sum the scores.
-
The constructor argument `field` determines whether 'team' or 'user' info is
extracted.
"""
@@ -100,85 +105,21 @@
def expand(self, pcoll):
return (pcoll
- | beam.Map(lambda info: (info[self.field], info['score']))
- | beam.CombinePerKey(sum_ints))
-
-
-def configure_bigquery_write():
- return [
- ('user', 'STRING', lambda e: e[0]),
- ('total_score', 'INTEGER', lambda e: e[1]),
- ]
-
-
-class WriteToBigQuery(beam.PTransform):
- """Generate, format, and write BigQuery table row information.
-
- Use provided information about the field names and types, as well as lambda
- functions that describe how to generate their values.
- """
-
- def __init__(self, table_name, dataset, field_info):
- """Initializes the transform.
-
- Args:
- table_name: Name of the BigQuery table to use.
- dataset: Name of the dataset to use.
- field_info: List of tuples that holds information about output table field
- definitions. The tuples are in the
- (field_name, field_type, field_fn) format, where field_name is
- the name of the field, field_type is the BigQuery type of the
- field and field_fn is a lambda function to generate the field
- value from the element.
- """
- super(WriteToBigQuery, self).__init__()
- self.table_name = table_name
- self.dataset = dataset
- self.field_info = field_info
-
- def get_schema(self):
- """Build the output table schema."""
- return ', '.join(
- '%s:%s' % (entry[0], entry[1]) for entry in self.field_info)
-
- def get_table(self, pipeline):
- """Utility to construct an output table reference."""
- project = pipeline.options.view_as(GoogleCloudOptions).project
- return '%s:%s.%s' % (project, self.dataset, self.table_name)
-
- class BuildRowFn(beam.DoFn):
- """Convert each key/score pair into a BigQuery TableRow as specified."""
- def __init__(self, field_info):
- super(WriteToBigQuery.BuildRowFn, self).__init__()
- self.field_info = field_info
-
- def process(self, element):
- row = {}
- for entry in self.field_info:
- row[entry[0]] = entry[2](element)
- yield row
-
- def expand(self, pcoll):
- table = self.get_table(pcoll.pipeline)
- return (
- pcoll
- | 'ConvertToRow' >> beam.ParDo(
- WriteToBigQuery.BuildRowFn(self.field_info))
- | beam.io.Write(beam.io.BigQuerySink(
- table,
- schema=self.get_schema(),
- create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
- write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)))
+ | beam.Map(lambda elem: (elem[self.field], elem['score']))
+ | beam.CombinePerKey(sum))
+# [END extract_and_sum_score]
class UserScore(beam.PTransform):
def expand(self, pcoll):
- return (pcoll
- | 'ParseGameEvent' >> beam.ParDo(ParseEventFn())
- # Extract and sum username/score pairs from the event data.
- | 'ExtractUserScore' >> ExtractAndSumScore('user'))
+ return (
+ pcoll
+ | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
+ # Extract and sum username/score pairs from the event data.
+ | 'ExtractAndSumScore' >> ExtractAndSumScore('user'))
+# [START main]
def run(argv=None):
"""Main entry point; defines and runs the user_score pipeline."""
parser = argparse.ArgumentParser()
@@ -186,31 +127,27 @@
# The default maps to two large Google Cloud Storage files (each ~12GB)
# holding two subsequent day's worth (roughly) of data.
parser.add_argument('--input',
- dest='input',
- default='gs://dataflow-samples/game/gaming_data*.csv',
+ type=str,
+ default='gs://apache-beam-samples/game/gaming_data*.csv',
help='Path to the data file(s) containing game data.')
- parser.add_argument('--dataset',
- dest='dataset',
+ parser.add_argument('--output',
+ type=str,
required=True,
- help='BigQuery Dataset to write tables to. '
- 'Must already exist.')
- parser.add_argument('--table_name',
- dest='table_name',
- default='user_score',
- help='The BigQuery table name. Should not already exist.')
- known_args, pipeline_args = parser.parse_known_args(argv)
+ help='Path to the output file(s).')
- pipeline_options = PipelineOptions(pipeline_args)
- p = beam.Pipeline(options=pipeline_options)
+ args, pipeline_args = parser.parse_known_args(argv)
- (p # pylint: disable=expression-not-assigned
- | ReadFromText(known_args.input) # Read events from a file and parse them.
- | UserScore()
- | WriteToBigQuery(
- known_args.table_name, known_args.dataset, configure_bigquery_write()))
+ with beam.Pipeline(argv=pipeline_args) as p:
+ def format_user_score_sums(user_score):
+ (user, score) = user_score
+ return 'user: %s, total_score: %s' % (user, score)
- result = p.run()
- result.wait_until_finish()
+ (p # pylint: disable=expression-not-assigned
+ | 'ReadInputText' >> beam.io.ReadFromText(args.input)
+ | 'UserScore' >> UserScore()
+ | 'FormatUserScoreSums' >> beam.Map(format_user_score_sums)
+ | 'WriteUserScoreSums' >> beam.io.WriteToText(args.output))
+# [END main]
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
index 5ff2b78..3f3ef03 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
@@ -99,26 +99,28 @@
help='Output file to write the resulting image to.')
known_args, pipeline_args = parser.parse_known_args(argv)
- p = beam.Pipeline(argv=pipeline_args)
- n = int(known_args.grid_size)
+ with beam.Pipeline(argv=pipeline_args) as p:
+ n = int(known_args.grid_size)
- coordinates = generate_julia_set_colors(p, complex(-.62772, .42193), n, 100)
+ coordinates = generate_julia_set_colors(p, complex(-.62772, .42193), n, 100)
- # Group each coordinate triplet by its x value, then write the coordinates to
- # the output file with an x-coordinate grouping per line.
- # pylint: disable=expression-not-assigned
- (coordinates
- | 'x coord key' >> beam.Map(lambda (x, y, i): (x, (x, y, i)))
- | 'x coord' >> beam.GroupByKey()
- | 'format' >> beam.Map(
- lambda (k, coords): ' '.join('(%s, %s, %s)' % coord for coord in coords))
- | WriteToText(known_args.coordinate_output))
- # pylint: enable=expression-not-assigned
- return p.run().wait_until_finish()
+ def x_coord_key(x_y_i):
+ (x, y, i) = x_y_i
+ return (x, (x, y, i))
- # Optionally render the image and save it to a file.
- # TODO(silviuc): Add this functionality.
- # if p.options.image_output is not None:
- # julia_set_image = generate_julia_set_visualization(
- # file_with_coordinates, n, 100)
- # save_julia_set_visualization(p.options.image_output, julia_set_image)
+ # Group each coordinate triplet by its x value, then write the coordinates
+ # to the output file with an x-coordinate grouping per line.
+ # pylint: disable=expression-not-assigned
+ (coordinates
+ | 'x coord key' >> beam.Map(x_coord_key)
+ | 'x coord' >> beam.GroupByKey()
+ | 'format' >> beam.Map(
+ lambda k_coords: ' '.join('(%s, %s, %s)' % c for c in k_coords[1]))
+ | WriteToText(known_args.coordinate_output))
+
+ # Optionally render the image and save it to a file.
+ # TODO(silviuc): Add this functionality.
+ # if p.options.image_output is not None:
+ # julia_set_image = generate_julia_set_visualization(
+ # file_with_coordinates, n, 100)
+ # save_julia_set_visualization(p.options.image_output, julia_set_image)
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py
index 17d9cf3..e498627 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py
@@ -23,8 +23,8 @@
import tempfile
import unittest
-
from apache_beam.examples.complete.juliaset.juliaset import juliaset
+from apache_beam.testing.util import open_shards
class JuliaSetTest(unittest.TestCase):
@@ -60,8 +60,8 @@
# Parse the results from the file, and ensure it was written in the proper
# format.
- with open(self.test_files['output_coord_file_name'] +
- '-00000-of-00001') as result_file:
+ with open_shards(self.test_files['output_coord_file_name'] +
+ '-*-of-*') as result_file:
output_lines = result_file.readlines()
# Should have a line for each x-coordinate.
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py
index 0db5431..1d521be 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py
@@ -49,10 +49,8 @@
import logging
-
from apache_beam.examples.complete.juliaset.juliaset import juliaset
-
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
juliaset.run()
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/setup.py b/sdks/python/apache_beam/examples/complete/juliaset/setup.py
index 589e47c..cbf5f3d 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/setup.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/setup.py
@@ -24,9 +24,10 @@
This behavior is triggered by specifying the --setup_file command line option
when running the workflow for remote execution.
"""
+from __future__ import print_function
-from distutils.command.build import build as _build
import subprocess
+from distutils.command.build import build as _build
import setuptools
@@ -76,14 +77,14 @@
pass
def RunCustomCommand(self, command_list):
- print 'Running command: %s' % command_list
+ print('Running command: %s' % command_list)
p = subprocess.Popen(
command_list,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# Can use communicate(input='y\n'.encode()) if the command run requires
# some confirmation.
stdout_data, _ = p.communicate()
- print 'Command output: %s' % stdout_data
+ print('Command output: %s' % stdout_data)
if p.returncode != 0:
raise RuntimeError(
'Command %s failed: exit code: %s' % (command_list, p.returncode))
diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py
index a98d906..065e4b3 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf.py
@@ -31,9 +31,9 @@
import apache_beam as beam
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
-from apache_beam.pvalue import AsSingleton
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.pvalue import AsSingleton
def read_documents(pipeline, uris):
@@ -68,7 +68,8 @@
# Create a collection of pairs mapping a URI to each of the words
# in the document associated with that that URI.
- def split_into_words((uri, line)):
+ def split_into_words(uri_line):
+ (uri, line) = uri_line
return [(uri, w.lower()) for w in re.findall(r'[A-Za-z\']+', line)]
uri_to_words = (
@@ -99,10 +100,12 @@
# Adjust the above collection to a mapping from (URI, word) pairs to counts
# into an isomorphic mapping from URI to (word, count) pairs, to prepare
# for a join by the URI key.
+ def shift_keys(uri_word_count):
+ return (uri_word_count[0][0], (uri_word_count[0][1], uri_word_count[1]))
+
uri_to_word_and_count = (
uri_and_word_to_count
- | 'ShiftKeys' >> beam.Map(
- lambda ((uri, word), count): (uri, (word, count))))
+ | 'ShiftKeys' >> beam.Map(shift_keys))
# Perform a CoGroupByKey (a sort of pre-join) on the prepared
# uri_to_word_total and uri_to_word_and_count tagged by 'word totals' and
@@ -125,7 +128,8 @@
# that word occurs in the document divided by the total number of words in
# the document.
- def compute_term_frequency((uri, count_and_total)):
+ def compute_term_frequency(uri_count_and_total):
+ (uri, count_and_total) = uri_count_and_total
word_and_count = count_and_total['word counts']
# We have an iterable for one element that we want extracted.
[word_total] = count_and_total['word totals']
@@ -165,7 +169,8 @@
# basic version that is the term frequency divided by the log of the
# document frequency.
- def compute_tf_idf((word, tf_and_df)):
+ def compute_tf_idf(word_tf_and_df):
+ (word, tf_and_df) = word_tf_and_df
[docf] = tf_and_df['df']
for uri, tf in tf_and_df['tf']:
yield word, (uri, tf * math.log(1 / docf))
@@ -191,17 +196,16 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # Read documents specified by the uris command line option.
- pcoll = read_documents(p, glob.glob(known_args.uris))
- # Compute TF-IDF information for each word.
- output = pcoll | TfIdf()
- # Write the output using a "Write" transform that has side effects.
- # pylint: disable=expression-not-assigned
- output | 'write' >> WriteToText(known_args.output)
- # Execute the pipeline and wait until it is completed.
- p.run().wait_until_finish()
+ # Read documents specified by the uris command line option.
+ pcoll = read_documents(p, glob.glob(known_args.uris))
+ # Compute TF-IDF information for each word.
+ output = pcoll | TfIdf()
+ # Write the output using a "Write" transform that has side effects.
+ # pylint: disable=expression-not-assigned
+ output | 'write' >> WriteToText(known_args.output)
+ # Execute the pipeline and wait until it is completed.
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py
index f177dfc..637d10a0 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf_test.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py
@@ -28,7 +28,7 @@
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
-
+from apache_beam.testing.util import open_shards
EXPECTED_RESULTS = set([
('ghi', '1.txt', 0.3662040962227032),
@@ -50,20 +50,24 @@
f.write(contents)
def test_tfidf_transform(self):
- p = TestPipeline()
- uri_to_line = p | 'create sample' >> beam.Create(
- [('1.txt', 'abc def ghi'),
- ('2.txt', 'abc def'),
- ('3.txt', 'abc')])
- result = (
- uri_to_line
- | tfidf.TfIdf()
- | beam.Map(lambda (word, (uri, tfidf)): (word, uri, tfidf)))
- assert_that(result, equal_to(EXPECTED_RESULTS))
- # Run the pipeline. Note that the assert_that above adds to the pipeline
- # a check that the result PCollection contains expected values. To actually
- # trigger the check the pipeline must be run.
- p.run()
+ with TestPipeline() as p:
+ def re_key(word_uri_tfidf):
+ (word, (uri, tfidf)) = word_uri_tfidf
+ return (word, uri, tfidf)
+
+ uri_to_line = p | 'create sample' >> beam.Create(
+ [('1.txt', 'abc def ghi'),
+ ('2.txt', 'abc def'),
+ ('3.txt', 'abc')])
+ result = (
+ uri_to_line
+ | tfidf.TfIdf()
+ | beam.Map(re_key))
+ assert_that(result, equal_to(EXPECTED_RESULTS))
+ # Run the pipeline. Note that the assert_that above adds to the pipeline
+ # a check that the result PCollection contains expected values.
+ # To actually trigger the check the pipeline must be run (e.g. by
+ # exiting the with context).
def test_basics(self):
# Setup the files with expected content.
@@ -76,8 +80,8 @@
'--output', os.path.join(temp_folder, 'result')])
# Parse result file and compare.
results = []
- with open(os.path.join(temp_folder,
- 'result-00000-of-00001')) as result_file:
+ with open_shards(os.path.join(
+ temp_folder, 'result-*-of-*')) as result_file:
for line in result_file:
match = re.search(EXPECTED_LINE_RE, line)
logging.info('Result line: %s', line)
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
index aa48e4e..dd827bc 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
@@ -49,12 +49,11 @@
from apache_beam import combiners
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.window import Sessions
from apache_beam.transforms.window import TimestampedValue
-from apache_beam.options.pipeline_options import PipelineOptions
-from apache_beam.options.pipeline_options import SetupOptions
-
ONE_HOUR_IN_SECONDS = 3600
THIRTY_DAYS_IN_SECONDS = 30 * 24 * ONE_HOUR_IN_SECONDS
@@ -159,14 +158,12 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- (p # pylint: disable=expression-not-assigned
- | ReadFromText(known_args.input)
- | ComputeTopSessions(known_args.sampling_threshold)
- | WriteToText(known_args.output))
-
- p.run()
+ (p # pylint: disable=expression-not-assigned
+ | ReadFromText(known_args.input)
+ | ComputeTopSessions(known_args.sampling_threshold)
+ | WriteToText(known_args.output))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
index 5fb6276..a0b368f 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
@@ -20,7 +20,6 @@
import json
import unittest
-
import apache_beam as beam
from apache_beam.examples.complete import top_wikipedia_sessions
from apache_beam.testing.test_pipeline import TestPipeline
@@ -52,12 +51,11 @@
]
def test_compute_top_sessions(self):
- p = TestPipeline()
- edits = p | beam.Create(self.EDITS)
- result = edits | top_wikipedia_sessions.ComputeTopSessions(1.0)
+ with TestPipeline() as p:
+ edits = p | beam.Create(self.EDITS)
+ result = edits | top_wikipedia_sessions.ComputeTopSessions(1.0)
- assert_that(result, equal_to(self.EXPECTED))
- p.run()
+ assert_that(result, equal_to(self.EXPECTED))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py b/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
index 400189e..3a8af67 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
@@ -42,86 +42,85 @@
'or DATASET.TABLE.'))
known_args, pipeline_args = parser.parse_known_args(argv)
- p = beam.Pipeline(argv=pipeline_args)
+ with beam.Pipeline(argv=pipeline_args) as p:
- from apache_beam.io.gcp.internal.clients import bigquery # pylint: disable=wrong-import-order, wrong-import-position
+ from apache_beam.io.gcp.internal.clients import bigquery # pylint: disable=wrong-import-order, wrong-import-position
- table_schema = bigquery.TableSchema()
+ table_schema = bigquery.TableSchema()
- # Fields that use standard types.
- kind_schema = bigquery.TableFieldSchema()
- kind_schema.name = 'kind'
- kind_schema.type = 'string'
- kind_schema.mode = 'nullable'
- table_schema.fields.append(kind_schema)
+ # Fields that use standard types.
+ kind_schema = bigquery.TableFieldSchema()
+ kind_schema.name = 'kind'
+ kind_schema.type = 'string'
+ kind_schema.mode = 'nullable'
+ table_schema.fields.append(kind_schema)
- full_name_schema = bigquery.TableFieldSchema()
- full_name_schema.name = 'fullName'
- full_name_schema.type = 'string'
- full_name_schema.mode = 'required'
- table_schema.fields.append(full_name_schema)
+ full_name_schema = bigquery.TableFieldSchema()
+ full_name_schema.name = 'fullName'
+ full_name_schema.type = 'string'
+ full_name_schema.mode = 'required'
+ table_schema.fields.append(full_name_schema)
- age_schema = bigquery.TableFieldSchema()
- age_schema.name = 'age'
- age_schema.type = 'integer'
- age_schema.mode = 'nullable'
- table_schema.fields.append(age_schema)
+ age_schema = bigquery.TableFieldSchema()
+ age_schema.name = 'age'
+ age_schema.type = 'integer'
+ age_schema.mode = 'nullable'
+ table_schema.fields.append(age_schema)
- gender_schema = bigquery.TableFieldSchema()
- gender_schema.name = 'gender'
- gender_schema.type = 'string'
- gender_schema.mode = 'nullable'
- table_schema.fields.append(gender_schema)
+ gender_schema = bigquery.TableFieldSchema()
+ gender_schema.name = 'gender'
+ gender_schema.type = 'string'
+ gender_schema.mode = 'nullable'
+ table_schema.fields.append(gender_schema)
- # A nested field
- phone_number_schema = bigquery.TableFieldSchema()
- phone_number_schema.name = 'phoneNumber'
- phone_number_schema.type = 'record'
- phone_number_schema.mode = 'nullable'
+ # A nested field
+ phone_number_schema = bigquery.TableFieldSchema()
+ phone_number_schema.name = 'phoneNumber'
+ phone_number_schema.type = 'record'
+ phone_number_schema.mode = 'nullable'
- area_code = bigquery.TableFieldSchema()
- area_code.name = 'areaCode'
- area_code.type = 'integer'
- area_code.mode = 'nullable'
- phone_number_schema.fields.append(area_code)
+ area_code = bigquery.TableFieldSchema()
+ area_code.name = 'areaCode'
+ area_code.type = 'integer'
+ area_code.mode = 'nullable'
+ phone_number_schema.fields.append(area_code)
- number = bigquery.TableFieldSchema()
- number.name = 'number'
- number.type = 'integer'
- number.mode = 'nullable'
- phone_number_schema.fields.append(number)
- table_schema.fields.append(phone_number_schema)
+ number = bigquery.TableFieldSchema()
+ number.name = 'number'
+ number.type = 'integer'
+ number.mode = 'nullable'
+ phone_number_schema.fields.append(number)
+ table_schema.fields.append(phone_number_schema)
- # A repeated field.
- children_schema = bigquery.TableFieldSchema()
- children_schema.name = 'children'
- children_schema.type = 'string'
- children_schema.mode = 'repeated'
- table_schema.fields.append(children_schema)
+ # A repeated field.
+ children_schema = bigquery.TableFieldSchema()
+ children_schema.name = 'children'
+ children_schema.type = 'string'
+ children_schema.mode = 'repeated'
+ table_schema.fields.append(children_schema)
- def create_random_record(record_id):
- return {'kind': 'kind' + record_id, 'fullName': 'fullName'+record_id,
- 'age': int(record_id) * 10, 'gender': 'male',
- 'phoneNumber': {
- 'areaCode': int(record_id) * 100,
- 'number': int(record_id) * 100000},
- 'children': ['child' + record_id + '1',
- 'child' + record_id + '2',
- 'child' + record_id + '3']
- }
+ def create_random_record(record_id):
+ return {'kind': 'kind' + record_id, 'fullName': 'fullName'+record_id,
+ 'age': int(record_id) * 10, 'gender': 'male',
+ 'phoneNumber': {
+ 'areaCode': int(record_id) * 100,
+ 'number': int(record_id) * 100000},
+ 'children': ['child' + record_id + '1',
+ 'child' + record_id + '2',
+ 'child' + record_id + '3']
+ }
- # pylint: disable=expression-not-assigned
- record_ids = p | 'CreateIDs' >> beam.Create(['1', '2', '3', '4', '5'])
- records = record_ids | 'CreateRecords' >> beam.Map(create_random_record)
- records | 'write' >> beam.io.Write(
- beam.io.BigQuerySink(
- known_args.output,
- schema=table_schema,
- create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
- write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
+ # pylint: disable=expression-not-assigned
+ record_ids = p | 'CreateIDs' >> beam.Create(['1', '2', '3', '4', '5'])
+ records = record_ids | 'CreateRecords' >> beam.Map(create_random_record)
+ records | 'write' >> beam.io.Write(
+ beam.io.BigQuerySink(
+ known_args.output,
+ schema=table_schema,
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
- # Run the pipeline (all operations are deferred until run() is called).
- p.run()
+ # Run the pipeline (all operations are deferred until run() is called).
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
index 6b28818..e16ae73 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
@@ -32,12 +32,11 @@
from random import randrange
import apache_beam as beam
-
from apache_beam.io import WriteToText
-from apache_beam.pvalue import AsList
-from apache_beam.pvalue import AsSingleton
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.pvalue import AsList
+from apache_beam.pvalue import AsSingleton
def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word):
@@ -88,32 +87,31 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- group_ids = []
- for i in xrange(0, int(known_args.num_groups)):
- group_ids.append('id' + str(i))
+ group_ids = []
+ for i in xrange(0, int(known_args.num_groups)):
+ group_ids.append('id' + str(i))
- query_corpus = 'select UNIQUE(corpus) from publicdata:samples.shakespeare'
- query_word = 'select UNIQUE(word) from publicdata:samples.shakespeare'
- ignore_corpus = known_args.ignore_corpus
- ignore_word = known_args.ignore_word
+ query_corpus = 'select UNIQUE(corpus) from publicdata:samples.shakespeare'
+ query_word = 'select UNIQUE(word) from publicdata:samples.shakespeare'
+ ignore_corpus = known_args.ignore_corpus
+ ignore_word = known_args.ignore_word
- pcoll_corpus = p | 'read corpus' >> beam.io.Read(
- beam.io.BigQuerySource(query=query_corpus))
- pcoll_word = p | 'read_words' >> beam.io.Read(
- beam.io.BigQuerySource(query=query_word))
- pcoll_ignore_corpus = p | 'create_ignore_corpus' >> beam.Create(
- [ignore_corpus])
- pcoll_ignore_word = p | 'create_ignore_word' >> beam.Create([ignore_word])
- pcoll_group_ids = p | 'create groups' >> beam.Create(group_ids)
+ pcoll_corpus = p | 'read corpus' >> beam.io.Read(
+ beam.io.BigQuerySource(query=query_corpus))
+ pcoll_word = p | 'read_words' >> beam.io.Read(
+ beam.io.BigQuerySource(query=query_word))
+ pcoll_ignore_corpus = p | 'create_ignore_corpus' >> beam.Create(
+ [ignore_corpus])
+ pcoll_ignore_word = p | 'create_ignore_word' >> beam.Create([ignore_word])
+ pcoll_group_ids = p | 'create groups' >> beam.Create(group_ids)
- pcoll_groups = create_groups(pcoll_group_ids, pcoll_corpus, pcoll_word,
- pcoll_ignore_corpus, pcoll_ignore_word)
+ pcoll_groups = create_groups(pcoll_group_ids, pcoll_corpus, pcoll_word,
+ pcoll_ignore_corpus, pcoll_ignore_word)
- # pylint:disable=expression-not-assigned
- pcoll_groups | WriteToText(known_args.output)
- p.run()
+ # pylint:disable=expression-not-assigned
+ pcoll_groups | WriteToText(known_args.output)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
index b11dc47..964b35b 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
@@ -30,25 +30,26 @@
class BigQuerySideInputTest(unittest.TestCase):
def test_create_groups(self):
- p = TestPipeline()
+ with TestPipeline() as p:
- group_ids_pcoll = p | 'CreateGroupIds' >> beam.Create(['A', 'B', 'C'])
- corpus_pcoll = p | 'CreateCorpus' >> beam.Create(
- [{'f': 'corpus1'}, {'f': 'corpus2'}, {'f': 'corpus3'}])
- words_pcoll = p | 'CreateWords' >> beam.Create(
- [{'f': 'word1'}, {'f': 'word2'}, {'f': 'word3'}])
- ignore_corpus_pcoll = p | 'CreateIgnoreCorpus' >> beam.Create(['corpus1'])
- ignore_word_pcoll = p | 'CreateIgnoreWord' >> beam.Create(['word1'])
+ group_ids_pcoll = p | 'CreateGroupIds' >> beam.Create(['A', 'B', 'C'])
+ corpus_pcoll = p | 'CreateCorpus' >> beam.Create(
+ [{'f': 'corpus1'}, {'f': 'corpus2'}, {'f': 'corpus3'}])
+ words_pcoll = p | 'CreateWords' >> beam.Create(
+ [{'f': 'word1'}, {'f': 'word2'}, {'f': 'word3'}])
+ ignore_corpus_pcoll = p | 'CreateIgnoreCorpus' >> beam.Create(['corpus1'])
+ ignore_word_pcoll = p | 'CreateIgnoreWord' >> beam.Create(['word1'])
- groups = bigquery_side_input.create_groups(group_ids_pcoll, corpus_pcoll,
- words_pcoll, ignore_corpus_pcoll,
- ignore_word_pcoll)
+ groups = bigquery_side_input.create_groups(group_ids_pcoll,
+ corpus_pcoll,
+ words_pcoll,
+ ignore_corpus_pcoll,
+ ignore_word_pcoll)
- assert_that(groups, equal_to(
- [('A', 'corpus2', 'word2'),
- ('B', 'corpus2', 'word2'),
- ('C', 'corpus2', 'word2')]))
- p.run()
+ assert_that(groups, equal_to(
+ [('A', 'corpus2', 'word2'),
+ ('B', 'corpus2', 'word2'),
+ ('C', 'corpus2', 'word2')]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
index ed0c79a..7b40353 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
@@ -58,7 +58,7 @@
lambda row: [(int(row['month']), 1)] if row['tornado'] else [])
| 'monthly count' >> beam.CombinePerKey(sum)
| 'format' >> beam.Map(
- lambda (k, v): {'month': k, 'tornado_count': v}))
+ lambda k_v: {'month': k_v[0], 'tornado_count': k_v[1]}))
def run(argv=None):
@@ -75,23 +75,21 @@
'or DATASET.TABLE.'))
known_args, pipeline_args = parser.parse_known_args(argv)
- p = beam.Pipeline(argv=pipeline_args)
+ with beam.Pipeline(argv=pipeline_args) as p:
- # Read the table rows into a PCollection.
- rows = p | 'read' >> beam.io.Read(beam.io.BigQuerySource(known_args.input))
- counts = count_tornadoes(rows)
+ # Read the table rows into a PCollection.
+ rows = p | 'read' >> beam.io.Read(beam.io.BigQuerySource(known_args.input))
+ counts = count_tornadoes(rows)
- # Write the output using a "Write" transform that has side effects.
- # pylint: disable=expression-not-assigned
- counts | 'write' >> beam.io.Write(
- beam.io.BigQuerySink(
- known_args.output,
- schema='month:INTEGER, tornado_count:INTEGER',
- create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
- write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
+ # Write the output using a "Write" transform that has side effects.
+ # pylint: disable=expression-not-assigned
+ counts | 'Write' >> beam.io.WriteToBigQuery(
+ known_args.output,
+ schema='month:INTEGER, tornado_count:INTEGER',
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)
- # Run the pipeline (all operations are deferred until run() is called).
- p.run().wait_until_finish()
+ # Run the pipeline (all operations are deferred until run() is called).
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
index 5d2ee7c..9612849 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
@@ -25,6 +25,7 @@
from nose.plugins.attrib import attr
from apache_beam.examples.cookbook import bigquery_tornadoes
+from apache_beam.io.gcp.tests import utils
from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher
from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
@@ -44,17 +45,24 @@
test_pipeline = TestPipeline(is_integration_test=True)
# Set extra options to the pipeline for test purpose
- output_table = ('BigQueryTornadoesIT'
- '.monthly_tornadoes_%s' % int(round(time.time() * 1000)))
+ project = test_pipeline.get_option('project')
+
+ dataset = 'BigQueryTornadoesIT'
+ table = 'monthly_tornadoes_%s' % int(round(time.time() * 1000))
+ output_table = '.'.join([dataset, table])
query = 'SELECT month, tornado_count FROM [%s]' % output_table
+
pipeline_verifiers = [PipelineStateMatcher(),
BigqueryMatcher(
- project=test_pipeline.get_option('project'),
+ project=project,
query=query,
checksum=self.DEFAULT_CHECKSUM)]
extra_opts = {'output': output_table,
'on_success_matcher': all_of(*pipeline_verifiers)}
+ # Register cleanup before pipeline execution.
+ self.addCleanup(utils.delete_bq_table, project, dataset, table)
+
# Get pipeline options from command argument: --test-pipeline-options,
# and start pipeline job by calling pipeline main function.
bigquery_tornadoes.run(
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
index c926df8..45dcaba 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
@@ -30,16 +30,15 @@
class BigQueryTornadoesTest(unittest.TestCase):
def test_basics(self):
- p = TestPipeline()
- rows = (p | 'create' >> beam.Create([
- {'month': 1, 'day': 1, 'tornado': False},
- {'month': 1, 'day': 2, 'tornado': True},
- {'month': 1, 'day': 3, 'tornado': True},
- {'month': 2, 'day': 1, 'tornado': True}]))
- results = bigquery_tornadoes.count_tornadoes(rows)
- assert_that(results, equal_to([{'month': 1, 'tornado_count': 2},
- {'month': 2, 'tornado_count': 1}]))
- p.run().wait_until_finish()
+ with TestPipeline() as p:
+ rows = (p | 'create' >> beam.Create([
+ {'month': 1, 'day': 1, 'tornado': False},
+ {'month': 1, 'day': 2, 'tornado': True},
+ {'month': 1, 'day': 3, 'tornado': True},
+ {'month': 2, 'day': 1, 'tornado': True}]))
+ results = bigquery_tornadoes.count_tornadoes(rows)
+ assert_that(results, equal_to([{'month': 1, 'tornado_count': 2},
+ {'month': 2, 'tornado_count': 1}]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py
index aeeb3c9..f97b0f2 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders.py
@@ -85,15 +85,13 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
- p = beam.Pipeline(argv=pipeline_args)
- (p # pylint: disable=expression-not-assigned
- | 'read' >> ReadFromText(known_args.input, coder=JsonCoder())
- | 'points' >> beam.FlatMap(compute_points)
- | beam.CombinePerKey(sum)
- | 'write' >> WriteToText(known_args.output, coder=JsonCoder()))
- p.run()
+ with beam.Pipeline(options=pipeline_options) as p:
+ (p # pylint: disable=expression-not-assigned
+ | 'read' >> ReadFromText(known_args.input, coder=JsonCoder())
+ | 'points' >> beam.FlatMap(compute_points)
+ | beam.CombinePerKey(sum)
+ | 'write' >> WriteToText(known_args.output, coder=JsonCoder()))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py
index f71dad8..988d3c9 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py
@@ -35,13 +35,13 @@
{'host': ['Brasil', 1], 'guest': ['Italy', 0]}]
def test_compute_points(self):
- p = TestPipeline()
- records = p | 'create' >> beam.Create(self.SAMPLE_RECORDS)
- result = (records
- | 'points' >> beam.FlatMap(coders.compute_points)
- | beam.CombinePerKey(sum))
- assert_that(result, equal_to([('Italy', 0), ('Brasil', 6), ('Germany', 3)]))
- p.run()
+ with TestPipeline() as p:
+ records = p | 'create' >> beam.Create(self.SAMPLE_RECORDS)
+ result = (records
+ | 'points' >> beam.FlatMap(coders.compute_points)
+ | beam.CombinePerKey(sum))
+ assert_that(result,
+ equal_to([('Italy', 0), ('Brasil', 6), ('Germany', 3)]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
index 609f2cd..db86003 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
@@ -30,10 +30,10 @@
from apache_beam.io import WriteToText
from apache_beam.options.pipeline_options import PipelineOptions
-
# pylint doesn't understand our pipeline syntax:
# pylint:disable=expression-not-assigned
+
class Count1(beam.PTransform):
"""Count as a subclass of PTransform, with an apply method."""
@@ -47,11 +47,10 @@
def run_count1(known_args, options):
"""Runs the first example pipeline."""
logging.info('Running first pipeline')
- p = beam.Pipeline(options=options)
- (p | beam.io.ReadFromText(known_args.input)
- | Count1()
- | beam.io.WriteToText(known_args.output))
- p.run().wait_until_finish()
+ with beam.Pipeline(options=options) as p:
+ (p | beam.io.ReadFromText(known_args.input)
+ | Count1()
+ | beam.io.WriteToText(known_args.output))
@beam.ptransform_fn
@@ -66,11 +65,10 @@
def run_count2(known_args, options):
"""Runs the second example pipeline."""
logging.info('Running second pipeline')
- p = beam.Pipeline(options=options)
- (p | ReadFromText(known_args.input)
- | Count2() # pylint: disable=no-value-for-parameter
- | WriteToText(known_args.output))
- p.run().wait_until_finish()
+ with beam.Pipeline(options=options) as p:
+ (p | ReadFromText(known_args.input)
+ | Count2() # pylint: disable=no-value-for-parameter
+ | WriteToText(known_args.output))
@beam.ptransform_fn
@@ -93,11 +91,10 @@
def run_count3(known_args, options):
"""Runs the third example pipeline."""
logging.info('Running third pipeline')
- p = beam.Pipeline(options=options)
- (p | ReadFromText(known_args.input)
- | Count3(2) # pylint: disable=no-value-for-parameter
- | WriteToText(known_args.output))
- p.run()
+ with beam.Pipeline(options=options) as p:
+ (p | ReadFromText(known_args.input)
+ | Count3(2) # pylint: disable=no-value-for-parameter
+ | WriteToText(known_args.output))
def get_args(argv):
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
index c7c6dba..7aaccb4 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
@@ -40,12 +40,11 @@
self.run_pipeline(custom_ptransform.Count3(factor), factor=factor)
def run_pipeline(self, count_implementation, factor=1):
- p = TestPipeline()
- words = p | beam.Create(['CAT', 'DOG', 'CAT', 'CAT', 'DOG'])
- result = words | count_implementation
- assert_that(
- result, equal_to([('CAT', (3 * factor)), ('DOG', (2 * factor))]))
- p.run()
+ with TestPipeline() as p:
+ words = p | beam.Create(['CAT', 'DOG', 'CAT', 'CAT', 'DOG'])
+ result = words | count_implementation
+ assert_that(
+ result, equal_to([('CAT', (3 * factor)), ('DOG', (2 * factor))]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
index 411feb8..7204e3b 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
@@ -32,12 +32,14 @@
The following options must be provided to run this pipeline in read-only mode:
``
---project YOUR_PROJECT_ID
+--dataset YOUR_DATASET
--kind YOUR_DATASTORE_KIND
--output [YOUR_LOCAL_FILE *or* gs://YOUR_OUTPUT_PATH]
--read_only
``
+Dataset maps to Project ID for v1 version of datastore.
+
Read-write Mode: In this mode, this example reads words from an input file,
converts them to Cloud Datastore ``Entity`` objects and writes them to
Cloud Datastore using the ``datastoreio.Write`` transform. The second pipeline
@@ -47,7 +49,7 @@
The following options must be provided to run this pipeline in read-write mode:
``
---project YOUR_PROJECT_ID
+--dataset YOUR_DATASET
--kind YOUR_DATASTORE_KIND
--output [YOUR_LOCAL_FILE *or* gs://YOUR_OUTPUT_PATH]
``
@@ -68,7 +70,8 @@
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.proto.datastore.v1 import query_pb2
-from googledatastore import helper as datastore_helper, PropertyFilter
+from googledatastore import helper as datastore_helper
+from googledatastore import PropertyFilter
import apache_beam as beam
from apache_beam.io import ReadFromText
@@ -76,7 +79,6 @@
from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
-from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
@@ -133,20 +135,17 @@
return entity
-def write_to_datastore(project, user_options, pipeline_options):
+def write_to_datastore(user_options, pipeline_options):
"""Creates a pipeline that writes entities to Cloud Datastore."""
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # pylint: disable=expression-not-assigned
- (p
- | 'read' >> ReadFromText(user_options.input)
- | 'create entity' >> beam.Map(
- EntityWrapper(user_options.namespace, user_options.kind,
- user_options.ancestor).make_entity)
- | 'write to datastore' >> WriteToDatastore(project))
-
- # Actually run the pipeline (all operations above are deferred).
- p.run().wait_until_finish()
+ # pylint: disable=expression-not-assigned
+ (p
+ | 'read' >> ReadFromText(user_options.input)
+ | 'create entity' >> beam.Map(
+ EntityWrapper(user_options.namespace, user_options.kind,
+ user_options.ancestor).make_entity)
+ | 'write to datastore' >> WriteToDatastore(user_options.dataset))
def make_ancestor_query(kind, namespace, ancestor):
@@ -169,7 +168,7 @@
return query
-def read_from_datastore(project, user_options, pipeline_options):
+def read_from_datastore(user_options, pipeline_options):
"""Creates a pipeline that reads entities from Cloud Datastore."""
p = beam.Pipeline(options=pipeline_options)
# Create a query to read entities from datastore.
@@ -178,25 +177,32 @@
# Read entities from Cloud Datastore into a PCollection.
lines = p | 'read from datastore' >> ReadFromDatastore(
- project, query, user_options.namespace)
+ user_options.dataset, query, user_options.namespace)
# Count the occurrences of each word.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
counts = (lines
| 'split' >> (beam.ParDo(WordExtractingDoFn())
.with_output_types(unicode))
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| 'group' >> beam.GroupByKey()
- | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones))))
+ | 'count' >> beam.Map(count_ones))
# Format the counts into a PCollection of strings.
- output = counts | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
+ output = counts | 'format' >> beam.Map(format_result)
# Write the output using a "Write" transform that has side effects.
# pylint: disable=expression-not-assigned
output | 'write' >> beam.io.WriteToText(file_path_prefix=user_options.output,
num_shards=user_options.num_shards)
- # Actually run the pipeline (all operations above are deferred).
result = p.run()
# Wait until completion, main thread would access post-completion job results.
result.wait_until_finish()
@@ -211,6 +217,9 @@
dest='input',
default='gs://dataflow-samples/shakespeare/kinglear.txt',
help='Input file to process.')
+ parser.add_argument('--dataset',
+ dest='dataset',
+ help='Dataset ID to read from Cloud Datastore.')
parser.add_argument('--kind',
dest='kind',
required=True,
@@ -241,15 +250,13 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- gcloud_options = pipeline_options.view_as(GoogleCloudOptions)
# Write to Datastore if `read_only` options is not specified.
if not known_args.read_only:
- write_to_datastore(gcloud_options.project, known_args, pipeline_options)
+ write_to_datastore(known_args, pipeline_options)
# Read entities from Datastore.
- result = read_from_datastore(gcloud_options.project, known_args,
- pipeline_options)
+ result = read_from_datastore(known_args, pipeline_options)
empty_lines_filter = MetricsFilter().with_name('empty_lines')
query_result = result.metrics().query(empty_lines_filter)
diff --git a/sdks/python/apache_beam/examples/cookbook/filters.py b/sdks/python/apache_beam/examples/cookbook/filters.py
index 374001c..1fbf763 100644
--- a/sdks/python/apache_beam/examples/cookbook/filters.py
+++ b/sdks/python/apache_beam/examples/cookbook/filters.py
@@ -86,20 +86,17 @@
help='Numeric value of month to filter on.')
known_args, pipeline_args = parser.parse_known_args(argv)
- p = beam.Pipeline(argv=pipeline_args)
+ with beam.Pipeline(argv=pipeline_args) as p:
- input_data = p | beam.io.Read(beam.io.BigQuerySource(known_args.input))
+ input_data = p | beam.io.Read(beam.io.BigQuerySource(known_args.input))
- # pylint: disable=expression-not-assigned
- (filter_cold_days(input_data, known_args.month_filter)
- | 'SaveToBQ' >> beam.io.Write(beam.io.BigQuerySink(
- known_args.output,
- schema='year:INTEGER,month:INTEGER,day:INTEGER,mean_temp:FLOAT',
- create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
- write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
-
- # Actually run the pipeline (all operations above are deferred).
- p.run()
+ # pylint: disable=expression-not-assigned
+ (filter_cold_days(input_data, known_args.month_filter)
+ | 'SaveToBQ' >> beam.io.Write(beam.io.BigQuerySink(
+ known_args.output,
+ schema='year:INTEGER,month:INTEGER,day:INTEGER,mean_temp:FLOAT',
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
index 6bdadae..d5dbecf 100644
--- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
+++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
@@ -35,10 +35,10 @@
from apache_beam import coders
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
-from apache_beam.typehints import typehints
-from apache_beam.typehints.decorators import with_output_types
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.typehints import typehints
+from apache_beam.typehints.decorators import with_output_types
class Player(object):
@@ -95,28 +95,27 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # Register the custom coder for the Player class, so that it will be used in
- # the computation.
- coders.registry.register_coder(Player, PlayerCoder)
+ # Register the custom coder for the Player class, so that it will be used in
+ # the computation.
+ coders.registry.register_coder(Player, PlayerCoder)
- (p # pylint: disable=expression-not-assigned
- | ReadFromText(known_args.input)
- # The get_players function is annotated with a type hint above, so the type
- # system knows the output type of the following operation is a key-value pair
- # of a Player and an int. Please see the documentation for details on
- # types that are inferred automatically as well as other ways to specify
- # type hints.
- | beam.Map(get_players)
- # The output type hint of the previous step is used to infer that the key
- # type of the following operation is the Player type. Since a custom coder
- # is registered for the Player class above, a PlayerCoder will be used to
- # encode Player objects as keys for this combine operation.
- | beam.CombinePerKey(sum)
- | beam.Map(lambda (k, v): '%s,%d' % (k.name, v))
- | WriteToText(known_args.output))
- return p.run()
+ (p # pylint: disable=expression-not-assigned
+ | ReadFromText(known_args.input)
+ # The get_players function is annotated with a type hint above, so the type
+ # system knows the output type of the following operation is a key-value
+ # pair of a Player and an int. Please see the documentation for details on
+ # types that are inferred automatically as well as other ways to specify
+ # type hints.
+ | beam.Map(get_players)
+ # The output type hint of the previous step is used to infer that the key
+ # type of the following operation is the Player type. Since a custom coder
+ # is registered for the Player class above, a PlayerCoder will be used to
+ # encode Player objects as keys for this combine operation.
+ | beam.CombinePerKey(sum)
+ | beam.Map(lambda k_v: '%s,%d' % (k_v[0].name, k_v[1]))
+ | WriteToText(known_args.output))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
index 4e87966..ed38b5d 100644
--- a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
@@ -22,7 +22,7 @@
import unittest
from apache_beam.examples.cookbook import group_with_coder
-
+from apache_beam.testing.util import open_shards
# Patch group_with_coder.PlayerCoder.decode(). To test that the PlayerCoder was
# used, we do not strip the prepended 'x:' string when decoding a Player object.
@@ -50,10 +50,10 @@
temp_path = self.create_temp_file(self.SAMPLE_RECORDS)
group_with_coder.run([
'--input=%s*' % temp_path,
- '--output=%s.result' % temp_path]).wait_until_finish()
+ '--output=%s.result' % temp_path])
# Parse result file and compare.
results = []
- with open(temp_path + '.result-00000-of-00001') as result_file:
+ with open_shards(temp_path + '.result-*-of-*') as result_file:
for line in result_file:
name, points = line.split(',')
results.append((name, int(points)))
@@ -71,10 +71,10 @@
group_with_coder.run([
'--no_pipeline_type_check',
'--input=%s*' % temp_path,
- '--output=%s.result' % temp_path]).wait_until_finish()
+ '--output=%s.result' % temp_path])
# Parse result file and compare.
results = []
- with open(temp_path + '.result-00000-of-00001') as result_file:
+ with open_shards(temp_path + '.result-*-of-*') as result_file:
for line in result_file:
name, points = line.split(',')
results.append((name, int(points)))
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
index 4f53c61..237d4ca 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
@@ -70,64 +70,75 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # Helper: read a tab-separated key-value mapping from a text file, escape all
- # quotes/backslashes, and convert it a PCollection of (key, value) pairs.
- def read_kv_textfile(label, textfile):
- return (p
- | 'Read: %s' % label >> ReadFromText(textfile)
- | 'Backslash: %s' % label >> beam.Map(
- lambda x: re.sub(r'\\', r'\\\\', x))
- | 'EscapeQuotes: %s' % label >> beam.Map(
- lambda x: re.sub(r'"', r'\"', x))
- | 'Split: %s' % label >> beam.Map(
- lambda x: re.split(r'\t+', x, 1)))
+ # Helper: read a tab-separated key-value mapping from a text file,
+ # escape all quotes/backslashes, and convert it a PCollection of
+ # (key, value) pairs.
+ def read_kv_textfile(label, textfile):
+ return (p
+ | 'Read: %s' % label >> ReadFromText(textfile)
+ | 'Backslash: %s' % label >> beam.Map(
+ lambda x: re.sub(r'\\', r'\\\\', x))
+ | 'EscapeQuotes: %s' % label >> beam.Map(
+ lambda x: re.sub(r'"', r'\"', x))
+ | 'Split: %s' % label >> beam.Map(
+ lambda x: re.split(r'\t+', x, 1)))
- # Read input databases.
- email = read_kv_textfile('email', known_args.input_email)
- phone = read_kv_textfile('phone', known_args.input_phone)
- snailmail = read_kv_textfile('snailmail', known_args.input_snailmail)
+ # Read input databases.
+ email = read_kv_textfile('email', known_args.input_email)
+ phone = read_kv_textfile('phone', known_args.input_phone)
+ snailmail = read_kv_textfile('snailmail', known_args.input_snailmail)
- # Group together all entries under the same name.
- grouped = (email, phone, snailmail) | 'group_by_name' >> beam.CoGroupByKey()
+ # Group together all entries under the same name.
+ grouped = (email, phone, snailmail) | 'group_by_name' >> beam.CoGroupByKey()
- # Prepare tab-delimited output; something like this:
- # "name"<TAB>"email_1,email_2"<TAB>"phone"<TAB>"first_snailmail_only"
- tsv_lines = grouped | beam.Map(
- lambda (name, (email, phone, snailmail)): '\t'.join(
+ # Prepare tab-delimited output; something like this:
+ # "name"<TAB>"email_1,email_2"<TAB>"phone"<TAB>"first_snailmail_only"
+ def format_as_tsv(name_email_phone_snailmail):
+ (name, (email, phone, snailmail)) = name_email_phone_snailmail
+ return '\t'.join(
['"%s"' % name,
'"%s"' % ','.join(email),
'"%s"' % ','.join(phone),
- '"%s"' % next(iter(snailmail), '')]))
+ '"%s"' % next(iter(snailmail), '')])
- # Compute some stats about our database of people.
- luddites = grouped | beam.Filter( # People without email.
- lambda (name, (email, phone, snailmail)): not next(iter(email), None))
- writers = grouped | beam.Filter( # People without phones.
- lambda (name, (email, phone, snailmail)): not next(iter(phone), None))
- nomads = grouped | beam.Filter( # People without addresses.
- lambda (name, (email, phone, snailmail)): not next(iter(snailmail), None))
+ tsv_lines = grouped | beam.Map(format_as_tsv)
- num_luddites = luddites | 'Luddites' >> beam.combiners.Count.Globally()
- num_writers = writers | 'Writers' >> beam.combiners.Count.Globally()
- num_nomads = nomads | 'Nomads' >> beam.combiners.Count.Globally()
+ # Compute some stats about our database of people.
+ def without_email(name_email_phone_snailmail):
+ (_, (email, _, _)) = name_email_phone_snailmail
+ return not next(iter(email), None)
- # Write tab-delimited output.
- # pylint: disable=expression-not-assigned
- tsv_lines | 'WriteTsv' >> WriteToText(known_args.output_tsv)
+ def without_phones(name_email_phone_snailmail):
+ (_, (_, phone, _)) = name_email_phone_snailmail
+ return not next(iter(phone), None)
- # TODO(silviuc): Move the assert_results logic to the unit test.
- if assert_results is not None:
- expected_luddites, expected_writers, expected_nomads = assert_results
- assert_that(num_luddites, equal_to([expected_luddites]),
- label='assert:luddites')
- assert_that(num_writers, equal_to([expected_writers]),
- label='assert:writers')
- assert_that(num_nomads, equal_to([expected_nomads]),
- label='assert:nomads')
- # Execute pipeline.
- return p.run()
+ def without_address(name_email_phone_snailmail):
+ (_, (_, _, snailmail)) = name_email_phone_snailmail
+ return not next(iter(snailmail), None)
+
+ luddites = grouped | beam.Filter(without_email) # People without email.
+ writers = grouped | beam.Filter(without_phones) # People without phones.
+ nomads = grouped | beam.Filter(without_address) # People without addresses.
+
+ num_luddites = luddites | 'Luddites' >> beam.combiners.Count.Globally()
+ num_writers = writers | 'Writers' >> beam.combiners.Count.Globally()
+ num_nomads = nomads | 'Nomads' >> beam.combiners.Count.Globally()
+
+ # Write tab-delimited output.
+ # pylint: disable=expression-not-assigned
+ tsv_lines | 'WriteTsv' >> WriteToText(known_args.output_tsv)
+
+ # TODO(silviuc): Move the assert_results logic to the unit test.
+ if assert_results is not None:
+ expected_luddites, expected_writers, expected_nomads = assert_results
+ assert_that(num_luddites, equal_to([expected_luddites]),
+ label='assert:luddites')
+ assert_that(num_writers, equal_to([expected_writers]),
+ label='assert:writers')
+ assert_that(num_nomads, equal_to([expected_nomads]),
+ label='assert:nomads')
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
index 09f71d3..32a3d51 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
@@ -22,6 +22,7 @@
import unittest
from apache_beam.examples.cookbook import mergecontacts
+from apache_beam.testing.util import open_shards
class MergeContactsTest(unittest.TestCase):
@@ -107,15 +108,14 @@
result_prefix = self.create_temp_file('')
- result = mergecontacts.run([
+ mergecontacts.run([
'--input_email=%s' % path_email,
'--input_phone=%s' % path_phone,
'--input_snailmail=%s' % path_snailmail,
'--output_tsv=%s.tsv' % result_prefix,
'--output_stats=%s.stats' % result_prefix], assert_results=(2, 1, 3))
- result.wait_until_finish()
- with open('%s.tsv-00000-of-00001' % result_prefix) as f:
+ with open_shards('%s.tsv-*-of-*' % result_prefix) as f:
contents = f.read()
self.assertEqual(self.EXPECTED_TSV, self.normalize_tsv_results(contents))
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
index 9759f48..e3df3a8 100644
--- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
+++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
@@ -119,11 +119,19 @@
"""
def expand(self, pcoll):
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
return (pcoll
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| 'group' >> beam.GroupByKey()
- | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones)))
- | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c)))
+ | 'count' >> beam.Map(count_ones)
+ | 'format' >> beam.Map(format_result))
def run(argv=None):
@@ -141,43 +149,41 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- lines = p | ReadFromText(known_args.input)
+ lines = p | ReadFromText(known_args.input)
- # with_outputs allows accessing the explicitly tagged outputs of a DoFn.
- split_lines_result = (lines
- | beam.ParDo(SplitLinesToWordsFn()).with_outputs(
- SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS,
- SplitLinesToWordsFn.OUTPUT_TAG_CHARACTER_COUNT,
- main='words'))
+ # with_outputs allows accessing the explicitly tagged outputs of a DoFn.
+ split_lines_result = (lines
+ | beam.ParDo(SplitLinesToWordsFn()).with_outputs(
+ SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS,
+ SplitLinesToWordsFn.OUTPUT_TAG_CHARACTER_COUNT,
+ main='words'))
- # split_lines_result is an object of type DoOutputsTuple. It supports
- # accessing result in alternative ways.
- words, _, _ = split_lines_result
- short_words = split_lines_result[
- SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS]
- character_count = split_lines_result.tag_character_count
+ # split_lines_result is an object of type DoOutputsTuple. It supports
+ # accessing result in alternative ways.
+ words, _, _ = split_lines_result
+ short_words = split_lines_result[
+ SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS]
+ character_count = split_lines_result.tag_character_count
- # pylint: disable=expression-not-assigned
- (character_count
- | 'pair_with_key' >> beam.Map(lambda x: ('chars_temp_key', x))
- | beam.GroupByKey()
- | 'count chars' >> beam.Map(lambda (_, counts): sum(counts))
- | 'write chars' >> WriteToText(known_args.output + '-chars'))
+ # pylint: disable=expression-not-assigned
+ (character_count
+ | 'pair_with_key' >> beam.Map(lambda x: ('chars_temp_key', x))
+ | beam.GroupByKey()
+ | 'count chars' >> beam.Map(lambda char_counts: sum(char_counts[1]))
+ | 'write chars' >> WriteToText(known_args.output + '-chars'))
- # pylint: disable=expression-not-assigned
- (short_words
- | 'count short words' >> CountWords()
- | 'write short words' >> WriteToText(
- known_args.output + '-short-words'))
+ # pylint: disable=expression-not-assigned
+ (short_words
+ | 'count short words' >> CountWords()
+ | 'write short words' >> WriteToText(
+ known_args.output + '-short-words'))
- # pylint: disable=expression-not-assigned
- (words
- | 'count words' >> CountWords()
- | 'write words' >> WriteToText(known_args.output + '-words'))
-
- return p.run()
+ # pylint: disable=expression-not-assigned
+ (words
+ | 'count words' >> CountWords()
+ | 'write words' >> WriteToText(known_args.output + '-words'))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
index 2c9111c..1051106 100644
--- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
@@ -23,6 +23,7 @@
import unittest
from apache_beam.examples.cookbook import multiple_output_pardo
+from apache_beam.testing.util import open_shards
class MultipleOutputParDo(unittest.TestCase):
@@ -37,9 +38,9 @@
f.write(contents)
return f.name
- def get_wordcount_results(self, temp_path):
+ def get_wordcount_results(self, result_path):
results = []
- with open(temp_path) as result_file:
+ with open_shards(result_path) as result_file:
for line in result_file:
match = re.search(r'([A-Za-z]+): ([0-9]+)', line)
if match is not None:
@@ -52,18 +53,18 @@
multiple_output_pardo.run([
'--input=%s*' % temp_path,
- '--output=%s' % result_prefix]).wait_until_finish()
+ '--output=%s' % result_prefix])
expected_char_count = len(''.join(self.SAMPLE_TEXT.split('\n')))
- with open(result_prefix + '-chars-00000-of-00001') as f:
+ with open_shards(result_prefix + '-chars-*-of-*') as f:
contents = f.read()
self.assertEqual(expected_char_count, int(contents))
short_words = self.get_wordcount_results(
- result_prefix + '-short-words-00000-of-00001')
+ result_prefix + '-short-words-*-of-*')
self.assertEqual(sorted(short_words), sorted(self.EXPECTED_SHORT_WORDS))
- words = self.get_wordcount_results(result_prefix + '-words-00000-of-00001')
+ words = self.get_wordcount_results(result_prefix + '-words-*-of-*')
self.assertEqual(sorted(words), sorted(self.EXPECTED_WORDS))
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 7259572..6cc96ef 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -31,10 +31,14 @@
"""
import apache_beam as beam
+from apache_beam.io import iobase
+from apache_beam.io.range_trackers import OffsetRangeTracker
from apache_beam.metrics import Metrics
+from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
+from apache_beam.transforms.core import PTransform
# Quiet some pylint warnings that happen because of the somewhat special
# format for the code snippets.
@@ -147,18 +151,15 @@
pipeline_options = PipelineOptions(argv)
my_options = pipeline_options.view_as(MyOptions)
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- (p
- | beam.io.ReadFromText(my_options.input)
- | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- | beam.Map(lambda x: (x, 1))
- | beam.combiners.Count.PerKey()
- | beam.io.WriteToText(my_options.output))
-
- result = p.run()
+ (p
+ | beam.io.ReadFromText(my_options.input)
+ | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+ | beam.Map(lambda x: (x, 1))
+ | beam.combiners.Count.PerKey()
+ | beam.io.WriteToText(my_options.output))
# [END model_pipelines]
- result.wait_until_finish()
def model_pcollection(argv):
@@ -178,21 +179,18 @@
my_options = pipeline_options.view_as(MyOptions)
# [START model_pcollection]
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- lines = (p
- | beam.Create([
- 'To be, or not to be: that is the question: ',
- 'Whether \'tis nobler in the mind to suffer ',
- 'The slings and arrows of outrageous fortune, ',
- 'Or to take arms against a sea of troubles, ']))
- # [END model_pcollection]
+ lines = (p
+ | beam.Create([
+ 'To be, or not to be: that is the question: ',
+ 'Whether \'tis nobler in the mind to suffer ',
+ 'The slings and arrows of outrageous fortune, ',
+ 'Or to take arms against a sea of troubles, ']))
+ # [END model_pcollection]
- (lines
- | beam.io.WriteToText(my_options.output))
-
- result = p.run()
- result.wait_until_finish()
+ (lines
+ | beam.io.WriteToText(my_options.output))
def pipeline_options_remote(argv):
@@ -297,12 +295,10 @@
known_args, pipeline_args = parser.parse_known_args(argv)
# Create the Pipeline with remaining arguments.
- p = beam.Pipeline(argv=pipeline_args)
- lines = p | 'ReadFromText' >> beam.io.ReadFromText(known_args.input)
- lines | 'WriteToText' >> beam.io.WriteToText(known_args.output)
- # [END pipeline_options_command_line]
-
- p.run().wait_until_finish()
+ with beam.Pipeline(argv=pipeline_args) as p:
+ lines = p | 'ReadFromText' >> beam.io.ReadFromText(known_args.input)
+ lines | 'WriteToText' >> beam.io.WriteToText(known_args.output)
+ # [END pipeline_options_command_line]
def pipeline_logging(lines, output):
@@ -329,13 +325,11 @@
# Remaining WordCount example code ...
# [END pipeline_logging]
- p = TestPipeline() # Use TestPipeline for testing.
- (p
- | beam.Create(lines)
- | beam.ParDo(ExtractWordsFn())
- | beam.io.WriteToText(output))
-
- p.run()
+ with TestPipeline() as p: # Use TestPipeline for testing.
+ (p
+ | beam.Create(lines)
+ | beam.ParDo(ExtractWordsFn())
+ | beam.io.WriteToText(output))
def pipeline_monitoring(renames):
@@ -385,20 +379,19 @@
pipeline_options = PipelineOptions()
options = pipeline_options.view_as(WordCountOptions)
- p = TestPipeline() # Use TestPipeline for testing.
+ with TestPipeline() as p: # Use TestPipeline for testing.
- # [START pipeline_monitoring_execution]
- (p
- # Read the lines of the input text.
- | 'ReadLines' >> beam.io.ReadFromText(options.input)
- # Count the words.
- | CountWords()
- # Write the formatted word counts to output.
- | 'WriteCounts' >> beam.io.WriteToText(options.output))
- # [END pipeline_monitoring_execution]
+ # [START pipeline_monitoring_execution]
+ (p
+ # Read the lines of the input text.
+ | 'ReadLines' >> beam.io.ReadFromText(options.input)
+ # Count the words.
+ | CountWords()
+ # Write the formatted word counts to output.
+ | 'WriteCounts' >> beam.io.WriteToText(options.output))
+ # [END pipeline_monitoring_execution]
- p.visit(SnippetUtils.RenameFiles(renames))
- p.run()
+ p.visit(SnippetUtils.RenameFiles(renames))
def examples_wordcount_minimal(renames):
@@ -443,7 +436,7 @@
# [END examples_wordcount_minimal_count]
# [START examples_wordcount_minimal_map]
- | beam.Map(lambda (word, count): '%s: %s' % (word, count))
+ | beam.Map(lambda word_count: '%s: %s' % (word_count[0], word_count[1]))
# [END examples_wordcount_minimal_map]
# [START examples_wordcount_minimal_write]
@@ -478,40 +471,88 @@
default='gs://my-bucket/input')
options = PipelineOptions(argv)
- p = beam.Pipeline(options=options)
- # [END examples_wordcount_wordcount_options]
+ with beam.Pipeline(options=options) as p:
+ # [END examples_wordcount_wordcount_options]
- lines = p | beam.io.ReadFromText(
- 'gs://dataflow-samples/shakespeare/kinglear.txt')
+ lines = p | beam.io.ReadFromText(
+ 'gs://dataflow-samples/shakespeare/kinglear.txt')
- # [START examples_wordcount_wordcount_composite]
- class CountWords(beam.PTransform):
+ # [START examples_wordcount_wordcount_composite]
+ class CountWords(beam.PTransform):
- def expand(self, pcoll):
- return (pcoll
- # Convert lines of text into individual words.
- | 'ExtractWords' >> beam.FlatMap(
- lambda x: re.findall(r'[A-Za-z\']+', x))
+ def expand(self, pcoll):
+ return (pcoll
+ # Convert lines of text into individual words.
+ | 'ExtractWords' >> beam.FlatMap(
+ lambda x: re.findall(r'[A-Za-z\']+', x))
- # Count the number of times each word occurs.
- | beam.combiners.Count.PerElement())
+ # Count the number of times each word occurs.
+ | beam.combiners.Count.PerElement())
- counts = lines | CountWords()
- # [END examples_wordcount_wordcount_composite]
+ counts = lines | CountWords()
+ # [END examples_wordcount_wordcount_composite]
- # [START examples_wordcount_wordcount_dofn]
- class FormatAsTextFn(beam.DoFn):
+ # [START examples_wordcount_wordcount_dofn]
+ class FormatAsTextFn(beam.DoFn):
- def process(self, element):
- word, count = element
- yield '%s: %s' % (word, count)
+ def process(self, element):
+ word, count = element
+ yield '%s: %s' % (word, count)
- formatted = counts | beam.ParDo(FormatAsTextFn())
- # [END examples_wordcount_wordcount_dofn]
+ formatted = counts | beam.ParDo(FormatAsTextFn())
+ # [END examples_wordcount_wordcount_dofn]
- formatted | beam.io.WriteToText('gs://my-bucket/counts.txt')
+ formatted | beam.io.WriteToText('gs://my-bucket/counts.txt')
+ p.visit(SnippetUtils.RenameFiles(renames))
+
+
+def examples_wordcount_templated(renames):
+ """Templated WordCount example snippet."""
+ import re
+
+ import apache_beam as beam
+ from apache_beam.io import ReadFromText
+ from apache_beam.io import WriteToText
+ from apache_beam.options.pipeline_options import PipelineOptions
+
+ # [START example_wordcount_templated]
+ class WordcountTemplatedOptions(PipelineOptions):
+ @classmethod
+ def _add_argparse_args(cls, parser):
+ # Use add_value_provider_argument for arguments to be templatable
+ # Use add_argument as usual for non-templatable arguments
+ parser.add_value_provider_argument(
+ '--input',
+ help='Path of the file to read from')
+ parser.add_argument(
+ '--output',
+ required=True,
+ help='Output file to write results to.')
+ pipeline_options = PipelineOptions(['--output', 'some/output_path'])
+ p = beam.Pipeline(options=pipeline_options)
+
+ wordcount_options = pipeline_options.view_as(WordcountTemplatedOptions)
+ lines = p | 'Read' >> ReadFromText(wordcount_options.input)
+ # [END example_wordcount_templated]
+
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
+ (
+ lines
+ | 'ExtractWords' >> beam.FlatMap(
+ lambda x: re.findall(r'[A-Za-z\']+', x))
+ | 'PairWithOnes' >> beam.Map(lambda x: (x, 1))
+ | 'Group' >> beam.GroupByKey()
+ | 'Sum' >> beam.Map(lambda word_ones: (word_ones[0], sum(word_ones[1])))
+ | 'Format' >> beam.Map(format_result)
+ | 'Write' >> WriteToText(wordcount_options.output)
+ )
+
p.visit(SnippetUtils.RenameFiles(renames))
- p.run().wait_until_finish()
+ result = p.run()
+ result.wait_until_finish()
def examples_wordcount_debugging(renames):
@@ -558,34 +599,72 @@
# [END example_wordcount_debugging_logging]
# [END example_wordcount_debugging_aggregators]
- p = TestPipeline() # Use TestPipeline for testing.
- filtered_words = (
- p
- | beam.io.ReadFromText(
- 'gs://dataflow-samples/shakespeare/kinglear.txt')
- | 'ExtractWords' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- | beam.combiners.Count.PerElement()
- | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
+ with TestPipeline() as p: # Use TestPipeline for testing.
+ filtered_words = (
+ p
+ | beam.io.ReadFromText(
+ 'gs://dataflow-samples/shakespeare/kinglear.txt')
+ | 'ExtractWords' >> beam.FlatMap(
+ lambda x: re.findall(r'[A-Za-z\']+', x))
+ | beam.combiners.Count.PerElement()
+ | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
- # [START example_wordcount_debugging_assert]
- beam.testing.util.assert_that(
- filtered_words, beam.testing.util.equal_to(
- [('Flourish', 3), ('stomach', 1)]))
- # [END example_wordcount_debugging_assert]
+ # [START example_wordcount_debugging_assert]
+ beam.testing.util.assert_that(
+ filtered_words, beam.testing.util.equal_to(
+ [('Flourish', 3), ('stomach', 1)]))
+ # [END example_wordcount_debugging_assert]
- output = (filtered_words
- | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
- | 'Write' >> beam.io.WriteToText('gs://my-bucket/counts.txt'))
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
+ output = (filtered_words
+ | 'format' >> beam.Map(format_result)
+ | 'Write' >> beam.io.WriteToText('gs://my-bucket/counts.txt'))
+
+ p.visit(SnippetUtils.RenameFiles(renames))
+
+
+def examples_ptransforms_templated(renames):
+ # [START examples_ptransforms_templated]
+ import apache_beam as beam
+ from apache_beam.io import WriteToText
+ from apache_beam.options.pipeline_options import PipelineOptions
+ from apache_beam.options.value_provider import StaticValueProvider
+
+ class TemplatedUserOptions(PipelineOptions):
+ @classmethod
+ def _add_argparse_args(cls, parser):
+ parser.add_value_provider_argument('--templated_int', type=int)
+
+ class MySumFn(beam.DoFn):
+ def __init__(self, templated_int):
+ self.templated_int = templated_int
+
+ def process(self, an_int):
+ yield self.templated_int.get() + an_int
+
+ pipeline_options = PipelineOptions()
+ p = beam.Pipeline(options=pipeline_options)
+
+ user_options = pipeline_options.view_as(TemplatedUserOptions)
+ my_sum_fn = MySumFn(user_options.templated_int)
+ sum = (p
+ | 'ReadCollection' >> beam.io.ReadFromText(
+ 'gs://some/integer_collection')
+ | 'StringToInt' >> beam.Map(lambda w: int(w))
+ | 'AddGivenInt' >> beam.ParDo(my_sum_fn)
+ | 'WriteResultingCollection' >> WriteToText('some/output_path'))
+ # [END examples_ptransforms_templated]
+
+ # Templates are not supported by DirectRunner (only by DataflowRunner)
+ # so a value must be provided at graph-construction time
+ my_sum_fn.templated_int = StaticValueProvider(int, 10)
p.visit(SnippetUtils.RenameFiles(renames))
- p.run()
-
-
-import apache_beam as beam
-from apache_beam.io import iobase
-from apache_beam.io.range_trackers import OffsetRangeTracker
-from apache_beam.transforms.core import PTransform
-from apache_beam.options.pipeline_options import PipelineOptions
+ result = p.run()
+ result.wait_until_finish()
# Defining a new source.
@@ -659,16 +738,14 @@
# Using the source in an example pipeline.
# [START model_custom_source_use_new_source]
- p = beam.Pipeline(options=PipelineOptions())
- numbers = p | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
- # [END model_custom_source_use_new_source]
+ with beam.Pipeline(options=PipelineOptions()) as p:
+ numbers = p | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
+ # [END model_custom_source_use_new_source]
- lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
- assert_that(
- lines, equal_to(
- ['line ' + str(number) for number in range(0, count)]))
-
- p.run().wait_until_finish()
+ lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
+ assert_that(
+ lines, equal_to(
+ ['line ' + str(number) for number in range(0, count)]))
# We recommend users to start Source classes with an underscore to discourage
# using the Source class directly when a PTransform for the source is
@@ -796,14 +873,12 @@
# Using the new sink in an example pipeline.
# [START model_custom_sink_use_new_sink]
- p = beam.Pipeline(options=PipelineOptions())
- kvs = p | 'CreateKVs' >> beam.Create(KVs)
+ with beam.Pipeline(options=PipelineOptions()) as p:
+ kvs = p | 'CreateKVs' >> beam.Create(KVs)
- kvs | 'WriteToSimpleKV' >> beam.io.Write(
- SimpleKVSink('http://url_to_simple_kv/', final_table_name))
- # [END model_custom_sink_use_new_sink]
-
- p.run().wait_until_finish()
+ kvs | 'WriteToSimpleKV' >> beam.io.Write(
+ SimpleKVSink('http://url_to_simple_kv/', final_table_name))
+ # [END model_custom_sink_use_new_sink]
# We recommend users to start Sink class names with an underscore to
# discourage using the Sink class directly when a PTransform for the sink is
@@ -828,13 +903,11 @@
final_table_name = final_table_name_with_ptransform
# [START model_custom_sink_use_ptransform]
- p = beam.Pipeline(options=PipelineOptions())
- kvs = p | 'CreateKVs' >> beam.core.Create(KVs)
- kvs | 'WriteToSimpleKV' >> WriteToKVSink(
- 'http://url_to_simple_kv/', final_table_name)
- # [END model_custom_sink_use_ptransform]
-
- p.run().wait_until_finish()
+ with beam.Pipeline(options=PipelineOptions()) as p:
+ kvs = p | 'CreateKVs' >> beam.core.Create(KVs)
+ kvs | 'WriteToSimpleKV' >> WriteToKVSink(
+ 'http://url_to_simple_kv/', final_table_name)
+ # [END model_custom_sink_use_ptransform]
def model_textio(renames):
@@ -847,37 +920,35 @@
from apache_beam.options.pipeline_options import PipelineOptions
# [START model_textio_read]
- p = beam.Pipeline(options=PipelineOptions())
- # [START model_pipelineio_read]
- lines = p | 'ReadFromText' >> beam.io.ReadFromText('path/to/input-*.csv')
- # [END model_pipelineio_read]
- # [END model_textio_read]
+ with beam.Pipeline(options=PipelineOptions()) as p:
+ # [START model_pipelineio_read]
+ lines = p | 'ReadFromText' >> beam.io.ReadFromText('path/to/input-*.csv')
+ # [END model_pipelineio_read]
+ # [END model_textio_read]
- # [START model_textio_write]
- filtered_words = lines | 'FilterWords' >> beam.FlatMap(filter_words)
- # [START model_pipelineio_write]
- filtered_words | 'WriteToText' >> beam.io.WriteToText(
- '/path/to/numbers', file_name_suffix='.csv')
- # [END model_pipelineio_write]
- # [END model_textio_write]
+ # [START model_textio_write]
+ filtered_words = lines | 'FilterWords' >> beam.FlatMap(filter_words)
+ # [START model_pipelineio_write]
+ filtered_words | 'WriteToText' >> beam.io.WriteToText(
+ '/path/to/numbers', file_name_suffix='.csv')
+ # [END model_pipelineio_write]
+ # [END model_textio_write]
- p.visit(SnippetUtils.RenameFiles(renames))
- p.run().wait_until_finish()
+ p.visit(SnippetUtils.RenameFiles(renames))
def model_textio_compressed(renames, expected):
"""Using a Read Transform to read compressed text files."""
- p = TestPipeline()
+ with TestPipeline() as p:
- # [START model_textio_write_compressed]
- lines = p | 'ReadFromText' >> beam.io.ReadFromText(
- '/path/to/input-*.csv.gz',
- compression_type=beam.io.filesystem.CompressionTypes.GZIP)
- # [END model_textio_write_compressed]
+ # [START model_textio_write_compressed]
+ lines = p | 'ReadFromText' >> beam.io.ReadFromText(
+ '/path/to/input-*.csv.gz',
+ compression_type=beam.io.filesystem.CompressionTypes.GZIP)
+ # [END model_textio_write_compressed]
- assert_that(lines, equal_to(expected))
- p.visit(SnippetUtils.RenameFiles(renames))
- p.run().wait_until_finish()
+ assert_that(lines, equal_to(expected))
+ p.visit(SnippetUtils.RenameFiles(renames))
def model_datastoreio():
@@ -983,47 +1054,44 @@
return (pcoll
| beam.FlatMap(lambda x: re.findall(r'\w+', x))
| beam.combiners.Count.PerElement()
- | beam.Map(lambda (word, c): '%s: %s' % (word, c)))
+ | beam.Map(lambda word_c: '%s: %s' % (word_c[0], word_c[1])))
# [END composite_ptransform_apply_method]
# [END composite_transform_example]
- p = TestPipeline() # Use TestPipeline for testing.
- (p
- | beam.Create(contents)
- | CountWords()
- | beam.io.WriteToText(output_path))
- p.run()
+ with TestPipeline() as p: # Use TestPipeline for testing.
+ (p
+ | beam.Create(contents)
+ | CountWords()
+ | beam.io.WriteToText(output_path))
def model_multiple_pcollections_flatten(contents, output_path):
"""Merging a PCollection with Flatten."""
some_hash_fn = lambda s: ord(s[0])
- import apache_beam as beam
- p = TestPipeline() # Use TestPipeline for testing.
partition_fn = lambda element, partitions: some_hash_fn(element) % partitions
+ import apache_beam as beam
+ with TestPipeline() as p: # Use TestPipeline for testing.
- # Partition into deciles
- partitioned = p | beam.Create(contents) | beam.Partition(partition_fn, 3)
- pcoll1 = partitioned[0]
- pcoll2 = partitioned[1]
- pcoll3 = partitioned[2]
+ # Partition into deciles
+ partitioned = p | beam.Create(contents) | beam.Partition(partition_fn, 3)
+ pcoll1 = partitioned[0]
+ pcoll2 = partitioned[1]
+ pcoll3 = partitioned[2]
- # Flatten them back into 1
+ # Flatten them back into 1
- # A collection of PCollection objects can be represented simply
- # as a tuple (or list) of PCollections.
- # (The SDK for Python has no separate type to store multiple
- # PCollection objects, whether containing the same or different
- # types.)
- # [START model_multiple_pcollections_flatten]
- merged = (
- (pcoll1, pcoll2, pcoll3)
- # A list of tuples can be "piped" directly into a Flatten transform.
- | beam.Flatten())
- # [END model_multiple_pcollections_flatten]
- merged | beam.io.WriteToText(output_path)
-
- p.run()
+ # A collection of PCollection objects can be represented simply
+ # as a tuple (or list) of PCollections.
+ # (The SDK for Python has no separate type to store multiple
+ # PCollection objects, whether containing the same or different
+ # types.)
+ # [START model_multiple_pcollections_flatten]
+ merged = (
+ (pcoll1, pcoll2, pcoll3)
+ # A list of tuples can be "piped" directly into a Flatten transform.
+ | beam.Flatten())
+ # [END model_multiple_pcollections_flatten]
+ merged | beam.io.WriteToText(output_path)
def model_multiple_pcollections_partition(contents, output_path):
@@ -1034,25 +1102,23 @@
"""Assume i in [0,100)."""
return i
import apache_beam as beam
- p = TestPipeline() # Use TestPipeline for testing.
+ with TestPipeline() as p: # Use TestPipeline for testing.
- students = p | beam.Create(contents)
+ students = p | beam.Create(contents)
- # [START model_multiple_pcollections_partition]
- def partition_fn(student, num_partitions):
- return int(get_percentile(student) * num_partitions / 100)
+ # [START model_multiple_pcollections_partition]
+ def partition_fn(student, num_partitions):
+ return int(get_percentile(student) * num_partitions / 100)
- by_decile = students | beam.Partition(partition_fn, 10)
- # [END model_multiple_pcollections_partition]
- # [START model_multiple_pcollections_partition_40th]
- fortieth_percentile = by_decile[4]
- # [END model_multiple_pcollections_partition_40th]
+ by_decile = students | beam.Partition(partition_fn, 10)
+ # [END model_multiple_pcollections_partition]
+ # [START model_multiple_pcollections_partition_40th]
+ fortieth_percentile = by_decile[4]
+ # [END model_multiple_pcollections_partition_40th]
- ([by_decile[d] for d in xrange(10) if d != 4] + [fortieth_percentile]
- | beam.Flatten()
- | beam.io.WriteToText(output_path))
-
- p.run()
+ ([by_decile[d] for d in xrange(10) if d != 4] + [fortieth_percentile]
+ | beam.Flatten()
+ | beam.io.WriteToText(output_path))
def model_group_by_key(contents, output_path):
@@ -1060,58 +1126,49 @@
import re
import apache_beam as beam
- p = TestPipeline() # Use TestPipeline for testing.
- words_and_counts = (
- p
- | beam.Create(contents)
- | beam.FlatMap(lambda x: re.findall(r'\w+', x))
- | 'one word' >> beam.Map(lambda w: (w, 1)))
- # GroupByKey accepts a PCollection of (w, 1) and
- # outputs a PCollection of (w, (1, 1, ...)).
- # (A key/value pair is just a tuple in Python.)
- # This is a somewhat forced example, since one could
- # simply use beam.combiners.Count.PerElement here.
- # [START model_group_by_key_transform]
- grouped_words = words_and_counts | beam.GroupByKey()
- # [END model_group_by_key_transform]
- (grouped_words
- | 'count words' >> beam.Map(lambda (word, counts): (word, len(counts)))
- | beam.io.WriteToText(output_path))
- p.run()
+ with TestPipeline() as p: # Use TestPipeline for testing.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
+ words_and_counts = (
+ p
+ | beam.Create(contents)
+ | beam.FlatMap(lambda x: re.findall(r'\w+', x))
+ | 'one word' >> beam.Map(lambda w: (w, 1)))
+ # GroupByKey accepts a PCollection of (w, 1) and
+ # outputs a PCollection of (w, (1, 1, ...)).
+ # (A key/value pair is just a tuple in Python.)
+ # This is a somewhat forced example, since one could
+ # simply use beam.combiners.Count.PerElement here.
+ # [START model_group_by_key_transform]
+ grouped_words = words_and_counts | beam.GroupByKey()
+ # [END model_group_by_key_transform]
+ (grouped_words
+ | 'count words' >> beam.Map(count_ones)
+ | beam.io.WriteToText(output_path))
-def model_co_group_by_key_tuple(email_list, phone_list, output_path):
+def model_co_group_by_key_tuple(emails, phones, output_path):
"""Applying a CoGroupByKey Transform to a tuple."""
import apache_beam as beam
- p = TestPipeline() # Use TestPipeline for testing.
# [START model_group_by_key_cogroupbykey_tuple]
- # Each data set is represented by key-value pairs in separate PCollections.
- # Both data sets share a common key type (in this example str).
- # The email_list contains values such as: ('joe', 'joe@example.com') with
- # multiple possible values for each key.
- # The phone_list contains values such as: ('mary': '111-222-3333') with
- # multiple possible values for each key.
- emails = p | 'email' >> beam.Create(email_list)
- phones = p | 'phone' >> beam.Create(phone_list)
# The result PCollection contains one key-value element for each key in the
# input PCollections. The key of the pair will be the key from the input and
# the value will be a dictionary with two entries: 'emails' - an iterable of
# all values for the current key in the emails PCollection and 'phones': an
# iterable of all values for the current key in the phones PCollection.
- # For instance, if 'emails' contained ('joe', 'joe@example.com') and
- # ('joe', 'joe@gmail.com'), then 'result' will contain the element
- # ('joe', {'emails': ['joe@example.com', 'joe@gmail.com'], 'phones': ...})
- result = {'emails': emails, 'phones': phones} | beam.CoGroupByKey()
+ results = ({'emails': emails, 'phones': phones}
+ | beam.CoGroupByKey())
- def join_info((name, info)):
- return '; '.join(['%s' % name,
- '%s' % ','.join(info['emails']),
- '%s' % ','.join(info['phones'])])
+ def join_info(name_info):
+ (name, info) = name_info
+ return '%s; %s; %s' %\
+ (name, sorted(info['emails']), sorted(info['phones']))
- contact_lines = result | beam.Map(join_info)
+ contact_lines = results | beam.Map(join_info)
# [END model_group_by_key_cogroupbykey_tuple]
contact_lines | beam.io.WriteToText(output_path)
- p.run()
def model_join_using_side_inputs(
@@ -1121,42 +1178,41 @@
import apache_beam as beam
from apache_beam.pvalue import AsIter
- p = TestPipeline() # Use TestPipeline for testing.
- # [START model_join_using_side_inputs]
- # This code performs a join by receiving the set of names as an input and
- # passing PCollections that contain emails and phone numbers as side inputs
- # instead of using CoGroupByKey.
- names = p | 'names' >> beam.Create(name_list)
- emails = p | 'email' >> beam.Create(email_list)
- phones = p | 'phone' >> beam.Create(phone_list)
+ with TestPipeline() as p: # Use TestPipeline for testing.
+ # [START model_join_using_side_inputs]
+ # This code performs a join by receiving the set of names as an input and
+ # passing PCollections that contain emails and phone numbers as side inputs
+ # instead of using CoGroupByKey.
+ names = p | 'names' >> beam.Create(name_list)
+ emails = p | 'email' >> beam.Create(email_list)
+ phones = p | 'phone' >> beam.Create(phone_list)
- def join_info(name, emails, phone_numbers):
- filtered_emails = []
- for name_in_list, email in emails:
- if name_in_list == name:
- filtered_emails.append(email)
+ def join_info(name, emails, phone_numbers):
+ filtered_emails = []
+ for name_in_list, email in emails:
+ if name_in_list == name:
+ filtered_emails.append(email)
- filtered_phone_numbers = []
- for name_in_list, phone_number in phone_numbers:
- if name_in_list == name:
- filtered_phone_numbers.append(phone_number)
+ filtered_phone_numbers = []
+ for name_in_list, phone_number in phone_numbers:
+ if name_in_list == name:
+ filtered_phone_numbers.append(phone_number)
- return '; '.join(['%s' % name,
- '%s' % ','.join(filtered_emails),
- '%s' % ','.join(filtered_phone_numbers)])
+ return '; '.join(['%s' % name,
+ '%s' % ','.join(filtered_emails),
+ '%s' % ','.join(filtered_phone_numbers)])
- contact_lines = names | 'CreateContacts' >> beam.core.Map(
- join_info, AsIter(emails), AsIter(phones))
- # [END model_join_using_side_inputs]
- contact_lines | beam.io.WriteToText(output_path)
- p.run()
+ contact_lines = names | 'CreateContacts' >> beam.core.Map(
+ join_info, AsIter(emails), AsIter(phones))
+ # [END model_join_using_side_inputs]
+ contact_lines | beam.io.WriteToText(output_path)
# [START model_library_transforms_keys]
class Keys(beam.PTransform):
def expand(self, pcoll):
- return pcoll | 'Keys' >> beam.Map(lambda (k, v): k)
+ return pcoll | 'Keys' >> beam.Map(lambda k_v: k_v[0])
# [END model_library_transforms_keys]
# pylint: enable=invalid-name
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index f7b51a7..505858a 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -30,15 +30,15 @@
from apache_beam import pvalue
from apache_beam import typehints
from apache_beam.coders.coders import ToStringCoder
-from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.examples.snippets import snippets
+from apache_beam.metrics import Metrics
+from apache_beam.metrics.metric import MetricsFilter
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.utils.windowed_value import WindowedValue
-# pylint: disable=expression-not-assigned
-from apache_beam.testing.test_pipeline import TestPipeline
-
# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
try:
@@ -119,7 +119,6 @@
self.assertEqual({'A', 'C'}, set(all_capitals))
def test_pardo_with_label(self):
- # pylint: disable=line-too-long
words = ['aa', 'bbc', 'defg']
# [START model_pardo_with_label]
result = words | 'CountUniqueLetters' >> beam.Map(
@@ -129,41 +128,41 @@
self.assertEqual({1, 2, 4}, set(result))
def test_pardo_side_input(self):
- p = TestPipeline()
- words = p | 'start' >> beam.Create(['a', 'bb', 'ccc', 'dddd'])
+ # pylint: disable=line-too-long
+ with TestPipeline() as p:
+ words = p | 'start' >> beam.Create(['a', 'bb', 'ccc', 'dddd'])
- # [START model_pardo_side_input]
- # Callable takes additional arguments.
- def filter_using_length(word, lower_bound, upper_bound=float('inf')):
- if lower_bound <= len(word) <= upper_bound:
- yield word
+ # [START model_pardo_side_input]
+ # Callable takes additional arguments.
+ def filter_using_length(word, lower_bound, upper_bound=float('inf')):
+ if lower_bound <= len(word) <= upper_bound:
+ yield word
- # Construct a deferred side input.
- avg_word_len = (words
- | beam.Map(len)
- | beam.CombineGlobally(beam.combiners.MeanCombineFn()))
+ # Construct a deferred side input.
+ avg_word_len = (words
+ | beam.Map(len)
+ | beam.CombineGlobally(beam.combiners.MeanCombineFn()))
- # Call with explicit side inputs.
- small_words = words | 'small' >> beam.FlatMap(filter_using_length, 0, 3)
+ # Call with explicit side inputs.
+ small_words = words | 'small' >> beam.FlatMap(filter_using_length, 0, 3)
- # A single deferred side input.
- larger_than_average = (words | 'large' >> beam.FlatMap(
- filter_using_length,
- lower_bound=pvalue.AsSingleton(avg_word_len)))
+ # A single deferred side input.
+ larger_than_average = (words | 'large' >> beam.FlatMap(
+ filter_using_length,
+ lower_bound=pvalue.AsSingleton(avg_word_len)))
- # Mix and match.
- small_but_nontrivial = words | beam.FlatMap(filter_using_length,
- lower_bound=2,
- upper_bound=pvalue.AsSingleton(
- avg_word_len))
- # [END model_pardo_side_input]
+ # Mix and match.
+ small_but_nontrivial = words | beam.FlatMap(
+ filter_using_length,
+ lower_bound=2,
+ upper_bound=pvalue.AsSingleton(avg_word_len))
+ # [END model_pardo_side_input]
- assert_that(small_words, equal_to(['a', 'bb', 'ccc']))
- assert_that(larger_than_average, equal_to(['ccc', 'dddd']),
- label='larger_than_average')
- assert_that(small_but_nontrivial, equal_to(['bb']),
- label='small_but_not_trivial')
- p.run()
+ assert_that(small_words, equal_to(['a', 'bb', 'ccc']))
+ assert_that(larger_than_average, equal_to(['ccc', 'dddd']),
+ label='larger_than_average')
+ assert_that(small_but_nontrivial, equal_to(['bb']),
+ label='small_but_not_trivial')
def test_pardo_side_input_dofn(self):
words = ['a', 'bb', 'ccc', 'dddd']
@@ -307,10 +306,9 @@
def test_runtime_checks_off(self):
# pylint: disable=expression-not-assigned
- p = TestPipeline()
- # [START type_hints_runtime_off]
- p | beam.Create(['a']) | beam.Map(lambda x: 3).with_output_types(str)
- p.run()
+ with TestPipeline() as p:
+ # [START type_hints_runtime_off]
+ p | beam.Create(['a']) | beam.Map(lambda x: 3).with_output_types(str)
# [END type_hints_runtime_off]
def test_runtime_checks_on(self):
@@ -323,47 +321,45 @@
# [END type_hints_runtime_on]
def test_deterministic_key(self):
- p = TestPipeline()
- lines = (p | beam.Create(
- ['banana,fruit,3', 'kiwi,fruit,2', 'kiwi,fruit,2', 'zucchini,veg,3']))
+ with TestPipeline() as p:
+ lines = (p | beam.Create(
+ ['banana,fruit,3', 'kiwi,fruit,2', 'kiwi,fruit,2', 'zucchini,veg,3']))
- # For pickling
- global Player # pylint: disable=global-variable-not-assigned
+ # For pickling
+ global Player # pylint: disable=global-variable-not-assigned
- # [START type_hints_deterministic_key]
- class Player(object):
- def __init__(self, team, name):
- self.team = team
- self.name = name
+ # [START type_hints_deterministic_key]
+ class Player(object):
+ def __init__(self, team, name):
+ self.team = team
+ self.name = name
- class PlayerCoder(beam.coders.Coder):
- def encode(self, player):
- return '%s:%s' % (player.team, player.name)
+ class PlayerCoder(beam.coders.Coder):
+ def encode(self, player):
+ return '%s:%s' % (player.team, player.name)
- def decode(self, s):
- return Player(*s.split(':'))
+ def decode(self, s):
+ return Player(*s.split(':'))
- def is_deterministic(self):
- return True
+ def is_deterministic(self):
+ return True
- beam.coders.registry.register_coder(Player, PlayerCoder)
+ beam.coders.registry.register_coder(Player, PlayerCoder)
- def parse_player_and_score(csv):
- name, team, score = csv.split(',')
- return Player(team, name), int(score)
+ def parse_player_and_score(csv):
+ name, team, score = csv.split(',')
+ return Player(team, name), int(score)
- totals = (
- lines
- | beam.Map(parse_player_and_score)
- | beam.CombinePerKey(sum).with_input_types(
- beam.typehints.Tuple[Player, int]))
- # [END type_hints_deterministic_key]
+ totals = (
+ lines
+ | beam.Map(parse_player_and_score)
+ | beam.CombinePerKey(sum).with_input_types(
+ beam.typehints.Tuple[Player, int]))
+ # [END type_hints_deterministic_key]
- assert_that(
- totals | beam.Map(lambda (k, v): (k.name, v)),
- equal_to([('banana', 3), ('kiwi', 4), ('zucchini', 3)]))
-
- p.run()
+ assert_that(
+ totals | beam.Map(lambda k_v: (k_v[0].name, k_v[1])),
+ equal_to([('banana', 3), ('kiwi', 4), ('zucchini', 3)]))
class SnippetsTest(unittest.TestCase):
@@ -638,7 +634,8 @@
def test_examples_wordcount(self):
pipelines = [snippets.examples_wordcount_minimal,
snippets.examples_wordcount_wordcount,
- snippets.pipeline_monitoring]
+ snippets.pipeline_monitoring,
+ snippets.examples_wordcount_templated]
for pipeline in pipelines:
temp_path = self.create_temp_file(
@@ -649,6 +646,17 @@
self.get_output(result_path),
['abc: 2', 'def: 1', 'ghi: 1', 'jkl: 1'])
+ def test_examples_ptransforms_templated(self):
+ pipelines = [snippets.examples_ptransforms_templated]
+
+ for pipeline in pipelines:
+ temp_path = self.create_temp_file('1\n 2\n 3')
+ result_path = self.create_temp_file()
+ pipeline({'read': temp_path, 'write': result_path})
+ self.assertEqual(
+ self.get_output(result_path),
+ ['11', '12', '13'])
+
def test_examples_wordcount_debugging(self):
temp_path = self.create_temp_file(
'Flourish Flourish Flourish stomach abc def')
@@ -686,12 +694,116 @@
self.assertEqual([str(s) for s in expected], self.get_output(result_path))
def test_model_co_group_by_key_tuple(self):
- email_list = [['a', 'a@example.com'], ['b', 'b@example.com']]
- phone_list = [['a', 'x4312'], ['b', 'x8452']]
- result_path = self.create_temp_file()
- snippets.model_co_group_by_key_tuple(email_list, phone_list, result_path)
- expect = ['a; a@example.com; x4312', 'b; b@example.com; x8452']
- self.assertEqual(expect, self.get_output(result_path))
+ with TestPipeline() as p:
+ # [START model_group_by_key_cogroupbykey_tuple_inputs]
+ emails_list = [
+ ('amy', 'amy@example.com'),
+ ('carl', 'carl@example.com'),
+ ('julia', 'julia@example.com'),
+ ('carl', 'carl@email.com'),
+ ]
+ phones_list = [
+ ('amy', '111-222-3333'),
+ ('james', '222-333-4444'),
+ ('amy', '333-444-5555'),
+ ('carl', '444-555-6666'),
+ ]
+
+ emails = p | 'CreateEmails' >> beam.Create(emails_list)
+ phones = p | 'CreatePhones' >> beam.Create(phones_list)
+ # [END model_group_by_key_cogroupbykey_tuple_inputs]
+
+ result_path = self.create_temp_file()
+ snippets.model_co_group_by_key_tuple(emails, phones, result_path)
+
+ # [START model_group_by_key_cogroupbykey_tuple_outputs]
+ results = [
+ ('amy', {
+ 'emails': ['amy@example.com'],
+ 'phones': ['111-222-3333', '333-444-5555']}),
+ ('carl', {
+ 'emails': ['carl@email.com', 'carl@example.com'],
+ 'phones': ['444-555-6666']}),
+ ('james', {
+ 'emails': [],
+ 'phones': ['222-333-4444']}),
+ ('julia', {
+ 'emails': ['julia@example.com'],
+ 'phones': []}),
+ ]
+ # [END model_group_by_key_cogroupbykey_tuple_outputs]
+ # [START model_group_by_key_cogroupbykey_tuple_formatted_outputs]
+ formatted_results = [
+ "amy; ['amy@example.com']; ['111-222-3333', '333-444-5555']",
+ "carl; ['carl@email.com', 'carl@example.com']; ['444-555-6666']",
+ "james; []; ['222-333-4444']",
+ "julia; ['julia@example.com']; []",
+ ]
+ # [END model_group_by_key_cogroupbykey_tuple_formatted_outputs]
+ expected_results = ['%s; %s; %s' % (name, info['emails'], info['phones'])
+ for name, info in results]
+ self.assertEqual(expected_results, formatted_results)
+ self.assertEqual(formatted_results, self.get_output(result_path))
+
+ def test_model_use_and_query_metrics(self):
+ """DebuggingWordCount example snippets."""
+
+ import re
+
+ p = TestPipeline() # Use TestPipeline for testing.
+ words = p | beam.Create(['albert', 'sam', 'mark', 'sarah',
+ 'swati', 'daniel', 'andrea'])
+
+ # pylint: disable=unused-variable
+ # [START metrics_usage_example]
+ class FilterTextFn(beam.DoFn):
+ """A DoFn that filters for a specific key based on a regex."""
+
+ def __init__(self, pattern):
+ self.pattern = pattern
+ # A custom metric can track values in your pipeline as it runs. Create
+ # custom metrics to count unmatched words, and know the distribution of
+ # word lengths in the input PCollection.
+ self.word_len_dist = Metrics.distribution(self.__class__,
+ 'word_len_dist')
+ self.unmatched_words = Metrics.counter(self.__class__,
+ 'unmatched_words')
+
+ def process(self, element):
+ word = element
+ self.word_len_dist.update(len(word))
+ if re.match(self.pattern, word):
+ yield element
+ else:
+ self.unmatched_words.inc()
+
+ filtered_words = (
+ words | 'FilterText' >> beam.ParDo(FilterTextFn('s.*')))
+ # [END metrics_usage_example]
+ # pylint: enable=unused-variable
+
+ # [START metrics_check_values_example]
+ result = p.run()
+ result.wait_until_finish()
+
+ custom_distribution = result.metrics().query(
+ MetricsFilter().with_name('word_len_dist'))['distributions']
+ custom_counter = result.metrics().query(
+ MetricsFilter().with_name('unmatched_words'))['counters']
+
+ if custom_distribution:
+ logging.info('The average word length was %d',
+ custom_distribution[0].committed.mean)
+ if custom_counter:
+ logging.info('There were %d words that did not match the filter.',
+ custom_counter[0].committed)
+ # [END metrics_check_values_example]
+
+ # There should be 4 words that did not match
+ self.assertEqual(custom_counter[0].committed, 4)
+ # The shortest word is 3 characters, the longest is 6
+ self.assertEqual(custom_distribution[0].committed.min, 3)
+ self.assertEqual(custom_distribution[0].committed.max, 6)
def test_model_join_using_side_inputs(self):
name_list = ['a', 'b']
@@ -776,14 +888,16 @@
def create_accumulator(self):
return (0.0, 0)
- def add_input(self, (sum, count), input):
+ def add_input(self, sum_count, input):
+ (sum, count) = sum_count
return sum + input, count + 1
def merge_accumulators(self, accumulators):
sums, counts = zip(*accumulators)
return sum(sums), sum(counts)
- def extract_output(self, (sum, count)):
+ def extract_output(self, sum_count):
+ (sum, count) = sum_count
return sum / count if count else float('NaN')
# [END combine_custom_average_define]
# [START combine_custom_average_execute]
@@ -802,109 +916,104 @@
self.assertEqual({('cat', 3), ('dog', 2)}, set(perkey_counts))
def test_setting_fixed_windows(self):
- p = TestPipeline()
- unkeyed_items = p | beam.Create([22, 33, 55, 100, 115, 120])
- items = (unkeyed_items
- | 'key' >> beam.Map(
- lambda x: beam.window.TimestampedValue(('k', x), x)))
- # [START setting_fixed_windows]
- from apache_beam import window
- fixed_windowed_items = (
- items | 'window' >> beam.WindowInto(window.FixedWindows(60)))
- # [END setting_fixed_windows]
- summed = (fixed_windowed_items
- | 'group' >> beam.GroupByKey()
- | 'combine' >> beam.CombineValues(sum))
- unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
- assert_that(unkeyed, equal_to([110, 215, 120]))
- p.run()
+ with TestPipeline() as p:
+ unkeyed_items = p | beam.Create([22, 33, 55, 100, 115, 120])
+ items = (unkeyed_items
+ | 'key' >> beam.Map(
+ lambda x: beam.window.TimestampedValue(('k', x), x)))
+ # [START setting_fixed_windows]
+ from apache_beam import window
+ fixed_windowed_items = (
+ items | 'window' >> beam.WindowInto(window.FixedWindows(60)))
+ # [END setting_fixed_windows]
+ summed = (fixed_windowed_items
+ | 'group' >> beam.GroupByKey()
+ | 'combine' >> beam.CombineValues(sum))
+ unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+ assert_that(unkeyed, equal_to([110, 215, 120]))
def test_setting_sliding_windows(self):
- p = TestPipeline()
- unkeyed_items = p | beam.Create([2, 16, 23])
- items = (unkeyed_items
- | 'key' >> beam.Map(
- lambda x: beam.window.TimestampedValue(('k', x), x)))
- # [START setting_sliding_windows]
- from apache_beam import window
- sliding_windowed_items = (
- items | 'window' >> beam.WindowInto(window.SlidingWindows(30, 5)))
- # [END setting_sliding_windows]
- summed = (sliding_windowed_items
- | 'group' >> beam.GroupByKey()
- | 'combine' >> beam.CombineValues(sum))
- unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
- assert_that(unkeyed,
- equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
- p.run()
+ with TestPipeline() as p:
+ unkeyed_items = p | beam.Create([2, 16, 23])
+ items = (unkeyed_items
+ | 'key' >> beam.Map(
+ lambda x: beam.window.TimestampedValue(('k', x), x)))
+ # [START setting_sliding_windows]
+ from apache_beam import window
+ sliding_windowed_items = (
+ items | 'window' >> beam.WindowInto(window.SlidingWindows(30, 5)))
+ # [END setting_sliding_windows]
+ summed = (sliding_windowed_items
+ | 'group' >> beam.GroupByKey()
+ | 'combine' >> beam.CombineValues(sum))
+ unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+ assert_that(unkeyed,
+ equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
def test_setting_session_windows(self):
- p = TestPipeline()
- unkeyed_items = p | beam.Create([2, 11, 16, 27])
- items = (unkeyed_items
- | 'key' >> beam.Map(
- lambda x: beam.window.TimestampedValue(('k', x), x)))
- # [START setting_session_windows]
- from apache_beam import window
- session_windowed_items = (
- items | 'window' >> beam.WindowInto(window.Sessions(10)))
- # [END setting_session_windows]
- summed = (session_windowed_items
- | 'group' >> beam.GroupByKey()
- | 'combine' >> beam.CombineValues(sum))
- unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
- assert_that(unkeyed,
- equal_to([29, 27]))
- p.run()
+ with TestPipeline() as p:
+ unkeyed_items = p | beam.Create([2, 11, 16, 27])
+ items = (unkeyed_items
+ | 'key' >> beam.Map(
+ lambda x: beam.window.TimestampedValue(('k', x), x)))
+ # [START setting_session_windows]
+ from apache_beam import window
+ session_windowed_items = (
+ items | 'window' >> beam.WindowInto(window.Sessions(10)))
+ # [END setting_session_windows]
+ summed = (session_windowed_items
+ | 'group' >> beam.GroupByKey()
+ | 'combine' >> beam.CombineValues(sum))
+ unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+ assert_that(unkeyed,
+ equal_to([29, 27]))
def test_setting_global_window(self):
- p = TestPipeline()
- unkeyed_items = p | beam.Create([2, 11, 16, 27])
- items = (unkeyed_items
- | 'key' >> beam.Map(
- lambda x: beam.window.TimestampedValue(('k', x), x)))
- # [START setting_global_window]
- from apache_beam import window
- session_windowed_items = (
- items | 'window' >> beam.WindowInto(window.GlobalWindows()))
- # [END setting_global_window]
- summed = (session_windowed_items
- | 'group' >> beam.GroupByKey()
- | 'combine' >> beam.CombineValues(sum))
- unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
- assert_that(unkeyed, equal_to([56]))
- p.run()
+ with TestPipeline() as p:
+ unkeyed_items = p | beam.Create([2, 11, 16, 27])
+ items = (unkeyed_items
+ | 'key' >> beam.Map(
+ lambda x: beam.window.TimestampedValue(('k', x), x)))
+ # [START setting_global_window]
+ from apache_beam import window
+ session_windowed_items = (
+ items | 'window' >> beam.WindowInto(window.GlobalWindows()))
+ # [END setting_global_window]
+ summed = (session_windowed_items
+ | 'group' >> beam.GroupByKey()
+ | 'combine' >> beam.CombineValues(sum))
+ unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+ assert_that(unkeyed, equal_to([56]))
def test_setting_timestamp(self):
- p = TestPipeline()
- unkeyed_items = p | beam.Create([12, 30, 60, 61, 66])
- items = (unkeyed_items | 'key' >> beam.Map(lambda x: ('k', x)))
+ with TestPipeline() as p:
+ unkeyed_items = p | beam.Create([12, 30, 60, 61, 66])
+ items = (unkeyed_items | 'key' >> beam.Map(lambda x: ('k', x)))
- def extract_timestamp_from_log_entry(entry):
- return entry[1]
+ def extract_timestamp_from_log_entry(entry):
+ return entry[1]
- # [START setting_timestamp]
- class AddTimestampDoFn(beam.DoFn):
+ # [START setting_timestamp]
+ class AddTimestampDoFn(beam.DoFn):
- def process(self, element):
- # Extract the numeric Unix seconds-since-epoch timestamp to be
- # associated with the current log entry.
- unix_timestamp = extract_timestamp_from_log_entry(element)
- # Wrap and emit the current entry and new timestamp in a
- # TimestampedValue.
- yield beam.window.TimestampedValue(element, unix_timestamp)
+ def process(self, element):
+ # Extract the numeric Unix seconds-since-epoch timestamp to be
+ # associated with the current log entry.
+ unix_timestamp = extract_timestamp_from_log_entry(element)
+ # Wrap and emit the current entry and new timestamp in a
+ # TimestampedValue.
+ yield beam.window.TimestampedValue(element, unix_timestamp)
- timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn())
- # [END setting_timestamp]
- fixed_windowed_items = (
- timestamped_items | 'window' >> beam.WindowInto(
- beam.window.FixedWindows(60)))
- summed = (fixed_windowed_items
- | 'group' >> beam.GroupByKey()
- | 'combine' >> beam.CombineValues(sum))
- unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
- assert_that(unkeyed, equal_to([42, 187]))
- p.run()
+ timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn())
+ # [END setting_timestamp]
+ fixed_windowed_items = (
+ timestamped_items | 'window' >> beam.WindowInto(
+ beam.window.FixedWindows(60)))
+ summed = (fixed_windowed_items
+ | 'group' >> beam.GroupByKey()
+ | 'combine' >> beam.CombineValues(sum))
+ unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+ assert_that(unkeyed, equal_to([42, 187]))
class PTransformTest(unittest.TestCase):
@@ -919,10 +1028,9 @@
return pcoll | beam.Map(lambda x: len(x))
# [END model_composite_transform]
- p = TestPipeline()
- lengths = p | beam.Create(["a", "ab", "abc"]) | ComputeWordLengths()
- assert_that(lengths, equal_to([1, 2, 3]))
- p.run()
+ with TestPipeline() as p:
+ lengths = p | beam.Create(["a", "ab", "abc"]) | ComputeWordLengths()
+ assert_that(lengths, equal_to([1, 2, 3]))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/streaming_wordcap.py b/sdks/python/apache_beam/examples/streaming_wordcap.py
index d0cc8a2..19f9e5f 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcap.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcap.py
@@ -41,22 +41,18 @@
help='Output PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
known_args, pipeline_args = parser.parse_known_args(argv)
- p = beam.Pipeline(argv=pipeline_args)
+ with beam.Pipeline(argv=pipeline_args) as p:
- # Read the text file[pattern] into a PCollection.
- lines = p | beam.io.Read(
- beam.io.PubSubSource(known_args.input_topic))
+ # Read the text file[pattern] into a PCollection.
+ lines = p | beam.io.ReadStringsFromPubSub(known_args.input_topic)
- # Capitalize the characters in each line.
- transformed = (lines
- | 'capitalize' >> (beam.Map(lambda x: x.upper())))
+ # Capitalize the characters in each line.
+ transformed = (lines
+ | 'capitalize' >> (beam.Map(lambda x: x.upper())))
- # Write to PubSub.
- # pylint: disable=expression-not-assigned
- transformed | beam.io.Write(
- beam.io.PubSubSink(known_args.output_topic))
-
- p.run().wait_until_finish()
+ # Write to PubSub.
+ # pylint: disable=expression-not-assigned
+ transformed | beam.io.WriteStringsToPubSub(known_args.output_topic)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py
index 4b6aecc..df8a99b 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcount.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcount.py
@@ -25,48 +25,56 @@
import argparse
import logging
-import re
-
import apache_beam as beam
import apache_beam.transforms.window as window
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import StandardOptions
+
+
+def split_fn(lines):
+ import re
+ return re.findall(r'[A-Za-z\']+', lines)
def run(argv=None):
"""Build and run the pipeline."""
-
parser = argparse.ArgumentParser()
parser.add_argument(
'--input_topic', required=True,
- help='Input PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
+ help=('Input PubSub topic of the form '
+ '"projects/<PROJECT>/topics/<TOPIC>".'))
parser.add_argument(
'--output_topic', required=True,
- help='Output PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
+ help=('Output PubSub topic of the form '
+ '"projects/<PROJECT>/topic/<TOPIC>".'))
known_args, pipeline_args = parser.parse_known_args(argv)
+ options = PipelineOptions(pipeline_args)
+ options.view_as(StandardOptions).streaming = True
- p = beam.Pipeline(argv=pipeline_args)
+ with beam.Pipeline(options=options) as p:
- # Read the text file[pattern] into a PCollection.
- lines = p | 'read' >> beam.io.Read(
- beam.io.PubSubSource(known_args.input_topic))
+ # Read from PubSub into a PCollection.
+ lines = p | beam.io.ReadStringsFromPubSub(known_args.input_topic)
- # Capitalize the characters in each line.
- transformed = (lines
- | 'Split' >> (
- beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- .with_output_types(unicode))
- | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
- | beam.WindowInto(window.FixedWindows(15, 0))
- | 'Group' >> beam.GroupByKey()
- | 'Count' >> beam.Map(lambda (word, ones): (word, sum(ones)))
- | 'Format' >> beam.Map(lambda tup: '%s: %d' % tup))
+ # Capitalize the characters in each line.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
- # Write to PubSub.
- # pylint: disable=expression-not-assigned
- transformed | 'pubsub_write' >> beam.io.Write(
- beam.io.PubSubSink(known_args.output_topic))
+ transformed = (lines
+ # Use a pre-defined function that imports the re package.
+ | 'Split' >> (
+ beam.FlatMap(split_fn).with_output_types(unicode))
+ | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
+ | beam.WindowInto(window.FixedWindows(15, 0))
+ | 'Group' >> beam.GroupByKey()
+ | 'Count' >> beam.Map(count_ones)
+ | 'Format' >> beam.Map(lambda tup: '%s: %d' % tup))
- p.run().wait_until_finish()
+ # Write to PubSub.
+ # pylint: disable=expression-not-assigned
+ transformed | beam.io.WriteStringsToPubSub(known_args.output_topic)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/windowed_wordcount.py b/sdks/python/apache_beam/examples/windowed_wordcount.py
new file mode 100644
index 0000000..4c7eee1
--- /dev/null
+++ b/sdks/python/apache_beam/examples/windowed_wordcount.py
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A streaming word-counting workflow.
+
+Important: streaming pipeline support in Python Dataflow is in development
+and is not yet available for use.
+"""
+
+from __future__ import absolute_import
+
+import argparse
+import logging
+
+import apache_beam as beam
+import apache_beam.transforms.window as window
+
+TABLE_SCHEMA = ('word:STRING, count:INTEGER, '
+ 'window_start:TIMESTAMP, window_end:TIMESTAMP')
+
+
+def find_words(element):
+ import re
+ return re.findall(r'[A-Za-z\']+', element)
+
+
+class FormatDoFn(beam.DoFn):
+ def process(self, element, window=beam.DoFn.WindowParam):
+ ts_format = '%Y-%m-%d %H:%M:%S.%f UTC'
+ window_start = window.start.to_utc_datetime().strftime(ts_format)
+ window_end = window.end.to_utc_datetime().strftime(ts_format)
+ return [{'word': element[0],
+ 'count': element[1],
+ 'window_start':window_start,
+ 'window_end':window_end}]
+
+
+def run(argv=None):
+ """Build and run the pipeline."""
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--input_topic', required=True,
+ help='Input PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
+ parser.add_argument(
+ '--output_table', required=True,
+ help=
+ ('Output BigQuery table for results specified as: PROJECT:DATASET.TABLE '
+ 'or DATASET.TABLE.'))
+ known_args, pipeline_args = parser.parse_known_args(argv)
+
+ with beam.Pipeline(argv=pipeline_args) as p:
+
+ # Read the text from PubSub messages
+ lines = p | beam.io.ReadStringsFromPubSub(known_args.input_topic)
+
+ # Capitalize the characters in each line.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
+ transformed = (lines
+ | 'Split' >> (beam.FlatMap(find_words)
+ .with_output_types(unicode))
+ | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
+ | beam.WindowInto(window.FixedWindows(2*60, 0))
+ | 'Group' >> beam.GroupByKey()
+ | 'Count' >> beam.Map(count_ones)
+ | 'Format' >> beam.ParDo(FormatDoFn()))
+
+ # Write to BigQuery.
+ # pylint: disable=expression-not-assigned
+ transformed | 'Write' >> beam.io.WriteToBigQuery(
+ known_args.output_table,
+ schema=TABLE_SCHEMA,
+ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ run()
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index e7e542a..b1c4a5e 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -88,21 +88,28 @@
lines = p | 'read' >> ReadFromText(known_args.input)
# Count the occurrences of each word.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
counts = (lines
| 'split' >> (beam.ParDo(WordExtractingDoFn())
.with_output_types(unicode))
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| 'group' >> beam.GroupByKey()
- | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones))))
+ | 'count' >> beam.Map(count_ones))
# Format the counts into a PCollection of strings.
- output = counts | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
+ output = counts | 'format' >> beam.Map(format_result)
# Write the output using a "Write" transform that has side effects.
# pylint: disable=expression-not-assigned
output | 'write' >> WriteToText(known_args.output)
- # Actually run the pipeline (all operations above are deferred).
result = p.run()
result.wait_until_finish()
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index ca9f7b6..6ff8f26 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -93,12 +93,16 @@
PCollection of (word, count) tuples.
"""
def expand(self, pcoll):
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
return (pcoll
| 'split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
.with_output_types(unicode))
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| 'group' >> beam.GroupByKey()
- | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones))))
+ | 'count' >> beam.Map(count_ones))
def run(argv=None):
@@ -118,35 +122,36 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # Read the text file[pattern] into a PCollection, count the occurrences of
- # each word and filter by a list of words.
- filtered_words = (
- p | 'read' >> ReadFromText(known_args.input)
- | CountWords()
- | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
+ # Read the text file[pattern] into a PCollection, count the occurrences of
+ # each word and filter by a list of words.
+ filtered_words = (
+ p | 'read' >> ReadFromText(known_args.input)
+ | CountWords()
+ | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
- # assert_that is a convenient PTransform that checks a PCollection has an
- # expected value. Asserts are best used in unit tests with small data sets but
- # is demonstrated here as a teaching tool.
- #
- # Note assert_that does not provide any output and that successful completion
- # of the Pipeline implies that the expectations were met. Learn more at
- # https://cloud.google.com/dataflow/pipelines/testing-your-pipeline on how to
- # test your pipeline.
- assert_that(
- filtered_words, equal_to([('Flourish', 3), ('stomach', 1)]))
+ # assert_that is a convenient PTransform that checks a PCollection has an
+ # expected value. Asserts are best used in unit tests with small data sets
+ # but is demonstrated here as a teaching tool.
+ #
+ # Note assert_that does not provide any output and that successful
+ # completion of the Pipeline implies that the expectations were met. Learn
+ # more at https://cloud.google.com/dataflow/pipelines/testing-your-pipeline
+ # on how to best test your pipeline.
+ assert_that(
+ filtered_words, equal_to([('Flourish', 3), ('stomach', 1)]))
- # Format the counts into a PCollection of strings and write the output using a
- # "Write" transform that has side effects.
- # pylint: disable=unused-variable
- output = (filtered_words
- | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
- | 'write' >> WriteToText(known_args.output))
+ # Format the counts into a PCollection of strings and write the output using
+ # a "Write" transform that has side effects.
+ # pylint: disable=unused-variable
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
- # Actually run the pipeline (all operations above are deferred).
- p.run().wait_until_finish()
+ output = (filtered_words
+ | 'format' >> beam.Map(format_result)
+ | 'write' >> WriteToText(known_args.output))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging_test.py b/sdks/python/apache_beam/examples/wordcount_debugging_test.py
index 900a8e7..92ee240 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging_test.py
@@ -23,6 +23,7 @@
import unittest
from apache_beam.examples import wordcount_debugging
+from apache_beam.testing.util import open_shards
class WordCountTest(unittest.TestCase):
@@ -36,7 +37,7 @@
def get_results(self, temp_path):
results = []
- with open(temp_path + '.result-00000-of-00001') as result_file:
+ with open_shards(temp_path + '.result-*-of-*') as result_file:
for line in result_file:
match = re.search(r'([A-Za-z]+): ([0-9]+)', line)
if match is not None:
diff --git a/sdks/python/apache_beam/examples/wordcount_fnapi.py b/sdks/python/apache_beam/examples/wordcount_fnapi.py
new file mode 100644
index 0000000..5e92a23
--- /dev/null
+++ b/sdks/python/apache_beam/examples/wordcount_fnapi.py
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A word-counting workflow using the experimental FnApi.
+
+For the stable wordcount example see wordcount.py.
+"""
+
+# TODO(BEAM-2887): Merge with wordcount.py.
+
+from __future__ import absolute_import
+
+import argparse
+import logging
+
+import apache_beam as beam
+from apache_beam.io import ReadFromText
+# TODO(BEAM-2887): Enable after the issue is fixed.
+# from apache_beam.io import WriteToText
+from apache_beam.metrics import Metrics
+from apache_beam.metrics.metric import MetricsFilter
+from apache_beam.options.pipeline_options import DebugOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+
+
+class WordExtractingDoFn(beam.DoFn):
+ """Parse each line of input text into words."""
+
+ def __init__(self):
+ super(WordExtractingDoFn, self).__init__()
+ self.words_counter = Metrics.counter(self.__class__, 'words')
+ self.word_lengths_counter = Metrics.counter(self.__class__, 'word_lengths')
+ self.word_lengths_dist = Metrics.distribution(
+ self.__class__, 'word_len_dist')
+ self.empty_line_counter = Metrics.counter(self.__class__, 'empty_lines')
+
+ def process(self, element):
+ """Returns an iterator over the words of this element.
+
+ The element is a line of text. If the line is blank, note that, too.
+
+ Args:
+ element: the element being processed
+
+ Returns:
+ The processed element.
+ """
+
+ # TODO(BEAM-3041): Move this import to top of the file after the fix.
+ # Portable containers does not support save main session, and importing here
+ # is required. This is only needed for running experimental jobs with FnApi.
+ import re
+
+ text_line = element.strip()
+ if not text_line:
+ self.empty_line_counter.inc(1)
+ words = re.findall(r'[A-Za-z\']+', text_line)
+ for w in words:
+ self.words_counter.inc()
+ self.word_lengths_counter.inc(len(w))
+ self.word_lengths_dist.update(len(w))
+ return words
+
+
+def run(argv=None):
+ """Main entry point; defines and runs the wordcount pipeline."""
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--input',
+ dest='input',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='Input file to process.')
+ parser.add_argument('--output',
+ dest='output',
+ required=True,
+ help='Output file to write results to.')
+ known_args, pipeline_args = parser.parse_known_args(argv)
+
+ pipeline_options = PipelineOptions(pipeline_args)
+ p = beam.Pipeline(options=pipeline_options)
+
+ # Ensure that the experiment flag is set explicitly by the user.
+ debug_options = pipeline_options.view_as(DebugOptions)
+ use_fn_api = (
+ debug_options.experiments and 'beam_fn_api' in debug_options.experiments)
+ assert use_fn_api, 'Enable beam_fn_api experiment, in order run this example.'
+
+ # Read the text file[pattern] into a PCollection.
+ lines = p | 'read' >> ReadFromText(known_args.input)
+
+ # Count the occurrences of each word.
+ def count_ones(word_ones):
+ (word, ones) = word_ones
+ return (word, sum(ones))
+
+ counts = (lines
+ | 'split' >> (beam.ParDo(WordExtractingDoFn())
+ .with_output_types(unicode))
+ | 'pair_with_one' >> beam.Map(lambda x: (x, 1))
+ | 'group' >> beam.GroupByKey()
+ | 'count' >> beam.Map(count_ones))
+
+ # Format the counts into a PCollection of strings.
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
+
+ # pylint: disable=unused-variable
+ output = counts | 'format' >> beam.Map(format_result)
+
+ # Write the output using a "Write" transform that has side effects.
+ # pylint: disable=expression-not-assigned
+
+ # TODO(BEAM-2887): Enable after the issue is fixed.
+ # output | 'write' >> WriteToText(known_args.output)
+
+ result = p.run()
+ result.wait_until_finish()
+
+ # Do not query metrics when creating a template which doesn't run
+ if (not hasattr(result, 'has_job') # direct runner
+ or result.has_job): # not just a template creation
+ empty_lines_filter = MetricsFilter().with_name('empty_lines')
+ query_result = result.metrics().query(empty_lines_filter)
+ if query_result['counters']:
+ empty_lines_counter = query_result['counters'][0]
+ logging.info('number of empty lines: %d', empty_lines_counter.committed)
+
+ word_lengths_filter = MetricsFilter().with_name('word_len_dist')
+ query_result = result.metrics().query(word_lengths_filter)
+ if query_result['distributions']:
+ word_lengths_dist = query_result['distributions'][0]
+ logging.info('average word length: %d', word_lengths_dist.committed.mean)
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ run()
diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py
index 4bee127..8532f49 100644
--- a/sdks/python/apache_beam/examples/wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_it_test.py
@@ -25,9 +25,11 @@
from nose.plugins.attrib import attr
from apache_beam.examples import wordcount
+from apache_beam.examples import wordcount_fnapi
from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher
from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.test_utils import delete_files
class WordCountIT(unittest.TestCase):
@@ -56,10 +58,24 @@
extra_opts = {'output': output,
'on_success_matcher': all_of(*pipeline_verifiers)}
+ # Register clean up before pipeline execution
+ self.addCleanup(delete_files, [output + '*'])
+
# Get pipeline options from command argument: --test-pipeline-options,
# and start pipeline job by calling pipeline main function.
wordcount.run(test_pipeline.get_full_options_as_args(**extra_opts))
+ @attr('IT')
+ def test_wordcount_fnapi_it(self):
+ test_pipeline = TestPipeline(is_integration_test=True)
+
+ # Get pipeline options from command argument: --test-pipeline-options,
+ # and start pipeline job by calling pipeline main function.
+ wordcount_fnapi.run(
+ test_pipeline.get_full_options_as_args(
+ experiment='beam_fn_api',
+ on_success_matcher=PipelineStateMatcher()))
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.DEBUG)
diff --git a/sdks/python/apache_beam/examples/wordcount_minimal.py b/sdks/python/apache_beam/examples/wordcount_minimal.py
index 5109c08..390c8c0 100644
--- a/sdks/python/apache_beam/examples/wordcount_minimal.py
+++ b/sdks/python/apache_beam/examples/wordcount_minimal.py
@@ -92,28 +92,29 @@
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = True
- p = beam.Pipeline(options=pipeline_options)
+ with beam.Pipeline(options=pipeline_options) as p:
- # Read the text file[pattern] into a PCollection.
- lines = p | 'read' >> ReadFromText(known_args.input)
+ # Read the text file[pattern] into a PCollection.
+ lines = p | ReadFromText(known_args.input)
- # Count the occurrences of each word.
- counts = (lines
- | 'split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- .with_output_types(unicode))
- | 'pair_with_one' >> beam.Map(lambda x: (x, 1))
- | 'group' >> beam.GroupByKey()
- | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones))))
+ # Count the occurrences of each word.
+ counts = (
+ lines
+ | 'Split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+ .with_output_types(unicode))
+ | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
+ | 'GroupAndSum' >> beam.CombinePerKey(sum))
- # Format the counts into a PCollection of strings.
- output = counts | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+ # Format the counts into a PCollection of strings.
+ def format_result(word_count):
+ (word, count) = word_count
+ return '%s: %s' % (word, count)
- # Write the output using a "Write" transform that has side effects.
- # pylint: disable=expression-not-assigned
- output | 'write' >> WriteToText(known_args.output)
+ output = counts | 'Format' >> beam.Map(format_result)
- # Actually run the pipeline (all operations above are deferred).
- p.run().wait_until_finish()
+ # Write the output using a "Write" transform that has side effects.
+ # pylint: disable=expression-not-assigned
+ output | WriteToText(known_args.output)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/wordcount_minimal_test.py b/sdks/python/apache_beam/examples/wordcount_minimal_test.py
index 82bace4..5ee7b78 100644
--- a/sdks/python/apache_beam/examples/wordcount_minimal_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_minimal_test.py
@@ -24,6 +24,7 @@
import unittest
from apache_beam.examples import wordcount_minimal
+from apache_beam.testing.util import open_shards
class WordCountMinimalTest(unittest.TestCase):
@@ -46,7 +47,7 @@
'--output=%s.result' % temp_path])
# Parse result file and compare.
results = []
- with open(temp_path + '.result-00000-of-00001') as result_file:
+ with open_shards(temp_path + '.result-*-of-*') as result_file:
for line in result_file:
match = re.search(r'([a-z]+): ([0-9]+)', line)
if match is not None:
diff --git a/sdks/python/apache_beam/examples/wordcount_test.py b/sdks/python/apache_beam/examples/wordcount_test.py
index 616540b..9834ba5 100644
--- a/sdks/python/apache_beam/examples/wordcount_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_test.py
@@ -24,6 +24,7 @@
import unittest
from apache_beam.examples import wordcount
+from apache_beam.testing.util import open_shards
class WordCountTest(unittest.TestCase):
@@ -45,7 +46,7 @@
'--output=%s.result' % temp_path])
# Parse result file and compare.
results = []
- with open(temp_path + '.result-00000-of-00001') as result_file:
+ with open_shards(temp_path + '.result-*-of-*') as result_file:
for line in result_file:
match = re.search(r'([a-z]+): ([0-9]+)', line)
if match is not None:
diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py
index 9f32092..35676f1 100644
--- a/sdks/python/apache_beam/internal/gcp/auth.py
+++ b/sdks/python/apache_beam/internal/gcp/auth.py
@@ -28,7 +28,6 @@
from apache_beam.utils import retry
-
# When we are running in GCE, we can authenticate with VM credentials.
is_running_in_gce = False
diff --git a/sdks/python/apache_beam/internal/gcp/json_value.py b/sdks/python/apache_beam/internal/gcp/json_value.py
index 59f8b60..7a5dc54 100644
--- a/sdks/python/apache_beam/internal/gcp/json_value.py
+++ b/sdks/python/apache_beam/internal/gcp/json_value.py
@@ -27,7 +27,6 @@
from apache_beam.options.value_provider import ValueProvider
-
_MAXINT64 = (1 << 63) - 1
_MININT64 = - (1 << 63)
@@ -41,11 +40,12 @@
obj: A basestring, bool, int, or float to be converted.
Returns:
- A dictionary containing the keys '@type' and 'value' with the value for
- the @type of appropriate type.
+ A dictionary containing the keys ``@type`` and ``value`` with the value for
+ the ``@type`` of appropriate type.
Raises:
- TypeError: if the Python object has a type that is not supported.
+ ~exceptions.TypeError: if the Python object has a type that is not
+ supported.
"""
if isinstance(obj, basestring):
type_name = 'Text'
@@ -66,21 +66,23 @@
Converts Python objects into extra_types.JsonValue objects.
Args:
- obj: Python object to be converted. Can be 'None'.
- with_type: If true then the basic types (string, int, float, bool) will
- be wrapped in @type/value dictionaries. Otherwise the straight value is
- encoded into a JsonValue.
+ obj: Python object to be converted. Can be :data:`None`.
+ with_type: If true then the basic types (``string``, ``int``, ``float``,
+ ``bool``) will be wrapped in ``@type:value`` dictionaries. Otherwise the
+ straight value is encoded into a ``JsonValue``.
Returns:
- A JsonValue object using JsonValue, JsonArray and JsonObject types for the
- corresponding values, lists, or dictionaries.
+ A ``JsonValue`` object using ``JsonValue``, ``JsonArray`` and ``JsonObject``
+ types for the corresponding values, lists, or dictionaries.
Raises:
- TypeError: if the Python object contains a type that is not supported.
+ ~exceptions.TypeError: if the Python object contains a type that is not
+ supported.
- The types supported are str, bool, list, tuple, dict, and None. The Dataflow
- API requires JsonValue(s) in many places, and it is quite convenient to be
- able to specify these hierarchical objects using Python syntax.
+ The types supported are ``str``, ``bool``, ``list``, ``tuple``, ``dict``, and
+ ``None``. The Dataflow API requires JsonValue(s) in many places, and it is
+ quite convenient to be able to specify these hierarchical objects using
+ Python syntax.
"""
if obj is None:
return extra_types.JsonValue(is_null=True)
@@ -121,21 +123,23 @@
def from_json_value(v):
"""For internal use only; no backwards-compatibility guarantees.
- Converts extra_types.JsonValue objects into Python objects.
+ Converts ``extra_types.JsonValue`` objects into Python objects.
Args:
- v: JsonValue object to be converted.
+ v: ``JsonValue`` object to be converted.
Returns:
A Python object structured as values, lists, and dictionaries corresponding
- to JsonValue, JsonArray and JsonObject types.
+ to ``JsonValue``, ``JsonArray`` and ``JsonObject`` types.
Raises:
- TypeError: if the JsonValue object contains a type that is not supported.
+ ~exceptions.TypeError: if the ``JsonValue`` object contains a type that is
+ not supported.
- The types supported are str, bool, list, dict, and None. The Dataflow API
- returns JsonValue(s) in many places and it is quite convenient to be able to
- convert these hierarchical objects to much simpler Python objects.
+ The types supported are ``str``, ``bool``, ``list``, ``dict``, and ``None``.
+ The Dataflow API returns JsonValue(s) in many places and it is quite
+ convenient to be able to convert these hierarchical objects to much simpler
+ Python objects.
"""
if isinstance(v, extra_types.JsonValue):
if v.string_value is not None:
diff --git a/sdks/python/apache_beam/internal/gcp/json_value_test.py b/sdks/python/apache_beam/internal/gcp/json_value_test.py
index b1fd63f..14223f1 100644
--- a/sdks/python/apache_beam/internal/gcp/json_value_test.py
+++ b/sdks/python/apache_beam/internal/gcp/json_value_test.py
@@ -21,9 +21,8 @@
from apache_beam.internal.gcp.json_value import from_json_value
from apache_beam.internal.gcp.json_value import to_json_value
-from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.options.value_provider import RuntimeValueProvider
-
+from apache_beam.options.value_provider import StaticValueProvider
# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index e049a71..102cf23 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -52,7 +52,7 @@
for k, v in outer.__dict__.items():
if v is nested_class:
return outer, k
- elif isinstance(v, (type, types.ClassType)) and hasattr(v, '__dict__'):
+ elif isinstance(v, type) and hasattr(v, '__dict__'):
res = _find_containing_class_inner(v)
if res: return res
diff --git a/sdks/python/apache_beam/internal/util.py b/sdks/python/apache_beam/internal/util.py
index dbbeafc..e4f230b 100644
--- a/sdks/python/apache_beam/internal/util.py
+++ b/sdks/python/apache_beam/internal/util.py
@@ -21,9 +21,9 @@
"""
import logging
-from multiprocessing.pool import ThreadPool
import threading
import weakref
+from multiprocessing.pool import ThreadPool
class ArgumentPlaceholder(object):
@@ -100,10 +100,10 @@
# Use a local iterator so that we don't modify values.
v_iter = iter(values)
new_args = [
- v_iter.next() if isinstance(arg, ArgumentPlaceholder) else arg
+ next(v_iter) if isinstance(arg, ArgumentPlaceholder) else arg
for arg in args]
new_kwargs = dict(
- (k, v_iter.next()) if isinstance(v, ArgumentPlaceholder) else (k, v)
+ (k, next(v_iter)) if isinstance(v, ArgumentPlaceholder) else (k, v)
for k, v in sorted(kwargs.iteritems()))
return (new_args, new_kwargs)
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 7df9983..30fc890 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -14,65 +14,124 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-"""Implements a source for reading Avro files."""
+"""``PTransforms`` for reading from and writing to Avro files.
+
+Provides two read ``PTransform``s, ``ReadFromAvro`` and ``ReadAllFromAvro``,
+that produces a ``PCollection`` of records.
+Each record of this ``PCollection`` will contain a single record read from
+an Avro file. Records that are of simple types will be mapped into
+corresponding Python types. Records that are of Avro type 'RECORD' will be
+mapped to Python dictionaries that comply with the schema contained in the
+Avro file that contains those records. In this case, keys of each dictionary
+will contain the corresponding field names and will be of type ``string``
+while the values of the dictionary will be of the type defined in the
+corresponding Avro schema.
+
+For example, if schema of the Avro file is the following.
+{"namespace": "example.avro","type": "record","name": "User","fields":
+[{"name": "name", "type": "string"},
+{"name": "favorite_number", "type": ["int", "null"]},
+{"name": "favorite_color", "type": ["string", "null"]}]}
+
+Then records generated by read transforms will be dictionaries of the
+following form.
+{u'name': u'Alyssa', u'favorite_number': 256, u'favorite_color': None}).
+
+Additionally, this module provides a write ``PTransform`` ``WriteToAvro``
+that can be used to write a given ``PCollection`` of Python objects to an
+Avro file.
+"""
import cStringIO
import os
import zlib
+from functools import partial
import avro
-from avro import datafile
from avro import io as avroio
+from avro import datafile
from avro import schema
import apache_beam as beam
-from apache_beam.io import filebasedsource
from apache_beam.io import filebasedsink
+from apache_beam.io import filebasedsource
from apache_beam.io import iobase
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.iobase import Read
from apache_beam.transforms import PTransform
-__all__ = ['ReadFromAvro', 'WriteToAvro']
+__all__ = ['ReadFromAvro', 'ReadAllFromAvro', 'WriteToAvro']
class ReadFromAvro(PTransform):
- """A ``PTransform`` for reading avro files."""
+ """A :class:`~apache_beam.transforms.ptransform.PTransform` for reading avro
+ files."""
def __init__(self, file_pattern=None, min_bundle_size=0, validate=True):
- """Initializes ``ReadFromAvro``.
+ """Initializes :class:`ReadFromAvro`.
- Uses source '_AvroSource' to read a set of Avro files defined by a given
- file pattern.
- If '/mypath/myavrofiles*' is a file-pattern that points to a set of Avro
- files, a ``PCollection`` for the records in these Avro files can be created
- in the following manner.
- p = df.Pipeline(argv=pipeline_args)
- records = p | 'Read' >> df.io.ReadFromAvro('/mypath/myavrofiles*')
+ Uses source :class:`~apache_beam.io._AvroSource` to read a set of Avro
+ files defined by a given file pattern.
- Each record of this ``PCollection`` will contain a single record read from a
- source. Records that are of simple types will be mapped into corresponding
- Python types. Records that are of Avro type 'RECORD' will be mapped to
- Python dictionaries that comply with the schema contained in the Avro file
- that contains those records. In this case, keys of each dictionary
- will contain the corresponding field names and will be of type ``string``
- while the values of the dictionary will be of the type defined in the
- corresponding Avro schema.
- For example, if schema of the Avro file is the following.
- {"namespace": "example.avro","type": "record","name": "User","fields":
- [{"name": "name", "type": "string"},
- {"name": "favorite_number", "type": ["int", "null"]},
- {"name": "favorite_color", "type": ["string", "null"]}]}
- Then records generated by ``AvroSource`` will be dictionaries of the
- following form.
+ If ``/mypath/myavrofiles*`` is a file-pattern that points to a set of Avro
+ files, a :class:`~apache_beam.pvalue.PCollection` for the records in
+ these Avro files can be created in the following manner.
+
+ .. testcode::
+
+ with beam.Pipeline() as p:
+ records = p | 'Read' >> beam.io.ReadFromAvro('/mypath/myavrofiles*')
+
+ .. NOTE: We're not actually interested in this error; but if we get here,
+ it means that the way of calling this transform hasn't changed.
+
+ .. testoutput::
+ :hide:
+
+ Traceback (most recent call last):
+ ...
+ IOError: No files found based on the file pattern
+
+ Each record of this :class:`~apache_beam.pvalue.PCollection` will contain
+ a single record read from a source. Records that are of simple types will be
+ mapped into corresponding Python types. Records that are of Avro type
+ ``RECORD`` will be mapped to Python dictionaries that comply with the schema
+ contained in the Avro file that contains those records. In this case, keys
+ of each dictionary will contain the corresponding field names and will be of
+ type :class:`str` while the values of the dictionary will be of the type
+ defined in the corresponding Avro schema.
+
+ For example, if schema of the Avro file is the following. ::
+
+ {
+ "namespace": "example.avro",
+ "type": "record",
+ "name": "User",
+ "fields": [
+
+ {"name": "name",
+ "type": "string"},
+
+ {"name": "favorite_number",
+ "type": ["int", "null"]},
+
+ {"name": "favorite_color",
+ "type": ["string", "null"]}
+
+ ]
+ }
+
+ Then records generated by :class:`~apache_beam.io._AvroSource` will be
+ dictionaries of the following form. ::
+
{u'name': u'Alyssa', u'favorite_number': 256, u'favorite_color': None}).
Args:
- file_pattern: the set of files to be read.
- min_bundle_size: the minimum size in bytes, to be considered when
- splitting the input into bundles.
- validate: flag to verify that the files exist during the pipeline
- creation time.
+ file_pattern (str): the file glob to read
+ min_bundle_size (int): the minimum size in bytes, to be considered when
+ splitting the input into bundles.
+ validate (bool): flag to verify that the files exist during the pipeline
+ creation time.
"""
super(ReadFromAvro, self).__init__()
self._source = _AvroSource(file_pattern, min_bundle_size, validate=validate)
@@ -84,6 +143,35 @@
return {'source_dd': self._source}
+class ReadAllFromAvro(PTransform):
+ """A ``PTransform`` for reading ``PCollection`` of Avro files.
+
+ Uses source '_AvroSource' to read a ``PCollection`` of Avro files or
+ file patterns and produce a ``PCollection`` of Avro records.
+ """
+
+ DEFAULT_DESIRED_BUNDLE_SIZE = 64 * 1024 * 1024 # 64MB
+
+ def __init__(self, min_bundle_size=0,
+ desired_bundle_size=DEFAULT_DESIRED_BUNDLE_SIZE):
+ """Initializes ``ReadAllFromAvro``.
+
+ Args:
+ min_bundle_size: the minimum size in bytes, to be considered when
+ splitting the input into bundles.
+ desired_bundle_size: the desired size in bytes, to be considered when
+ splitting the input into bundles.
+ """
+ source_from_file = partial(
+ _create_avro_source, min_bundle_size=min_bundle_size)
+ self._read_all_files = filebasedsource.ReadAllFiles(
+ True, CompressionTypes.AUTO, desired_bundle_size, min_bundle_size,
+ source_from_file)
+
+ def expand(self, pvalue):
+ return pvalue | 'ReadAllFiles' >> self._read_all_files
+
+
class _AvroUtils(object):
@staticmethod
@@ -176,6 +264,12 @@
data = f.read(buf_size)
+def _create_avro_source(file_pattern=None, min_bundle_size=None):
+ return _AvroSource(
+ file_pattern=file_pattern, min_bundle_size=min_bundle_size,
+ validate=False)
+
+
class _AvroBlock(object):
"""Represents a block of an Avro file."""
diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py
index 6dcf121..8a34427 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -21,28 +21,26 @@
import tempfile
import unittest
+import avro.datafile
+import avro.schema
+from avro.datafile import DataFileWriter
+from avro.io import DatumWriter
+import hamcrest as hc
+
import apache_beam as beam
-from apache_beam.io import iobase
+from apache_beam import Create
from apache_beam.io import avroio
from apache_beam.io import filebasedsource
+from apache_beam.io import iobase
from apache_beam.io import source_test_utils
+from apache_beam.io.avroio import _AvroSink as AvroSink # For testing
+from apache_beam.io.avroio import _AvroSource as AvroSource # For testing
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display_test import DisplayDataItemMatcher
-# Importing following private class for testing purposes.
-from apache_beam.io.avroio import _AvroSource as AvroSource
-from apache_beam.io.avroio import _AvroSink as AvroSink
-
-import avro.datafile
-from avro.datafile import DataFileWriter
-from avro.io import DatumWriter
-import avro.schema
-import hamcrest as hc
-
-
# Import snappy optionally; some tests will be skipped when import fails.
try:
import snappy # pylint: disable=import-error
@@ -346,11 +344,41 @@
source_test_utils.read_from_source(source, None, None)
self.assertEqual(0, exn.exception.message.find('Unexpected sync marker'))
- def test_source_transform(self):
+ def test_read_from_avro(self):
path = self._write_data()
with TestPipeline() as p:
assert_that(p | avroio.ReadFromAvro(path), equal_to(self.RECORDS))
+ def test_read_all_from_avro_single_file(self):
+ path = self._write_data()
+ with TestPipeline() as p:
+ assert_that(p | Create([path]) | avroio.ReadAllFromAvro(),
+ equal_to(self.RECORDS))
+
+ def test_read_all_from_avro_many_single_files(self):
+ path1 = self._write_data()
+ path2 = self._write_data()
+ path3 = self._write_data()
+ with TestPipeline() as p:
+ assert_that(p | Create([path1, path2, path3]) | avroio.ReadAllFromAvro(),
+ equal_to(self.RECORDS * 3))
+
+ def test_read_all_from_avro_file_pattern(self):
+ file_pattern = self._write_pattern(5)
+ with TestPipeline() as p:
+ assert_that(p | Create([file_pattern]) | avroio.ReadAllFromAvro(),
+ equal_to(self.RECORDS * 5))
+
+ def test_read_all_from_avro_many_file_patterns(self):
+ file_pattern1 = self._write_pattern(5)
+ file_pattern2 = self._write_pattern(2)
+ file_pattern3 = self._write_pattern(3)
+ with TestPipeline() as p:
+ assert_that(p
+ | Create([file_pattern1, file_pattern2, file_pattern3])
+ | avroio.ReadAllFromAvro(),
+ equal_to(self.RECORDS * 10))
+
def test_sink_transform(self):
with tempfile.NamedTemporaryFile() as dst:
path = dst.name
diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py
index 4a8f519..0f7dd54 100644
--- a/sdks/python/apache_beam/io/concat_source_test.py
+++ b/sdks/python/apache_beam/io/concat_source_test.py
@@ -21,7 +21,6 @@
import unittest
import apache_beam as beam
-
from apache_beam.io import iobase
from apache_beam.io import range_trackers
from apache_beam.io import source_test_utils
diff --git a/sdks/python/apache_beam/io/filebasedsink.py b/sdks/python/apache_beam/io/filebasedsink.py
index 76c09fc..ba1a495 100644
--- a/sdks/python/apache_beam/io/filebasedsink.py
+++ b/sdks/python/apache_beam/io/filebasedsink.py
@@ -30,10 +30,10 @@
from apache_beam.io.filesystem import BeamIOError
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.filesystems import FileSystems
-from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.options.value_provider import ValueProvider
from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import ValueProvider
from apache_beam.options.value_provider import check_accessible
+from apache_beam.transforms.display import DisplayDataItem
DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN'
@@ -44,12 +44,13 @@
"""A sink to a GCS or local files.
To implement a file-based sink, extend this class and override
- either ``write_record()`` or ``write_encoded_record()``.
+ either :meth:`.write_record()` or :meth:`.write_encoded_record()`.
- If needed, also overwrite ``open()`` and/or ``close()`` to customize the
- file handling or write headers and footers.
+ If needed, also overwrite :meth:`.open()` and/or :meth:`.close()` to customize
+ the file handling or write headers and footers.
- The output of this write is a PCollection of all written shards.
+ The output of this write is a :class:`~apache_beam.pvalue.PCollection` of
+ all written shards.
"""
# Max number of threads to be used for renaming.
@@ -65,9 +66,12 @@
compression_type=CompressionTypes.AUTO):
"""
Raises:
- TypeError: if file path parameters are not a string or ValueProvider,
- or if compression_type is not member of CompressionTypes.
- ValueError: if shard_name_template is not of expected format.
+ ~exceptions.TypeError: if file path parameters are not a :class:`str` or
+ :class:`~apache_beam.options.value_provider.ValueProvider`, or if
+ **compression_type** is not member of
+ :class:`~apache_beam.io.filesystem.CompressionTypes`.
+ ~exceptions.ValueError: if **shard_name_template** is not of expected
+ format.
"""
if not isinstance(file_path_prefix, (basestring, ValueProvider)):
raise TypeError('file_path_prefix must be a string or ValueProvider;'
diff --git a/sdks/python/apache_beam/io/filebasedsink_test.py b/sdks/python/apache_beam/io/filebasedsink_test.py
index 1f6aeee..156251a 100644
--- a/sdks/python/apache_beam/io/filebasedsink_test.py
+++ b/sdks/python/apache_beam/io/filebasedsink_test.py
@@ -31,12 +31,11 @@
import apache_beam as beam
from apache_beam.coders import coders
from apache_beam.io import filebasedsink
+from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.options.value_provider import StaticValueProvider
-
# TODO: Refactor code so all io tests are using same library
# TestCaseWithTempDirCleanup class.
@@ -146,9 +145,8 @@
sink = MyFileBasedSink(
temp_path, file_name_suffix='.output', coder=coders.ToStringCoder()
)
- p = TestPipeline()
- p | beam.Create([]) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
- p.run()
+ with TestPipeline() as p:
+ p | beam.Create([]) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
self.assertEqual(
open(temp_path + '-00000-of-00001.output').read(), '[start][end]')
@@ -160,9 +158,8 @@
file_name_suffix=StaticValueProvider(value_type=str, value='.output'),
coder=coders.ToStringCoder()
)
- p = TestPipeline()
- p | beam.Create([]) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
- p.run()
+ with TestPipeline() as p:
+ p | beam.Create([]) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
self.assertEqual(
open(temp_path.get() + '-00000-of-00001.output').read(), '[start][end]')
@@ -174,10 +171,8 @@
num_shards=3,
shard_name_template='_NN_SSS_',
coder=coders.ToStringCoder())
- p = TestPipeline()
- p | beam.Create(['a', 'b']) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
-
- p.run()
+ with TestPipeline() as p:
+ p | beam.Create(['a', 'b']) | beam.io.Write(sink) # pylint: disable=expression-not-assigned
concat = ''.join(
open(temp_path + '_03_%03d_.output' % shard_num).read()
diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py
index bb9efc4..052c2f3 100644
--- a/sdks/python/apache_beam/io/filebasedsource.py
+++ b/sdks/python/apache_beam/io/filebasedsource.py
@@ -17,13 +17,15 @@
"""A framework for developing sources for new file types.
-To create a source for a new file type a sub-class of ``FileBasedSource`` should
-be created. Sub-classes of ``FileBasedSource`` must implement the method
-``FileBasedSource.read_records()``. Please read the documentation of that method
-for more details.
+To create a source for a new file type a sub-class of :class:`FileBasedSource`
+should be created. Sub-classes of :class:`FileBasedSource` must implement the
+method :meth:`FileBasedSource.read_records()`. Please read the documentation of
+that method for more details.
-For an example implementation of ``FileBasedSource`` see ``avroio.AvroSource``.
+For an example implementation of :class:`FileBasedSource` see
+:class:`~apache_beam.io._AvroSource`.
"""
+import uuid
from apache_beam.internal import pickler
from apache_beam.io import concat_source
@@ -31,10 +33,18 @@
from apache_beam.io import range_trackers
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.filesystems import FileSystems
-from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.options.value_provider import ValueProvider
+from apache_beam.io.range_trackers import OffsetRange
from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import ValueProvider
from apache_beam.options.value_provider import check_accessible
+from apache_beam.transforms.core import DoFn
+from apache_beam.transforms.core import FlatMap
+from apache_beam.transforms.core import GroupByKey
+from apache_beam.transforms.core import Map
+from apache_beam.transforms.core import ParDo
+from apache_beam.transforms.core import PTransform
+from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms.trigger import DefaultTrigger
MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25
@@ -42,7 +52,8 @@
class FileBasedSource(iobase.BoundedSource):
- """A ``BoundedSource`` for reading a file glob of a given type."""
+ """A :class:`~apache_beam.io.iobase.BoundedSource` for reading a file glob of
+ a given type."""
MIN_NUMBER_OF_FILES_TO_STAT = 100
MIN_FRACTION_OF_FILES_TO_STAT = 0.01
@@ -53,31 +64,40 @@
compression_type=CompressionTypes.AUTO,
splittable=True,
validate=True):
- """Initializes ``FileBasedSource``.
+ """Initializes :class:`FileBasedSource`.
Args:
- file_pattern: the file glob to read a string or a ValueProvider
- (placeholder to inject a runtime value).
- min_bundle_size: minimum size of bundles that should be generated when
- performing initial splitting on this source.
- compression_type: compression type to use
- splittable: whether FileBasedSource should try to logically split a single
- file into data ranges so that different parts of the same file
- can be read in parallel. If set to False, FileBasedSource will
- prevent both initial and dynamic splitting of sources for
- single files. File patterns that represent multiple files may
- still get split into sources for individual files. Even if set
- to True by the user, FileBasedSource may choose to not split
- the file, for example, for compressed files where currently
- it is not possible to efficiently read a data range without
- decompressing the whole file.
- validate: Boolean flag to verify that the files exist during the pipeline
- creation time.
+ file_pattern (str): the file glob to read a string or a
+ :class:`~apache_beam.options.value_provider.ValueProvider`
+ (placeholder to inject a runtime value).
+ min_bundle_size (str): minimum size of bundles that should be generated
+ when performing initial splitting on this source.
+ compression_type (str): Used to handle compressed output files.
+ Typical value is :attr:`CompressionTypes.AUTO
+ <apache_beam.io.filesystem.CompressionTypes.AUTO>`,
+ in which case the final file path's extension will be used to detect
+ the compression.
+ splittable (bool): whether :class:`FileBasedSource` should try to
+ logically split a single file into data ranges so that different parts
+ of the same file can be read in parallel. If set to :data:`False`,
+ :class:`FileBasedSource` will prevent both initial and dynamic splitting
+ of sources for single files. File patterns that represent multiple files
+ may still get split into sources for individual files. Even if set to
+ :data:`True` by the user, :class:`FileBasedSource` may choose to not
+ split the file, for example, for compressed files where currently it is
+ not possible to efficiently read a data range without decompressing the
+ whole file.
+ validate (bool): Boolean flag to verify that the files exist during the
+ pipeline creation time.
+
Raises:
- TypeError: when compression_type is not valid or if file_pattern is not a
- string or a ValueProvider.
- ValueError: when compression and splittable files are specified.
- IOError: when the file pattern specified yields an empty result.
+ ~exceptions.TypeError: when **compression_type** is not valid or if
+ **file_pattern** is not a :class:`str` or a
+ :class:`~apache_beam.options.value_provider.ValueProvider`.
+ ~exceptions.ValueError: when compression and splittable files are
+ specified.
+ ~exceptions.IOError: when the file pattern specified yields an empty
+ result.
"""
if not isinstance(file_pattern, (basestring, ValueProvider)):
@@ -95,12 +115,7 @@
raise TypeError('compression_type must be CompressionType object but '
'was %s' % type(compression_type))
self._compression_type = compression_type
- if compression_type in (CompressionTypes.UNCOMPRESSED,
- CompressionTypes.AUTO):
- self._splittable = splittable
- else:
- # We can't split compressed files efficiently so turn off splitting.
- self._splittable = False
+ self._splittable = splittable
if validate and file_pattern.is_accessible():
self._validate()
@@ -132,13 +147,10 @@
continue # Ignoring empty file.
# We determine splittability of this specific file.
- splittable = self.splittable
- if (splittable and
- self._compression_type == CompressionTypes.AUTO):
- compression_type = CompressionTypes.detect_compression_type(
- file_name)
- if compression_type != CompressionTypes.UNCOMPRESSED:
- splittable = False
+ splittable = (
+ self.splittable and
+ _determine_splittability_from_compression_type(
+ file_name, self._compression_type))
single_file_source = _SingleFileSource(
file_based_source_ref, file_name,
@@ -211,6 +223,14 @@
return self._splittable
+def _determine_splittability_from_compression_type(
+ file_path, compression_type):
+ if compression_type == CompressionTypes.AUTO:
+ compression_type = CompressionTypes.detect_compression_type(file_path)
+
+ return compression_type == CompressionTypes.UNCOMPRESSED
+
+
class _SingleFileSource(iobase.BoundedSource):
"""Denotes a source for a specific file type."""
@@ -244,24 +264,21 @@
stop_offset = self._stop_offset
if self._splittable:
- bundle_size = max(desired_bundle_size, self._min_bundle_size)
-
- bundle_start = start_offset
- while bundle_start < stop_offset:
- bundle_stop = min(bundle_start + bundle_size, stop_offset)
+ splits = OffsetRange(start_offset, stop_offset).split(
+ desired_bundle_size, self._min_bundle_size)
+ for split in splits:
yield iobase.SourceBundle(
- bundle_stop - bundle_start,
+ split.stop - split.start,
_SingleFileSource(
# Copying this so that each sub-source gets a fresh instance.
pickler.loads(pickler.dumps(self._file_based_source)),
self._file_name,
- bundle_start,
- bundle_stop,
+ split.start,
+ split.stop,
min_bundle_size=self._min_bundle_size,
splittable=self._splittable),
- bundle_start,
- bundle_stop)
- bundle_start = bundle_stop
+ split.start,
+ split.stop)
else:
# Returning a single sub-source with end offset set to OFFSET_INFINITY (so
# that all data of the source gets read) since this source is
@@ -308,3 +325,112 @@
def default_output_coder(self):
return self._file_based_source.default_output_coder()
+
+
+class _ExpandIntoRanges(DoFn):
+
+ def __init__(
+ self, splittable, compression_type, desired_bundle_size, min_bundle_size):
+ self._desired_bundle_size = desired_bundle_size
+ self._min_bundle_size = min_bundle_size
+ self._splittable = splittable
+ self._compression_type = compression_type
+
+ def process(self, element, *args, **kwargs):
+ match_results = FileSystems.match([element])
+ for metadata in match_results[0].metadata_list:
+ splittable = (
+ self._splittable and
+ _determine_splittability_from_compression_type(
+ metadata.path, self._compression_type))
+
+ if splittable:
+ for split in OffsetRange(
+ 0, metadata.size_in_bytes).split(
+ self._desired_bundle_size, self._min_bundle_size):
+ yield (metadata, split)
+ else:
+ yield (metadata, OffsetRange(
+ 0, range_trackers.OffsetRangeTracker.OFFSET_INFINITY))
+
+
+# Replace following with a generic reshard transform once
+# https://issues.apache.org/jira/browse/BEAM-1872 is implemented.
+class _Reshard(PTransform):
+
+ def expand(self, pvalue):
+ keyed_pc = (pvalue
+ | 'AssignKey' >> Map(lambda x: (uuid.uuid4(), x)))
+ if keyed_pc.windowing.windowfn.is_merging():
+ raise ValueError('Transform ReadAllFiles cannot be used in the presence '
+ 'of merging windows')
+ if not isinstance(keyed_pc.windowing.triggerfn, DefaultTrigger):
+ raise ValueError('Transform ReadAllFiles cannot be used in the presence '
+ 'of non-trivial triggers')
+
+ return (keyed_pc | 'GroupByKey' >> GroupByKey()
+ # Using FlatMap below due to the possibility of key collisions.
+ | 'DropKey' >> FlatMap(lambda k_values: k_values[1]))
+
+
+class _ReadRange(DoFn):
+
+ def __init__(self, source_from_file):
+ self._source_from_file = source_from_file
+
+ def process(self, element, *args, **kwargs):
+ metadata, range = element
+ source = self._source_from_file(metadata.path)
+ # Following split() operation has to be performed to create a proper
+ # _SingleFileSource. Otherwise what we have is a ConcatSource that contains
+ # a single _SingleFileSource. ConcatSource.read() expects a RangeTraker for
+ # sub-source range and reads full sub-sources (not byte ranges).
+ source = list(source.split(float('inf')))[0].source
+ for record in source.read(range.new_tracker()):
+ yield record
+
+
+class ReadAllFiles(PTransform):
+ """A Read transform that reads a PCollection of files.
+
+ Pipeline authors should not use this directly. This is to be used by Read
+ PTransform authors who wishes to implement file-based Read transforms that
+ read a PCollection of files.
+ """
+
+ def __init__(
+ self, splittable, compression_type, desired_bundle_size, min_bundle_size,
+ source_from_file):
+ """
+ Args:
+ splittable: If True, files won't be split into sub-ranges. If False, files
+ may or may not be split into data ranges.
+ compression_type: A ``CompressionType`` object that specifies the
+ compression type of the files that will be processed. If
+ ``CompressionType.AUTO``, system will try to automatically
+ determine the compression type based on the extension of
+ files.
+ desired_bundle_size: the desired size of data ranges that should be
+ generated when splitting a file into data ranges.
+ min_bundle_size: minimum size of data ranges that should be generated when
+ splitting a file into data ranges.
+ source_from_file: a function that produces a ``BoundedSource`` given a
+ file name. System will use this function to generate
+ ``BoundedSource`` objects for file paths. Note that file
+ paths passed to this will be for individual files, not
+ for file patterns even if the ``PCollection`` of files
+ processed by the transform consist of file patterns.
+ """
+ self._splittable = splittable
+ self._compression_type = compression_type
+ self._desired_bundle_size = desired_bundle_size
+ self._min_bundle_size = min_bundle_size
+ self._source_from_file = source_from_file
+
+ def expand(self, pvalue):
+ return (pvalue
+ | 'ExpandIntoRanges' >> ParDo(_ExpandIntoRanges(
+ self._splittable, self._compression_type,
+ self._desired_bundle_size, self._min_bundle_size))
+ | 'Reshard' >> _Reshard()
+ | 'ReadRange' >> ParDo(_ReadRange(self._source_from_file)))
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py
index afb340d..0999510 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -20,8 +20,8 @@
import gzip
import logging
import math
-import random
import os
+import random
import tempfile
import unittest
@@ -31,15 +31,13 @@
from apache_beam.io import filebasedsource
from apache_beam.io import iobase
from apache_beam.io import range_trackers
-from apache_beam.io.filesystem import CompressionTypes
-
# importing following private classes for testing
from apache_beam.io.concat_source import ConcatSource
from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource
-
from apache_beam.io.filebasedsource import FileBasedSource
-from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.io.filesystem import CompressionTypes
from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py
index db6a1d0..69049ae 100644
--- a/sdks/python/apache_beam/io/filesystem.py
+++ b/sdks/python/apache_beam/io/filesystem.py
@@ -21,10 +21,12 @@
import abc
import bz2
import cStringIO
-import os
-import zlib
import logging
+import os
import time
+import zlib
+
+from apache_beam.utils.plugin import BeamPlugin
logger = logging.getLogger(__name__)
@@ -185,21 +187,26 @@
del buf # Free up some possibly large and no-longer-needed memory.
self._read_buffer.write(decompressed)
else:
- # EOF reached.
- # Verify completeness and no corruption and flush (if needed by
- # the underlying algorithm).
- if self._compression_type == CompressionTypes.BZIP2:
- # Having unused_data past end of stream would imply file corruption.
- assert not self._decompressor.unused_data, 'Possible file corruption.'
- try:
- # EOF implies that the underlying BZIP2 stream must also have
- # reached EOF. We expect this to raise an EOFError and we catch it
- # below. Any other kind of error though would be problematic.
- self._decompressor.decompress('dummy')
- assert False, 'Possible file corruption.'
- except EOFError:
- pass # All is as expected!
+ # EOF of current stream reached.
+ #
+ # Any uncompressed data at the end of the stream of a gzip or bzip2
+ # file that is not corrupted points to a concatenated compressed
+ # file. We read concatenated files by recursively creating decompressor
+ # objects for the unused compressed data.
+ if (self._compression_type == CompressionTypes.BZIP2 or
+ self._compression_type == CompressionTypes.GZIP):
+ if self._decompressor.unused_data != '':
+ buf = self._decompressor.unused_data
+ self._decompressor = (
+ bz2.BZ2Decompressor()
+ if self._compression_type == CompressionTypes.BZIP2
+ else zlib.decompressobj(self._gzip_mask))
+ decompressed = self._decompressor.decompress(buf)
+ self._read_buffer.write(decompressed)
+ continue
else:
+ # Gzip and bzip2 formats do not require flushing remaining data in the
+ # decompressor into the read buffer when fully decompressing files.
self._read_buffer.write(self._decompressor.flush())
# Record that we have hit the end of file, so we won't unnecessarily
@@ -292,23 +299,28 @@
"""Set the file's current offset.
Seeking behavior:
- * seeking from the end (SEEK_END) the whole file is decompressed once to
- determine it's size. Therefore it is preferred to use
- SEEK_SET or SEEK_CUR to avoid the processing overhead
- * seeking backwards from the current position rewinds the file to 0
+
+ * seeking from the end :data:`os.SEEK_END` the whole file is decompressed
+ once to determine it's size. Therefore it is preferred to use
+ :data:`os.SEEK_SET` or :data:`os.SEEK_CUR` to avoid the processing
+ overhead
+ * seeking backwards from the current position rewinds the file to ``0``
and decompresses the chunks to the requested offset
* seeking is only supported in files opened for reading
- * if the new offset is out of bound, it is adjusted to either 0 or EOF.
+ * if the new offset is out of bound, it is adjusted to either ``0`` or
+ ``EOF``.
Args:
- offset: seek offset in the uncompressed content represented as number
- whence: seek mode. Supported modes are os.SEEK_SET (absolute seek),
- os.SEEK_CUR (seek relative to the current position), and os.SEEK_END
- (seek relative to the end, offset should be negative).
+ offset (int): seek offset in the uncompressed content represented as
+ number
+ whence (int): seek mode. Supported modes are :data:`os.SEEK_SET`
+ (absolute seek), :data:`os.SEEK_CUR` (seek relative to the current
+ position), and :data:`os.SEEK_END` (seek relative to the end, offset
+ should be negative).
Raises:
- IOError: When this buffer is closed.
- ValueError: When whence is invalid or the file is not seekable
+ ~exceptions.IOError: When this buffer is closed.
+ ~exceptions.ValueError: When whence is invalid or the file is not seekable
"""
if whence == os.SEEK_SET:
absolute_offset = offset
@@ -409,7 +421,7 @@
self.exception_details = exception_details
-class FileSystem(object):
+class FileSystem(BeamPlugin):
"""A class that defines the functions that can be performed on a filesystem.
All methods are abstract and they are for file system providers to
@@ -429,16 +441,6 @@
return compression_type
@classmethod
- def get_all_subclasses(cls):
- """Get all the subclasses of the FileSystem class
- """
- all_subclasses = []
- for subclass in cls.__subclasses__():
- all_subclasses.append(subclass)
- all_subclasses.extend(subclass.get_all_subclasses())
- return all_subclasses
-
- @classmethod
def scheme(cls):
"""URI scheme for the FileSystem
"""
diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py
index 607393d..1c6cdd7 100644
--- a/sdks/python/apache_beam/io/filesystem_test.py
+++ b/sdks/python/apache_beam/io/filesystem_test.py
@@ -20,11 +20,12 @@
import bz2
import gzip
import os
-import unittest
import tempfile
+import unittest
from StringIO import StringIO
-from apache_beam.io.filesystem import CompressedFile, CompressionTypes
+from apache_beam.io.filesystem import CompressedFile
+from apache_beam.io.filesystem import CompressionTypes
class TestCompressedFile(unittest.TestCase):
diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py
index e039686..f9ce553 100644
--- a/sdks/python/apache_beam/io/filesystems.py
+++ b/sdks/python/apache_beam/io/filesystems.py
@@ -22,7 +22,6 @@
from apache_beam.io.filesystem import BeamIOError
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.filesystem import FileSystem
-
# All filesystem implements should be added here
# pylint: disable=wrong-import-position, unused-import
from apache_beam.io.localfilesystem import LocalFileSystem
diff --git a/sdks/python/apache_beam/io/filesystems_test.py b/sdks/python/apache_beam/io/filesystems_test.py
index 9a6f013..eaaa40f 100644
--- a/sdks/python/apache_beam/io/filesystems_test.py
+++ b/sdks/python/apache_beam/io/filesystems_test.py
@@ -18,12 +18,12 @@
"""Unit tests for LocalFileSystem."""
-import unittest
-
import filecmp
import os
import shutil
import tempfile
+import unittest
+
import mock
from apache_beam.io import localfilesystem
diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py
index 201c798..ee79ae5 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery.py
@@ -114,11 +114,14 @@
from apache_beam.internal.gcp import auth
from apache_beam.internal.gcp.json_value import from_json_value
from apache_beam.internal.gcp.json_value import to_json_value
+from apache_beam.io.gcp.internal.clients import bigquery
+from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
+from apache_beam.transforms import DoFn
+from apache_beam.transforms import ParDo
+from apache_beam.transforms import PTransform
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.utils import retry
-from apache_beam.options.pipeline_options import GoogleCloudOptions
-from apache_beam.io.gcp.internal.clients import bigquery
# Protect against environments where bigquery library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
@@ -134,6 +137,7 @@
'BigQueryDisposition',
'BigQuerySource',
'BigQuerySink',
+ 'WriteToBigQuery',
]
JSON_COMPLIANCE_ERROR = 'NAN, INF and -INF values are not JSON compliant.'
@@ -326,45 +330,49 @@
def __init__(self, table=None, dataset=None, project=None, query=None,
validate=False, coder=None, use_standard_sql=False,
flatten_results=True):
- """Initialize a BigQuerySource.
+ """Initialize a :class:`BigQuerySource`.
Args:
- table: The ID of a BigQuery table. If specified all data of the table
- will be used as input of the current source. The ID must contain only
- letters (a-z, A-Z), numbers (0-9), or underscores (_). If dataset
- and query arguments are None then the table argument must contain the
- entire table reference specified as: 'DATASET.TABLE' or
- 'PROJECT:DATASET.TABLE'.
- dataset: The ID of the dataset containing this table or null if the table
- reference is specified entirely by the table argument or a query is
- specified.
- project: The ID of the project containing this table or null if the table
- reference is specified entirely by the table argument or a query is
- specified.
- query: A query to be used instead of arguments table, dataset, and
+ table (str): The ID of a BigQuery table. If specified all data of the
+ table will be used as input of the current source. The ID must contain
+ only letters ``a-z``, ``A-Z``, numbers ``0-9``, or underscores
+ ``_``. If dataset and query arguments are :data:`None` then the table
+ argument must contain the entire table reference specified as:
+ ``'DATASET.TABLE'`` or ``'PROJECT:DATASET.TABLE'``.
+ dataset (str): The ID of the dataset containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument or a query is specified.
+ project (str): The ID of the project containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument or a query is specified.
+ query (str): A query to be used instead of arguments table, dataset, and
project.
- validate: If true, various checks will be done when source gets
- initialized (e.g., is table present?). This should be True for most
- scenarios in order to catch errors as early as possible (pipeline
- construction instead of pipeline execution). It should be False if the
- table is created during pipeline execution by a previous step.
- coder: The coder for the table rows if serialized to disk. If None, then
- the default coder is RowAsDictJsonCoder, which will interpret every line
- in a file as a JSON serialized dictionary. This argument needs a value
- only in special cases when returning table rows as dictionaries is not
- desirable.
- use_standard_sql: Specifies whether to use BigQuery's standard
- SQL dialect for this query. The default value is False. If set to True,
- the query will use BigQuery's updated SQL dialect with improved
- standards compliance. This parameter is ignored for table inputs.
- flatten_results: Flattens all nested and repeated fields in the
- query results. The default value is true.
+ validate (bool): If :data:`True`, various checks will be done when source
+ gets initialized (e.g., is table present?). This should be
+ :data:`True` for most scenarios in order to catch errors as early as
+ possible (pipeline construction instead of pipeline execution). It
+ should be :data:`False` if the table is created during pipeline
+ execution by a previous step.
+ coder (~apache_beam.coders.coders.Coder): The coder for the table
+ rows if serialized to disk. If :data:`None`, then the default coder is
+ :class:`~apache_beam.io.gcp.bigquery.RowAsDictJsonCoder`,
+ which will interpret every line in a file as a JSON serialized
+ dictionary. This argument needs a value only in special cases when
+ returning table rows as dictionaries is not desirable.
+ use_standard_sql (bool): Specifies whether to use BigQuery's standard SQL
+ dialect for this query. The default value is :data:`False`.
+ If set to :data:`True`, the query will use BigQuery's updated SQL
+ dialect with improved standards compliance.
+ This parameter is ignored for table inputs.
+ flatten_results (bool): Flattens all nested and repeated fields in the
+ query results. The default value is :data:`True`.
Raises:
- ValueError: if any of the following is true
- (1) the table reference as a string does not match the expected format
- (2) neither a table nor a query is specified
- (3) both a table and a query is specified.
+ ~exceptions.ValueError: if any of the following is true:
+
+ 1) the table reference as a string does not match the expected format
+ 2) neither a table nor a query is specified
+ 3) both a table and a query is specified.
"""
# Import here to avoid adding the dependency for local running scenarios.
@@ -435,46 +443,62 @@
"""Initialize a BigQuerySink.
Args:
- table: The ID of the table. The ID must contain only letters
- (a-z, A-Z), numbers (0-9), or underscores (_). If dataset argument is
- None then the table argument must contain the entire table reference
- specified as: 'DATASET.TABLE' or 'PROJECT:DATASET.TABLE'.
- dataset: The ID of the dataset containing this table or null if the table
- reference is specified entirely by the table argument.
- project: The ID of the project containing this table or null if the table
- reference is specified entirely by the table argument.
- schema: The schema to be used if the BigQuery table to write has to be
- created. This can be either specified as a 'bigquery.TableSchema' object
- or a single string of the form 'field1:type1,field2:type2,field3:type3'
- that defines a comma separated list of fields. Here 'type' should
- specify the BigQuery type of the field. Single string based schemas do
- not support nested fields, repeated fields, or specifying a BigQuery
- mode for fields (mode will always be set to 'NULLABLE').
- create_disposition: A string describing what happens if the table does not
- exist. Possible values are:
- - BigQueryDisposition.CREATE_IF_NEEDED: create if does not exist.
- - BigQueryDisposition.CREATE_NEVER: fail the write if does not exist.
- write_disposition: A string describing what happens if the table has
- already some data. Possible values are:
- - BigQueryDisposition.WRITE_TRUNCATE: delete existing rows.
- - BigQueryDisposition.WRITE_APPEND: add to existing rows.
- - BigQueryDisposition.WRITE_EMPTY: fail the write if table not empty.
- validate: If true, various checks will be done when sink gets
- initialized (e.g., is table present given the disposition arguments?).
- This should be True for most scenarios in order to catch errors as early
- as possible (pipeline construction instead of pipeline execution). It
- should be False if the table is created during pipeline execution by a
- previous step.
- coder: The coder for the table rows if serialized to disk. If None, then
- the default coder is RowAsDictJsonCoder, which will interpret every
- element written to the sink as a dictionary that will be JSON serialized
- as a line in a file. This argument needs a value only in special cases
- when writing table rows as dictionaries is not desirable.
+ table (str): The ID of the table. The ID must contain only letters
+ ``a-z``, ``A-Z``, numbers ``0-9``, or underscores ``_``. If
+ **dataset** argument is :data:`None` then the table argument must
+ contain the entire table reference specified as: ``'DATASET.TABLE'`` or
+ ``'PROJECT:DATASET.TABLE'``.
+ dataset (str): The ID of the dataset containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument.
+ project (str): The ID of the project containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument.
+ schema (str): The schema to be used if the BigQuery table to write has
+ to be created. This can be either specified as a
+ :class:`~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema` object or a single string of the form
+ ``'field1:type1,field2:type2,field3:type3'`` that defines a comma
+ separated list of fields. Here ``'type'`` should specify the BigQuery
+ type of the field. Single string based schemas do not support nested
+ fields, repeated fields, or specifying a BigQuery mode for fields (mode
+ will always be set to ``'NULLABLE'``).
+ create_disposition (BigQueryDisposition): A string describing what
+ happens if the table does not exist. Possible values are:
+
+ * :attr:`BigQueryDisposition.CREATE_IF_NEEDED`: create if does not
+ exist.
+ * :attr:`BigQueryDisposition.CREATE_NEVER`: fail the write if does not
+ exist.
+
+ write_disposition (BigQueryDisposition): A string describing what
+ happens if the table has already some data. Possible values are:
+
+ * :attr:`BigQueryDisposition.WRITE_TRUNCATE`: delete existing rows.
+ * :attr:`BigQueryDisposition.WRITE_APPEND`: add to existing rows.
+ * :attr:`BigQueryDisposition.WRITE_EMPTY`: fail the write if table not
+ empty.
+
+ validate (bool): If :data:`True`, various checks will be done when sink
+ gets initialized (e.g., is table present given the disposition
+ arguments?). This should be :data:`True` for most scenarios in order to
+ catch errors as early as possible (pipeline construction instead of
+ pipeline execution). It should be :data:`False` if the table is created
+ during pipeline execution by a previous step.
+ coder (~apache_beam.coders.coders.Coder): The coder for the
+ table rows if serialized to disk. If :data:`None`, then the default
+ coder is :class:`~apache_beam.io.gcp.bigquery.RowAsDictJsonCoder`,
+ which will interpret every element written to the sink as a dictionary
+ that will be JSON serialized as a line in a file. This argument needs a
+ value only in special cases when writing table rows as dictionaries is
+ not desirable.
Raises:
- TypeError: if the schema argument is not a string or a TableSchema object.
- ValueError: if the table reference as a string does not match the expected
- format.
+ ~exceptions.TypeError: if the schema argument is not a :class:`str` or a
+ :class:`~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema` object.
+ ~exceptions.ValueError: if the table reference as a string does not
+ match the expected format.
"""
# Import here to avoid adding the dependency for local running scenarios.
@@ -813,6 +837,7 @@
request = bigquery.BigqueryTablesInsertRequest(
projectId=project_id, datasetId=dataset_id, table=table)
response = self.client.tables.Insert(request)
+ logging.debug("Created the table with id %s", table_id)
# The response is a bigquery.Table instance.
return response
@@ -997,12 +1022,23 @@
if found_table and write_disposition != BigQueryDisposition.WRITE_TRUNCATE:
return found_table
else:
+ created_table = self._create_table(project_id=project_id,
+ dataset_id=dataset_id,
+ table_id=table_id,
+ schema=schema or found_table.schema)
# if write_disposition == BigQueryDisposition.WRITE_TRUNCATE we delete
# the table before this point.
- return self._create_table(project_id=project_id,
- dataset_id=dataset_id,
- table_id=table_id,
- schema=schema or found_table.schema)
+ if write_disposition == BigQueryDisposition.WRITE_TRUNCATE:
+ # BigQuery can route data to the old table for 2 mins max so wait
+ # that much time before creating the table and writing it
+ logging.warning('Sleeping for 150 seconds before the write as ' +
+ 'BigQuery inserts can be routed to deleted table ' +
+ 'for 2 mins after the delete and create.')
+ # TODO(BEAM-2673): Remove this sleep by migrating to load api
+ time.sleep(150)
+ return created_table
+ else:
+ return created_table
def run_query(self, project_id, query, use_legacy_sql, flatten_results,
dry_run=False):
@@ -1134,3 +1170,269 @@
else:
result[field.name] = self._convert_cell_value_to_dict(value, field)
return result
+
+
+class BigQueryWriteFn(DoFn):
+ """A ``DoFn`` that streams writes to BigQuery once the table is created.
+ """
+
+ def __init__(self, table_id, dataset_id, project_id, batch_size, schema,
+ create_disposition, write_disposition, client):
+ """Initialize a WriteToBigQuery transform.
+
+ Args:
+ table_id: The ID of the table. The ID must contain only letters
+ (a-z, A-Z), numbers (0-9), or underscores (_). If dataset argument is
+ None then the table argument must contain the entire table reference
+ specified as: 'DATASET.TABLE' or 'PROJECT:DATASET.TABLE'.
+ dataset_id: The ID of the dataset containing this table or null if the
+ table reference is specified entirely by the table argument.
+ project_id: The ID of the project containing this table or null if the
+ table reference is specified entirely by the table argument.
+ batch_size: Number of rows to be written to BQ per streaming API insert.
+ schema: The schema to be used if the BigQuery table to write has to be
+ created. This can be either specified as a 'bigquery.TableSchema' object
+ or a single string of the form 'field1:type1,field2:type2,field3:type3'
+ that defines a comma separated list of fields. Here 'type' should
+ specify the BigQuery type of the field. Single string based schemas do
+ not support nested fields, repeated fields, or specifying a BigQuery
+ mode for fields (mode will always be set to 'NULLABLE').
+ create_disposition: A string describing what happens if the table does not
+ exist. Possible values are:
+ - BigQueryDisposition.CREATE_IF_NEEDED: create if does not exist.
+ - BigQueryDisposition.CREATE_NEVER: fail the write if does not exist.
+ write_disposition: A string describing what happens if the table has
+ already some data. Possible values are:
+ - BigQueryDisposition.WRITE_TRUNCATE: delete existing rows.
+ - BigQueryDisposition.WRITE_APPEND: add to existing rows.
+ - BigQueryDisposition.WRITE_EMPTY: fail the write if table not empty.
+ For streaming pipelines WriteTruncate can not be used.
+ test_client: Override the default bigquery client used for testing.
+ """
+ self.table_id = table_id
+ self.dataset_id = dataset_id
+ self.project_id = project_id
+ self.schema = schema
+ self.client = client
+ self.create_disposition = create_disposition
+ self.write_disposition = write_disposition
+ self._rows_buffer = []
+ # The default batch size is 500
+ self._max_batch_size = batch_size or 500
+
+ @staticmethod
+ def get_table_schema(schema):
+ """Transform the table schema into a bigquery.TableSchema instance.
+
+ Args:
+ schema: The schema to be used if the BigQuery table to write has to be
+ created. This is a dictionary object created in the WriteToBigQuery
+ transform.
+ Returns:
+ table_schema: The schema to be used if the BigQuery table to write has
+ to be created but in the bigquery.TableSchema format.
+ """
+ if schema is None:
+ return schema
+ elif isinstance(schema, dict):
+ return parse_table_schema_from_json(json.dumps(schema))
+ else:
+ raise TypeError('Unexpected schema argument: %s.' % schema)
+
+ def start_bundle(self):
+ self._rows_buffer = []
+ self.table_schema = self.get_table_schema(self.schema)
+
+ self.bigquery_wrapper = BigQueryWrapper(client=self.client)
+ self.bigquery_wrapper.get_or_create_table(
+ self.project_id, self.dataset_id, self.table_id, self.table_schema,
+ self.create_disposition, self.write_disposition)
+
+ def process(self, element, unused_create_fn_output=None):
+ self._rows_buffer.append(element)
+ if len(self._rows_buffer) >= self._max_batch_size:
+ self._flush_batch()
+
+ def finish_bundle(self):
+ if self._rows_buffer:
+ self._flush_batch()
+ self._rows_buffer = []
+
+ def _flush_batch(self):
+ # Flush the current batch of rows to BigQuery.
+ passed, errors = self.bigquery_wrapper.insert_rows(
+ project_id=self.project_id, dataset_id=self.dataset_id,
+ table_id=self.table_id, rows=self._rows_buffer)
+ if not passed:
+ raise RuntimeError('Could not successfully insert rows to BigQuery'
+ ' table [%s:%s.%s]. Errors: %s'%
+ (self.project_id, self.dataset_id,
+ self.table_id, errors))
+ logging.debug("Successfully wrote %d rows.", len(self._rows_buffer))
+ self._rows_buffer = []
+
+
+class WriteToBigQuery(PTransform):
+
+ def __init__(self, table, dataset=None, project=None, schema=None,
+ create_disposition=BigQueryDisposition.CREATE_IF_NEEDED,
+ write_disposition=BigQueryDisposition.WRITE_APPEND,
+ batch_size=None, test_client=None):
+ """Initialize a WriteToBigQuery transform.
+
+ Args:
+ table (str): The ID of the table. The ID must contain only letters
+ ``a-z``, ``A-Z``, numbers ``0-9``, or underscores ``_``. If dataset
+ argument is :data:`None` then the table argument must contain the
+ entire table reference specified as: ``'DATASET.TABLE'`` or
+ ``'PROJECT:DATASET.TABLE'``.
+ dataset (str): The ID of the dataset containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument.
+ project (str): The ID of the project containing this table or
+ :data:`None` if the table reference is specified entirely by the table
+ argument.
+ schema (str): The schema to be used if the BigQuery table to write has to
+ be created. This can be either specified as a
+ :class:`~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema`
+ object or a single string of the form
+ ``'field1:type1,field2:type2,field3:type3'`` that defines a comma
+ separated list of fields. Here ``'type'`` should specify the BigQuery
+ type of the field. Single string based schemas do not support nested
+ fields, repeated fields, or specifying a BigQuery mode for fields
+ (mode will always be set to ``'NULLABLE'``).
+ create_disposition (BigQueryDisposition): A string describing what
+ happens if the table does not exist. Possible values are:
+
+ * :attr:`BigQueryDisposition.CREATE_IF_NEEDED`: create if does not
+ exist.
+ * :attr:`BigQueryDisposition.CREATE_NEVER`: fail the write if does not
+ exist.
+
+ write_disposition (BigQueryDisposition): A string describing what happens
+ if the table has already some data. Possible values are:
+
+ * :attr:`BigQueryDisposition.WRITE_TRUNCATE`: delete existing rows.
+ * :attr:`BigQueryDisposition.WRITE_APPEND`: add to existing rows.
+ * :attr:`BigQueryDisposition.WRITE_EMPTY`: fail the write if table not
+ empty.
+
+ For streaming pipelines WriteTruncate can not be used.
+
+ batch_size (int): Number of rows to be written to BQ per streaming API
+ insert.
+ test_client: Override the default bigquery client used for testing.
+ """
+ self.table_reference = _parse_table_reference(table, dataset, project)
+ self.create_disposition = BigQueryDisposition.validate_create(
+ create_disposition)
+ self.write_disposition = BigQueryDisposition.validate_write(
+ write_disposition)
+ self.schema = schema
+ self.batch_size = batch_size
+ self.test_client = test_client
+
+ @staticmethod
+ def get_table_schema_from_string(schema):
+ """Transform the string table schema into a
+ :class:`~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema` instance.
+
+ Args:
+ schema (str): The sting schema to be used if the BigQuery table to write
+ has to be created.
+
+ Returns:
+ ~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema:
+ The schema to be used if the BigQuery table to write has to be created
+ but in the :class:`~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema` format.
+ """
+ table_schema = bigquery.TableSchema()
+ schema_list = [s.strip() for s in schema.split(',')]
+ for field_and_type in schema_list:
+ field_name, field_type = field_and_type.split(':')
+ field_schema = bigquery.TableFieldSchema()
+ field_schema.name = field_name
+ field_schema.type = field_type
+ field_schema.mode = 'NULLABLE'
+ table_schema.fields.append(field_schema)
+ return table_schema
+
+ @staticmethod
+ def table_schema_to_dict(table_schema):
+ """Create a dictionary representation of table schema for serialization
+ """
+ def get_table_field(field):
+ """Create a dictionary representation of a table field
+ """
+ result = {}
+ result['name'] = field.name
+ result['type'] = field.type
+ result['mode'] = getattr(field, 'mode', 'NULLABLE')
+ if hasattr(field, 'description') and field.description is not None:
+ result['description'] = field.description
+ if hasattr(field, 'fields') and field.fields:
+ result['fields'] = [get_table_field(f) for f in field.fields]
+ return result
+
+ if not isinstance(table_schema, bigquery.TableSchema):
+ raise ValueError("Table schema must be of the type bigquery.TableSchema")
+ schema = {'fields': []}
+ for field in table_schema.fields:
+ schema['fields'].append(get_table_field(field))
+ return schema
+
+ @staticmethod
+ def get_dict_table_schema(schema):
+ """Transform the table schema into a dictionary instance.
+
+ Args:
+ schema (~apache_beam.io.gcp.internal.clients.bigquery.\
+bigquery_v2_messages.TableSchema):
+ The schema to be used if the BigQuery table to write has to be created.
+ This can either be a dict or string or in the TableSchema format.
+
+ Returns:
+ Dict[str, Any]: The schema to be used if the BigQuery table to write has
+ to be created but in the dictionary format.
+ """
+ if isinstance(schema, dict):
+ return schema
+ elif schema is None:
+ return schema
+ elif isinstance(schema, basestring):
+ table_schema = WriteToBigQuery.get_table_schema_from_string(schema)
+ return WriteToBigQuery.table_schema_to_dict(table_schema)
+ elif isinstance(schema, bigquery.TableSchema):
+ return WriteToBigQuery.table_schema_to_dict(schema)
+ else:
+ raise TypeError('Unexpected schema argument: %s.' % schema)
+
+ def expand(self, pcoll):
+ if self.table_reference.projectId is None:
+ self.table_reference.projectId = pcoll.pipeline.options.view_as(
+ GoogleCloudOptions).project
+ bigquery_write_fn = BigQueryWriteFn(
+ table_id=self.table_reference.tableId,
+ dataset_id=self.table_reference.datasetId,
+ project_id=self.table_reference.projectId,
+ batch_size=self.batch_size,
+ schema=self.get_dict_table_schema(self.schema),
+ create_disposition=self.create_disposition,
+ write_disposition=self.write_disposition,
+ client=self.test_client)
+ return pcoll | 'WriteToBigQuery' >> ParDo(bigquery_write_fn)
+
+ def display_data(self):
+ res = {}
+ if self.table_reference is not None:
+ tableSpec = '{}.{}'.format(self.table_reference.datasetId,
+ self.table_reference.tableId)
+ if self.table_reference.projectId is not None:
+ tableSpec = '{}:{}'.format(self.table_reference.projectId,
+ tableSpec)
+ res['table'] = DisplayDataItem(tableSpec, label='Table')
+ return res
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py
index a26050c..8490481 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py
@@ -27,14 +27,14 @@
import mock
import apache_beam as beam
+from apache_beam.internal.gcp.json_value import to_json_value
from apache_beam.io.gcp.bigquery import RowAsDictJsonCoder
from apache_beam.io.gcp.bigquery import TableRowJsonCoder
from apache_beam.io.gcp.bigquery import parse_table_schema_from_json
from apache_beam.io.gcp.internal.clients import bigquery
-from apache_beam.internal.gcp.json_value import to_json_value
+from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.options.pipeline_options import PipelineOptions
# Protect against environments where bigquery library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
@@ -650,7 +650,8 @@
self.assertFalse(client.tables.Delete.called)
self.assertFalse(client.tables.Insert.called)
- def test_table_with_write_disposition_truncate(self):
+ @mock.patch('time.sleep', return_value=None)
+ def test_table_with_write_disposition_truncate(self, _patched_sleep):
client = mock.Mock()
table = bigquery.Table(
tableReference=bigquery.TableReference(
@@ -824,6 +825,265 @@
self.assertEqual(new_dataset.datasetReference.datasetId, 'dataset_id')
+@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
+class WriteToBigQuery(unittest.TestCase):
+
+ def test_dofn_client_start_bundle_called(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ self.assertTrue(client.tables.Get.called)
+
+ def test_dofn_client_start_bundle_create_called(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = None
+ client.tables.Insert.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ self.assertTrue(client.tables.Get.called)
+ self.assertTrue(client.tables.Insert.called)
+
+ def test_dofn_client_process_performs_batching(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ client.tabledata.InsertAll.return_value = \
+ bigquery.TableDataInsertAllResponse(insertErrors=[])
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ fn.process({'month': 1})
+
+ self.assertTrue(client.tables.Get.called)
+ # InsertRows not called as batch size is not hit yet
+ self.assertFalse(client.tabledata.InsertAll.called)
+
+ def test_dofn_client_process_flush_called(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ client.tabledata.InsertAll.return_value = (
+ bigquery.TableDataInsertAllResponse(insertErrors=[]))
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ fn.process({'month': 1})
+ fn.process({'month': 2})
+ self.assertTrue(client.tables.Get.called)
+ # InsertRows called as batch size is hit
+ self.assertTrue(client.tabledata.InsertAll.called)
+
+ def test_dofn_client_finish_bundle_flush_called(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ client.tabledata.InsertAll.return_value = \
+ bigquery.TableDataInsertAllResponse(insertErrors=[])
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ fn.process({'month': 1})
+
+ self.assertTrue(client.tables.Get.called)
+ # InsertRows not called as batch size is not hit
+ self.assertFalse(client.tabledata.InsertAll.called)
+
+ fn.finish_bundle()
+ # InsertRows called in finish bundle
+ self.assertTrue(client.tabledata.InsertAll.called)
+
+ def test_dofn_client_no_records(self):
+ client = mock.Mock()
+ client.tables.Get.return_value = bigquery.Table(
+ tableReference=bigquery.TableReference(
+ projectId='project_id', datasetId='dataset_id', tableId='table_id'))
+ client.tabledata.InsertAll.return_value = \
+ bigquery.TableDataInsertAllResponse(insertErrors=[])
+ create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
+ write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND
+ schema = {'fields': [
+ {'name': 'month', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+
+ fn = beam.io.gcp.bigquery.BigQueryWriteFn(
+ table_id='table_id',
+ dataset_id='dataset_id',
+ project_id='project_id',
+ batch_size=2,
+ schema=schema,
+ create_disposition=create_disposition,
+ write_disposition=write_disposition,
+ client=client)
+
+ fn.start_bundle()
+ self.assertTrue(client.tables.Get.called)
+ # InsertRows not called as batch size is not hit
+ self.assertFalse(client.tabledata.InsertAll.called)
+
+ fn.finish_bundle()
+ # InsertRows not called in finish bundle as no records
+ self.assertFalse(client.tabledata.InsertAll.called)
+
+ def test_noop_schema_parsing(self):
+ expected_table_schema = None
+ table_schema = beam.io.gcp.bigquery.BigQueryWriteFn.get_table_schema(
+ schema=None)
+ self.assertEqual(expected_table_schema, table_schema)
+
+ def test_dict_schema_parsing(self):
+ schema = {'fields': [
+ {'name': 's', 'type': 'STRING', 'mode': 'NULLABLE'},
+ {'name': 'n', 'type': 'INTEGER', 'mode': 'NULLABLE'},
+ {'name': 'r', 'type': 'RECORD', 'mode': 'NULLABLE', 'fields': [
+ {'name': 'x', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}]}
+ table_schema = beam.io.gcp.bigquery.BigQueryWriteFn.get_table_schema(schema)
+ string_field = bigquery.TableFieldSchema(
+ name='s', type='STRING', mode='NULLABLE')
+ nested_field = bigquery.TableFieldSchema(
+ name='x', type='INTEGER', mode='NULLABLE')
+ number_field = bigquery.TableFieldSchema(
+ name='n', type='INTEGER', mode='NULLABLE')
+ record_field = bigquery.TableFieldSchema(
+ name='r', type='RECORD', mode='NULLABLE', fields=[nested_field])
+ expected_table_schema = bigquery.TableSchema(
+ fields=[string_field, number_field, record_field])
+ self.assertEqual(expected_table_schema, table_schema)
+
+ def test_string_schema_parsing(self):
+ schema = 's:STRING, n:INTEGER'
+ expected_dict_schema = {'fields': [
+ {'name': 's', 'type': 'STRING', 'mode': 'NULLABLE'},
+ {'name': 'n', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+ dict_schema = (
+ beam.io.gcp.bigquery.WriteToBigQuery.get_dict_table_schema(schema))
+ self.assertEqual(expected_dict_schema, dict_schema)
+
+ def test_table_schema_parsing(self):
+ string_field = bigquery.TableFieldSchema(
+ name='s', type='STRING', mode='NULLABLE')
+ nested_field = bigquery.TableFieldSchema(
+ name='x', type='INTEGER', mode='NULLABLE')
+ number_field = bigquery.TableFieldSchema(
+ name='n', type='INTEGER', mode='NULLABLE')
+ record_field = bigquery.TableFieldSchema(
+ name='r', type='RECORD', mode='NULLABLE', fields=[nested_field])
+ schema = bigquery.TableSchema(
+ fields=[string_field, number_field, record_field])
+ expected_dict_schema = {'fields': [
+ {'name': 's', 'type': 'STRING', 'mode': 'NULLABLE'},
+ {'name': 'n', 'type': 'INTEGER', 'mode': 'NULLABLE'},
+ {'name': 'r', 'type': 'RECORD', 'mode': 'NULLABLE', 'fields': [
+ {'name': 'x', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}]}
+ dict_schema = (
+ beam.io.gcp.bigquery.WriteToBigQuery.get_dict_table_schema(schema))
+ self.assertEqual(expected_dict_schema, dict_schema)
+
+ def test_table_schema_parsing_end_to_end(self):
+ string_field = bigquery.TableFieldSchema(
+ name='s', type='STRING', mode='NULLABLE')
+ nested_field = bigquery.TableFieldSchema(
+ name='x', type='INTEGER', mode='NULLABLE')
+ number_field = bigquery.TableFieldSchema(
+ name='n', type='INTEGER', mode='NULLABLE')
+ record_field = bigquery.TableFieldSchema(
+ name='r', type='RECORD', mode='NULLABLE', fields=[nested_field])
+ schema = bigquery.TableSchema(
+ fields=[string_field, number_field, record_field])
+ table_schema = beam.io.gcp.bigquery.BigQueryWriteFn.get_table_schema(
+ beam.io.gcp.bigquery.WriteToBigQuery.get_dict_table_schema(schema))
+ self.assertEqual(table_schema, schema)
+
+ def test_none_schema_parsing(self):
+ schema = None
+ expected_dict_schema = None
+ dict_schema = (
+ beam.io.gcp.bigquery.WriteToBigQuery.get_dict_table_schema(schema))
+ self.assertEqual(expected_dict_schema, dict_schema)
+
+ def test_noop_dict_schema_parsing(self):
+ schema = {'fields': [
+ {'name': 's', 'type': 'STRING', 'mode': 'NULLABLE'},
+ {'name': 'n', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}
+ expected_dict_schema = schema
+ dict_schema = (
+ beam.io.gcp.bigquery.WriteToBigQuery.get_dict_table_schema(schema))
+ self.assertEqual(expected_dict_schema, dict_schema)
+
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
unittest.main()
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler.py
new file mode 100644
index 0000000..7d94f24
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler.py
@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility functions & classes that are _not_ specific to the datastore client.
+#
+# For internal use only; no backwards-compatibility guarantees.
+
+import random
+
+from apache_beam.io.gcp.datastore.v1 import util
+
+
+class AdaptiveThrottler(object):
+ """Implements adaptive throttling.
+
+ See
+ https://landing.google.com/sre/book/chapters/handling-overload.html#client-side-throttling-a7sYUg
+ for a full discussion of the use case and algorithm applied.
+ """
+
+ # The target minimum number of requests per samplePeriodMs, even if no
+ # requests succeed. Must be greater than 0, else we could throttle to zero.
+ # Because every decision is probabilistic, there is no guarantee that the
+ # request rate in any given interval will not be zero. (This is the +1 from
+ # the formula in
+ # https://landing.google.com/sre/book/chapters/handling-overload.html )
+ MIN_REQUESTS = 1
+
+ def __init__(self, window_ms, bucket_ms, overload_ratio):
+ """Initializes AdaptiveThrottler.
+
+ Args:
+ window_ms: int, length of history to consider, in ms, to set
+ throttling.
+ bucket_ms: int, granularity of time buckets that we store data in, in
+ ms.
+ overload_ratio: float, the target ratio between requests sent and
+ successful requests. This is "K" in the formula in
+ https://landing.google.com/sre/book/chapters/handling-overload.html.
+ """
+ self._all_requests = util.MovingSum(window_ms, bucket_ms)
+ self._successful_requests = util.MovingSum(window_ms, bucket_ms)
+ self._overload_ratio = float(overload_ratio)
+ self._random = random.Random()
+
+ def _throttling_probability(self, now):
+ if not self._all_requests.has_data(now):
+ return 0
+ all_requests = self._all_requests.sum(now)
+ successful_requests = self._successful_requests.sum(now)
+ return max(
+ 0, (all_requests - self._overload_ratio * successful_requests)
+ / (all_requests + AdaptiveThrottler.MIN_REQUESTS))
+
+ def throttle_request(self, now):
+ """Determines whether one RPC attempt should be throttled.
+
+ This should be called once each time the caller intends to send an RPC; if
+ it returns true, drop or delay that request (calling this function again
+ after the delay).
+
+ Args:
+ now: int, time in ms since the epoch
+ Returns:
+ bool, True if the caller should throttle or delay the request.
+ """
+ throttling_probability = self._throttling_probability(now)
+ self._all_requests.add(now, 1)
+ return self._random.uniform(0, 1) < throttling_probability
+
+ def successful_request(self, now):
+ """Notifies the throttler of a successful request.
+
+ Must be called once for each request (for which throttle_request was
+ previously called) that succeeded.
+
+ Args:
+ now: int, time in ms since the epoch
+ """
+ self._successful_requests.add(now, 1)
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
new file mode 100644
index 0000000..1ac2393
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from mock import patch
+
+from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler
+
+
+class AdaptiveThrottlerTest(unittest.TestCase):
+
+ START_TIME = 1500000000000
+ SAMPLE_PERIOD = 60000
+ BUCKET = 1000
+ OVERLOAD_RATIO = 2
+
+ def setUp(self):
+ self._throttler = AdaptiveThrottler(
+ AdaptiveThrottlerTest.SAMPLE_PERIOD, AdaptiveThrottlerTest.BUCKET,
+ AdaptiveThrottlerTest.OVERLOAD_RATIO)
+
+ # As far as practical, keep these tests aligned with
+ # AdaptiveThrottlerTest.java.
+
+ def test_no_initial_throttling(self):
+ self.assertEqual(0, self._throttler._throttling_probability(
+ AdaptiveThrottlerTest.START_TIME))
+
+ def test_no_throttling_if_no_errors(self):
+ for t in range(AdaptiveThrottlerTest.START_TIME,
+ AdaptiveThrottlerTest.START_TIME + 20):
+ self.assertFalse(self._throttler.throttle_request(t))
+ self._throttler.successful_request(t)
+ self.assertEqual(0, self._throttler._throttling_probability(
+ AdaptiveThrottlerTest.START_TIME + 20))
+
+ def test_no_throttling_after_errors_expire(self):
+ for t in range(AdaptiveThrottlerTest.START_TIME,
+ AdaptiveThrottlerTest.START_TIME
+ + AdaptiveThrottlerTest.SAMPLE_PERIOD, 100):
+ self._throttler.throttle_request(t)
+ # And no sucessful_request
+ self.assertLess(0, self._throttler._throttling_probability(
+ AdaptiveThrottlerTest.START_TIME + AdaptiveThrottlerTest.SAMPLE_PERIOD
+ ))
+ for t in range(AdaptiveThrottlerTest.START_TIME
+ + AdaptiveThrottlerTest.SAMPLE_PERIOD,
+ AdaptiveThrottlerTest.START_TIME
+ + AdaptiveThrottlerTest.SAMPLE_PERIOD*2, 100):
+ self._throttler.throttle_request(t)
+ self._throttler.successful_request(t)
+
+ self.assertEqual(0, self._throttler._throttling_probability(
+ AdaptiveThrottlerTest.START_TIME +
+ AdaptiveThrottlerTest.SAMPLE_PERIOD*2))
+
+ @patch('random.Random')
+ def test_throttling_after_errors(self, mock_random):
+ mock_random().uniform.side_effect = [x/10.0 for x in range(0, 10)]*2
+ self._throttler = AdaptiveThrottler(
+ AdaptiveThrottlerTest.SAMPLE_PERIOD, AdaptiveThrottlerTest.BUCKET,
+ AdaptiveThrottlerTest.OVERLOAD_RATIO)
+ for t in range(AdaptiveThrottlerTest.START_TIME,
+ AdaptiveThrottlerTest.START_TIME + 20):
+ throttled = self._throttler.throttle_request(t)
+ # 1/3rd of requests succeeding.
+ if t % 3 == 1:
+ self._throttler.successful_request(t)
+
+ if t > AdaptiveThrottlerTest.START_TIME + 10:
+ # Roughly 1/3rd succeeding, 1/3rd failing, 1/3rd throttled.
+ self.assertAlmostEqual(
+ 0.33, self._throttler._throttling_probability(t), delta=0.1)
+ # Given the mocked random numbers, we expect 10..13 to be throttled and
+ # 14+ to be unthrottled.
+ self.assertEqual(t < AdaptiveThrottlerTest.START_TIME + 14, throttled)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
index c606133..13209c1 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
@@ -18,6 +18,21 @@
"""A connector for reading from and writing to Google Cloud Datastore"""
import logging
+import time
+
+from apache_beam.io.gcp.datastore.v1 import helper
+from apache_beam.io.gcp.datastore.v1 import query_splitter
+from apache_beam.io.gcp.datastore.v1 import util
+from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler
+from apache_beam.metrics.metric import Metrics
+from apache_beam.transforms import Create
+from apache_beam.transforms import DoFn
+from apache_beam.transforms import FlatMap
+from apache_beam.transforms import GroupByKey
+from apache_beam.transforms import Map
+from apache_beam.transforms import ParDo
+from apache_beam.transforms import PTransform
+from apache_beam.transforms.util import Values
# Protect against environments where datastore library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
@@ -28,16 +43,6 @@
pass
# pylint: enable=wrong-import-order, wrong-import-position
-from apache_beam.io.gcp.datastore.v1 import helper
-from apache_beam.io.gcp.datastore.v1 import query_splitter
-from apache_beam.transforms import Create
-from apache_beam.transforms import DoFn
-from apache_beam.transforms import FlatMap
-from apache_beam.transforms import GroupByKey
-from apache_beam.transforms import Map
-from apache_beam.transforms import PTransform
-from apache_beam.transforms import ParDo
-from apache_beam.transforms.util import Values
__all__ = ['ReadFromDatastore', 'WriteToDatastore', 'DeleteFromDatastore']
@@ -84,10 +89,10 @@
_DEFAULT_BUNDLE_SIZE_BYTES = 64 * 1024 * 1024
def __init__(self, project, query, namespace=None, num_splits=0):
- """Initialize the ReadFromDatastore transform.
+ """Initialize the `ReadFromDatastore` transform.
Args:
- project: The Project ID
+ project: The ID of the project to read from.
query: Cloud Datastore query to be read from.
namespace: An optional namespace.
num_splits: Number of splits for the query.
@@ -313,8 +318,15 @@
supported, as the commits are retried when failures occur.
"""
- # Max allowed Datastore write batch size.
- _WRITE_BATCH_SIZE = 500
+ _WRITE_BATCH_INITIAL_SIZE = 200
+ # Max allowed Datastore writes per batch, and max bytes per batch.
+ # Note that the max bytes per batch set here is lower than the 10MB limit
+ # actually enforced by the API, to leave space for the CommitRequest wrapper
+ # around the mutations.
+ _WRITE_BATCH_MAX_SIZE = 500
+ _WRITE_BATCH_MAX_BYTES_SIZE = 9000000
+ _WRITE_BATCH_MIN_SIZE = 10
+ _WRITE_BATCH_TARGET_LATENCY_MS = 5000
def __init__(self, project, mutation_fn):
"""Initializes a Mutate transform.
@@ -338,46 +350,122 @@
return {'project': self._project,
'mutation_fn': self._mutation_fn.__class__.__name__}
+ class _DynamicBatchSizer(object):
+ """Determines request sizes for future Datastore RPCS."""
+ def __init__(self):
+ self._commit_time_per_entity_ms = util.MovingSum(window_ms=120000,
+ bucket_ms=10000)
+
+ def get_batch_size(self, now):
+ """Returns the recommended size for datastore RPCs at this time."""
+ if not self._commit_time_per_entity_ms.has_data(now):
+ return _Mutate._WRITE_BATCH_INITIAL_SIZE
+
+ recent_mean_latency_ms = (self._commit_time_per_entity_ms.sum(now)
+ / self._commit_time_per_entity_ms.count(now))
+ return max(_Mutate._WRITE_BATCH_MIN_SIZE,
+ min(_Mutate._WRITE_BATCH_MAX_SIZE,
+ _Mutate._WRITE_BATCH_TARGET_LATENCY_MS
+ / max(recent_mean_latency_ms, 1)
+ ))
+
+ def report_latency(self, now, latency_ms, num_mutations):
+ """Reports the latency of an RPC to Datastore.
+
+ Args:
+ now: double, completion time of the RPC as seconds since the epoch.
+ latency_ms: double, the observed latency in milliseconds for this RPC.
+ num_mutations: int, number of mutations contained in the RPC.
+ """
+ self._commit_time_per_entity_ms.add(now, latency_ms / num_mutations)
+
class DatastoreWriteFn(DoFn):
"""A ``DoFn`` that write mutations to Datastore.
Mutations are written in batches, where the maximum batch size is
- `Mutate._WRITE_BATCH_SIZE`.
+ `_Mutate._WRITE_BATCH_SIZE`.
Commits are non-transactional. If a commit fails because of a conflict over
an entity group, the commit will be retried. This means that the mutation
should be idempotent (`upsert` and `delete` mutations) to prevent duplicate
data or errors.
"""
- def __init__(self, project):
+ def __init__(self, project, fixed_batch_size=None):
+ """
+ Args:
+ project: str, the cloud project id.
+ fixed_batch_size: int, for testing only, this forces all batches of
+ writes to be a fixed size, for easier unittesting.
+ """
self._project = project
self._datastore = None
- self._mutations = []
+ self._fixed_batch_size = fixed_batch_size
+ self._rpc_successes = Metrics.counter(
+ _Mutate.DatastoreWriteFn, "datastoreRpcSuccesses")
+ self._rpc_errors = Metrics.counter(
+ _Mutate.DatastoreWriteFn, "datastoreRpcErrors")
+ self._throttled_secs = Metrics.counter(
+ _Mutate.DatastoreWriteFn, "cumulativeThrottlingSeconds")
+ self._throttler = AdaptiveThrottler(window_ms=120000, bucket_ms=1000,
+ overload_ratio=1.25)
+
+ def _update_rpc_stats(self, successes=0, errors=0, throttled_secs=0):
+ self._rpc_successes.inc(successes)
+ self._rpc_errors.inc(errors)
+ self._throttled_secs.inc(throttled_secs)
def start_bundle(self):
self._mutations = []
+ self._mutations_size = 0
self._datastore = helper.get_datastore(self._project)
+ if self._fixed_batch_size:
+ self._target_batch_size = self._fixed_batch_size
+ else:
+ self._batch_sizer = _Mutate._DynamicBatchSizer()
+ self._target_batch_size = self._batch_sizer.get_batch_size(
+ time.time()*1000)
def process(self, element):
+ size = element.ByteSize()
+ if (self._mutations and
+ size + self._mutations_size > _Mutate._WRITE_BATCH_MAX_BYTES_SIZE):
+ self._flush_batch()
self._mutations.append(element)
- if len(self._mutations) >= _Mutate._WRITE_BATCH_SIZE:
+ self._mutations_size += size
+ if len(self._mutations) >= self._target_batch_size:
self._flush_batch()
def finish_bundle(self):
if self._mutations:
self._flush_batch()
- self._mutations = []
def _flush_batch(self):
# Flush the current batch of mutations to Cloud Datastore.
- helper.write_mutations(self._datastore, self._project, self._mutations)
- logging.debug("Successfully wrote %d mutations.", len(self._mutations))
+ _, latency_ms = helper.write_mutations(
+ self._datastore, self._project, self._mutations,
+ self._throttler, self._update_rpc_stats,
+ throttle_delay=_Mutate._WRITE_BATCH_TARGET_LATENCY_MS/1000)
+ logging.debug("Successfully wrote %d mutations in %dms.",
+ len(self._mutations), latency_ms)
+
+ if not self._fixed_batch_size:
+ now = time.time()*1000
+ self._batch_sizer.report_latency(now, latency_ms, len(self._mutations))
+ self._target_batch_size = self._batch_sizer.get_batch_size(now)
+
self._mutations = []
+ self._mutations_size = 0
class WriteToDatastore(_Mutate):
"""A ``PTransform`` to write a ``PCollection[Entity]`` to Cloud Datastore."""
+
def __init__(self, project):
+ """Initialize the `WriteToDatastore` transform.
+
+ Args:
+ project: The ID of the project to write to.
+ """
# Import here to avoid adding the dependency for local running scenarios.
try:
@@ -404,6 +492,12 @@
class DeleteFromDatastore(_Mutate):
"""A ``PTransform`` to delete a ``PCollection[Key]`` from Cloud Datastore."""
def __init__(self, project):
+ """Initialize the `DeleteFromDatastore` transform.
+
+ Args:
+ project: The ID of the project from which the entities will be deleted.
+ """
+
super(DeleteFromDatastore, self).__init__(
project, DeleteFromDatastore.to_delete_mutation)
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
index 6adc08a..e131f93 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
@@ -15,16 +15,21 @@
# limitations under the License.
#
+from __future__ import print_function
+
import unittest
-from mock import MagicMock, call, patch
+from mock import MagicMock
+from mock import call
+from mock import patch
from apache_beam.io.gcp.datastore.v1 import fake_datastore
from apache_beam.io.gcp.datastore.v1 import helper
from apache_beam.io.gcp.datastore.v1 import query_splitter
-from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate
from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore
from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore
+from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate
+
# Protect against environments where datastore library is not available.
# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
@@ -155,15 +160,15 @@
self.check_DatastoreWriteFn(0)
def test_DatastoreWriteFn_with_one_batch(self):
- num_entities_to_write = _Mutate._WRITE_BATCH_SIZE * 1 - 50
+ num_entities_to_write = _Mutate._WRITE_BATCH_INITIAL_SIZE * 1 - 50
self.check_DatastoreWriteFn(num_entities_to_write)
def test_DatastoreWriteFn_with_multiple_batches(self):
- num_entities_to_write = _Mutate._WRITE_BATCH_SIZE * 3 + 50
+ num_entities_to_write = _Mutate._WRITE_BATCH_INITIAL_SIZE * 3 + 50
self.check_DatastoreWriteFn(num_entities_to_write)
def test_DatastoreWriteFn_with_batch_size_exact_multiple(self):
- num_entities_to_write = _Mutate._WRITE_BATCH_SIZE * 2
+ num_entities_to_write = _Mutate._WRITE_BATCH_INITIAL_SIZE * 2
self.check_DatastoreWriteFn(num_entities_to_write)
def check_DatastoreWriteFn(self, num_entities):
@@ -180,7 +185,8 @@
self._mock_datastore.commit.side_effect = (
fake_datastore.create_commit(actual_mutations))
- datastore_write_fn = _Mutate.DatastoreWriteFn(self._PROJECT)
+ datastore_write_fn = _Mutate.DatastoreWriteFn(
+ self._PROJECT, fixed_batch_size=_Mutate._WRITE_BATCH_INITIAL_SIZE)
datastore_write_fn.start_bundle()
for mutation in expected_mutations:
@@ -188,8 +194,26 @@
datastore_write_fn.finish_bundle()
self.assertEqual(actual_mutations, expected_mutations)
- self.assertEqual((num_entities - 1) / _Mutate._WRITE_BATCH_SIZE + 1,
- self._mock_datastore.commit.call_count)
+ self.assertEqual(
+ (num_entities - 1) / _Mutate._WRITE_BATCH_INITIAL_SIZE + 1,
+ self._mock_datastore.commit.call_count)
+
+ def test_DatastoreWriteLargeEntities(self):
+ """100*100kB entities gets split over two Commit RPCs."""
+ with patch.object(helper, 'get_datastore',
+ return_value=self._mock_datastore):
+ entities = [e.entity for e in fake_datastore.create_entities(100)]
+
+ datastore_write_fn = _Mutate.DatastoreWriteFn(
+ self._PROJECT, fixed_batch_size=_Mutate._WRITE_BATCH_INITIAL_SIZE)
+ datastore_write_fn.start_bundle()
+ for entity in entities:
+ datastore_helper.add_properties(
+ entity, {'large': u'A' * 100000}, exclude_from_indexes=True)
+ datastore_write_fn.process(WriteToDatastore.to_upsert_mutation(entity))
+ datastore_write_fn.finish_bundle()
+
+ self.assertEqual(2, self._mock_datastore.commit.call_count)
def verify_unique_keys(self, queries):
"""A helper function that verifies if all the queries have unique keys."""
@@ -217,7 +241,7 @@
elif req == kind_stat_req:
return kind_stat_resp
else:
- print kind_stat_req
+ print(kind_stat_req)
raise ValueError("Unknown req: %s" % req)
self._mock_datastore.run_query.side_effect = fake_run_query
@@ -242,5 +266,41 @@
return split_queries
+@unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
+class DynamicWriteBatcherTest(unittest.TestCase):
+
+ def setUp(self):
+ self._batcher = _Mutate._DynamicBatchSizer()
+
+ # If possible, keep these test cases aligned with the Java test cases in
+ # DatastoreV1Test.java
+ def test_no_data(self):
+ self.assertEquals(_Mutate._WRITE_BATCH_INITIAL_SIZE,
+ self._batcher.get_batch_size(0))
+
+ def test_fast_queries(self):
+ self._batcher.report_latency(0, 1000, 200)
+ self._batcher.report_latency(0, 1000, 200)
+ self.assertEquals(_Mutate._WRITE_BATCH_MAX_SIZE,
+ self._batcher.get_batch_size(0))
+
+ def test_slow_queries(self):
+ self._batcher.report_latency(0, 10000, 200)
+ self._batcher.report_latency(0, 10000, 200)
+ self.assertEquals(100, self._batcher.get_batch_size(0))
+
+ def test_size_not_below_minimum(self):
+ self._batcher.report_latency(0, 30000, 50)
+ self._batcher.report_latency(0, 30000, 50)
+ self.assertEquals(_Mutate._WRITE_BATCH_MIN_SIZE,
+ self._batcher.get_batch_size(0))
+
+ def test_sliding_window(self):
+ self._batcher.report_latency(0, 30000, 50)
+ self._batcher.report_latency(50000, 5000, 200)
+ self._batcher.report_latency(100000, 5000, 200)
+ self.assertEquals(200, self._batcher.get_batch_size(150000))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
index f977536..b86a2fa 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
@@ -19,7 +19,16 @@
For internal use only; no backwards-compatibility guarantees.
"""
+
+import errno
+import logging
import sys
+import time
+from socket import error as SocketError
+
+# pylint: disable=ungrouped-imports
+from apache_beam.internal.gcp import auth
+from apache_beam.utils import retry
# Protect against environments where datastore library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
@@ -36,8 +45,7 @@
pass
# pylint: enable=wrong-import-order, wrong-import-position
-from apache_beam.internal.gcp import auth
-from apache_beam.utils import retry
+# pylint: enable=ungrouped-imports
def key_comparator(k1, k2):
@@ -130,6 +138,11 @@
err_code == code_pb2.UNAVAILABLE or
err_code == code_pb2.UNKNOWN or
err_code == code_pb2.INTERNAL)
+
+ if isinstance(exception, SocketError):
+ return (exception.errno == errno.ECONNRESET or
+ exception.errno == errno.ETIMEDOUT)
+
return False
@@ -158,13 +171,28 @@
return key.path[-1].HasField('id') or key.path[-1].HasField('name')
-def write_mutations(datastore, project, mutations):
+def write_mutations(datastore, project, mutations, throttler,
+ rpc_stats_callback=None, throttle_delay=1):
"""A helper function to write a batch of mutations to Cloud Datastore.
If a commit fails, it will be retried upto 5 times. All mutations in the
batch will be committed again, even if the commit was partially successful.
If the retry limit is exceeded, the last exception from Cloud Datastore will
be raised.
+
+ Args:
+ datastore: googledatastore.connection.Datastore
+ project: str, project id
+ mutations: list of google.cloud.proto.datastore.v1.datastore_pb2.Mutation
+ rpc_stats_callback: a function to call with arguments `successes` and
+ `failures` and `throttled_secs`; this is called to record successful
+ and failed RPCs to Datastore and time spent waiting for throttling.
+ throttler: AdaptiveThrottler, to use to select requests to be throttled.
+ throttle_delay: float, time in seconds to sleep when throttled.
+
+ Returns a tuple of:
+ CommitResponse, the response from Datastore;
+ int, the latency of the successful RPC in milliseconds.
"""
commit_request = datastore_pb2.CommitRequest()
commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL
@@ -174,10 +202,30 @@
@retry.with_exponential_backoff(num_retries=5,
retry_filter=retry_on_rpc_error)
- def commit(req):
- datastore.commit(req)
+ def commit(request):
+ # Client-side throttling.
+ while throttler.throttle_request(time.time()*1000):
+ logging.info("Delaying request for %ds due to previous failures",
+ throttle_delay)
+ time.sleep(throttle_delay)
+ rpc_stats_callback(throttled_secs=throttle_delay)
- commit(commit_request)
+ try:
+ start_time = time.time()
+ response = datastore.commit(request)
+ end_time = time.time()
+
+ rpc_stats_callback(successes=1)
+ throttler.successful_request(start_time*1000)
+ commit_time_ms = int((end_time-start_time)*1000)
+ return response, commit_time_ms
+ except (RPCError, SocketError):
+ if rpc_stats_callback:
+ rpc_stats_callback(errors=1)
+ raise
+
+ response, commit_time_ms = commit(commit_request)
+ return response, commit_time_ms
def make_latest_timestamp_query(namespace):
@@ -229,7 +277,7 @@
self._project = project
self._namespace = namespace
self._start_cursor = None
- self._limit = self._query.limit.value or sys.maxint
+ self._limit = self._query.limit.value or sys.maxsize
self._req = make_request(project, namespace, query)
@retry.with_exponential_backoff(num_retries=5,
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
index a804c09..90a3668 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
@@ -16,16 +16,19 @@
#
"""Tests for datastore helper."""
+import errno
+import random
import sys
import unittest
+from socket import error as SocketError
from mock import MagicMock
+# pylint: disable=ungrouped-imports
from apache_beam.io.gcp.datastore.v1 import fake_datastore
from apache_beam.io.gcp.datastore.v1 import helper
from apache_beam.testing.test_utils import patch_retry
-
# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
try:
@@ -39,6 +42,7 @@
except ImportError:
datastore_helper = None
# pylint: enable=wrong-import-order, wrong-import-position
+# pylint: enable=ungrouped-imports
@unittest.skipIf(datastore_helper is None, 'GCP dependencies are not installed')
@@ -49,6 +53,16 @@
self._query = query_pb2.Query()
self._query.kind.add().name = 'dummy_kind'
patch_retry(self, helper)
+ self._retriable_errors = [
+ RPCError("dummy", code_pb2.INTERNAL, "failed"),
+ SocketError(errno.ECONNRESET, "Connection Reset"),
+ SocketError(errno.ETIMEDOUT, "Timed out")
+ ]
+
+ self._non_retriable_errors = [
+ RPCError("dummy", code_pb2.UNAUTHENTICATED, "failed"),
+ SocketError(errno.EADDRNOTAVAIL, "Address not available")
+ ]
def permanent_retriable_datastore_failure(self, req):
raise RPCError("dummy", code_pb2.UNAVAILABLE, "failed")
@@ -56,12 +70,12 @@
def transient_retriable_datastore_failure(self, req):
if self._transient_fail_count:
self._transient_fail_count -= 1
- raise RPCError("dummy", code_pb2.INTERNAL, "failed")
+ raise random.choice(self._retriable_errors)
else:
return datastore_pb2.RunQueryResponse()
def non_retriable_datastore_failure(self, req):
- raise RPCError("dummy", code_pb2.UNAUTHENTICATED, "failed")
+ raise random.choice(self._non_retriable_errors)
def test_query_iterator(self):
self._mock_datastore.run_query.side_effect = (
@@ -76,7 +90,7 @@
self.transient_retriable_datastore_failure)
query_iterator = helper.QueryIterator("project", None, self._query,
self._mock_datastore)
- fail_count = 2
+ fail_count = 5
self._transient_fail_count = fail_count
for _ in query_iterator:
pass
@@ -89,7 +103,8 @@
self.non_retriable_datastore_failure)
query_iterator = helper.QueryIterator("project", None, self._query,
self._mock_datastore)
- self.assertRaises(RPCError, iter(query_iterator).next)
+ self.assertRaises(tuple(map(type, self._non_retriable_errors)),
+ iter(query_iterator).next)
self.assertEqual(1, len(self._mock_datastore.run_query.call_args_list))
def test_query_iterator_with_single_batch(self):
@@ -139,7 +154,7 @@
self.assertEqual(entity, entities[i].entity)
i += 1
- limit = query.limit.value if query.HasField('limit') else sys.maxint
+ limit = query.limit.value if query.HasField('limit') else sys.maxsize
self.assertEqual(i, min(num_entities, limit))
def test_is_key_valid(self):
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/util.py b/sdks/python/apache_beam/io/gcp/datastore/v1/util.py
new file mode 100644
index 0000000..5670a24
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/util.py
@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility functions & classes that are _not_ specific to the datastore client.
+#
+# For internal use only; no backwards-compatibility guarantees.
+
+import math
+
+
+class MovingSum(object):
+ """Class that keeps track of a rolling window sum.
+
+ For use in tracking recent performance of the connector.
+
+ Intended to be similar to
+ org.apache.beam.sdk.util.MovingFunction(..., Sum.ofLongs()), but for
+ convenience we expose the count of entries as well so this doubles as a
+ moving average tracker.
+ """
+
+ def __init__(self, window_ms, bucket_ms):
+ if window_ms <= bucket_ms or bucket_ms <= 0:
+ raise ValueError("window_ms > bucket_ms > 0 please")
+ self._num_buckets = int(math.ceil(window_ms / bucket_ms))
+ self._bucket_ms = bucket_ms
+ self._Reset(now=0) # initialize the moving window members
+
+ def _Reset(self, now):
+ self._current_index = 0 # pointer into self._buckets
+ self._current_ms_since_epoch = math.floor(
+ now / self._bucket_ms) * self._bucket_ms
+
+ # _buckets is a list where each element is a list [sum, num_samples]
+ # This is a circular buffer where
+ # [_current_index] represents the time range
+ # [_current_ms_since_epoch, _current_ms_since_epoch+_bucket_ms)
+ # [_current_index-1] represents immediatly prior time range
+ # [_current_ms_since_epoch-_bucket_ms, _current_ms_since_epoch)
+ # etc, wrapping around from the start to the end of the array, so
+ # [_current_index+1] is the element representing the oldest bucket.
+ self._buckets = [[0, 0] for _ in range(0, self._num_buckets)]
+
+ def _Flush(self, now):
+ """
+
+ Args:
+ now: int, milliseconds since epoch
+ """
+ if now >= (self._current_ms_since_epoch
+ + self._bucket_ms * self._num_buckets):
+ # Time moved forward so far that all currently held data is outside of
+ # the window. It is faster to simply reset our data.
+ self._Reset(now)
+ return
+
+ while now > self._current_ms_since_epoch + self._bucket_ms:
+ # Advance time by one _bucket_ms, setting the new bucket's counts to 0.
+ self._current_ms_since_epoch += self._bucket_ms
+ self._current_index = (self._current_index+1) % self._num_buckets
+ self._buckets[self._current_index] = [0, 0]
+ # Intentional dead reckoning here; we don't care about staying precisely
+ # aligned with multiples of _bucket_ms since the epoch, we just need our
+ # buckets to represent the most recent _window_ms time window.
+
+ def sum(self, now):
+ self._Flush(now)
+ return sum(bucket[0] for bucket in self._buckets)
+
+ def add(self, now, inc):
+ self._Flush(now)
+ bucket = self._buckets[self._current_index]
+ bucket[0] += inc
+ bucket[1] += 1
+
+ def count(self, now):
+ self._Flush(now)
+ return sum(bucket[1] for bucket in self._buckets)
+
+ def has_data(self, now):
+ return self.count(now) > 0
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
new file mode 100644
index 0000000..8f17c21
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Tests for util.py."""
+import unittest
+
+from apache_beam.io.gcp.datastore.v1 import util
+
+
+class MovingSumTest(unittest.TestCase):
+
+ TIMESTAMP = 1500000000
+
+ def test_bad_bucket_size(self):
+ with self.assertRaises(ValueError):
+ _ = util.MovingSum(1, 0)
+
+ def test_bad_window_size(self):
+ with self.assertRaises(ValueError):
+ _ = util.MovingSum(1, 2)
+
+ def test_no_data(self):
+ ms = util.MovingSum(10, 1)
+ self.assertEqual(0, ms.sum(MovingSumTest.TIMESTAMP))
+ self.assertEqual(0, ms.count(MovingSumTest.TIMESTAMP))
+ self.assertFalse(ms.has_data(MovingSumTest.TIMESTAMP))
+
+ def test_one_data_point(self):
+ ms = util.MovingSum(10, 1)
+ ms.add(MovingSumTest.TIMESTAMP, 5)
+ self.assertEqual(5, ms.sum(MovingSumTest.TIMESTAMP))
+ self.assertEqual(1, ms.count(MovingSumTest.TIMESTAMP))
+ self.assertTrue(ms.has_data(MovingSumTest.TIMESTAMP))
+
+ def test_aggregates_within_window(self):
+ ms = util.MovingSum(10, 1)
+ ms.add(MovingSumTest.TIMESTAMP, 5)
+ ms.add(MovingSumTest.TIMESTAMP+1, 3)
+ ms.add(MovingSumTest.TIMESTAMP+2, 7)
+ self.assertEqual(15, ms.sum(MovingSumTest.TIMESTAMP+3))
+ self.assertEqual(3, ms.count(MovingSumTest.TIMESTAMP+3))
+
+ def test_data_expires_from_moving_window(self):
+ ms = util.MovingSum(5, 1)
+ ms.add(MovingSumTest.TIMESTAMP, 5)
+ ms.add(MovingSumTest.TIMESTAMP+3, 3)
+ ms.add(MovingSumTest.TIMESTAMP+6, 7)
+ self.assertEqual(10, ms.sum(MovingSumTest.TIMESTAMP+7))
+ self.assertEqual(2, ms.count(MovingSumTest.TIMESTAMP+7))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
index 923fc7d..88a601f 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
@@ -21,6 +21,7 @@
import unittest
import mock
+
from apache_beam.io.filesystem import BeamIOError
from apache_beam.io.filesystem import FileMetadata
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index 7e21586..448a0c9 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -32,6 +32,8 @@
import time
import traceback
+import httplib2
+
from apache_beam.utils import retry
__all__ = ['GcsIO']
@@ -68,6 +70,10 @@
# +---------------+------------+-------------+-------------+-------------+
DEFAULT_READ_BUFFER_SIZE = 16 * 1024 * 1024
+# This is the number of seconds the library will wait for GCS operations to
+# complete.
+DEFAULT_HTTP_TIMEOUT_SECONDS = 60
+
# This is the number of seconds the library will wait for a partial-file read
# operation from GCS to complete before retrying.
DEFAULT_READ_SEGMENT_TIMEOUT_SECONDS = 60
@@ -99,7 +105,8 @@
def __new__(cls, storage_client=None):
if storage_client:
- return super(GcsIO, cls).__new__(cls, storage_client)
+ # This path is only used for testing.
+ return super(GcsIO, cls).__new__(cls)
else:
# Create a single storage client for each thread. We would like to avoid
# creating more than one storage client for each thread, since each
@@ -108,7 +115,9 @@
local_state = threading.local()
if getattr(local_state, 'gcsio_instance', None) is None:
credentials = auth.get_service_credentials()
- storage_client = storage.StorageV1(credentials=credentials)
+ storage_client = storage.StorageV1(
+ credentials=credentials,
+ http=httplib2.Http(timeout=DEFAULT_HTTP_TIMEOUT_SECONDS))
local_state.gcsio_instance = (
super(GcsIO, cls).__new__(cls, storage_client))
local_state.gcsio_instance.client = storage_client
@@ -129,16 +138,16 @@
"""Open a GCS file path for reading or writing.
Args:
- filename: GCS file path in the form gs://<bucket>/<object>.
- mode: 'r' for reading or 'w' for writing.
- read_buffer_size: Buffer size to use during read operations.
- mime_type: Mime type to set for write operations.
+ filename (str): GCS file path in the form ``gs://<bucket>/<object>``.
+ mode (str): ``'r'`` for reading or ``'w'`` for writing.
+ read_buffer_size (int): Buffer size to use during read operations.
+ mime_type (str): Mime type to set for write operations.
Returns:
- file object.
+ GCS file object.
Raises:
- ValueError: Invalid open file mode.
+ ~exceptions.ValueError: Invalid open file mode.
"""
if mode == 'r' or mode == 'rb':
return GcsBufferedReader(self.client, filename, mode=mode,
@@ -392,7 +401,7 @@
if fnmatch.fnmatch(item.name, name_pattern):
file_name = 'gs://%s/%s' % (item.bucket, item.name)
file_sizes[file_name] = item.size
- counter += 1
+ counter += 1
if limit is not None and counter >= limit:
break
if counter % 10000 == 0:
@@ -464,7 +473,7 @@
def __next__(self):
"""Read one line delimited by '\\n' from the file.
"""
- return self.next()
+ return next(self)
def next(self):
"""Read one line delimited by '\\n' from the file.
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py b/sdks/python/apache_beam/io/gcp/gcsio_test.py
index 73d2213..06a8227 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py
@@ -641,6 +641,7 @@
'apple/fish/cat',
'apple/fish/cart',
'apple/fish/carl',
+ 'apple/fish/handle',
'apple/dish/bat',
'apple/dish/cat',
'apple/dish/carl',
@@ -661,6 +662,7 @@
'apple/fish/cat',
'apple/fish/cart',
'apple/fish/carl',
+ 'apple/fish/handle',
'apple/dish/bat',
'apple/dish/cat',
'apple/dish/carl',
@@ -691,6 +693,12 @@
'apple/fish/bambi',
'apple/fish/balloon',
]),
+ ('gs://gcsio-test/apple/f*/b*', [
+ 'apple/fish/blubber',
+ 'apple/fish/blowfish',
+ 'apple/fish/bambi',
+ 'apple/fish/balloon',
+ ]),
('gs://gcsio-test/apple/dish/[cb]at', [
'apple/dish/bat',
'apple/dish/cat',
@@ -726,6 +734,7 @@
('apple/dish/bat', 13),
('apple/dish/cat', 14),
('apple/dish/carl', 15),
+ ('apple/fish/handle', 16),
]
for (object_name, size) in object_names:
file_name = 'gs://%s/%s' % (bucket_name, object_name)
@@ -739,7 +748,11 @@
('gs://gcsio-test/apple/fish/car?', [
('apple/fish/cart', 11),
('apple/fish/carl', 12),
- ])
+ ]),
+ ('gs://gcsio-test/*/f*/car?', [
+ ('apple/fish/cart', 11),
+ ('apple/fish/carl', 12),
+ ]),
]
for file_pattern, expected_object_names in test_cases:
expected_file_sizes = {'gs://%s/%s' % (bucket_name, o): s
@@ -747,6 +760,13 @@
self.assertEqual(
self.gcs.size_of_files_in_glob(file_pattern), expected_file_sizes)
+ # Check if limits are followed correctly
+ limit = 1
+ for file_pattern, expected_object_names in test_cases:
+ expected_num_items = min(len(expected_object_names), limit)
+ self.assertEqual(
+ len(self.gcs.glob(file_pattern, limit)), expected_num_items)
+
def test_size_of_files_in_glob_limited(self):
bucket_name = 'gcsio-test'
object_names = [
diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py
index 4045428..3e741cd 100644
--- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py
+++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py
@@ -25,7 +25,6 @@
from apitools.base.py import encoding
from apitools.base.py import extra_types
-
package = 'bigquery'
@@ -1906,5 +1905,3 @@
query = _messages.StringField(1)
userDefinedFunctionResources = _messages.MessageField('UserDefinedFunctionResource', 2, repeated=True)
-
-
diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py
index dc9e5e6..3c180a6 100644
--- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py
+++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py
@@ -26,7 +26,6 @@
from apitools.base.py import encoding
from apitools.base.py import extra_types
-
package = 'storage'
@@ -1916,5 +1915,3 @@
prefix = _messages.StringField(6)
projection = _messages.EnumField('ProjectionValueValuesEnum', 7)
versions = _messages.BooleanField(8)
-
-
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index 103fce0..98aa884 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -18,39 +18,149 @@
Cloud Pub/Sub sources and sinks are currently supported only in streaming
pipelines, during remote execution.
+
+This API is currently under development and is subject to change.
"""
from __future__ import absolute_import
+import re
+
from apache_beam import coders
+from apache_beam.io.iobase import Read
+from apache_beam.io.iobase import Write
from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
+from apache_beam.transforms import Map
+from apache_beam.transforms import PTransform
+from apache_beam.transforms import core
+from apache_beam.transforms import window
from apache_beam.transforms.display import DisplayDataItem
-__all__ = ['PubSubSink', 'PubSubSource']
+__all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub']
-class PubSubSource(dataflow_io.NativeSource):
- """Source for reading from a given Cloud Pub/Sub topic.
+class ReadStringsFromPubSub(PTransform):
+ """A ``PTransform`` for reading utf-8 string payloads from Cloud Pub/Sub."""
+
+ def __init__(self, topic=None, subscription=None, id_label=None):
+ """Initializes ``ReadStringsFromPubSub``.
+
+ Attributes:
+ topic: Cloud Pub/Sub topic in the form "projects/<project>/topics/
+ <topic>". If provided, subscription must be None.
+ subscription: Existing Cloud Pub/Sub subscription to use in the
+ form "projects/<project>/subscriptions/<subscription>". If not
+ specified, a temporary subscription will be created from the specified
+ topic. If provided, topic must be None.
+ id_label: The attribute on incoming Pub/Sub messages to use as a unique
+ record identifier. When specified, the value of this attribute (which
+ can be any string that uniquely identifies the record) will be used for
+ deduplication of messages. If not provided, we cannot guarantee
+ that no duplicate data will be delivered on the Pub/Sub stream. In this
+ case, deduplication of the stream will be strictly best effort.
+ """
+ super(ReadStringsFromPubSub, self).__init__()
+ self._source = _PubSubPayloadSource(
+ topic,
+ subscription=subscription,
+ id_label=id_label)
+
+ def get_windowing(self, unused_inputs):
+ return core.Windowing(window.GlobalWindows())
+
+ def expand(self, pvalue):
+ pcoll = pvalue.pipeline | Read(self._source)
+ pcoll.element_type = bytes
+ pcoll = pcoll | 'DecodeString' >> Map(lambda b: b.decode('utf-8'))
+ pcoll.element_type = unicode
+ return pcoll
+
+
+class WriteStringsToPubSub(PTransform):
+ """A ``PTransform`` for writing utf-8 string payloads to Cloud Pub/Sub."""
+
+ def __init__(self, topic):
+ """Initializes ``WriteStringsToPubSub``.
+
+ Attributes:
+ topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
+ """
+ super(WriteStringsToPubSub, self).__init__()
+ self._sink = _PubSubPayloadSink(topic)
+
+ def expand(self, pcoll):
+ pcoll = pcoll | 'EncodeString' >> Map(lambda s: s.encode('utf-8'))
+ pcoll.element_type = bytes
+ return pcoll | Write(self._sink)
+
+
+PROJECT_ID_REGEXP = '[a-z][-a-z0-9:.]{4,61}[a-z0-9]'
+SUBSCRIPTION_REGEXP = 'projects/([^/]+)/subscriptions/(.+)'
+TOPIC_REGEXP = 'projects/([^/]+)/topics/(.+)'
+
+
+def parse_topic(full_topic):
+ match = re.match(TOPIC_REGEXP, full_topic)
+ if not match:
+ raise ValueError(
+ 'PubSub topic must be in the form "projects/<project>/topics'
+ '/<topic>" (got %r).' % full_topic)
+ project, topic_name = match.group(1), match.group(2)
+ if not re.match(PROJECT_ID_REGEXP, project):
+ raise ValueError('Invalid PubSub project name: %r.' % project)
+ return project, topic_name
+
+
+def parse_subscription(full_subscription):
+ match = re.match(SUBSCRIPTION_REGEXP, full_subscription)
+ if not match:
+ raise ValueError(
+ 'PubSub subscription must be in the form "projects/<project>'
+ '/subscriptions/<subscription>" (got %r).' % full_subscription)
+ project, subscription_name = match.group(1), match.group(2)
+ if not re.match(PROJECT_ID_REGEXP, project):
+ raise ValueError('Invalid PubSub project name: %r.' % project)
+ return project, subscription_name
+
+
+class _PubSubPayloadSource(dataflow_io.NativeSource):
+ """Source for the payload of a message as bytes from a Cloud Pub/Sub topic.
Attributes:
- topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
- subscription: Optional existing Cloud Pub/Sub subscription to use in the
- form "projects/<project>/subscriptions/<subscription>".
+ topic: Cloud Pub/Sub topic in the form "projects/<project>/topics/<topic>".
+ If provided, subscription must be None.
+ subscription: Existing Cloud Pub/Sub subscription to use in the
+ form "projects/<project>/subscriptions/<subscription>". If not specified,
+ a temporary subscription will be created from the specified topic. If
+ provided, topic must be None.
id_label: The attribute on incoming Pub/Sub messages to use as a unique
record identifier. When specified, the value of this attribute (which can
be any string that uniquely identifies the record) will be used for
deduplication of messages. If not provided, Dataflow cannot guarantee
that no duplicate data will be delivered on the Pub/Sub stream. In this
case, deduplication of the stream will be strictly best effort.
- coder: The Coder to use for decoding incoming Pub/Sub messages.
"""
- def __init__(self, topic, subscription=None, id_label=None,
- coder=coders.StrUtf8Coder()):
- self.topic = topic
- self.subscription = subscription
+ def __init__(self, topic=None, subscription=None, id_label=None):
+ # We are using this coder explicitly for portability reasons of PubsubIO
+ # across implementations in languages.
+ self.coder = coders.BytesCoder()
+ self.full_topic = topic
+ self.full_subscription = subscription
+ self.topic_name = None
+ self.subscription_name = None
self.id_label = id_label
- self.coder = coder
+
+ # Perform some validation on the topic and subscription.
+ if not (topic or subscription):
+ raise ValueError('Either a topic or subscription must be provided.')
+ if topic and subscription:
+ raise ValueError('Only one of topic or subscription should be provided.')
+
+ if topic:
+ self.project, self.topic_name = parse_topic(topic)
+ if subscription:
+ self.project, self.subscription_name = parse_subscription(subscription)
@property
def format(self):
@@ -62,23 +172,30 @@
DisplayDataItem(self.id_label,
label='ID Label Attribute').drop_if_none(),
'topic':
- DisplayDataItem(self.topic,
- label='Pubsub Topic'),
+ DisplayDataItem(self.full_topic,
+ label='Pubsub Topic').drop_if_none(),
'subscription':
- DisplayDataItem(self.subscription,
+ DisplayDataItem(self.full_subscription,
label='Pubsub Subscription').drop_if_none()}
def reader(self):
raise NotImplementedError(
- 'PubSubSource is not supported in local execution.')
+ 'PubSubPayloadSource is not supported in local execution.')
+
+ def is_bounded(self):
+ return False
-class PubSubSink(dataflow_io.NativeSink):
- """Sink for writing to a given Cloud Pub/Sub topic."""
+class _PubSubPayloadSink(dataflow_io.NativeSink):
+ """Sink for the payload of a message as bytes to a Cloud Pub/Sub topic."""
- def __init__(self, topic, coder=coders.StrUtf8Coder()):
- self.topic = topic
- self.coder = coder
+ def __init__(self, topic):
+ # we are using this coder explicitly for portability reasons of PubsubIO
+ # across implementations in languages.
+ self.coder = coders.BytesCoder()
+ self.full_topic = topic
+
+ self.project, self.topic_name = parse_topic(topic)
@property
def format(self):
@@ -86,8 +203,8 @@
return 'pubsub'
def display_data(self):
- return {'topic': DisplayDataItem(self.topic, label='Pubsub Topic')}
+ return {'topic': DisplayDataItem(self.full_topic, label='Pubsub Topic')}
def writer(self):
raise NotImplementedError(
- 'PubSubSink is not supported in local execution.')
+ 'PubSubPayloadSink is not supported in local execution.')
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index 1642a95..0c4ba02 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -22,39 +22,116 @@
import hamcrest as hc
-from apache_beam.io.gcp.pubsub import PubSubSink
-from apache_beam.io.gcp.pubsub import PubSubSource
+from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub
+from apache_beam.io.gcp.pubsub import WriteStringsToPubSub
+from apache_beam.io.gcp.pubsub import _PubSubPayloadSink
+from apache_beam.io.gcp.pubsub import _PubSubPayloadSource
+from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display_test import DisplayDataItemMatcher
+# Protect against environments where the PubSub library is not available.
+# pylint: disable=wrong-import-order, wrong-import-position
+try:
+ from google.cloud import pubsub
+except ImportError:
+ pubsub = None
+# pylint: enable=wrong-import-order, wrong-import-position
+
+@unittest.skipIf(pubsub is None, 'GCP dependencies are not installed')
+class TestReadStringsFromPubSub(unittest.TestCase):
+ def test_expand_with_topic(self):
+ p = TestPipeline()
+ pcoll = p | ReadStringsFromPubSub('projects/fakeprj/topics/a_topic',
+ None, 'a_label')
+ # Ensure that the output type is str
+ self.assertEqual(unicode, pcoll.element_type)
+
+ # Ensure that the properties passed through correctly
+ source = pcoll.producer.transform._source
+ self.assertEqual('a_topic', source.topic_name)
+ self.assertEqual('a_label', source.id_label)
+
+ def test_expand_with_subscription(self):
+ p = TestPipeline()
+ pcoll = p | ReadStringsFromPubSub(
+ None, 'projects/fakeprj/subscriptions/a_subscription', 'a_label')
+ # Ensure that the output type is str
+ self.assertEqual(unicode, pcoll.element_type)
+
+ # Ensure that the properties passed through correctly
+ source = pcoll.producer.transform._source
+ self.assertEqual('a_subscription', source.subscription_name)
+ self.assertEqual('a_label', source.id_label)
+
+ def test_expand_with_no_topic_or_subscription(self):
+ with self.assertRaisesRegexp(
+ ValueError, "Either a topic or subscription must be provided."):
+ ReadStringsFromPubSub(None, None, 'a_label')
+
+ def test_expand_with_both_topic_and_subscription(self):
+ with self.assertRaisesRegexp(
+ ValueError, "Only one of topic or subscription should be provided."):
+ ReadStringsFromPubSub('a_topic', 'a_subscription', 'a_label')
+
+
+@unittest.skipIf(pubsub is None, 'GCP dependencies are not installed')
+class TestWriteStringsToPubSub(unittest.TestCase):
+ def test_expand(self):
+ p = TestPipeline()
+ pdone = (p
+ | ReadStringsFromPubSub('projects/fakeprj/topics/baz')
+ | WriteStringsToPubSub('projects/fakeprj/topics/a_topic'))
+
+ # Ensure that the properties passed through correctly
+ self.assertEqual('a_topic', pdone.producer.transform.dofn.topic_name)
+
+
+@unittest.skipIf(pubsub is None, 'GCP dependencies are not installed')
class TestPubSubSource(unittest.TestCase):
-
- def test_display_data(self):
- source = PubSubSource('a_topic', 'a_subscription', 'a_label')
+ def test_display_data_topic(self):
+ source = _PubSubPayloadSource(
+ 'projects/fakeprj/topics/a_topic',
+ None,
+ 'a_label')
dd = DisplayData.create_from(source)
expected_items = [
- DisplayDataItemMatcher('topic', 'a_topic'),
- DisplayDataItemMatcher('subscription', 'a_subscription'),
+ DisplayDataItemMatcher(
+ 'topic', 'projects/fakeprj/topics/a_topic'),
+ DisplayDataItemMatcher('id_label', 'a_label')]
+
+ hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
+
+ def test_display_data_subscription(self):
+ source = _PubSubPayloadSource(
+ None,
+ 'projects/fakeprj/subscriptions/a_subscription',
+ 'a_label')
+ dd = DisplayData.create_from(source)
+ expected_items = [
+ DisplayDataItemMatcher(
+ 'subscription', 'projects/fakeprj/subscriptions/a_subscription'),
DisplayDataItemMatcher('id_label', 'a_label')]
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
def test_display_data_no_subscription(self):
- source = PubSubSource('a_topic')
+ source = _PubSubPayloadSource('projects/fakeprj/topics/a_topic')
dd = DisplayData.create_from(source)
expected_items = [
- DisplayDataItemMatcher('topic', 'a_topic')]
+ DisplayDataItemMatcher('topic', 'projects/fakeprj/topics/a_topic')]
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
+@unittest.skipIf(pubsub is None, 'GCP dependencies are not installed')
class TestPubSubSink(unittest.TestCase):
def test_display_data(self):
- sink = PubSubSink('a_topic')
+ sink = _PubSubPayloadSink('projects/fakeprj/topics/a_topic')
dd = DisplayData.create_from(sink)
expected_items = [
- DisplayDataItemMatcher('topic', 'a_topic')]
+ DisplayDataItemMatcher('topic', 'projects/fakeprj/topics/a_topic')]
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
index 844cbc5..d6f0e97 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
@@ -92,9 +92,9 @@
page_token = None
results = []
while True:
- rows, _, page_token = query.fetch_data(page_token=page_token)
- results.extend(rows)
- if not page_token:
+ for row in query.fetch_data(page_token=page_token):
+ results.append(row)
+ if results:
break
return results
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
index f12293e..a097718 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
@@ -21,7 +21,8 @@
import unittest
from hamcrest import assert_that as hc_assert_that
-from mock import Mock, patch
+from mock import Mock
+from mock import patch
from apache_beam.io.gcp.tests import bigquery_matcher as bq_verifier
from apache_beam.testing.test_utils import patch_retry
@@ -53,7 +54,7 @@
matcher = bq_verifier.BigqueryMatcher(
'mock_project',
'mock_query',
- 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+ '59f9d6bdee30d67ea73b8aded121c3a0280f9cd8')
hc_assert_that(self._mock_result, matcher)
@patch.object(bigquery, 'Client')
diff --git a/sdks/python/apache_beam/io/gcp/tests/utils.py b/sdks/python/apache_beam/io/gcp/tests/utils.py
new file mode 100644
index 0000000..b4b4ba8
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/tests/utils.py
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""Utility methods for testing on GCP."""
+
+import logging
+
+from apache_beam.utils import retry
+
+# Protect against environments where bigquery library is not available.
+try:
+ from google.cloud import bigquery
+except ImportError:
+ bigquery = None
+
+
+class GcpTestIOError(retry.PermanentException):
+ """Basic GCP IO error for testing. Function that raises this error should
+ not be retried."""
+ pass
+
+
+@retry.with_exponential_backoff(
+ num_retries=3,
+ retry_filter=retry.retry_on_server_errors_filter)
+def delete_bq_table(project, dataset, table):
+ """Delete a Biqquery table.
+
+ Args:
+ project: Name of the project.
+ dataset: Name of the dataset where table is.
+ table: Name of the table.
+ """
+ logging.info('Clean up a Bigquery table with project: %s, dataset: %s, '
+ 'table: %s.', project, dataset, table)
+ bq_dataset = bigquery.Client(project=project).dataset(dataset)
+ if not bq_dataset.exists():
+ raise GcpTestIOError('Failed to cleanup. Bigquery dataset %s doesn\'t '
+ 'exist in project %s.' % (dataset, project))
+ bq_table = bq_dataset.table(table)
+ if not bq_table.exists():
+ raise GcpTestIOError('Failed to cleanup. Bigquery table %s doesn\'t '
+ 'exist in project %s, dataset %s.' %
+ (table, project, dataset))
+ bq_table.delete()
+ if bq_table.exists():
+ raise RuntimeError('Failed to cleanup. Bigquery table %s still exists '
+ 'after cleanup.' % table)
diff --git a/sdks/python/apache_beam/io/gcp/tests/utils_test.py b/sdks/python/apache_beam/io/gcp/tests/utils_test.py
new file mode 100644
index 0000000..ac09e44
--- /dev/null
+++ b/sdks/python/apache_beam/io/gcp/tests/utils_test.py
@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unittest for GCP testing utils."""
+
+import logging
+import unittest
+
+from mock import Mock
+from mock import patch
+
+from apache_beam.io.gcp.tests import utils
+from apache_beam.testing.test_utils import patch_retry
+
+# Protect against environments where bigquery library is not available.
+try:
+ from google.cloud import bigquery
+except ImportError:
+ bigquery = None
+
+
+@unittest.skipIf(bigquery is None, 'Bigquery dependencies are not installed.')
+class UtilsTest(unittest.TestCase):
+
+ def setUp(self):
+ self._mock_result = Mock()
+ patch_retry(self, utils)
+
+ @patch.object(bigquery, 'Client')
+ def test_delete_table_succeeds(self, mock_client):
+ mock_dataset = Mock()
+ mock_client.return_value.dataset = mock_dataset
+ mock_dataset.return_value.exists.return_value = True
+
+ mock_table = Mock()
+ mock_dataset.return_value.table = mock_table
+ mock_table.return_value.exists.side_effect = [True, False]
+
+ utils.delete_bq_table('unused_project',
+ 'unused_dataset',
+ 'unused_table')
+
+ @patch.object(bigquery, 'Client')
+ def test_delete_table_fails_dataset_not_exist(self, mock_client):
+ mock_dataset = Mock()
+ mock_client.return_value.dataset = mock_dataset
+ mock_dataset.return_value.exists.return_value = False
+
+ with self.assertRaises(Exception) as e:
+ utils.delete_bq_table('unused_project',
+ 'unused_dataset',
+ 'unused_table')
+ self.assertTrue(
+ e.exception.message.startswith('Failed to cleanup. Bigquery dataset '
+ 'unused_dataset doesn\'t exist'))
+
+ @patch.object(bigquery, 'Client')
+ def test_delete_table_fails_table_not_exist(self, mock_client):
+ mock_dataset = Mock()
+ mock_client.return_value.dataset = mock_dataset
+ mock_dataset.return_value.exists.return_value = True
+
+ mock_table = Mock()
+ mock_dataset.return_value.table = mock_table
+ mock_table.return_value.exists.return_value = False
+
+ with self.assertRaises(Exception) as e:
+ utils.delete_bq_table('unused_project',
+ 'unused_dataset',
+ 'unused_table')
+ self.assertTrue(
+ e.exception.message.startswith('Failed to cleanup. Bigquery table '
+ 'unused_table doesn\'t exist'))
+
+ @patch.object(bigquery, 'Client')
+ def test_delete_table_fails_service_error(self, mock_client):
+ mock_dataset = Mock()
+ mock_client.return_value.dataset = mock_dataset
+ mock_dataset.return_value.exists.return_value = True
+
+ mock_table = Mock()
+ mock_dataset.return_value.table = mock_table
+ mock_table.return_value.exists.return_value = True
+
+ with self.assertRaises(Exception) as e:
+ utils.delete_bq_table('unused_project',
+ 'unused_dataset',
+ 'unused_table')
+ self.assertTrue(
+ e.exception.message.startswith('Failed to cleanup. Bigquery table '
+ 'unused_table still exists'))
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index 7e40d83..7cffa7f 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -29,21 +29,22 @@
the sink.
"""
-from collections import namedtuple
-
import logging
import random
import uuid
+from collections import namedtuple
-from apache_beam import pvalue
from apache_beam import coders
+from apache_beam import pvalue
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.pvalue import AsIter
from apache_beam.pvalue import AsSingleton
from apache_beam.transforms import core
from apache_beam.transforms import ptransform
from apache_beam.transforms import window
-from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms.display import HasDisplayData
+from apache_beam.utils import urns
from apache_beam.utils.windowed_value import WindowedValue
__all__ = ['BoundedSource', 'RangeTracker', 'Read', 'Sink', 'Write', 'Writer']
@@ -70,7 +71,13 @@
'weight source start_position stop_position')
-class BoundedSource(HasDisplayData):
+class SourceBase(HasDisplayData, urns.RunnerApiFn):
+ """Base class for all sources that can be passed to beam.io.Read(...).
+ """
+ urns.RunnerApiFn.register_pickle_urn(urns.PICKLED_SOURCE)
+
+
+class BoundedSource(SourceBase):
"""A source that reads a finite amount of input records.
This class defines following operations which can be used to read the source
@@ -189,6 +196,9 @@
"""
return coders.registry.get_coder(object)
+ def is_bounded(self):
+ return True
+
class RangeTracker(object):
"""A thread safe object used by Dataflow source framework.
@@ -820,6 +830,24 @@
label='Read Source'),
'source_dd': self.source}
+ def to_runner_api_parameter(self, context):
+ return (urns.READ_TRANSFORM,
+ beam_runner_api_pb2.ReadPayload(
+ source=self.source.to_runner_api(context),
+ is_bounded=beam_runner_api_pb2.IsBounded.BOUNDED
+ if self.source.is_bounded()
+ else beam_runner_api_pb2.IsBounded.UNBOUNDED))
+
+ @staticmethod
+ def from_runner_api_parameter(parameter, context):
+ return Read(SourceBase.from_runner_api(parameter.source, context))
+
+
+ptransform.PTransform.register_urn(
+ urns.READ_TRANSFORM,
+ beam_runner_api_pb2.ReadPayload,
+ Read.from_runner_api_parameter)
+
class Write(ptransform.PTransform):
"""A ``PTransform`` that writes to a sink.
@@ -985,3 +1013,75 @@
if self.counter >= self.count:
self.counter -= self.count
yield self.counter, element
+
+
+class RestrictionTracker(object):
+ """Manages concurrent access to a restriction.
+
+ Experimental; no backwards-compatibility guarantees.
+
+ Keeps track of the restrictions claimed part for a Splittable DoFn.
+
+ See following documents for more details.
+ * https://s.apache.org/splittable-do-fn
+ * https://s.apache.org/splittable-do-fn-python-sdk
+ """
+
+ def current_restriction(self):
+ """Returns the current restriction.
+
+ Returns a restriction accurately describing the full range of work the
+ current ``DoFn.process()`` call will do, including already completed work.
+
+ The current restriction returned by method may be updated dynamically due
+ to due to concurrent invocation of other methods of the
+ ``RestrictionTracker``, For example, ``checkpoint()``.
+
+ ** Thread safety **
+
+ Methods of the class ``RestrictionTracker`` including this method may get
+ invoked by different threads, hence must be made thread-safe, e.g. by using
+ a single lock object.
+ """
+ raise NotImplementedError
+
+ def checkpoint(self):
+ """Performs a checkpoint of the current restriction.
+
+ Signals that the current ``DoFn.process()`` call should terminate as soon as
+ possible. After this method returns, the tracker MUST refuse all future
+ claim calls, and ``RestrictionTracker.check_done()`` MUST succeed.
+
+ This invocation modifies the value returned by ``current_restriction()``
+ invocation and returns a restriction representing the rest of the work. The
+ old value of ``current_restriction()`` is equivalent to the new value of
+ ``current_restriction()`` and the return value of this method invocation
+ combined.
+
+ ** Thread safety **
+
+ Methods of the class ``RestrictionTracker`` including this method may get
+ invoked by different threads, hence must be made thread-safe, e.g. by using
+ a single lock object.
+ """
+
+ raise NotImplementedError
+
+ def check_done(self):
+ """Checks whether the restriction has been fully processed.
+
+ Called by the runner after iterator returned by ``DoFn.process()`` has been
+ fully read.
+
+ Returns: ``True`` if current restriction has been fully processed.
+ Raises ValueError: if there is still any unclaimed work remaining in the
+ restriction invoking this method. Exception raised must have an
+ informative error message.
+
+ ** Thread safety **
+
+ Methods of the class ``RestrictionTracker`` including this method may get
+ invoked by different threads, hence must be made thread-safe, e.g. by using
+ a single lock object.
+ """
+ raise NotImplementedError
diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py b/sdks/python/apache_beam/io/localfilesystem_test.py
index 04cf5b7..8c34ecd 100644
--- a/sdks/python/apache_beam/io/localfilesystem_test.py
+++ b/sdks/python/apache_beam/io/localfilesystem_test.py
@@ -18,12 +18,12 @@
"""Unit tests for LocalFileSystem."""
-import unittest
-
import filecmp
import os
import shutil
import tempfile
+import unittest
+
import mock
from apache_beam.io import localfilesystem
diff --git a/sdks/python/apache_beam/io/range_trackers.py b/sdks/python/apache_beam/io/range_trackers.py
index 9cb36e7..1339b91 100644
--- a/sdks/python/apache_beam/io/range_trackers.py
+++ b/sdks/python/apache_beam/io/range_trackers.py
@@ -28,6 +28,48 @@
'OrderedPositionRangeTracker', 'UnsplittableRangeTracker']
+class OffsetRange(object):
+
+ def __init__(self, start, stop):
+ if start >= stop:
+ raise ValueError(
+ 'Start offset must be smaller than the stop offset. '
+ 'Received %d and %d respectively.', start, stop)
+ self.start = start
+ self.stop = stop
+
+ def __eq__(self, other):
+ if not isinstance(other, OffsetRange):
+ return False
+
+ return self.start == other.start and self.stop == other.stop
+
+ def __ne__(self, other):
+ if not isinstance(other, OffsetRange):
+ return True
+
+ return not (self.start == other.start and self.stop == other.stop)
+
+ def split(self, desired_num_offsets_per_split, min_num_offsets_per_split=1):
+ current_split_start = self.start
+ max_split_size = max(desired_num_offsets_per_split,
+ min_num_offsets_per_split)
+ while current_split_start < self.stop:
+ current_split_stop = min(current_split_start + max_split_size, self.stop)
+ remaining = self.stop - current_split_stop
+
+ # Avoiding a small split at the end.
+ if (remaining < desired_num_offsets_per_split / 4 or
+ remaining < min_num_offsets_per_split):
+ current_split_stop = self.stop
+
+ yield OffsetRange(current_split_start, current_split_stop)
+ current_split_start = current_split_stop
+
+ def new_tracker(self):
+ return OffsetRangeTracker(self.start, self.stop)
+
+
class OffsetRangeTracker(iobase.RangeTracker):
"""A 'RangeTracker' for non-negative positions of type 'long'."""
@@ -193,136 +235,6 @@
self._split_points_unclaimed_callback = callback
-class GroupedShuffleRangeTracker(iobase.RangeTracker):
- """For internal use only; no backwards-compatibility guarantees.
-
- A 'RangeTracker' for positions used by'GroupedShuffleReader'.
-
- These positions roughly correspond to hashes of keys. In case of hash
- collisions, multiple groups can have the same position. In that case, the
- first group at a particular position is considered a split point (because
- it is the first to be returned when reading a position range starting at this
- position), others are not.
- """
-
- def __init__(self, decoded_start_pos, decoded_stop_pos):
- super(GroupedShuffleRangeTracker, self).__init__()
- self._decoded_start_pos = decoded_start_pos
- self._decoded_stop_pos = decoded_stop_pos
- self._decoded_last_group_start = None
- self._last_group_was_at_a_split_point = False
- self._split_points_seen = 0
- self._lock = threading.Lock()
-
- def start_position(self):
- return self._decoded_start_pos
-
- def stop_position(self):
- return self._decoded_stop_pos
-
- def last_group_start(self):
- return self._decoded_last_group_start
-
- def _validate_decoded_group_start(self, decoded_group_start, split_point):
- if self.start_position() and decoded_group_start < self.start_position():
- raise ValueError('Trying to return record at %r which is before the'
- ' starting position at %r' %
- (decoded_group_start, self.start_position()))
-
- if (self.last_group_start() and
- decoded_group_start < self.last_group_start()):
- raise ValueError('Trying to return group at %r which is before the'
- ' last-returned group at %r' %
- (decoded_group_start, self.last_group_start()))
- if (split_point and self.last_group_start() and
- self.last_group_start() == decoded_group_start):
- raise ValueError('Trying to return a group at a split point with '
- 'same position as the previous group: both at %r, '
- 'last group was %sat a split point.' %
- (decoded_group_start,
- ('' if self._last_group_was_at_a_split_point
- else 'not ')))
- if not split_point:
- if self.last_group_start() is None:
- raise ValueError('The first group [at %r] must be at a split point' %
- decoded_group_start)
- if self.last_group_start() != decoded_group_start:
- # This case is not a violation of general RangeTracker semantics, but it
- # is contrary to how GroupingShuffleReader in particular works. Hitting
- # it would mean it's behaving unexpectedly.
- raise ValueError('Trying to return a group not at a split point, but '
- 'with a different position than the previous group: '
- 'last group was %r at %r, current at a %s split'
- ' point.' %
- (self.last_group_start()
- , decoded_group_start
- , ('' if self._last_group_was_at_a_split_point
- else 'non-')))
-
- def try_claim(self, decoded_group_start):
- with self._lock:
- self._validate_decoded_group_start(decoded_group_start, True)
- if (self.stop_position()
- and decoded_group_start >= self.stop_position()):
- return False
-
- self._decoded_last_group_start = decoded_group_start
- self._last_group_was_at_a_split_point = True
- self._split_points_seen += 1
- return True
-
- def set_current_position(self, decoded_group_start):
- with self._lock:
- self._validate_decoded_group_start(decoded_group_start, False)
- self._decoded_last_group_start = decoded_group_start
- self._last_group_was_at_a_split_point = False
-
- def try_split(self, decoded_split_position):
- with self._lock:
- if self.last_group_start() is None:
- logging.info('Refusing to split %r at %r: unstarted'
- , self, decoded_split_position)
- return
-
- if decoded_split_position <= self.last_group_start():
- logging.info('Refusing to split %r at %r: already past proposed split '
- 'position'
- , self, decoded_split_position)
- return
-
- if ((self.stop_position()
- and decoded_split_position >= self.stop_position())
- or (self.start_position()
- and decoded_split_position <= self.start_position())):
- logging.error('Refusing to split %r at %r: proposed split position out '
- 'of range', self, decoded_split_position)
- return
-
- logging.debug('Agreeing to split %r at %r'
- , self, decoded_split_position)
- self._decoded_stop_pos = decoded_split_position
-
- # Since GroupedShuffleRangeTracker cannot determine relative sizes of the
- # two splits, returning 0.5 as the fraction below so that the framework
- # assumes the splits to be of the same size.
- return self._decoded_stop_pos, 0.5
-
- def fraction_consumed(self):
- # GroupingShuffle sources have special support on the service and the
- # service will estimate progress from positions for us.
- raise RuntimeError('GroupedShuffleRangeTracker does not measure fraction'
- ' consumed due to positions being opaque strings'
- ' that are interpreted by the service')
-
- def split_points(self):
- with self._lock:
- splits_points_consumed = (
- 0 if self._split_points_seen <= 1 else (self._split_points_seen - 1))
-
- return (splits_points_consumed,
- iobase.RangeTracker.SPLIT_POINTS_UNKNOWN)
-
-
class OrderedPositionRangeTracker(iobase.RangeTracker):
"""
An abstract base class for range trackers whose positions are comparable.
@@ -405,17 +317,19 @@
class UnsplittableRangeTracker(iobase.RangeTracker):
"""A RangeTracker that always ignores split requests.
- This can be used to make a given ``RangeTracker`` object unsplittable by
- ignoring all calls to ``try_split()``. All other calls will be delegated to
- the given ``RangeTracker``.
+ This can be used to make a given
+ :class:`~apache_beam.io.iobase.RangeTracker` object unsplittable by
+ ignoring all calls to :meth:`.try_split()`. All other calls will be delegated
+ to the given :class:`~apache_beam.io.iobase.RangeTracker`.
"""
def __init__(self, range_tracker):
"""Initializes UnsplittableRangeTracker.
Args:
- range_tracker: a ``RangeTracker`` to which all method calls expect calls
- to ``try_split()`` will be delegated.
+ range_tracker (~apache_beam.io.iobase.RangeTracker): a
+ :class:`~apache_beam.io.iobase.RangeTracker` to which all method
+ calls expect calls to :meth:`.try_split()` will be delegated.
"""
assert isinstance(range_tracker, iobase.RangeTracker)
self._range_tracker = range_tracker
diff --git a/sdks/python/apache_beam/io/range_trackers_test.py b/sdks/python/apache_beam/io/range_trackers_test.py
index edb6386..762d654 100644
--- a/sdks/python/apache_beam/io/range_trackers_test.py
+++ b/sdks/python/apache_beam/io/range_trackers_test.py
@@ -17,15 +17,49 @@
"""Unit tests for the range_trackers module."""
-import array
import copy
import logging
import math
import unittest
-
-from apache_beam.io import iobase
from apache_beam.io import range_trackers
+from apache_beam.io.range_trackers import OffsetRange
+
+
+class OffsetRangeTest(unittest.TestCase):
+
+ def test_create(self):
+ OffsetRange(0, 10)
+ OffsetRange(10, 100)
+
+ with self.assertRaises(ValueError):
+ OffsetRange(10, 9)
+
+ def test_split_respects_desired_num_splits(self):
+ range = OffsetRange(10, 100)
+ splits = list(range.split(desired_num_offsets_per_split=25))
+ self.assertEqual(4, len(splits))
+ self.assertIn(OffsetRange(10, 35), splits)
+ self.assertIn(OffsetRange(35, 60), splits)
+ self.assertIn(OffsetRange(60, 85), splits)
+ self.assertIn(OffsetRange(85, 100), splits)
+
+ def test_split_respects_min_num_splits(self):
+ range = OffsetRange(10, 100)
+ splits = list(range.split(desired_num_offsets_per_split=5,
+ min_num_offsets_per_split=25))
+ self.assertEqual(3, len(splits))
+ self.assertIn(OffsetRange(10, 35), splits)
+ self.assertIn(OffsetRange(35, 60), splits)
+ self.assertIn(OffsetRange(60, 100), splits)
+
+ def test_split_no_small_split_at_end(self):
+ range = OffsetRange(10, 90)
+ splits = list(range.split(desired_num_offsets_per_split=25))
+ self.assertEqual(3, len(splits))
+ self.assertIn(OffsetRange(10, 35), splits)
+ self.assertIn(OffsetRange(35, 60), splits)
+ self.assertIn(OffsetRange(60, 90), splits)
class OffsetRangeTrackerTest(unittest.TestCase):
@@ -189,189 +223,6 @@
(3, 41))
-class GroupedShuffleRangeTrackerTest(unittest.TestCase):
-
- def bytes_to_position(self, bytes_array):
- return array.array('B', bytes_array).tostring()
-
- def test_try_return_record_in_infinite_range(self):
- tracker = range_trackers.GroupedShuffleRangeTracker('', '')
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 3])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 5])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 6, 8, 10])))
-
- def test_try_return_record_finite_range(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([1, 0, 0]), self.bytes_to_position([5, 0, 0]))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 3])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 5])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 6, 8, 10])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([4, 255, 255, 255])))
- # Should fail for positions that are lexicographically equal to or larger
- # than the defined stop position.
- self.assertFalse(copy.copy(tracker).try_claim(
- self.bytes_to_position([5, 0, 0])))
- self.assertFalse(copy.copy(tracker).try_claim(
- self.bytes_to_position([5, 0, 1])))
- self.assertFalse(copy.copy(tracker).try_claim(
- self.bytes_to_position([6, 0, 0])))
-
- def test_try_return_record_with_non_split_point(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([1, 0, 0]), self.bytes_to_position([5, 0, 0]))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 3])))
- tracker.set_current_position(self.bytes_to_position([1, 2, 3]))
- tracker.set_current_position(self.bytes_to_position([1, 2, 3]))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 5])))
- tracker.set_current_position(self.bytes_to_position([1, 2, 5]))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 6, 8, 10])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([4, 255, 255, 255])))
-
- def test_first_record_non_split_point(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([3, 0, 0]), self.bytes_to_position([5, 0, 0]))
- with self.assertRaises(ValueError):
- tracker.set_current_position(self.bytes_to_position([3, 4, 5]))
-
- def test_non_split_point_record_with_different_position(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([3, 0, 0]), self.bytes_to_position([5, 0, 0]))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([3, 4, 5])))
- with self.assertRaises(ValueError):
- tracker.set_current_position(self.bytes_to_position([3, 4, 6]))
-
- def test_try_return_record_before_start(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([3, 0, 0]), self.bytes_to_position([5, 0, 0]))
- with self.assertRaises(ValueError):
- tracker.try_claim(self.bytes_to_position([1, 2, 3]))
-
- def test_try_return_non_monotonic(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([3, 0, 0]), self.bytes_to_position([5, 0, 0]))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([3, 4, 5])))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([3, 4, 6])))
- with self.assertRaises(ValueError):
- tracker.try_claim(self.bytes_to_position([3, 2, 1]))
-
- def test_try_return_identical_positions(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([3, 0, 0]), self.bytes_to_position([5, 0, 0]))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 4, 5])))
- with self.assertRaises(ValueError):
- tracker.try_claim(self.bytes_to_position([3, 4, 5]))
-
- def test_try_split_at_position_infinite_range(self):
- tracker = range_trackers.GroupedShuffleRangeTracker('', '')
- # Should fail before first record is returned.
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
-
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 3])))
-
- # Should now succeed.
- self.assertIsNotNone(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
- # Should not split at same or larger position.
- self.assertIsNone(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
- self.assertIsNone(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6, 7])))
- self.assertIsNone(tracker.try_split(
- self.bytes_to_position([4, 5, 6, 7])))
-
- # Should split at smaller position.
- self.assertIsNotNone(tracker.try_split(
- self.bytes_to_position([3, 2, 1])))
-
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([2, 3, 4])))
-
- # Should not split at a position we're already past.
- self.assertIsNone(tracker.try_split(
- self.bytes_to_position([2, 3, 4])))
- self.assertIsNone(tracker.try_split(
- self.bytes_to_position([2, 3, 3])))
-
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 2, 0])))
- self.assertFalse(tracker.try_claim(
- self.bytes_to_position([3, 2, 1])))
-
- def test_try_test_split_at_position_finite_range(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([0, 0, 0]),
- self.bytes_to_position([10, 20, 30]))
- # Should fail before first record is returned.
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([0, 0, 0])))
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
-
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([1, 2, 3])))
-
- # Should now succeed.
- self.assertTrue(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
- # Should not split at same or larger position.
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6])))
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([3, 4, 5, 6, 7])))
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([4, 5, 6, 7])))
-
- # Should split at smaller position.
- self.assertTrue(tracker.try_split(
- self.bytes_to_position([3, 2, 1])))
- # But not at a position at or before last returned record.
- self.assertFalse(tracker.try_split(
- self.bytes_to_position([1, 2, 3])))
-
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([2, 3, 4])))
- self.assertTrue(tracker.try_claim(
- self.bytes_to_position([3, 2, 0])))
- self.assertFalse(tracker.try_claim(
- self.bytes_to_position([3, 2, 1])))
-
- def test_split_points(self):
- tracker = range_trackers.GroupedShuffleRangeTracker(
- self.bytes_to_position([1, 0, 0]),
- self.bytes_to_position([5, 0, 0]))
- self.assertEqual(tracker.split_points(),
- (0, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([1, 2, 3])))
- self.assertEqual(tracker.split_points(),
- (0, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([1, 2, 5])))
- self.assertEqual(tracker.split_points(),
- (1, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([3, 6, 8])))
- self.assertEqual(tracker.split_points(),
- (2, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
- self.assertTrue(tracker.try_claim(self.bytes_to_position([4, 255, 255])))
- self.assertEqual(tracker.split_points(),
- (3, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
- self.assertFalse(tracker.try_claim(self.bytes_to_position([5, 1, 0])))
- self.assertEqual(tracker.split_points(),
- (3, iobase.RangeTracker.SPLIT_POINTS_UNKNOWN))
-
-
class OrderedPositionRangeTrackerTest(unittest.TestCase):
class DoubleRangeTracker(range_trackers.OrderedPositionRangeTracker):
diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py
index a144a8a..e4d2f6f 100644
--- a/sdks/python/apache_beam/io/source_test_utils.py
+++ b/sdks/python/apache_beam/io/source_test_utils.py
@@ -44,15 +44,16 @@
* apache_beam.io.avroio_test.py
"""
-from collections import namedtuple
import logging
import threading
import weakref
-
+from collections import namedtuple
from multiprocessing.pool import ThreadPool
+
from apache_beam.io import iobase
-__all__ = ['read_from_source', 'assert_sources_equal_reference_source',
+__all__ = ['read_from_source',
+ 'assert_sources_equal_reference_source',
'assert_reentrant_reads_succeed',
'assert_split_at_fraction_behavior',
'assert_split_at_fraction_binary',
@@ -80,12 +81,13 @@
Only reads elements within the given position range.
Args:
- source: ``iobase.BoundedSource`` implementation.
- start_position: start position for reading.
- stop_position: stop position for reading.
+ source (~apache_beam.io.iobase.BoundedSource):
+ :class:`~apache_beam.io.iobase.BoundedSource` implementation.
+ start_position (int): start position for reading.
+ stop_position (int): stop position for reading.
Returns:
- the set of values read from the sources.
+ List[str]: the set of values read from the sources.
"""
values = []
range_tracker = source.get_range_tracker(start_position, stop_position)
@@ -108,21 +110,25 @@
def assert_sources_equal_reference_source(reference_source_info, sources_info):
"""Tests if a reference source is equal to a given set of sources.
- Given a reference source (a ``BoundedSource`` and a position range) and a
- list of sources, assert that the union of the records
- read from the list of sources is equal to the records read from the
+ Given a reference source (a :class:`~apache_beam.io.iobase.BoundedSource`
+ and a position range) and a list of sources, assert that the union of the
+ records read from the list of sources is equal to the records read from the
reference source.
Args:
- reference_source_info: a three-tuple that gives the reference
- ``iobase.BoundedSource``, position to start reading
- at, and position to stop reading at.
- sources_info: a set of sources. Each source is a three-tuple that is of
- the same format described above.
+ reference_source_info\
+ (Tuple[~apache_beam.io.iobase.BoundedSource, int, int]):
+ a three-tuple that gives the reference
+ :class:`~apache_beam.io.iobase.BoundedSource`, position to start
+ reading at, and position to stop reading at.
+ sources_info\
+ (Iterable[Tuple[~apache_beam.io.iobase.BoundedSource, int, int]]):
+ a set of sources. Each source is a three-tuple that is of the same
+ format described above.
Raises:
- ValueError: if the set of data produced by the reference source and the
- given set of sources are not equivalent.
+ ~exceptions.ValueError: if the set of data produced by the reference source
+ and the given set of sources are not equivalent.
"""
@@ -172,18 +178,20 @@
def assert_reentrant_reads_succeed(source_info):
"""Tests if a given source can be read in a reentrant manner.
- Assume that given source produces the set of values {v1, v2, v3, ... vn}. For
- i in range [1, n-1] this method performs a reentrant read after reading i
- elements and verifies that both the original and reentrant read produce the
- expected set of values.
+ Assume that given source produces the set of values ``{v1, v2, v3, ... vn}``.
+ For ``i`` in range ``[1, n-1]`` this method performs a reentrant read after
+ reading ``i`` elements and verifies that both the original and reentrant read
+ produce the expected set of values.
Args:
- source_info: a three-tuple that gives the reference
- ``iobase.BoundedSource``, position to start reading at, and a
- position to stop reading at.
+ source_info (Tuple[~apache_beam.io.iobase.BoundedSource, int, int]):
+ a three-tuple that gives the reference
+ :class:`~apache_beam.io.iobase.BoundedSource`, position to start reading
+ at, and a position to stop reading at.
+
Raises:
- ValueError: if source is too trivial or reentrant read result in an
- incorrect read.
+ ~exceptions.ValueError: if source is too trivial or reentrant read result
+ in an incorrect read.
"""
source, start_position, stop_position = source_info
@@ -228,21 +236,25 @@
split_fraction, expected_outcome):
"""Verifies the behaviour of splitting a source at a given fraction.
- Asserts that splitting a ``BoundedSource`` either fails after reading
- ``num_items_to_read_before_split`` items, or succeeds in a way that is
- consistent according to ``assertSplitAtFractionSucceedsAndConsistent()``.
+ Asserts that splitting a :class:`~apache_beam.io.iobase.BoundedSource` either
+ fails after reading **num_items_to_read_before_split** items, or succeeds in
+ a way that is consistent according to
+ :func:`assert_split_at_fraction_succeeds_and_consistent()`.
Args:
- source: the source to perform dynamic splitting on.
- num_items_to_read_before_split: number of items to read before splitting.
- split_fraction: fraction to split at.
- expected_outcome: a value from 'ExpectedSplitOutcome'.
+ source (~apache_beam.io.iobase.BoundedSource): the source to perform
+ dynamic splitting on.
+ num_items_to_read_before_split (int): number of items to read before
+ splitting.
+ split_fraction (float): fraction to split at.
+ expected_outcome (int): a value from
+ :class:`~apache_beam.io.source_test_utils.ExpectedSplitOutcome`.
Returns:
- a tuple that gives the number of items produced by reading the two ranges
- produced after dynamic splitting. If splitting did not occur, the first
- value of the tuple will represent the full set of records read by the
- source while the second value of the tuple will be '-1'.
+ Tuple[int, int]: a tuple that gives the number of items produced by reading
+ the two ranges produced after dynamic splitting. If splitting did not
+ occur, the first value of the tuple will represent the full set of records
+ read by the source while the second value of the tuple will be ``-1``.
"""
assert isinstance(source, iobase.BoundedSource)
expected_items = read_from_source(source, None, None)
@@ -503,12 +515,13 @@
Verifies multi threaded splitting as well.
Args:
- source: the source to perform dynamic splitting on.
- perform_multi_threaded_test: if true performs a multi-threaded test
- otherwise this test is skipped.
+ source (~apache_beam.io.iobase.BoundedSource): the source to perform
+ dynamic splitting on.
+ perform_multi_threaded_test (bool): if :data:`True` performs a
+ multi-threaded test, otherwise this test is skipped.
Raises:
- ValueError: if the exhaustive splitting test fails.
+ ~exceptions.ValueError: if the exhaustive splitting test fails.
"""
expected_items = read_from_source(source, start_position, stop_position)
diff --git a/sdks/python/apache_beam/io/source_test_utils_test.py b/sdks/python/apache_beam/io/source_test_utils_test.py
index 00522c9..af2d4b8 100644
--- a/sdks/python/apache_beam/io/source_test_utils_test.py
+++ b/sdks/python/apache_beam/io/source_test_utils_test.py
@@ -19,8 +19,8 @@
import tempfile
import unittest
-from apache_beam.io.filebasedsource_test import LineSource
import apache_beam.io.source_test_utils as source_test_utils
+from apache_beam.io.filebasedsource_test import LineSource
class SourceTestUtilsTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py
index 10d401b..8f885e5 100644
--- a/sdks/python/apache_beam/io/sources_test.py
+++ b/sdks/python/apache_beam/io/sources_test.py
@@ -23,7 +23,6 @@
import unittest
import apache_beam as beam
-
from apache_beam import coders
from apache_beam.io import iobase
from apache_beam.io import range_trackers
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index 60e1512..4a4bd3a 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -19,19 +19,22 @@
from __future__ import absolute_import
+
import logging
+from functools import partial
from apache_beam.coders import coders
-from apache_beam.io import filebasedsource
from apache_beam.io import filebasedsink
+from apache_beam.io import filebasedsource
from apache_beam.io import iobase
+from apache_beam.io.filebasedsource import ReadAllFiles
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.iobase import Read
from apache_beam.io.iobase import Write
from apache_beam.transforms import PTransform
from apache_beam.transforms.display import DisplayDataItem
-__all__ = ['ReadFromText', 'WriteToText']
+__all__ = ['ReadFromText', 'ReadAllFromText', 'WriteToText']
class _TextSource(filebasedsource.FileBasedSource):
@@ -75,6 +78,10 @@
'size of data %d.', value, len(self._data))
self._position = value
+ def reset(self):
+ self.data = ''
+ self.position = 0
+
def __init__(self,
file_pattern,
min_bundle_size,
@@ -83,7 +90,26 @@
coder,
buffer_size=DEFAULT_READ_BUFFER_SIZE,
validate=True,
- skip_header_lines=0):
+ skip_header_lines=0,
+ header_processor_fns=(None, None)):
+ """Initialize a _TextSource
+
+ Args:
+ header_processor_fns (tuple): a tuple of a `header_matcher` function
+ and a `header_processor` function. The `header_matcher` should
+ return `True` for all lines at the start of the file that are part
+ of the file header and `False` otherwise. These header lines will
+ not be yielded when reading records and instead passed into
+ `header_processor` to be handled. If `skip_header_lines` and a
+ `header_matcher` are both provided, the value of `skip_header_lines`
+ lines will be skipped and the header will be processed from
+ there.
+ Raises:
+ ValueError: if skip_lines is negative.
+
+ Please refer to documentation in class `ReadFromText` for the rest
+ of the arguments.
+ """
super(_TextSource, self).__init__(file_pattern, min_bundle_size,
compression_type=compression_type,
validate=validate)
@@ -100,6 +126,7 @@
'Skipping %d header lines. Skipping large number of header '
'lines might significantly slow down processing.')
self._skip_header_lines = skip_header_lines
+ self._header_matcher, self._header_processor = header_processor_fns
def display_data(self):
parent_dd = super(_TextSource, self).display_data()
@@ -127,18 +154,17 @@
range_tracker.set_split_points_unclaimed_callback(split_points_unclaimed)
with self.open_file(file_name) as file_to_read:
- position_after_skipping_header_lines = self._skip_lines(
- file_to_read, read_buffer,
- self._skip_header_lines) if self._skip_header_lines else 0
- start_offset = max(start_offset, position_after_skipping_header_lines)
- if start_offset > position_after_skipping_header_lines:
+ position_after_processing_header_lines = (
+ self._process_header(file_to_read, read_buffer))
+ start_offset = max(start_offset, position_after_processing_header_lines)
+ if start_offset > position_after_processing_header_lines:
# Seeking to one position before the start index and ignoring the
# current line. If start_position is at beginning if the line, that line
# belongs to the current bundle, hence ignoring that is incorrect.
# Seeking to one byte before prevents that.
file_to_read.seek(start_offset - 1)
- read_buffer = _TextSource.ReadBuffer('', 0)
+ read_buffer.reset()
sep_bounds = self._find_separator_bounds(file_to_read, read_buffer)
if not sep_bounds:
# Could not find a separator after (start_offset - 1). This means that
@@ -149,7 +175,7 @@
read_buffer.data = read_buffer.data[sep_end:]
next_record_start_position = start_offset - 1 + sep_end
else:
- next_record_start_position = position_after_skipping_header_lines
+ next_record_start_position = position_after_processing_header_lines
while range_tracker.try_claim(next_record_start_position):
record, num_bytes_to_next_record = self._read_record(file_to_read,
@@ -172,6 +198,34 @@
if num_bytes_to_next_record < 0:
break
+ def _process_header(self, file_to_read, read_buffer):
+ # Returns a tuple containing the position in file after processing header
+ # records and a list of decoded header lines that match
+ # 'header_matcher'.
+ header_lines = []
+ position = self._skip_lines(
+ file_to_read, read_buffer,
+ self._skip_header_lines) if self._skip_header_lines else 0
+ if self._header_matcher:
+ while True:
+ record, num_bytes_to_next_record = self._read_record(file_to_read,
+ read_buffer)
+ decoded_line = self._coder.decode(record)
+ if not self._header_matcher(decoded_line):
+ # We've read past the header section at this point, so go back a line.
+ file_to_read.seek(position)
+ read_buffer.reset()
+ break
+ header_lines.append(decoded_line)
+ if num_bytes_to_next_record < 0:
+ break
+ position += num_bytes_to_next_record
+
+ if self._header_processor:
+ self._header_processor(header_lines)
+
+ return position
+
def _find_separator_bounds(self, file_to_read, read_buffer):
# Determines the start and end positions within 'read_buffer.data' of the
# next separator starting from position 'read_buffer.position'.
@@ -342,8 +396,21 @@
file_handle.write('\n')
-class ReadFromText(PTransform):
- """A PTransform for reading text files.
+def _create_text_source(
+ file_pattern=None, min_bundle_size=None, compression_type=None,
+ strip_trailing_newlines=None, coder=None, skip_header_lines=None):
+ return _TextSource(
+ file_pattern=file_pattern, min_bundle_size=min_bundle_size,
+ compression_type=compression_type,
+ strip_trailing_newlines=strip_trailing_newlines,
+ coder=coder, validate=False, skip_header_lines=skip_header_lines)
+
+
+class ReadAllFromText(PTransform):
+ """A ``PTransform`` for reading a ``PCollection`` of text files.
+
+ Reads a ``PCollection`` of text files or file patterns and and produces a
+ ``PCollection`` of strings.
Parses a text file as newline-delimited elements, by default assuming
UTF-8 encoding. Supports newline delimiters '\\n' and '\\r\\n'.
@@ -351,6 +418,67 @@
This implementation only supports reading text encoded using UTF-8 or ASCII.
This does not support other encodings such as UTF-16 or UTF-32.
"""
+
+ DEFAULT_DESIRED_BUNDLE_SIZE = 64 * 1024 * 1024 # 64MB
+
+ def __init__(
+ self,
+ min_bundle_size=0,
+ desired_bundle_size=DEFAULT_DESIRED_BUNDLE_SIZE,
+ compression_type=CompressionTypes.AUTO,
+ strip_trailing_newlines=True,
+ coder=coders.StrUtf8Coder(),
+ skip_header_lines=0,
+ **kwargs):
+ """Initialize the ``ReadAllFromText`` transform.
+
+ Args:
+ min_bundle_size: Minimum size of bundles that should be generated when
+ splitting this source into bundles. See ``FileBasedSource`` for more
+ details.
+ desired_bundle_size: Desired size of bundles that should be generated when
+ splitting this source into bundles. See ``FileBasedSource`` for more
+ details.
+ compression_type: Used to handle compressed input files. Typical value
+ is ``CompressionTypes.AUTO``, in which case the underlying file_path's
+ extension will be used to detect the compression.
+ strip_trailing_newlines: Indicates whether this source should remove
+ the newline char in each line it reads before decoding that line.
+ validate: flag to verify that the files exist during the pipeline
+ creation time.
+ skip_header_lines: Number of header lines to skip. Same number is skipped
+ from each source file. Must be 0 or higher. Large number of skipped
+ lines might impact performance.
+ coder: Coder used to decode each line.
+ """
+ super(ReadAllFromText, self).__init__(**kwargs)
+ source_from_file = partial(
+ _create_text_source, min_bundle_size=min_bundle_size,
+ compression_type=compression_type,
+ strip_trailing_newlines=strip_trailing_newlines, coder=coder,
+ skip_header_lines=skip_header_lines)
+ self._desired_bundle_size = desired_bundle_size
+ self._min_bundle_size = min_bundle_size
+ self._compression_type = compression_type
+ self._read_all_files = ReadAllFiles(
+ True, compression_type, desired_bundle_size, min_bundle_size,
+ source_from_file)
+
+ def expand(self, pvalue):
+ return pvalue | 'ReadAllFiles' >> self._read_all_files
+
+
+class ReadFromText(PTransform):
+ r"""A :class:`~apache_beam.transforms.ptransform.PTransform` for reading text
+ files.
+
+ Parses a text file as newline-delimited elements, by default assuming
+ ``UTF-8`` encoding. Supports newline delimiters ``\n`` and ``\r\n``.
+
+ This implementation only supports reading text encoded using ``UTF-8`` or
+ ``ASCII``.
+ This does not support other encodings such as ``UTF-16`` or ``UTF-32``.
+ """
def __init__(
self,
file_pattern=None,
@@ -361,26 +489,28 @@
validate=True,
skip_header_lines=0,
**kwargs):
- """Initialize the ReadFromText transform.
+ """Initialize the :class:`ReadFromText` transform.
Args:
- file_pattern: The file path to read from as a local file path or a GCS
- ``gs://`` path. The path can contain glob characters
- ``(*, ?, and [...] sets)``.
- min_bundle_size: Minimum size of bundles that should be generated when
- splitting this source into bundles. See ``FileBasedSource`` for more
+ file_pattern (str): The file path to read from as a local file path or a
+ GCS ``gs://`` path. The path can contain glob characters
+ (``*``, ``?``, and ``[...]`` sets).
+ min_bundle_size (int): Minimum size of bundles that should be generated
+ when splitting this source into bundles. See
+ :class:`~apache_beam.io.filebasedsource.FileBasedSource` for more
details.
- compression_type: Used to handle compressed input files. Typical value
- is CompressionTypes.AUTO, in which case the underlying file_path's
- extension will be used to detect the compression.
- strip_trailing_newlines: Indicates whether this source should remove
- the newline char in each line it reads before decoding that line.
- validate: flag to verify that the files exist during the pipeline
+ compression_type (str): Used to handle compressed input files.
+ Typical value is :attr:`CompressionTypes.AUTO
+ <apache_beam.io.filesystem.CompressionTypes.AUTO>`, in which case the
+ underlying file_path's extension will be used to detect the compression.
+ strip_trailing_newlines (bool): Indicates whether this source should
+ remove the newline char in each line it reads before decoding that line.
+ validate (bool): flag to verify that the files exist during the pipeline
creation time.
- skip_header_lines: Number of header lines to skip. Same number is skipped
- from each source file. Must be 0 or higher. Large number of skipped
- lines might impact performance.
- coder: Coder used to decode each line.
+ skip_header_lines (int): Number of header lines to skip. Same number is
+ skipped from each source file. Must be 0 or higher. Large number of
+ skipped lines might impact performance.
+ coder (~apache_beam.coders.coders.Coder): Coder used to decode each line.
"""
super(ReadFromText, self).__init__(**kwargs)
@@ -394,49 +524,54 @@
class WriteToText(PTransform):
- """A PTransform for writing to text files."""
+ """A :class:`~apache_beam.transforms.ptransform.PTransform` for writing to
+ text files."""
- def __init__(self,
- file_path_prefix,
- file_name_suffix='',
- append_trailing_newlines=True,
- num_shards=0,
- shard_name_template=None,
- coder=coders.ToStringCoder(),
- compression_type=CompressionTypes.AUTO,
- header=None):
- """Initialize a WriteToText PTransform.
+ def __init__(
+ self,
+ file_path_prefix,
+ file_name_suffix='',
+ append_trailing_newlines=True,
+ num_shards=0,
+ shard_name_template=None,
+ coder=coders.ToStringCoder(),
+ compression_type=CompressionTypes.AUTO,
+ header=None):
+ r"""Initialize a :class:`WriteToText` transform.
Args:
- file_path_prefix: The file path to write to. The files written will begin
- with this prefix, followed by a shard identifier (see num_shards), and
- end in a common extension, if given by file_name_suffix. In most cases,
- only this argument is specified and num_shards, shard_name_template, and
- file_name_suffix use default values.
- file_name_suffix: Suffix for the files written.
- append_trailing_newlines: indicate whether this sink should write an
- additional newline char after writing each element.
- num_shards: The number of files (shards) used for output. If not set, the
- service will decide on the optimal number of shards.
+ file_path_prefix (str): The file path to write to. The files written will
+ begin with this prefix, followed by a shard identifier (see
+ **num_shards**), and end in a common extension, if given by
+ **file_name_suffix**. In most cases, only this argument is specified and
+ **num_shards**, **shard_name_template**, and **file_name_suffix** use
+ default values.
+ file_name_suffix (str): Suffix for the files written.
+ append_trailing_newlines (bool): indicate whether this sink should write
+ an additional newline char after writing each element.
+ num_shards (int): The number of files (shards) used for output.
+ If not set, the service will decide on the optimal number of shards.
Constraining the number of shards is likely to reduce
the performance of a pipeline. Setting this value is not recommended
unless you require a specific number of output files.
- shard_name_template: A template string containing placeholders for
- the shard number and shard count. Currently only '' and
- '-SSSSS-of-NNNNN' are patterns accepted by the service.
+ shard_name_template (str): A template string containing placeholders for
+ the shard number and shard count. Currently only ``''`` and
+ ``'-SSSSS-of-NNNNN'`` are patterns accepted by the service.
When constructing a filename for a particular shard number, the
- upper-case letters 'S' and 'N' are replaced with the 0-padded shard
- number and shard count respectively. This argument can be '' in which
- case it behaves as if num_shards was set to 1 and only one file will be
- generated. The default pattern used is '-SSSSS-of-NNNNN'.
- coder: Coder used to encode each line.
- compression_type: Used to handle compressed output files. Typical value
- is CompressionTypes.AUTO, in which case the final file path's
- extension (as determined by file_path_prefix, file_name_suffix,
- num_shards and shard_name_template) will be used to detect the
- compression.
- header: String to write at beginning of file as a header. If not None and
- append_trailing_newlines is set, '\n' will be added.
+ upper-case letters ``S`` and ``N`` are replaced with the ``0``-padded
+ shard number and shard count respectively. This argument can be ``''``
+ in which case it behaves as if num_shards was set to 1 and only one file
+ will be generated. The default pattern used is ``'-SSSSS-of-NNNNN'``.
+ coder (~apache_beam.coders.coders.Coder): Coder used to encode each line.
+ compression_type (str): Used to handle compressed output files.
+ Typical value is :class:`CompressionTypes.AUTO
+ <apache_beam.io.filesystem.CompressionTypes.AUTO>`, in which case the
+ final file path's extension (as determined by **file_path_prefix**,
+ **file_name_suffix**, **num_shards** and **shard_name_template**) will
+ be used to detect the compression.
+ header (str): String to write at beginning of file as a header.
+ If not :data:`None` and **append_trailing_newlines** is set, ``\n`` will
+ be added.
"""
self._sink = _TextSink(file_path_prefix, file_name_suffix,
diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py
index 9a4ec47..324f52a 100644
--- a/sdks/python/apache_beam/io/textio_test.py
+++ b/sdks/python/apache_beam/io/textio_test.py
@@ -27,60 +27,27 @@
import unittest
import apache_beam as beam
-from apache_beam.io import iobase
import apache_beam.io.source_test_utils as source_test_utils
-
-# Importing following private classes for testing.
-from apache_beam.io.textio import _TextSink as TextSink
-from apache_beam.io.textio import _TextSource as TextSource
-
-from apache_beam.io.textio import ReadFromText
-from apache_beam.io.textio import WriteToText
-
from apache_beam import coders
+from apache_beam.io import ReadAllFromText
+from apache_beam.io import iobase
from apache_beam.io.filebasedsource_test import EOL
from apache_beam.io.filebasedsource_test import write_data
from apache_beam.io.filebasedsource_test import write_pattern
from apache_beam.io.filesystem import CompressionTypes
-
+from apache_beam.io.textio import _TextSink as TextSink
+from apache_beam.io.textio import _TextSource as TextSource
+# Importing following private classes for testing.
+from apache_beam.io.textio import ReadFromText
+from apache_beam.io.textio import WriteToText
from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.test_utils import TempDir
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
+from apache_beam.transforms.core import Create
-# TODO: Refactor code so all io tests are using same library
-# TestCaseWithTempDirCleanup class.
-class _TestCaseWithTempDirCleanUp(unittest.TestCase):
- """Base class for TestCases that deals with TempDir clean-up.
-
- Inherited test cases will call self._new_tempdir() to start a temporary dir
- which will be deleted at the end of the tests (when tearDown() is called).
- """
-
- def setUp(self):
- self._tempdirs = []
-
- def tearDown(self):
- for path in self._tempdirs:
- if os.path.exists(path):
- shutil.rmtree(path)
- self._tempdirs = []
-
- def _new_tempdir(self):
- result = tempfile.mkdtemp()
- self._tempdirs.append(result)
- return result
-
- def _create_temp_file(self, name='', suffix=''):
- if not name:
- name = tempfile.template
- file_name = tempfile.NamedTemporaryFile(
- delete=False, prefix=name,
- dir=self._new_tempdir(), suffix=suffix).name
- return file_name
-
-
-class TextSourceTest(_TestCaseWithTempDirCleanUp):
+class TextSourceTest(unittest.TestCase):
# Number of records that will be written by most tests.
DEFAULT_NUM_RECORDS = 100
@@ -94,7 +61,7 @@
source = TextSource(file_or_pattern, 0, compression,
True, coders.StrUtf8Coder(), buffer_size)
range_tracker = source.get_range_tracker(None, None)
- read_data = [record for record in source.read(range_tracker)]
+ read_data = list(source.read(range_tracker))
self.assertItemsEqual(expected_data, read_data)
def test_read_single_file(self):
@@ -216,7 +183,7 @@
False, coders.StrUtf8Coder())
range_tracker = source.get_range_tracker(None, None)
- read_data = [record for record in source.read(range_tracker)]
+ read_data = list(source.read(range_tracker))
self.assertItemsEqual([line + '\n' for line in written_data], read_data)
def test_read_single_file_without_striping_eol_crlf(self):
@@ -227,7 +194,7 @@
False, coders.StrUtf8Coder())
range_tracker = source.get_range_tracker(None, None)
- read_data = [record for record in source.read(range_tracker)]
+ read_data = list(source.read(range_tracker))
self.assertItemsEqual([line + '\r\n' for line in written_data], read_data)
def test_read_file_pattern_with_empty_files(self):
@@ -248,7 +215,7 @@
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=33)]
+ splits = list(source.split(desired_bundle_size=33))
reference_source_info = (source, None, None)
sources_info = ([
@@ -257,12 +224,37 @@
source_test_utils.assert_sources_equal_reference_source(
reference_source_info, sources_info)
+ def test_header_processing(self):
+ file_name, expected_data = write_data(10)
+ assert len(expected_data) == 10
+
+ def header_matcher(line):
+ return line in expected_data[:5]
+
+ header_lines = []
+
+ def store_header(lines):
+ for line in lines:
+ header_lines.append(line)
+
+ source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
+ coders.StrUtf8Coder(),
+ header_processor_fns=(header_matcher, store_header))
+ splits = list(source.split(desired_bundle_size=100000))
+ assert len(splits) == 1
+ range_tracker = splits[0].source.get_range_tracker(
+ splits[0].start_position, splits[0].stop_position)
+ read_data = list(source.read_records(file_name, range_tracker))
+
+ self.assertItemsEqual(expected_data[:5], header_lines)
+ self.assertItemsEqual(expected_data[5:], read_data)
+
def test_progress(self):
file_name, expected_data = write_data(10)
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=100000)]
+ splits = list(source.split(desired_bundle_size=100000))
assert len(splits) == 1
fraction_consumed_report = []
split_points_report = []
@@ -297,7 +289,7 @@
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=100000)]
+ splits = list(source.split(desired_bundle_size=100000))
assert len(splits) == 1
source_test_utils.assert_reentrant_reads_succeed(
(splits[0].source, splits[0].start_position, splits[0].stop_position))
@@ -307,7 +299,7 @@
assert len(expected_data) == 5
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=100000)]
+ splits = list(source.split(desired_bundle_size=100000))
assert len(splits) == 1
source_test_utils.assert_split_at_fraction_exhaustive(
splits[0].source, splits[0].start_position, splits[0].stop_position)
@@ -317,7 +309,7 @@
assert len(expected_data) == 15
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=100000)]
+ splits = list(source.split(desired_bundle_size=100000))
assert len(splits) == 1
source_test_utils.assert_split_at_fraction_exhaustive(
splits[0].source, splits[0].start_position, splits[0].stop_position,
@@ -328,13 +320,13 @@
assert len(expected_data) == 5
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=100000)]
+ splits = list(source.split(desired_bundle_size=100000))
assert len(splits) == 1
source_test_utils.assert_split_at_fraction_exhaustive(
splits[0].source, splits[0].start_position, splits[0].stop_position,
perform_multi_threaded_test=False)
- def test_dataflow_single_file(self):
+ def test_read_from_text_single_file(self):
file_name, expected_data = write_data(5)
assert len(expected_data) == 5
pipeline = TestPipeline()
@@ -342,7 +334,53 @@
assert_that(pcoll, equal_to(expected_data))
pipeline.run()
- def test_dataflow_single_file_with_coder(self):
+ def test_read_all_single_file(self):
+ file_name, expected_data = write_data(5)
+ assert len(expected_data) == 5
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Create' >> Create(
+ [file_name]) |'ReadAll' >> ReadAllFromText()
+ assert_that(pcoll, equal_to(expected_data))
+ pipeline.run()
+
+ def test_read_all_many_single_files(self):
+ file_name1, expected_data1 = write_data(5)
+ assert len(expected_data1) == 5
+ file_name2, expected_data2 = write_data(10)
+ assert len(expected_data2) == 10
+ file_name3, expected_data3 = write_data(15)
+ assert len(expected_data3) == 15
+ expected_data = []
+ expected_data.extend(expected_data1)
+ expected_data.extend(expected_data2)
+ expected_data.extend(expected_data3)
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Create' >> Create(
+ [file_name1, file_name2, file_name3]) |'ReadAll' >> ReadAllFromText()
+ assert_that(pcoll, equal_to(expected_data))
+ pipeline.run()
+
+ def test_read_all_unavailable_files_ignored(self):
+ file_name1, expected_data1 = write_data(5)
+ assert len(expected_data1) == 5
+ file_name2, expected_data2 = write_data(10)
+ assert len(expected_data2) == 10
+ file_name3, expected_data3 = write_data(15)
+ assert len(expected_data3) == 15
+ file_name4 = "/unavailable_file"
+ expected_data = []
+ expected_data.extend(expected_data1)
+ expected_data.extend(expected_data2)
+ expected_data.extend(expected_data3)
+ pipeline = TestPipeline()
+ pcoll = (pipeline
+ | 'Create' >> Create(
+ [file_name1, file_name2, file_name3, file_name4])
+ |'ReadAll' >> ReadAllFromText())
+ assert_that(pcoll, equal_to(expected_data))
+ pipeline.run()
+
+ def test_read_from_text_single_file_with_coder(self):
class DummyCoder(coders.Coder):
def encode(self, x):
raise ValueError
@@ -357,7 +395,7 @@
assert_that(pcoll, equal_to([record * 2 for record in expected_data]))
pipeline.run()
- def test_dataflow_file_pattern(self):
+ def test_read_from_text_file_pattern(self):
pattern, expected_data = write_pattern([5, 3, 12, 8, 8, 4])
assert len(expected_data) == 40
pipeline = TestPipeline()
@@ -365,101 +403,270 @@
assert_that(pcoll, equal_to(expected_data))
pipeline.run()
+ def test_read_all_file_pattern(self):
+ pattern, expected_data = write_pattern([5, 3, 12, 8, 8, 4])
+ assert len(expected_data) == 40
+ pipeline = TestPipeline()
+ pcoll = (pipeline
+ | 'Create' >> Create([pattern])
+ |'ReadAll' >> ReadAllFromText())
+ assert_that(pcoll, equal_to(expected_data))
+ pipeline.run()
+
+ def test_read_all_many_file_patterns(self):
+ pattern1, expected_data1 = write_pattern([5, 3, 12, 8, 8, 4])
+ assert len(expected_data1) == 40
+ pattern2, expected_data2 = write_pattern([3, 7, 9])
+ assert len(expected_data2) == 19
+ pattern3, expected_data3 = write_pattern([11, 20, 5, 5])
+ assert len(expected_data3) == 41
+ expected_data = []
+ expected_data.extend(expected_data1)
+ expected_data.extend(expected_data2)
+ expected_data.extend(expected_data3)
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Create' >> Create(
+ [pattern1, pattern2, pattern3]) |'ReadAll' >> ReadAllFromText()
+ assert_that(pcoll, equal_to(expected_data))
+ pipeline.run()
+
def test_read_auto_bzip2(self):
_, lines = write_data(15)
- file_name = self._create_temp_file(suffix='.bz2')
- with bz2.BZ2File(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file(suffix='.bz2')
+ with bz2.BZ2File(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(file_name)
- assert_that(pcoll, equal_to(lines))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(file_name)
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
def test_read_auto_gzip(self):
_, lines = write_data(15)
- file_name = self._create_temp_file(suffix='.gz')
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file(suffix='.gz')
- with gzip.GzipFile(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(file_name)
- assert_that(pcoll, equal_to(lines))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(file_name)
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
def test_read_bzip2(self):
_, lines = write_data(15)
- file_name = self._create_temp_file()
- with bz2.BZ2File(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with bz2.BZ2File(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(
- file_name,
- compression_type=CompressionTypes.BZIP2)
- assert_that(pcoll, equal_to(lines))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ compression_type=CompressionTypes.BZIP2)
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
+
+ def test_read_corrupted_bzip2_fails(self):
+ _, lines = write_data(15)
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with bz2.BZ2File(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
+
+ with open(file_name, 'wb') as f:
+ f.write('corrupt')
+
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ compression_type=CompressionTypes.BZIP2)
+ assert_that(pcoll, equal_to(lines))
+ with self.assertRaises(Exception):
+ pipeline.run()
+
+ def test_read_bzip2_concat(self):
+ with TempDir() as tempdir:
+ bzip2_file_name1 = tempdir.create_temp_file()
+ lines = ['a', 'b', 'c']
+ with bz2.BZ2File(bzip2_file_name1, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ bzip2_file_name2 = tempdir.create_temp_file()
+ lines = ['p', 'q', 'r']
+ with bz2.BZ2File(bzip2_file_name2, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ bzip2_file_name3 = tempdir.create_temp_file()
+ lines = ['x', 'y', 'z']
+ with bz2.BZ2File(bzip2_file_name3, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ final_bzip2_file = tempdir.create_temp_file()
+ with open(bzip2_file_name1, 'rb') as src, open(
+ final_bzip2_file, 'wb') as dst:
+ dst.writelines(src.readlines())
+
+ with open(bzip2_file_name2, 'rb') as src, open(
+ final_bzip2_file, 'ab') as dst:
+ dst.writelines(src.readlines())
+
+ with open(bzip2_file_name3, 'rb') as src, open(
+ final_bzip2_file, 'ab') as dst:
+ dst.writelines(src.readlines())
+
+ pipeline = TestPipeline()
+ lines = pipeline | 'ReadFromText' >> beam.io.ReadFromText(
+ final_bzip2_file,
+ compression_type=beam.io.filesystem.CompressionTypes.BZIP2)
+
+ expected = ['a', 'b', 'c', 'p', 'q', 'r', 'x', 'y', 'z']
+ assert_that(lines, equal_to(expected))
+ pipeline.run()
def test_read_gzip(self):
_, lines = write_data(15)
- file_name = self._create_temp_file()
- with gzip.GzipFile(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(
- file_name,
- 0, CompressionTypes.GZIP,
- True, coders.StrUtf8Coder())
- assert_that(pcoll, equal_to(lines))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ 0, CompressionTypes.GZIP,
+ True, coders.StrUtf8Coder())
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
+
+ def test_read_corrupted_gzip_fails(self):
+ _, lines = write_data(15)
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
+
+ with open(file_name, 'wb') as f:
+ f.write('corrupt')
+
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ 0, CompressionTypes.GZIP,
+ True, coders.StrUtf8Coder())
+ assert_that(pcoll, equal_to(lines))
+
+ with self.assertRaises(Exception):
+ pipeline.run()
+
+ def test_read_gzip_concat(self):
+ with TempDir() as tempdir:
+ gzip_file_name1 = tempdir.create_temp_file()
+ lines = ['a', 'b', 'c']
+ with gzip.open(gzip_file_name1, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ gzip_file_name2 = tempdir.create_temp_file()
+ lines = ['p', 'q', 'r']
+ with gzip.open(gzip_file_name2, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ gzip_file_name3 = tempdir.create_temp_file()
+ lines = ['x', 'y', 'z']
+ with gzip.open(gzip_file_name3, 'wb') as dst:
+ data = '\n'.join(lines) + '\n'
+ dst.write(data)
+
+ final_gzip_file = tempdir.create_temp_file()
+ with open(gzip_file_name1, 'rb') as src, \
+ open(final_gzip_file, 'wb') as dst:
+ dst.writelines(src.readlines())
+
+ with open(gzip_file_name2, 'rb') as src, \
+ open(final_gzip_file, 'ab') as dst:
+ dst.writelines(src.readlines())
+
+ with open(gzip_file_name3, 'rb') as src, \
+ open(final_gzip_file, 'ab') as dst:
+ dst.writelines(src.readlines())
+
+ pipeline = TestPipeline()
+ lines = pipeline | 'ReadFromText' >> beam.io.ReadFromText(
+ final_gzip_file,
+ compression_type=beam.io.filesystem.CompressionTypes.GZIP)
+
+ expected = ['a', 'b', 'c', 'p', 'q', 'r', 'x', 'y', 'z']
+ assert_that(lines, equal_to(expected))
+
+ def test_read_all_gzip(self):
+ _, lines = write_data(100)
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
+ pipeline = TestPipeline()
+ pcoll = (pipeline
+ | Create([file_name])
+ | 'ReadAll' >> ReadAllFromText(
+ compression_type=CompressionTypes.GZIP))
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
def test_read_gzip_large(self):
_, lines = write_data(10000)
- file_name = self._create_temp_file()
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
- with gzip.GzipFile(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(
- file_name,
- 0, CompressionTypes.GZIP,
- True, coders.StrUtf8Coder())
- assert_that(pcoll, equal_to(lines))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ 0, CompressionTypes.GZIP,
+ True, coders.StrUtf8Coder())
+ assert_that(pcoll, equal_to(lines))
+ pipeline.run()
def test_read_gzip_large_after_splitting(self):
_, lines = write_data(10000)
- file_name = self._create_temp_file()
- with gzip.GzipFile(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- source = TextSource(file_name, 0, CompressionTypes.GZIP, True,
- coders.StrUtf8Coder())
- splits = [split for split in source.split(desired_bundle_size=1000)]
+ source = TextSource(file_name, 0, CompressionTypes.GZIP, True,
+ coders.StrUtf8Coder())
+ splits = list(source.split(desired_bundle_size=1000))
- if len(splits) > 1:
- raise ValueError('FileBasedSource generated more than one initial split '
- 'for a compressed file.')
+ if len(splits) > 1:
+ raise ValueError('FileBasedSource generated more than one initial '
+ 'split for a compressed file.')
- reference_source_info = (source, None, None)
- sources_info = ([
- (split.source, split.start_position, split.stop_position) for
- split in splits])
- source_test_utils.assert_sources_equal_reference_source(
- reference_source_info, sources_info)
+ reference_source_info = (source, None, None)
+ sources_info = ([
+ (split.source, split.start_position, split.stop_position) for
+ split in splits])
+ source_test_utils.assert_sources_equal_reference_source(
+ reference_source_info, sources_info)
def test_read_gzip_empty_file(self):
- file_name = self._create_temp_file()
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(
- file_name,
- 0, CompressionTypes.GZIP,
- True, coders.StrUtf8Coder())
- assert_that(pcoll, equal_to([]))
- pipeline.run()
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name,
+ 0, CompressionTypes.GZIP,
+ True, coders.StrUtf8Coder())
+ assert_that(pcoll, equal_to([]))
+ pipeline.run()
def _remove_lines(self, lines, sublist_lengths, num_to_remove):
"""Utility function to remove num_to_remove lines from each sublist.
@@ -492,7 +699,7 @@
skip_header_lines=skip_header_lines)
range_tracker = source.get_range_tracker(None, None)
- return [record for record in source.read(range_tracker)]
+ return list(source.read(range_tracker))
def test_read_skip_header_single(self):
file_name, expected_data = write_data(TextSourceTest.DEFAULT_NUM_RECORDS)
@@ -537,23 +744,24 @@
def test_read_gzip_with_skip_lines(self):
_, lines = write_data(15)
- file_name = self._create_temp_file()
- with gzip.GzipFile(file_name, 'wb') as f:
- f.write('\n'.join(lines))
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file()
+ with gzip.GzipFile(file_name, 'wb') as f:
+ f.write('\n'.join(lines))
- pipeline = TestPipeline()
- pcoll = pipeline | 'Read' >> ReadFromText(
- file_name, 0, CompressionTypes.GZIP,
- True, coders.StrUtf8Coder(), skip_header_lines=2)
- assert_that(pcoll, equal_to(lines[2:]))
- pipeline.run()
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromText(
+ file_name, 0, CompressionTypes.GZIP,
+ True, coders.StrUtf8Coder(), skip_header_lines=2)
+ assert_that(pcoll, equal_to(lines[2:]))
+ pipeline.run()
def test_read_after_splitting_skip_header(self):
file_name, expected_data = write_data(100)
assert len(expected_data) == 100
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder(), skip_header_lines=2)
- splits = [split for split in source.split(desired_bundle_size=33)]
+ splits = list(source.split(desired_bundle_size=33))
reference_source_info = (source, None, None)
sources_info = ([
@@ -569,13 +777,26 @@
self.assertEqual(reference_lines, split_lines)
-class TextSinkTest(_TestCaseWithTempDirCleanUp):
+class TextSinkTest(unittest.TestCase):
def setUp(self):
super(TextSinkTest, self).setUp()
self.lines = ['Line %d' % d for d in range(100)]
+ self.tempdir = tempfile.mkdtemp()
self.path = self._create_temp_file()
+ def tearDown(self):
+ if os.path.exists(self.tempdir):
+ shutil.rmtree(self.tempdir)
+
+ def _create_temp_file(self, name='', suffix=''):
+ if not name:
+ name = tempfile.template
+ file_name = tempfile.NamedTemporaryFile(
+ delete=False, prefix=name,
+ dir=self.tempdir, suffix=suffix).name
+ return file_name
+
def _write_lines(self, sink, lines):
f = sink.open(self.path)
for line in lines:
diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py
index d7eb932..5af0716 100644
--- a/sdks/python/apache_beam/io/tfrecordio.py
+++ b/sdks/python/apache_beam/io/tfrecordio.py
@@ -21,14 +21,15 @@
import logging
import struct
+import crcmod
+
from apache_beam import coders
-from apache_beam.io import filebasedsource
from apache_beam.io import filebasedsink
+from apache_beam.io import filebasedsource
from apache_beam.io.filesystem import CompressionTypes
from apache_beam.io.iobase import Read
from apache_beam.io.iobase import Write
from apache_beam.transforms import PTransform
-import crcmod
__all__ = ['ReadFromTFRecord', 'WriteToTFRecord']
diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py
index 3c70ade..f7a160a 100644
--- a/sdks/python/apache_beam/io/tfrecordio_test.py
+++ b/sdks/python/apache_beam/io/tfrecordio_test.py
@@ -27,19 +27,19 @@
import tempfile
import unittest
+import crcmod
+
import apache_beam as beam
from apache_beam import coders
from apache_beam.io.filesystem import CompressionTypes
+from apache_beam.io.tfrecordio import ReadFromTFRecord
+from apache_beam.io.tfrecordio import WriteToTFRecord
from apache_beam.io.tfrecordio import _TFRecordSink
from apache_beam.io.tfrecordio import _TFRecordSource
from apache_beam.io.tfrecordio import _TFRecordUtil
-from apache_beam.io.tfrecordio import ReadFromTFRecord
-from apache_beam.io.tfrecordio import WriteToTFRecord
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
-import crcmod
-
try:
import tensorflow as tf # pylint: disable=import-error
diff --git a/sdks/python/apache_beam/io/vcfio.py b/sdks/python/apache_beam/io/vcfio.py
new file mode 100644
index 0000000..b877a32
--- /dev/null
+++ b/sdks/python/apache_beam/io/vcfio.py
@@ -0,0 +1,436 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A source for reading from VCF files (version 4.x).
+
+The 4.2 spec is available at https://samtools.github.io/hts-specs/VCFv4.2.pdf.
+"""
+
+from __future__ import absolute_import
+
+from collections import namedtuple
+
+import vcf
+
+from apache_beam.coders import coders
+from apache_beam.io import filebasedsource
+from apache_beam.io.filesystem import CompressionTypes
+from apache_beam.io.iobase import Read
+from apache_beam.io.textio import _TextSource as TextSource
+from apache_beam.transforms import PTransform
+
+__all__ = ['ReadFromVcf', 'Variant', 'VariantCall', 'VariantInfo']
+
+
+# Stores data about variant INFO fields. The type of 'data' is specified in the
+# VCF headers. 'field_count' is a string that specifies the number of fields
+# that the data type contains. Its value can either be a number representing a
+# constant number of fields, `None` indicating that the value is not set
+# (equivalent to '.' in the VCF file) or one of:
+# - 'A': one value per alternate allele.
+# - 'G': one value for each possible genotype.
+# - 'R': one value for each possible allele (including the reference).
+VariantInfo = namedtuple('VariantInfo', ['data', 'field_count'])
+MISSING_FIELD_VALUE = '.' # Indicates field is missing in VCF record.
+PASS_FILTER = 'PASS' # Indicates that all filters have been passed.
+END_INFO_KEY = 'END' # The info key that explicitly specifies end of a record.
+GENOTYPE_FORMAT_KEY = 'GT' # The genotype format key in a call.
+PHASESET_FORMAT_KEY = 'PS' # The phaseset format key.
+DEFAULT_PHASESET_VALUE = '*' # Default phaseset value if call is phased, but
+ # no 'PS' is present.
+MISSING_GENOTYPE_VALUE = -1 # Genotype to use when '.' is used in GT field.
+
+
+class Variant(object):
+ """A class to store info about a genomic variant.
+
+ Each object corresponds to a single record in a VCF file.
+ """
+
+ def __init__(self,
+ reference_name=None,
+ start=None,
+ end=None,
+ reference_bases=None,
+ alternate_bases=None,
+ names=None,
+ quality=None,
+ filters=None,
+ info=None,
+ calls=None):
+ """Initialize the :class:`Variant` object.
+
+ Args:
+ reference_name (str): The reference on which this variant occurs
+ (such as `chr20` or `X`). .
+ start (int): The position at which this variant occurs (0-based).
+ Corresponds to the first base of the string of reference bases.
+ end (int): The end position (0-based) of this variant. Corresponds to the
+ first base after the last base in the reference allele.
+ reference_bases (str): The reference bases for this variant.
+ alternate_bases (List[str]): The bases that appear instead of the
+ reference bases.
+ names (List[str]): Names for the variant, for example a RefSNP ID.
+ quality (float): Phred-scaled quality score (-10log10 prob(call is wrong))
+ Higher values imply better quality.
+ filters (List[str]): A list of filters (normally quality filters) this
+ variant has failed. `PASS` indicates this variant has passed all
+ filters.
+ info (dict): A map of additional variant information. The key is specified
+ in the VCF record and the value is of type ``VariantInfo``.
+ calls (list of :class:`VariantCall`): The variant calls for this variant.
+ Each one represents the determination of genotype with respect to this
+ variant.
+ """
+ self.reference_name = reference_name
+ self.start = start
+ self.end = end
+ self.reference_bases = reference_bases
+ self.alternate_bases = alternate_bases or []
+ self.names = names or []
+ self.quality = quality
+ self.filters = filters or []
+ self.info = info or {}
+ self.calls = calls or []
+
+ def __eq__(self, other):
+ return (isinstance(other, Variant) and
+ vars(self) == vars(other))
+
+ def __repr__(self):
+ return ', '.join(
+ [str(s) for s in [self.reference_name,
+ self.start,
+ self.end,
+ self.reference_bases,
+ self.alternate_bases,
+ self.names,
+ self.quality,
+ self.filters,
+ self.info,
+ self.calls]])
+
+ def __lt__(self, other):
+ if not isinstance(other, Variant):
+ return NotImplemented
+
+ # Elements should first be sorted by reference_name, start, end.
+ # Ordering of other members is not important, but must be
+ # deterministic.
+ if self.reference_name != other.reference_name:
+ return self.reference_name < other.reference_name
+ elif self.start != other.start:
+ return self.start < other.start
+ elif self.end != other.end:
+ return self.end < other.end
+
+ self_vars = vars(self)
+ other_vars = vars(other)
+ for key in sorted(self_vars):
+ if self_vars[key] != other_vars[key]:
+ return self_vars[key] < other_vars[key]
+
+ return False
+
+ def __le__(self, other):
+ if not isinstance(other, Variant):
+ return NotImplemented
+
+ return self < other or self == other
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __gt__(self, other):
+ if not isinstance(other, Variant):
+ return NotImplemented
+
+ return other < self
+
+ def __ge__(self, other):
+ if not isinstance(other, Variant):
+ return NotImplemented
+
+ return other <= self
+
+
+class VariantCall(object):
+ """A class to store info about a variant call.
+
+ A call represents the determination of genotype with respect to a particular
+ variant. It may include associated information such as quality and phasing.
+ """
+
+ def __init__(self, name=None, genotype=None, phaseset=None, info=None):
+ """Initialize the :class:`VariantCall` object.
+
+ Args:
+ name (str): The name of the call.
+ genotype (List[int]): The genotype of this variant call as specified by
+ the VCF schema. The values are either `0` representing the reference,
+ or a 1-based index into alternate bases. Ordering is only important if
+ `phaseset` is present. If a genotype is not called (that is, a `.` is
+ present in the GT string), -1 is used
+ phaseset (str): If this field is present, this variant call's genotype
+ ordering implies the phase of the bases and is consistent with any other
+ variant calls in the same reference sequence which have the same
+ phaseset value. If the genotype data was phased but no phase set was
+ specified, this field will be set to `*`.
+ info (dict): A map of additional variant call information. The key is
+ specified in the VCF record and the type of the value is specified by
+ the VCF header FORMAT.
+ """
+ self.name = name
+ self.genotype = genotype or []
+ self.phaseset = phaseset
+ self.info = info or {}
+
+ def __eq__(self, other):
+ return ((self.name, self.genotype, self.phaseset, self.info) ==
+ (other.name, other.genotype, other.phaseset, other.info))
+
+ def __repr__(self):
+ return ', '.join(
+ [str(s) for s in [self.name, self.genotype, self.phaseset, self.info]])
+
+
+class _VcfSource(filebasedsource.FileBasedSource):
+ """A source for reading VCF files.
+
+ Parses VCF files (version 4) using PyVCF library. If file_pattern specifies
+ multiple files, then the header from each file is used separately to parse
+ the content. However, the output will be a uniform PCollection of
+ :class:`Variant` objects.
+ """
+
+ DEFAULT_VCF_READ_BUFFER_SIZE = 65536 # 64kB
+
+ def __init__(self,
+ file_pattern,
+ compression_type=CompressionTypes.AUTO,
+ buffer_size=DEFAULT_VCF_READ_BUFFER_SIZE,
+ validate=True):
+ super(_VcfSource, self).__init__(file_pattern,
+ compression_type=compression_type,
+ validate=validate)
+
+ self._header_lines_per_file = {}
+ self._compression_type = compression_type
+ self._buffer_size = buffer_size
+
+ def read_records(self, file_name, range_tracker):
+ record_iterator = _VcfSource._VcfRecordIterator(
+ file_name,
+ range_tracker,
+ self._pattern,
+ self._compression_type,
+ buffer_size=self._buffer_size,
+ skip_header_lines=0)
+
+ # Convert iterator to generator to abstract behavior
+ for line in record_iterator:
+ yield line
+
+ class _VcfRecordIterator(object):
+ """An Iterator for processing a single VCF file."""
+
+ def __init__(self,
+ file_name,
+ range_tracker,
+ file_pattern,
+ compression_type,
+ **kwargs):
+ self._header_lines = []
+ self._last_record = None
+ self._file_name = file_name
+
+ text_source = TextSource(
+ file_pattern,
+ 0, # min_bundle_size
+ compression_type,
+ True, # strip_trailing_newlines
+ coders.StrUtf8Coder(), # coder
+ validate=False,
+ header_processor_fns=(lambda x: x.startswith('#'),
+ self._store_header_lines),
+ **kwargs)
+
+ self._text_lines = text_source.read_records(self._file_name,
+ range_tracker)
+ try:
+ self._vcf_reader = vcf.Reader(fsock=self._create_generator())
+ except SyntaxError as e:
+ raise ValueError('Invalid VCF header %s' % str(e))
+
+ def _store_header_lines(self, header_lines):
+ self._header_lines = header_lines
+
+ def _create_generator(self):
+ header_processed = False
+ for text_line in self._text_lines:
+ if not header_processed and self._header_lines:
+ for header in self._header_lines:
+ self._last_record = header
+ yield self._last_record
+ header_processed = True
+ # PyVCF has explicit str() calls when parsing INFO fields, which fails
+ # with UTF-8 decoded strings. Encode the line back to UTF-8.
+ self._last_record = text_line.encode('utf-8')
+ yield self._last_record
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ try:
+ record = next(self._vcf_reader)
+ return self._convert_to_variant_record(record, self._vcf_reader.infos,
+ self._vcf_reader.formats)
+ except (LookupError, ValueError) as e:
+ raise ValueError('Invalid record in VCF file. Error: %s' % str(e))
+
+ def _convert_to_variant_record(self, record, infos, formats):
+ """Converts the PyVCF record to a :class:`Variant` object.
+
+ Args:
+ record (:class:`~vcf.model._Record`): An object containing info about a
+ variant.
+ infos (dict): The PyVCF dict storing INFO extracted from the VCF header.
+ The key is the info key and the value is :class:`~vcf.parser._Info`.
+ formats (dict): The PyVCF dict storing FORMAT extracted from the VCF
+ header. The key is the FORMAT key and the value is
+ :class:`~vcf.parser._Format`.
+ Returns:
+ A :class:`Variant` object from the given record.
+ """
+ variant = Variant()
+ variant.reference_name = record.CHROM
+ variant.start = record.start
+ variant.end = record.end
+ variant.reference_bases = (
+ record.REF if record.REF != MISSING_FIELD_VALUE else None)
+ # ALT fields are classes in PyVCF (e.g. Substitution), so need convert
+ # them to their string representations.
+ variant.alternate_bases.extend(
+ [str(r) for r in record.ALT if r] if record.ALT else [])
+ variant.names.extend(record.ID.split(';') if record.ID else [])
+ variant.quality = record.QUAL
+ # PyVCF uses None for '.' and an empty list for 'PASS'.
+ if record.FILTER is not None:
+ variant.filters.extend(
+ record.FILTER if record.FILTER else [PASS_FILTER])
+ for k, v in record.INFO.iteritems():
+ # Special case: END info value specifies end of the record, so adjust
+ # variant.end and do not include it as part of variant.info.
+ if k == END_INFO_KEY:
+ variant.end = v
+ continue
+ field_count = None
+ if k in infos:
+ field_count = self._get_field_count_as_string(infos[k].num)
+ variant.info[k] = VariantInfo(data=v, field_count=field_count)
+ for sample in record.samples:
+ call = VariantCall()
+ call.name = sample.sample
+ for allele in sample.gt_alleles or [MISSING_GENOTYPE_VALUE]:
+ if allele is None:
+ allele = MISSING_GENOTYPE_VALUE
+ call.genotype.append(int(allele))
+ phaseset_from_format = (getattr(sample.data, PHASESET_FORMAT_KEY)
+ if PHASESET_FORMAT_KEY in sample.data._fields
+ else None)
+ # Note: Call is considered phased if it contains the 'PS' key regardless
+ # of whether it uses '|'.
+ if phaseset_from_format or sample.phased:
+ call.phaseset = (str(phaseset_from_format) if phaseset_from_format
+ else DEFAULT_PHASESET_VALUE)
+ for field in sample.data._fields:
+ # Genotype and phaseset (if present) are already included.
+ if field in (GENOTYPE_FORMAT_KEY, PHASESET_FORMAT_KEY):
+ continue
+ data = getattr(sample.data, field)
+ # Convert single values to a list for cases where the number of fields
+ # is unknown. This is to ensure consistent types across all records.
+ # Note: this is already done for INFO fields in PyVCF.
+ if (field in formats and
+ formats[field].num is None and
+ isinstance(data, (int, float, long, basestring, bool))):
+ data = [data]
+ call.info[field] = data
+ variant.calls.append(call)
+ return variant
+
+ def _get_field_count_as_string(self, field_count):
+ """Returns the string representation of field_count from PyVCF.
+
+ PyVCF converts field counts to an integer with some predefined constants
+ as specified in the vcf.parser.field_counts dict (e.g. 'A' is -1). This
+ method converts them back to their string representation to avoid having
+ direct dependency on the arbitrary PyVCF constants.
+ Args:
+ field_count (int): An integer representing the number of fields in INFO
+ as specified by PyVCF.
+ Returns:
+ A string representation of field_count (e.g. '-1' becomes 'A').
+ Raises:
+ ValueError: if the field_count is not valid.
+ """
+ if field_count is None:
+ return None
+ elif field_count >= 0:
+ return str(field_count)
+ field_count_to_string = {v: k for k, v in vcf.parser.field_counts.items()}
+ if field_count in field_count_to_string:
+ return field_count_to_string[field_count]
+ else:
+ raise ValueError('Invalid value for field_count: %d' % field_count)
+
+
+class ReadFromVcf(PTransform):
+ """A :class:`~apache_beam.transforms.ptransform.PTransform` for reading VCF
+ files.
+
+ Parses VCF files (version 4) using PyVCF library. If file_pattern specifies
+ multiple files, then the header from each file is used separately to parse
+ the content. However, the output will be a PCollection of
+ :class:`Variant` objects.
+ """
+
+ def __init__(
+ self,
+ file_pattern=None,
+ compression_type=CompressionTypes.AUTO,
+ validate=True,
+ **kwargs):
+ """Initialize the :class:`ReadFromVcf` transform.
+
+ Args:
+ file_pattern (str): The file path to read from either as a single file or
+ a glob pattern.
+ compression_type (str): Used to handle compressed input files.
+ Typical value is :attr:`CompressionTypes.AUTO
+ <apache_beam.io.filesystem.CompressionTypes.AUTO>`, in which case the
+ underlying file_path's extension will be used to detect the compression.
+ validate (bool): flag to verify that the files exist during the pipeline
+ creation time.
+ """
+ super(ReadFromVcf, self).__init__(**kwargs)
+ self._source = _VcfSource(
+ file_pattern, compression_type, validate=validate)
+
+ def expand(self, pvalue):
+ return pvalue.pipeline | Read(self._source)
diff --git a/sdks/python/apache_beam/io/vcfio_test.py b/sdks/python/apache_beam/io/vcfio_test.py
new file mode 100644
index 0000000..871b6e9
--- /dev/null
+++ b/sdks/python/apache_beam/io/vcfio_test.py
@@ -0,0 +1,519 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Tests for vcfio module."""
+
+import logging
+import os
+import unittest
+from itertools import permutations
+
+import apache_beam.io.source_test_utils as source_test_utils
+from apache_beam.io.vcfio import _VcfSource as VcfSource
+from apache_beam.io.vcfio import DEFAULT_PHASESET_VALUE
+from apache_beam.io.vcfio import MISSING_GENOTYPE_VALUE
+from apache_beam.io.vcfio import ReadFromVcf
+from apache_beam.io.vcfio import Variant
+from apache_beam.io.vcfio import VariantCall
+from apache_beam.io.vcfio import VariantInfo
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.test_utils import TempDir
+from apache_beam.testing.util import BeamAssertException
+from apache_beam.testing.util import assert_that
+
+# Note: mixing \n and \r\n to verify both behaviors.
+_SAMPLE_HEADER_LINES = [
+ '##fileformat=VCFv4.2\n',
+ '##INFO=<ID=NS,Number=1,Type=Integer,Description="Number samples">\n',
+ '##INFO=<ID=AF,Number=A,Type=Float,Description="Allele Frequency">\n',
+ '##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">\r\n',
+ '##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">\n',
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT Sample1 Sample2\r\n',
+]
+
+_SAMPLE_TEXT_LINES = [
+ '20 14370 . G A 29 PASS AF=0.5 GT:GQ 0|0:48 1|0:48\n',
+ '20 17330 . T A 3 q10 AF=0.017 GT:GQ 0|0:49 0|1:3\n',
+ '20 1110696 . A G,T 67 PASS AF=0.3,0.7 GT:GQ 1|2:21 2|1:2\n',
+ '20 1230237 . T . 47 PASS . GT:GQ 0|0:54 0|0:48\n',
+ '19 1234567 . GTCT G,GTACT 50 PASS . GT:GQ 0/1:35 0/2:17\n',
+ '20 1234 rs123 C A,T 50 PASS AF=0.5 GT:GQ 0/0:48 1/0:20\n',
+ '19 123 rs1234 GTC . 40 q10;s50 NS=2 GT:GQ 1|0:48 0/1:.\n',
+ '19 12 . C <SYMBOLIC> 49 q10 AF=0.5 GT:GQ 0|1:45 .:.\n'
+]
+
+
+def get_full_file_path(file_name):
+ """Returns the full path of the specified ``file_name`` from ``data``."""
+ return os.path.join(
+ os.path.dirname(__file__), '..', 'testing', 'data', 'vcf', file_name)
+
+
+def get_full_dir():
+ """Returns the full path of the ``data`` directory."""
+ return os.path.join(os.path.dirname(__file__), '..', 'testing', 'data', 'vcf')
+
+
+# Helper method for comparing variants.
+def _variant_comparator(v1, v2):
+ if v1.reference_name == v2.reference_name:
+ if v1.start == v2.start:
+ return cmp(v1.end, v2.end)
+ return cmp(v1.start, v2.start)
+ return cmp(v1.reference_name, v2.reference_name)
+
+
+# Helper method for verifying equal count on PCollection.
+def _count_equals_to(expected_count):
+ def _count_equal(actual_list):
+ actual_count = len(actual_list)
+ if expected_count != actual_count:
+ raise BeamAssertException(
+ 'Expected %d not equal actual %d' % (expected_count, actual_count))
+ return _count_equal
+
+
+class VcfSourceTest(unittest.TestCase):
+
+ # Distribution should skip tests that need VCF files due to large size
+ VCF_FILE_DIR_MISSING = not os.path.exists(get_full_dir())
+
+ def _create_temp_vcf_file(self, lines, tempdir):
+ return tempdir.create_temp_file(suffix='.vcf', lines=lines)
+
+ def _read_records(self, file_or_pattern):
+ return source_test_utils.read_from_source(VcfSource(file_or_pattern))
+
+ def _create_temp_file_and_read_records(self, lines):
+ with TempDir() as tempdir:
+ file_name = tempdir.create_temp_file(suffix='.vcf', lines=lines)
+ return self._read_records(file_name)
+
+ def _assert_variants_equal(self, actual, expected):
+ self.assertEqual(
+ sorted(expected),
+ sorted(actual))
+
+ def _get_sample_variant_1(self):
+ """Get first sample variant.
+
+ Features:
+ multiple alternates
+ not phased
+ multiple names
+ """
+ vcf_line = ('20 1234 rs123;rs2 C A,T 50 PASS AF=0.5,0.1;NS=1 '
+ 'GT:GQ 0/0:48 1/0:20\n')
+ variant = Variant(
+ reference_name='20', start=1233, end=1234, reference_bases='C',
+ alternate_bases=['A', 'T'], names=['rs123', 'rs2'], quality=50,
+ filters=['PASS'],
+ info={'AF': VariantInfo(data=[0.5, 0.1], field_count='A'),
+ 'NS': VariantInfo(data=1, field_count='1')})
+ variant.calls.append(
+ VariantCall(name='Sample1', genotype=[0, 0], info={'GQ': 48}))
+ variant.calls.append(
+ VariantCall(name='Sample2', genotype=[1, 0], info={'GQ': 20}))
+ return variant, vcf_line
+
+ def _get_sample_variant_2(self):
+ """Get second sample variant.
+
+ Features:
+ multiple references
+ no alternate
+ phased
+ multiple filters
+ missing format field
+ """
+ vcf_line = (
+ '19 123 rs1234 GTC . 40 q10;s50 NS=2 GT:GQ 1|0:48 0/1:.\n')
+ variant = Variant(
+ reference_name='19', start=122, end=125, reference_bases='GTC',
+ alternate_bases=[], names=['rs1234'], quality=40,
+ filters=['q10', 's50'],
+ info={'NS': VariantInfo(data=2, field_count='1')})
+ variant.calls.append(
+ VariantCall(name='Sample1', genotype=[1, 0],
+ phaseset=DEFAULT_PHASESET_VALUE,
+ info={'GQ': 48}))
+ variant.calls.append(
+ VariantCall(name='Sample2', genotype=[0, 1], info={'GQ': None}))
+ return variant, vcf_line
+
+ def _get_sample_variant_3(self):
+ """Get third sample variant.
+
+ Features:
+ symbolic alternate
+ no calls for sample 2
+ """
+ vcf_line = (
+ '19 12 . C <SYMBOLIC> 49 q10 AF=0.5 GT:GQ 0|1:45 .:.\n')
+ variant = Variant(
+ reference_name='19', start=11, end=12, reference_bases='C',
+ alternate_bases=['<SYMBOLIC>'], quality=49, filters=['q10'],
+ info={'AF': VariantInfo(data=[0.5], field_count='A')})
+ variant.calls.append(
+ VariantCall(name='Sample1', genotype=[0, 1],
+ phaseset=DEFAULT_PHASESET_VALUE,
+ info={'GQ': 45}))
+ variant.calls.append(
+ VariantCall(name='Sample2', genotype=[MISSING_GENOTYPE_VALUE],
+ info={'GQ': None}))
+ return variant, vcf_line
+
+ def test_sort_variants(self):
+ sorted_variants = [
+ Variant(reference_name='a', start=20, end=22),
+ Variant(reference_name='a', start=20, end=22, quality=20),
+ Variant(reference_name='b', start=20, end=22),
+ Variant(reference_name='b', start=21, end=22),
+ Variant(reference_name='b', start=21, end=23)]
+
+ for permutation in permutations(sorted_variants):
+ self.assertEqual(sorted(permutation), sorted_variants)
+
+ def test_variant_equality(self):
+ base_variant = Variant(reference_name='a', start=20, end=22,
+ reference_bases='a', alternate_bases=['g', 't'],
+ names=['variant'], quality=9, filters=['q10'],
+ info={'key': 'value'},
+ calls=[VariantCall(genotype=[0, 0])])
+ equal_variant = Variant(reference_name='a', start=20, end=22,
+ reference_bases='a', alternate_bases=['g', 't'],
+ names=['variant'], quality=9, filters=['q10'],
+ info={'key': 'value'},
+ calls=[VariantCall(genotype=[0, 0])])
+ different_calls = Variant(reference_name='a', start=20, end=22,
+ reference_bases='a', alternate_bases=['g', 't'],
+ names=['variant'], quality=9, filters=['q10'],
+ info={'key': 'value'},
+ calls=[VariantCall(genotype=[1, 0])])
+ missing_field = Variant(reference_name='a', start=20, end=22,
+ reference_bases='a', alternate_bases=['g', 't'],
+ names=['variant'], quality=9, filters=['q10'],
+ info={'key': 'value'})
+
+ self.assertEqual(base_variant, equal_variant)
+ self.assertNotEqual(base_variant, different_calls)
+ self.assertNotEqual(base_variant, missing_field)
+
+ @unittest.skipIf(VCF_FILE_DIR_MISSING, 'VCF test file directory is missing')
+ def test_read_single_file_large(self):
+ test_data_conifgs = [
+ {'file': 'valid-4.0.vcf', 'num_records': 5},
+ {'file': 'valid-4.0.vcf.gz', 'num_records': 5},
+ {'file': 'valid-4.0.vcf.bz2', 'num_records': 5},
+ {'file': 'valid-4.1-large.vcf', 'num_records': 9882},
+ {'file': 'valid-4.2.vcf', 'num_records': 13},
+ ]
+ for config in test_data_conifgs:
+ read_data = self._read_records(
+ get_full_file_path(config['file']))
+ self.assertEqual(config['num_records'], len(read_data))
+
+ @unittest.skipIf(VCF_FILE_DIR_MISSING, 'VCF test file directory is missing')
+ def test_read_file_pattern_large(self):
+ read_data = self._read_records(
+ os.path.join(get_full_dir(), 'valid-*.vcf'))
+ self.assertEqual(9900, len(read_data))
+ read_data_gz = self._read_records(
+ os.path.join(get_full_dir(), 'valid-*.vcf.gz'))
+ self.assertEqual(9900, len(read_data_gz))
+
+ def test_single_file_no_records(self):
+ self.assertEqual(
+ [], self._create_temp_file_and_read_records(['']))
+ self.assertEqual(
+ [], self._create_temp_file_and_read_records(['\n', '\r\n', '\n']))
+ self.assertEqual(
+ [], self._create_temp_file_and_read_records(_SAMPLE_HEADER_LINES))
+
+ def test_single_file_verify_details(self):
+ variant_1, vcf_line_1 = self._get_sample_variant_1()
+ read_data = self._create_temp_file_and_read_records(
+ _SAMPLE_HEADER_LINES + [vcf_line_1])
+ self.assertEqual(1, len(read_data))
+ self.assertEqual(variant_1, read_data[0])
+ variant_2, vcf_line_2 = self._get_sample_variant_2()
+ variant_3, vcf_line_3 = self._get_sample_variant_3()
+ read_data = self._create_temp_file_and_read_records(
+ _SAMPLE_HEADER_LINES + [vcf_line_1, vcf_line_2, vcf_line_3])
+ self.assertEqual(3, len(read_data))
+ self._assert_variants_equal([variant_1, variant_2, variant_3], read_data)
+
+ def test_file_pattern_verify_details(self):
+ variant_1, vcf_line_1 = self._get_sample_variant_1()
+ variant_2, vcf_line_2 = self._get_sample_variant_2()
+ variant_3, vcf_line_3 = self._get_sample_variant_3()
+ with TempDir() as tempdir:
+ self._create_temp_vcf_file(_SAMPLE_HEADER_LINES + [vcf_line_1], tempdir)
+ self._create_temp_vcf_file((_SAMPLE_HEADER_LINES +
+ [vcf_line_2, vcf_line_3]),
+ tempdir)
+ read_data = self._read_records(os.path.join(tempdir.get_path(), '*.vcf'))
+ self.assertEqual(3, len(read_data))
+ self._assert_variants_equal([variant_1, variant_2, variant_3], read_data)
+
+ @unittest.skipIf(VCF_FILE_DIR_MISSING, 'VCF test file directory is missing')
+ def test_read_after_splitting(self):
+ file_name = get_full_file_path('valid-4.1-large.vcf')
+ source = VcfSource(file_name)
+ splits = [p for p in source.split(desired_bundle_size=500)]
+ self.assertGreater(len(splits), 1)
+ sources_info = ([
+ (split.source, split.start_position, split.stop_position) for
+ split in splits])
+ self.assertGreater(len(sources_info), 1)
+ split_records = []
+ for source_info in sources_info:
+ split_records.extend(source_test_utils.read_from_source(*source_info))
+ self.assertEqual(9882, len(split_records))
+
+ def test_invalid_file(self):
+ invalid_file_contents = [
+ # Malfromed record.
+ [
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '1 1 '
+ ],
+ # Missing "GT:GQ" format, but GQ is provided.
+ [
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '19 123 rs12345 T C 50 q10 AF=0.2;NS=2 GT 1|0:48'
+ ],
+ # GT is not an integer.
+ [
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '19 123 rs12345 T C 50 q10 AF=0.2;NS=2 GT A|0'
+ ],
+ # Malformed FILTER.
+ [
+ '##FILTER=<ID=PASS,Description="All filters passed">\n',
+ '##FILTER=<ID=LowQual,Descri\n',
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '19 123 rs12345 T C 50 q10 AF=0.2;NS=2 GT:GQ 1|0:48',
+ ],
+ # Invalid Number value for INFO.
+ [
+ '##INFO=<ID=G,Number=U,Type=String,Description="InvalidNumber">\n',
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '19 123 rs12345 T C 50 q10 AF=0.2;NS=2 GT:GQ 1|0:48\n',
+ ],
+ # POS should be an integer.
+ [
+ '##FILTER=<ID=PASS,Description="All filters passed">\n',
+ '##FILTER=<ID=LowQual,Descri\n',
+ '#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SampleName\n',
+ '19 abc rs12345 T C 50 q10 AF=0.2;NS=2 GT:GQ 1|0:48\n',
+ ],
+ ]
+ for content in invalid_file_contents:
+ try:
+ with TempDir() as tempdir:
+ self._read_records(self._create_temp_vcf_file(content, tempdir))
+ self.fail('Invalid VCF file must throw an exception')
+ except ValueError:
+ pass
+ # Try with multiple files (any one of them will throw an exception).
+ with TempDir() as tempdir:
+ for content in invalid_file_contents:
+ self._create_temp_vcf_file(content, tempdir)
+ try:
+ self._read_records(os.path.join(tempdir.get_path(), '*.vcf'))
+ self.fail('Invalid VCF file must throw an exception.')
+ except ValueError:
+ pass
+
+ def test_no_samples(self):
+ header_line = '#CHROM POS ID REF ALT QUAL FILTER INFO\n'
+ record_line = '19 123 . G A . PASS AF=0.2'
+ expected_variant = Variant(
+ reference_name='19', start=122, end=123, reference_bases='G',
+ alternate_bases=['A'], filters=['PASS'],
+ info={'AF': VariantInfo(data=[0.2], field_count='A')})
+ read_data = self._create_temp_file_and_read_records(
+ _SAMPLE_HEADER_LINES[:-1] + [header_line, record_line])
+ self.assertEqual(1, len(read_data))
+ self.assertEqual(expected_variant, read_data[0])
+
+ def test_no_info(self):
+ record_line = 'chr19 123 . . . . . . GT . .'
+ expected_variant = Variant(reference_name='chr19', start=122, end=123)
+ expected_variant.calls.append(
+ VariantCall(name='Sample1', genotype=[MISSING_GENOTYPE_VALUE]))
+ expected_variant.calls.append(
+ VariantCall(name='Sample2', genotype=[MISSING_GENOTYPE_VALUE]))
+ read_data = self._create_temp_file_and_read_records(
+ _SAMPLE_HEADER_LINES + [record_line])
+ self.assertEqual(1, len(read_data))
+ self.assertEqual(expected_variant, read_data[0])
+
+ def test_info_numbers_and_types(self):
+ info_headers = [
+ '##INFO=<ID=HA,Number=A,Type=String,Description="StringInfo_A">\n',
+ '##INFO=<ID=HG,Number=G,Type=Integer,Description="IntInfo_G">\n',
+ '##INFO=<ID=HR,Number=R,Type=Character,Description="ChrInfo_R">\n',
+ '##INFO=<ID=HF,Number=0,Type=Flag,Description="FlagInfo">\n',
+ '##INFO=<ID=HU,Number=.,Type=Float,Description="FloatInfo_variable">\n']
+ record_lines = [
+ '19 2 . A T,C . . HA=a1,a2;HG=1,2,3;HR=a,b,c;HF;HU=0.1 GT 1/0 0/1\n',
+ '19 124 . A T . . HG=3,4,5;HR=d,e;HU=1.1,1.2 GT 0/0 0/1']
+ variant_1 = Variant(
+ reference_name='19', start=1, end=2, reference_bases='A',
+ alternate_bases=['T', 'C'],
+ info={'HA': VariantInfo(data=['a1', 'a2'], field_count='A'),
+ 'HG': VariantInfo(data=[1, 2, 3], field_count='G'),
+ 'HR': VariantInfo(data=['a', 'b', 'c'], field_count='R'),
+ 'HF': VariantInfo(data=True, field_count='0'),
+ 'HU': VariantInfo(data=[0.1], field_count=None)})
+ variant_1.calls.append(VariantCall(name='Sample1', genotype=[1, 0]))
+ variant_1.calls.append(VariantCall(name='Sample2', genotype=[0, 1]))
+ variant_2 = Variant(
+ reference_name='19', start=123, end=124, reference_bases='A',
+ alternate_bases=['T'],
+ info={'HG': VariantInfo(data=[3, 4, 5], field_count='G'),
+ 'HR': VariantInfo(data=['d', 'e'], field_count='R'),
+ 'HU': VariantInfo(data=[1.1, 1.2], field_count=None)})
+ variant_2.calls.append(VariantCall(name='Sample1', genotype=[0, 0]))
+ variant_2.calls.append(VariantCall(name='Sample2', genotype=[0, 1]))
+ read_data = self._create_temp_file_and_read_records(
+ info_headers + _SAMPLE_HEADER_LINES[1:] + record_lines)
+ self.assertEqual(2, len(read_data))
+ self._assert_variants_equal([variant_1, variant_2], read_data)
+
+ def test_end_info_key(self):
+ phaseset_header_line = (
+ '##INFO=<ID=END,Number=1,Type=Integer,Description="End of record.">\n')
+ record_lines = ['19 123 . A . . . END=1111 GT 1/0 0/1\n',
+ '19 123 . A . . . . GT 0/1 1/1\n']
+ variant_1 = Variant(
+ reference_name='19', start=122, end=1111, reference_bases='A')
+ variant_1.calls.append(VariantCall(name='Sample1', genotype=[1, 0]))
+ variant_1.calls.append(VariantCall(name='Sample2', genotype=[0, 1]))
+ variant_2 = Variant(
+ reference_name='19', start=122, end=123, reference_bases='A')
+ variant_2.calls.append(VariantCall(name='Sample1', genotype=[0, 1]))
+ variant_2.calls.append(VariantCall(name='Sample2', genotype=[1, 1]))
+ read_data = self._create_temp_file_and_read_records(
+ [phaseset_header_line] + _SAMPLE_HEADER_LINES[1:] + record_lines)
+ self.assertEqual(2, len(read_data))
+ self._assert_variants_equal([variant_1, variant_2], read_data)
+
+ def test_custom_phaseset(self):
+ phaseset_header_line = (
+ '##FORMAT=<ID=PS,Number=1,Type=Integer,Description="Phaseset">\n')
+ record_lines = ['19 123 . A T . . . GT:PS 1|0:1111 0/1:.\n',
+ '19 121 . A T . . . GT:PS 1|0:2222 0/1:2222\n']
+ variant_1 = Variant(
+ reference_name='19', start=122, end=123, reference_bases='A',
+ alternate_bases=['T'])
+ variant_1.calls.append(
+ VariantCall(name='Sample1', genotype=[1, 0], phaseset='1111'))
+ variant_1.calls.append(VariantCall(name='Sample2', genotype=[0, 1]))
+ variant_2 = Variant(
+ reference_name='19', start=120, end=121, reference_bases='A',
+ alternate_bases=['T'])
+ variant_2.calls.append(
+ VariantCall(name='Sample1', genotype=[1, 0], phaseset='2222'))
+ variant_2.calls.append(
+ VariantCall(name='Sample2', genotype=[0, 1], phaseset='2222'))
+ read_data = self._create_temp_file_and_read_records(
+ [phaseset_header_line] + _SAMPLE_HEADER_LINES[1:] + record_lines)
+ self.assertEqual(2, len(read_data))
+ self._assert_variants_equal([variant_1, variant_2], read_data)
+
+ def test_format_numbers(self):
+ format_headers = [
+ '##FORMAT=<ID=FU,Number=.,Type=String,Description="Format_variable">\n',
+ '##FORMAT=<ID=F1,Number=1,Type=Integer,Description="Format_one">\n',
+ '##FORMAT=<ID=F2,Number=2,Type=Character,Description="Format_two">\n']
+ record_lines = [
+ '19 2 . A T,C . . . GT:FU:F1:F2 1/0:a1:3:a,b 0/1:a2,a3:4:b,c\n']
+ expected_variant = Variant(
+ reference_name='19', start=1, end=2, reference_bases='A',
+ alternate_bases=['T', 'C'])
+ expected_variant.calls.append(VariantCall(
+ name='Sample1',
+ genotype=[1, 0],
+ info={'FU': ['a1'], 'F1': 3, 'F2': ['a', 'b']}))
+ expected_variant.calls.append(VariantCall(
+ name='Sample2',
+ genotype=[0, 1],
+ info={'FU': ['a2', 'a3'], 'F1': 4, 'F2': ['b', 'c']}))
+ read_data = self._create_temp_file_and_read_records(
+ format_headers + _SAMPLE_HEADER_LINES[1:] + record_lines)
+ self.assertEqual(1, len(read_data))
+ self.assertEqual(expected_variant, read_data[0])
+
+ def test_pipeline_read_single_file(self):
+ with TempDir() as tempdir:
+ file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES +
+ _SAMPLE_TEXT_LINES, tempdir)
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromVcf(file_name)
+ assert_that(pcoll, _count_equals_to(len(_SAMPLE_TEXT_LINES)))
+ pipeline.run()
+
+ @unittest.skipIf(VCF_FILE_DIR_MISSING, 'VCF test file directory is missing')
+ def test_pipeline_read_single_file_large(self):
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromVcf(
+ get_full_file_path('valid-4.0.vcf'))
+ assert_that(pcoll, _count_equals_to(5))
+ pipeline.run()
+
+ @unittest.skipIf(VCF_FILE_DIR_MISSING, 'VCF test file directory is missing')
+ def test_pipeline_read_file_pattern_large(self):
+ pipeline = TestPipeline()
+ pcoll = pipeline | 'Read' >> ReadFromVcf(
+ os.path.join(get_full_dir(), 'valid-*.vcf'))
+ assert_that(pcoll, _count_equals_to(9900))
+ pipeline.run()
+
+ def test_read_reentrant_without_splitting(self):
+ with TempDir() as tempdir:
+ file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES +
+ _SAMPLE_TEXT_LINES, tempdir)
+ source = VcfSource(file_name)
+ source_test_utils.assert_reentrant_reads_succeed((source, None, None))
+
+ def test_read_reentrant_after_splitting(self):
+ with TempDir() as tempdir:
+ file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES +
+ _SAMPLE_TEXT_LINES, tempdir)
+ source = VcfSource(file_name)
+ splits = [split for split in source.split(desired_bundle_size=100000)]
+ assert len(splits) == 1
+ source_test_utils.assert_reentrant_reads_succeed(
+ (splits[0].source, splits[0].start_position, splits[0].stop_position))
+
+ def test_dynamic_work_rebalancing(self):
+ with TempDir() as tempdir:
+ file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES +
+ _SAMPLE_TEXT_LINES, tempdir)
+ source = VcfSource(file_name)
+ splits = [split for split in source.split(desired_bundle_size=100000)]
+ assert len(splits) == 1
+ source_test_utils.assert_split_at_fraction_exhaustive(
+ splits[0].source, splits[0].start_position, splits[0].stop_position)
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/metrics/cells_test.py b/sdks/python/apache_beam/metrics/cells_test.py
index a4c8a43..c0664ab 100644
--- a/sdks/python/apache_beam/metrics/cells_test.py
+++ b/sdks/python/apache_beam/metrics/cells_test.py
@@ -18,10 +18,10 @@
import threading
import unittest
+from apache_beam.metrics.cells import CellCommitState
from apache_beam.metrics.cells import CounterCell
from apache_beam.metrics.cells import DistributionCell
from apache_beam.metrics.cells import DistributionData
-from apache_beam.metrics.cells import CellCommitState
class TestCounterCell(unittest.TestCase):
diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py
index 675e49c..1704b98 100644
--- a/sdks/python/apache_beam/metrics/execution.py
+++ b/sdks/python/apache_beam/metrics/execution.py
@@ -29,10 +29,11 @@
- MetricsContainer - Holds the metrics of a single step and a single
unit-of-commit (bundle).
"""
-from collections import defaultdict
import threading
+from collections import defaultdict
-from apache_beam.metrics.cells import CounterCell, DistributionCell
+from apache_beam.metrics.cells import CounterCell
+from apache_beam.metrics.cells import DistributionCell
class MetricKey(object):
diff --git a/sdks/python/apache_beam/metrics/execution_test.py b/sdks/python/apache_beam/metrics/execution_test.py
index 54569c1..855f54c 100644
--- a/sdks/python/apache_beam/metrics/execution_test.py
+++ b/sdks/python/apache_beam/metrics/execution_test.py
@@ -18,10 +18,10 @@
import unittest
from apache_beam.metrics.cells import CellCommitState
-from apache_beam.metrics.execution import MetricsContainer
-from apache_beam.metrics.execution import ScopedMetricsContainer
-from apache_beam.metrics.execution import MetricsEnvironment
from apache_beam.metrics.execution import MetricKey
+from apache_beam.metrics.execution import MetricsContainer
+from apache_beam.metrics.execution import MetricsEnvironment
+from apache_beam.metrics.execution import ScopedMetricsContainer
from apache_beam.metrics.metric import Metrics
from apache_beam.metrics.metricbase import MetricName
@@ -29,9 +29,9 @@
class TestMetricsContainer(unittest.TestCase):
def test_create_new_counter(self):
mc = MetricsContainer('astep')
- self.assertFalse(mc.counters.has_key(MetricName('namespace', 'name')))
+ self.assertFalse(MetricName('namespace', 'name') in mc.counters)
mc.get_counter(MetricName('namespace', 'name'))
- self.assertTrue(mc.counters.has_key(MetricName('namespace', 'name')))
+ self.assertTrue(MetricName('namespace', 'name') in mc.counters)
def test_scoped_container(self):
c1 = MetricsContainer('mystep')
diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py
index f99c0c4..8fbf980 100644
--- a/sdks/python/apache_beam/metrics/metric.py
+++ b/sdks/python/apache_beam/metrics/metric.py
@@ -27,7 +27,8 @@
import inspect
from apache_beam.metrics.execution import MetricsEnvironment
-from apache_beam.metrics.metricbase import Counter, Distribution
+from apache_beam.metrics.metricbase import Counter
+from apache_beam.metrics.metricbase import Distribution
from apache_beam.metrics.metricbase import MetricName
__all__ = ['Metrics', 'MetricsFilter']
diff --git a/sdks/python/apache_beam/metrics/metric_test.py b/sdks/python/apache_beam/metrics/metric_test.py
index 56b7468..ef98b2d 100644
--- a/sdks/python/apache_beam/metrics/metric_test.py
+++ b/sdks/python/apache_beam/metrics/metric_test.py
@@ -21,9 +21,9 @@
from apache_beam.metrics.execution import MetricKey
from apache_beam.metrics.execution import MetricsContainer
from apache_beam.metrics.execution import MetricsEnvironment
+from apache_beam.metrics.metric import MetricResults
from apache_beam.metrics.metric import Metrics
from apache_beam.metrics.metric import MetricsFilter
-from apache_beam.metrics.metric import MetricResults
from apache_beam.metrics.metricbase import MetricName
@@ -98,6 +98,22 @@
with self.assertRaises(ValueError):
Metrics.get_namespace(object())
+ def test_counter_empty_name(self):
+ with self.assertRaises(ValueError):
+ Metrics.counter("namespace", "")
+
+ def test_counter_empty_namespace(self):
+ with self.assertRaises(ValueError):
+ Metrics.counter("", "names")
+
+ def test_distribution_empty_name(self):
+ with self.assertRaises(ValueError):
+ Metrics.distribution("namespace", "")
+
+ def test_distribution_empty_namespace(self):
+ with self.assertRaises(ValueError):
+ Metrics.distribution("", "names")
+
def test_create_counter_distribution(self):
MetricsEnvironment.set_current_container(MetricsContainer('mystep'))
counter_ns = 'aCounterNamespace'
diff --git a/sdks/python/apache_beam/metrics/metricbase.py b/sdks/python/apache_beam/metrics/metricbase.py
index 699f29cb..9b19189 100644
--- a/sdks/python/apache_beam/metrics/metricbase.py
+++ b/sdks/python/apache_beam/metrics/metricbase.py
@@ -47,6 +47,10 @@
namespace: A string with the namespace of a metric.
name: A string with the name of a metric.
"""
+ if not namespace:
+ raise ValueError('Metric namespace must be non-empty')
+ if not name:
+ raise ValueError('Metric name must be non-empty')
self.namespace = namespace
self.name = name
diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index 777926a..5278b8a 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -19,11 +19,10 @@
import argparse
-from apache_beam.transforms.display import HasDisplayData
-from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.options.value_provider import ValueProvider
-
+from apache_beam.transforms.display import HasDisplayData
__all__ = [
'PipelineOptions',
@@ -51,7 +50,7 @@
"""
def _f(value):
- _f.func_name = value_type.__name__
+ _f.__name__ = value_type.__name__
return StaticValueProvider(value_type, value)
return _f
@@ -278,14 +277,6 @@
action='store_true',
help='Whether to enable streaming mode.')
- # TODO(BEAM-1265): Remove this error, once at least one runner supports
- # streaming pipelines.
- def validate(self, validator):
- errors = []
- if self.view_as(StandardOptions).streaming:
- errors.append('Streaming pipelines are not supported.')
- return errors
-
class TypeOptions(PipelineOptions):
@@ -322,6 +313,13 @@
help='DirectRunner uses stacked WindowedValues within a Bundle for '
'memory optimization. Set --no_direct_runner_use_stacked_bundle to '
'avoid it.')
+ parser.add_argument(
+ '--direct_runner_bundle_retry',
+ action='store_true',
+ default=False,
+ help=
+ ('Whether to allow bundle retries. If True the maximum'
+ 'number of attempts to process a bundle is 4. '))
class GoogleCloudOptions(PipelineOptions):
@@ -376,6 +374,13 @@
parser.add_argument('--template_location',
default=None,
help='Save job to specified local or GCS location.')
+ parser.add_argument(
+ '--label', '--labels',
+ dest='labels',
+ action='append',
+ default=None,
+ help='Labels that will be applied to this Dataflow job. Labels are key '
+ 'value pairs separated by = (e.g. --label key=value).')
def validate(self, validator):
errors = []
@@ -473,7 +478,14 @@
parser.add_argument(
'--use_public_ips',
default=None,
- help='Whether to assign public IP addresses to the worker machines.')
+ action='store_true',
+ help='Whether to assign public IP addresses to the worker VMs.')
+ parser.add_argument(
+ '--no_use_public_ips',
+ dest='use_public_ips',
+ default=None,
+ action='store_false',
+ help='Whether to assign only private IP addresses to the worker VMs.')
def validate(self, validator):
errors = []
@@ -553,6 +565,18 @@
'worker will install the resulting package before running any custom '
'code.'))
parser.add_argument(
+ '--beam_plugin', '--beam_plugin',
+ dest='beam_plugins',
+ action='append',
+ default=None,
+ help=
+ ('Bootstrap the python process before executing any code by importing '
+ 'all the plugins used in the pipeline. Please pass a comma separated'
+ 'list of import paths to be included. This is currently an '
+ 'experimental flag and provides no stability. Multiple '
+ '--beam_plugin options can be specified if more than one plugin '
+ 'is needed.'))
+ parser.add_argument(
'--save_main_session',
default=False,
action='store_true',
@@ -578,11 +602,12 @@
default=None,
help=
('Local path to a Python package file. The file is expected to be (1) '
- 'a package tarball (".tar") or (2) a compressed package tarball '
- '(".tar.gz") which can be installed using the "pip install" command '
- 'of the standard pip package. Multiple --extra_package options can '
- 'be specified if more than one package is needed. During job '
- 'submission, the files will be staged in the staging area '
+ 'a package tarball (".tar"), (2) a compressed package tarball '
+ '(".tar.gz"), (3) a Wheel file (".whl") or (4) a compressed package '
+ 'zip file (".zip") which can be installed using the "pip install" '
+ 'command of the standard pip package. Multiple --extra_package '
+ 'options can be specified if more than one package is needed. During '
+ 'job submission, the files will be staged in the staging area '
'(--staging_location option) and the workers will install them in '
'same order they were specified on the command line.'))
@@ -598,6 +623,11 @@
help=('Verify state/output of e2e test pipeline. This is pickled '
'version of the matcher which should extends '
'hamcrest.core.base_matcher.BaseMatcher.'))
+ parser.add_argument(
+ '--dry_run',
+ default=False,
+ help=('Used in unit testing runners without submitting the '
+ 'actual job.'))
def validate(self, validator):
errors = []
diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py
index 1a644b4..66c69bd 100644
--- a/sdks/python/apache_beam/options/pipeline_options_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_test.py
@@ -21,11 +21,12 @@
import unittest
import hamcrest as hc
+
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.options.pipeline_options import PipelineOptions
-from apache_beam.options.value_provider import StaticValueProvider
-from apache_beam.options.value_provider import RuntimeValueProvider
class PipelineOptionsTest(unittest.TestCase):
@@ -192,47 +193,52 @@
options = PipelineOptions(['--redefined_flag'])
self.assertTrue(options.get_all_options()['redefined_flag'])
+ # TODO(BEAM-1319): Require unique names only within a test.
+ # For now, <file name acronym>_vp_arg<number> will be the convention
+ # to name value-provider arguments in tests, as opposed to
+ # <file name acronym>_non_vp_arg<number> for non-value-provider arguments.
+ # The number will grow per file as tests are added.
def test_value_provider_options(self):
class UserOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- '--vp_arg',
+ '--pot_vp_arg1',
help='This flag is a value provider')
parser.add_value_provider_argument(
- '--vp_arg2',
+ '--pot_vp_arg2',
default=1,
type=int)
parser.add_argument(
- '--non_vp_arg',
+ '--pot_non_vp_arg1',
default=1,
type=int
)
# Provide values: if not provided, the option becomes of the type runtime vp
- options = UserOptions(['--vp_arg', 'hello'])
- self.assertIsInstance(options.vp_arg, StaticValueProvider)
- self.assertIsInstance(options.vp_arg2, RuntimeValueProvider)
- self.assertIsInstance(options.non_vp_arg, int)
+ options = UserOptions(['--pot_vp_arg1', 'hello'])
+ self.assertIsInstance(options.pot_vp_arg1, StaticValueProvider)
+ self.assertIsInstance(options.pot_vp_arg2, RuntimeValueProvider)
+ self.assertIsInstance(options.pot_non_vp_arg1, int)
# Values can be overwritten
- options = UserOptions(vp_arg=5,
- vp_arg2=StaticValueProvider(value_type=str,
- value='bye'),
- non_vp_arg=RuntimeValueProvider(
+ options = UserOptions(pot_vp_arg1=5,
+ pot_vp_arg2=StaticValueProvider(value_type=str,
+ value='bye'),
+ pot_non_vp_arg1=RuntimeValueProvider(
option_name='foo',
value_type=int,
default_value=10))
- self.assertEqual(options.vp_arg, 5)
- self.assertTrue(options.vp_arg2.is_accessible(),
- '%s is not accessible' % options.vp_arg2)
- self.assertEqual(options.vp_arg2.get(), 'bye')
- self.assertFalse(options.non_vp_arg.is_accessible())
+ self.assertEqual(options.pot_vp_arg1, 5)
+ self.assertTrue(options.pot_vp_arg2.is_accessible(),
+ '%s is not accessible' % options.pot_vp_arg2)
+ self.assertEqual(options.pot_vp_arg2.get(), 'bye')
+ self.assertFalse(options.pot_non_vp_arg1.is_accessible())
with self.assertRaises(RuntimeError):
- options.non_vp_arg.get()
+ options.pot_non_vp_arg1.get()
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/options/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py
index 28fcbe3..8ff66c7 100644
--- a/sdks/python/apache_beam/options/pipeline_options_validator_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py
@@ -20,10 +20,11 @@
import logging
import unittest
+from hamcrest.core.base_matcher import BaseMatcher
+
from apache_beam.internal import pickler
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator
-from hamcrest.core.base_matcher import BaseMatcher
# Mock runners to use for validations.
@@ -300,14 +301,6 @@
errors = validator.validate()
self.assertFalse(errors)
- def test_streaming(self):
- pipeline_options = PipelineOptions(['--streaming'])
- runner = MockRunners.TestDataflowRunner()
- validator = PipelineOptionsValidator(pipeline_options, runner)
- errors = validator.validate()
-
- self.assertIn('Streaming pipelines are not supported.', errors)
-
def test_test_matcher(self):
def get_validator(matcher):
options = ['--project=example:example',
diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py
index 40bddba..fe4614d 100644
--- a/sdks/python/apache_beam/options/value_provider.py
+++ b/sdks/python/apache_beam/options/value_provider.py
@@ -23,7 +23,6 @@
from apache_beam import error
-
__all__ = [
'ValueProvider',
'StaticValueProvider',
@@ -82,7 +81,6 @@
value = self.default_value
return value
- # TODO(BEAM-1999): Remove _unused_options_id
@classmethod
def set_runtime_options(cls, pipeline_options):
RuntimeValueProvider.runtime_options = pipeline_options
diff --git a/sdks/python/apache_beam/options/value_provider_test.py b/sdks/python/apache_beam/options/value_provider_test.py
index 3a45e8b..17e9590 100644
--- a/sdks/python/apache_beam/options/value_provider_test.py
+++ b/sdks/python/apache_beam/options/value_provider_test.py
@@ -24,72 +24,77 @@
from apache_beam.options.value_provider import StaticValueProvider
+# TODO(BEAM-1319): Require unique names only within a test.
+# For now, <file name acronym>_vp_arg<number> will be the convention
+# to name value-provider arguments in tests, as opposed to
+# <file name acronym>_non_vp_arg<number> for non-value-provider arguments.
+# The number will grow per file as tests are added.
class ValueProviderTests(unittest.TestCase):
def test_static_value_provider_keyword_argument(self):
class UserDefinedOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- '--vp_arg',
+ '--vpt_vp_arg1',
help='This keyword argument is a value provider',
default='some value')
- options = UserDefinedOptions(['--vp_arg', 'abc'])
- self.assertTrue(isinstance(options.vp_arg, StaticValueProvider))
- self.assertTrue(options.vp_arg.is_accessible())
- self.assertEqual(options.vp_arg.get(), 'abc')
+ options = UserDefinedOptions(['--vpt_vp_arg1', 'abc'])
+ self.assertTrue(isinstance(options.vpt_vp_arg1, StaticValueProvider))
+ self.assertTrue(options.vpt_vp_arg1.is_accessible())
+ self.assertEqual(options.vpt_vp_arg1.get(), 'abc')
def test_runtime_value_provider_keyword_argument(self):
class UserDefinedOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- '--vp_arg',
+ '--vpt_vp_arg2',
help='This keyword argument is a value provider')
options = UserDefinedOptions()
- self.assertTrue(isinstance(options.vp_arg, RuntimeValueProvider))
- self.assertFalse(options.vp_arg.is_accessible())
+ self.assertTrue(isinstance(options.vpt_vp_arg2, RuntimeValueProvider))
+ self.assertFalse(options.vpt_vp_arg2.is_accessible())
with self.assertRaises(RuntimeError):
- options.vp_arg.get()
+ options.vpt_vp_arg2.get()
def test_static_value_provider_positional_argument(self):
class UserDefinedOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- 'vp_pos_arg',
+ 'vpt_vp_arg3',
help='This positional argument is a value provider',
default='some value')
options = UserDefinedOptions(['abc'])
- self.assertTrue(isinstance(options.vp_pos_arg, StaticValueProvider))
- self.assertTrue(options.vp_pos_arg.is_accessible())
- self.assertEqual(options.vp_pos_arg.get(), 'abc')
+ self.assertTrue(isinstance(options.vpt_vp_arg3, StaticValueProvider))
+ self.assertTrue(options.vpt_vp_arg3.is_accessible())
+ self.assertEqual(options.vpt_vp_arg3.get(), 'abc')
def test_runtime_value_provider_positional_argument(self):
class UserDefinedOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- 'vp_pos_arg',
+ 'vpt_vp_arg4',
help='This positional argument is a value provider')
options = UserDefinedOptions([])
- self.assertTrue(isinstance(options.vp_pos_arg, RuntimeValueProvider))
- self.assertFalse(options.vp_pos_arg.is_accessible())
+ self.assertTrue(isinstance(options.vpt_vp_arg4, RuntimeValueProvider))
+ self.assertFalse(options.vpt_vp_arg4.is_accessible())
with self.assertRaises(RuntimeError):
- options.vp_pos_arg.get()
+ options.vpt_vp_arg4.get()
def test_static_value_provider_type_cast(self):
class UserDefinedOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- '--vp_arg',
+ '--vpt_vp_arg5',
type=int,
help='This flag is a value provider')
- options = UserDefinedOptions(['--vp_arg', '123'])
- self.assertTrue(isinstance(options.vp_arg, StaticValueProvider))
- self.assertTrue(options.vp_arg.is_accessible())
- self.assertEqual(options.vp_arg.get(), 123)
+ options = UserDefinedOptions(['--vpt_vp_arg5', '123'])
+ self.assertTrue(isinstance(options.vpt_vp_arg5, StaticValueProvider))
+ self.assertTrue(options.vpt_vp_arg5.is_accessible())
+ self.assertEqual(options.vpt_vp_arg5.get(), 123)
def test_set_runtime_option(self):
# define ValueProvider ptions, with and without default values
@@ -97,25 +102,25 @@
@classmethod
def _add_argparse_args(cls, parser):
parser.add_value_provider_argument(
- '--vp_arg',
+ '--vpt_vp_arg6',
help='This keyword argument is a value provider') # set at runtime
parser.add_value_provider_argument( # not set, had default int
- '-v', '--vp_arg2', # with short form
+ '-v', '--vpt_vp_arg7', # with short form
default=123,
type=int)
parser.add_value_provider_argument( # not set, had default str
- '--vp-arg3', # with dash in name
+ '--vpt_vp-arg8', # with dash in name
default='123',
type=str)
parser.add_value_provider_argument( # not set and no default
- '--vp_arg4',
+ '--vpt_vp_arg9',
type=float)
parser.add_value_provider_argument( # positional argument set
- 'vp_pos_arg', # default & runtime ignored
+ 'vpt_vp_arg10', # default & runtime ignored
help='This positional argument is a value provider',
type=float,
default=5.4)
@@ -123,23 +128,23 @@
# provide values at graph-construction time
# (options not provided here become of the type RuntimeValueProvider)
options = UserDefinedOptions1(['1.2'])
- self.assertFalse(options.vp_arg.is_accessible())
- self.assertFalse(options.vp_arg2.is_accessible())
- self.assertFalse(options.vp_arg3.is_accessible())
- self.assertFalse(options.vp_arg4.is_accessible())
- self.assertTrue(options.vp_pos_arg.is_accessible())
+ self.assertFalse(options.vpt_vp_arg6.is_accessible())
+ self.assertFalse(options.vpt_vp_arg7.is_accessible())
+ self.assertFalse(options.vpt_vp_arg8.is_accessible())
+ self.assertFalse(options.vpt_vp_arg9.is_accessible())
+ self.assertTrue(options.vpt_vp_arg10.is_accessible())
# provide values at job-execution time
# (options not provided here will use their default, if they have one)
- RuntimeValueProvider.set_runtime_options({'vp_arg': 'abc',
- 'vp_pos_arg':'3.2'})
- self.assertTrue(options.vp_arg.is_accessible())
- self.assertEqual(options.vp_arg.get(), 'abc')
- self.assertTrue(options.vp_arg2.is_accessible())
- self.assertEqual(options.vp_arg2.get(), 123)
- self.assertTrue(options.vp_arg3.is_accessible())
- self.assertEqual(options.vp_arg3.get(), '123')
- self.assertTrue(options.vp_arg4.is_accessible())
- self.assertIsNone(options.vp_arg4.get())
- self.assertTrue(options.vp_pos_arg.is_accessible())
- self.assertEqual(options.vp_pos_arg.get(), 1.2)
+ RuntimeValueProvider.set_runtime_options({'vpt_vp_arg6': 'abc',
+ 'vpt_vp_arg10':'3.2'})
+ self.assertTrue(options.vpt_vp_arg6.is_accessible())
+ self.assertEqual(options.vpt_vp_arg6.get(), 'abc')
+ self.assertTrue(options.vpt_vp_arg7.is_accessible())
+ self.assertEqual(options.vpt_vp_arg7.get(), 123)
+ self.assertTrue(options.vpt_vp_arg8.is_accessible())
+ self.assertEqual(options.vpt_vp_arg8.get(), '123')
+ self.assertTrue(options.vpt_vp_arg9.is_accessible())
+ self.assertIsNone(options.vpt_vp_arg9.get())
+ self.assertTrue(options.vpt_vp_arg10.is_accessible())
+ self.assertEqual(options.vpt_vp_arg10.get(), 1.2)
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 5048534..62626a3 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -15,37 +15,38 @@
# limitations under the License.
#
-"""Pipeline, the top-level Dataflow object.
+"""Pipeline, the top-level Beam object.
A pipeline holds a DAG of data transforms. Conceptually the nodes of the DAG
-are transforms (PTransform objects) and the edges are values (mostly PCollection
+are transforms (:class:`~apache_beam.transforms.ptransform.PTransform` objects)
+and the edges are values (mostly :class:`~apache_beam.pvalue.PCollection`
objects). The transforms take as inputs one or more PValues and output one or
-more PValues.
+more :class:`~apache_beam.pvalue.PValue` s.
The pipeline offers functionality to traverse the graph. The actual operation
to be executed for each node visited is specified through a runner object.
-Typical usage:
+Typical usage::
# Create a pipeline object using a local runner for execution.
- p = beam.Pipeline('DirectRunner')
+ with beam.Pipeline('DirectRunner') as p:
- # Add to the pipeline a "Create" transform. When executed this
- # transform will produce a PCollection object with the specified values.
- pcoll = p | 'Create' >> beam.Create([1, 2, 3])
+ # Add to the pipeline a "Create" transform. When executed this
+ # transform will produce a PCollection object with the specified values.
+ pcoll = p | 'Create' >> beam.Create([1, 2, 3])
- # Another transform could be applied to pcoll, e.g., writing to a text file.
- # For other transforms, refer to transforms/ directory.
- pcoll | 'Write' >> beam.io.WriteToText('./output')
+ # Another transform could be applied to pcoll, e.g., writing to a text file.
+ # For other transforms, refer to transforms/ directory.
+ pcoll | 'Write' >> beam.io.WriteToText('./output')
- # run() will execute the DAG stored in the pipeline. The execution of the
- # nodes visited is done using the specified local runner.
- p.run()
+ # run() will execute the DAG stored in the pipeline. The execution of the
+ # nodes visited is done using the specified local runner.
"""
from __future__ import absolute_import
+import abc
import collections
import logging
import os
@@ -54,49 +55,58 @@
from apache_beam import pvalue
from apache_beam.internal import pickler
-from apache_beam.runners import create_runner
-from apache_beam.runners import PipelineRunner
-from apache_beam.transforms import ptransform
-from apache_beam.typehints import typehints
-from apache_beam.typehints import TypeCheckError
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.options.pipeline_options import StandardOptions
from apache_beam.options.pipeline_options import TypeOptions
from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator
+from apache_beam.pvalue import PCollection
+from apache_beam.runners import PipelineRunner
+from apache_beam.runners import create_runner
+from apache_beam.transforms import ptransform
+from apache_beam.typehints import TypeCheckError
+from apache_beam.typehints import typehints
+from apache_beam.utils import urns
from apache_beam.utils.annotations import deprecated
-
__all__ = ['Pipeline']
class Pipeline(object):
- """A pipeline object that manages a DAG of PValues and their PTransforms.
+ """A pipeline object that manages a DAG of
+ :class:`~apache_beam.pvalue.PValue` s and their
+ :class:`~apache_beam.transforms.ptransform.PTransform` s.
- Conceptually the PValues are the DAG's nodes and the PTransforms computing
- the PValues are the edges.
+ Conceptually the :class:`~apache_beam.pvalue.PValue` s are the DAG's nodes and
+ the :class:`~apache_beam.transforms.ptransform.PTransform` s computing
+ the :class:`~apache_beam.pvalue.PValue` s are the edges.
All the transforms applied to the pipeline must have distinct full labels.
If same transform instance needs to be applied then the right shift operator
- should be used to designate new names (e.g. `input | "label" >> my_tranform`).
+ should be used to designate new names
+ (e.g. ``input | "label" >> my_tranform``).
"""
def __init__(self, runner=None, options=None, argv=None):
"""Initialize a pipeline object.
Args:
- runner: An object of type 'PipelineRunner' that will be used to execute
- the pipeline. For registered runners, the runner name can be specified,
- otherwise a runner object must be supplied.
- options: A configured 'PipelineOptions' object containing arguments
- that should be used for running the Dataflow job.
- argv: a list of arguments (such as sys.argv) to be used for building a
- 'PipelineOptions' object. This will only be used if argument 'options'
- is None.
+ runner (~apache_beam.runners.runner.PipelineRunner): An object of
+ type :class:`~apache_beam.runners.runner.PipelineRunner` that will be
+ used to execute the pipeline. For registered runners, the runner name
+ can be specified, otherwise a runner object must be supplied.
+ options (~apache_beam.options.pipeline_options.PipelineOptions):
+ A configured
+ :class:`~apache_beam.options.pipeline_options.PipelineOptions` object
+ containing arguments that should be used for running the Beam job.
+ argv (List[str]): a list of arguments (such as :data:`sys.argv`)
+ to be used for building a
+ :class:`~apache_beam.options.pipeline_options.PipelineOptions` object.
+ This will only be used if argument **options** is :data:`None`.
Raises:
- ValueError: if either the runner or options argument is not of the
- expected type.
+ ~exceptions.ValueError: if either the runner or options argument is not
+ of the expected type.
"""
if options is not None:
if isinstance(options, PipelineOptions):
@@ -158,6 +168,159 @@
"""Returns the root transform of the transform stack."""
return self.transforms_stack[0]
+ def _remove_labels_recursively(self, applied_transform):
+ for part in applied_transform.parts:
+ if part.full_label in self.applied_labels:
+ self.applied_labels.remove(part.full_label)
+ if part.parts:
+ for part2 in part.parts:
+ self._remove_labels_recursively(part2)
+
+ def _replace(self, override):
+
+ assert isinstance(override, PTransformOverride)
+ matcher = override.get_matcher()
+
+ output_map = {}
+ output_replacements = {}
+ input_replacements = {}
+
+ class TransformUpdater(PipelineVisitor): # pylint: disable=used-before-assignment
+ """"A visitor that replaces the matching PTransforms."""
+
+ def __init__(self, pipeline):
+ self.pipeline = pipeline
+
+ def _replace_if_needed(self, transform_node):
+ if matcher(transform_node):
+ replacement_transform = override.get_replacement_transform(
+ transform_node.transform)
+ inputs = transform_node.inputs
+ # TODO: Support replacing PTransforms with multiple inputs.
+ if len(inputs) > 1:
+ raise NotImplementedError(
+ 'PTransform overriding is only supported for PTransforms that '
+ 'have a single input. Tried to replace input of '
+ 'AppliedPTransform %r that has %d inputs',
+ transform_node, len(inputs))
+ transform_node.transform = replacement_transform
+ self.pipeline.transforms_stack.append(transform_node)
+
+ # Keeping the same label for the replaced node but recursively
+ # removing labels of child transforms since they will be replaced
+ # during the expand below.
+ self.pipeline._remove_labels_recursively(transform_node)
+
+ new_output = replacement_transform.expand(inputs[0])
+ if new_output.producer is None:
+ # When current transform is a primitive, we set the producer here.
+ new_output.producer = transform_node
+
+ # We only support replacing transforms with a single output with
+ # another transform that produces a single output.
+ # TODO: Support replacing PTransforms with multiple outputs.
+ if (len(transform_node.outputs) > 1 or
+ not isinstance(transform_node.outputs[None], PCollection) or
+ not isinstance(new_output, PCollection)):
+ raise NotImplementedError(
+ 'PTransform overriding is only supported for PTransforms that '
+ 'have a single output. Tried to replace output of '
+ 'AppliedPTransform %r with %r.'
+ , transform_node, new_output)
+
+ # Recording updated outputs. This cannot be done in the same visitor
+ # since if we dynamically update output type here, we'll run into
+ # errors when visiting child nodes.
+ output_map[transform_node.outputs[None]] = new_output
+
+ self.pipeline.transforms_stack.pop()
+
+ def enter_composite_transform(self, transform_node):
+ self._replace_if_needed(transform_node)
+
+ def visit_transform(self, transform_node):
+ self._replace_if_needed(transform_node)
+
+ self.visit(TransformUpdater(self))
+
+ # Adjusting inputs and outputs
+ class InputOutputUpdater(PipelineVisitor): # pylint: disable=used-before-assignment
+ """"A visitor that records input and output values to be replaced.
+
+ Input and output values that should be updated are recorded in maps
+ input_replacements and output_replacements respectively.
+
+ We cannot update input and output values while visiting since that results
+ in validation errors.
+ """
+
+ def __init__(self, pipeline):
+ self.pipeline = pipeline
+
+ def enter_composite_transform(self, transform_node):
+ self.visit_transform(transform_node)
+
+ def visit_transform(self, transform_node):
+ if (None in transform_node.outputs and
+ transform_node.outputs[None] in output_map):
+ output_replacements[transform_node] = (
+ output_map[transform_node.outputs[None]])
+
+ replace_input = False
+ for input in transform_node.inputs:
+ if input in output_map:
+ replace_input = True
+ break
+
+ if replace_input:
+ new_input = [
+ input if not input in output_map else output_map[input]
+ for input in transform_node.inputs]
+ input_replacements[transform_node] = new_input
+
+ self.visit(InputOutputUpdater(self))
+
+ for transform in output_replacements:
+ transform.replace_output(output_replacements[transform])
+
+ for transform in input_replacements:
+ transform.inputs = input_replacements[transform]
+
+ def _check_replacement(self, override):
+ matcher = override.get_matcher()
+
+ class ReplacementValidator(PipelineVisitor):
+ def visit_transform(self, transform_node):
+ if matcher(transform_node):
+ raise RuntimeError('Transform node %r was not replaced as expected.',
+ transform_node)
+
+ self.visit(ReplacementValidator())
+
+ def replace_all(self, replacements):
+ """ Dynamically replaces PTransforms in the currently populated hierarchy.
+
+ Currently this only works for replacements where input and output types
+ are exactly the same.
+
+ TODO: Update this to also work for transform overrides where input and
+ output types are different.
+
+ Args:
+ replacements (List[~apache_beam.pipeline.PTransformOverride]): a list of
+ :class:`~apache_beam.pipeline.PTransformOverride` objects.
+ """
+ for override in replacements:
+ assert isinstance(override, PTransformOverride)
+ self._replace(override)
+
+ # Checking if the PTransforms have been successfully replaced. This will
+ # result in a failure if a PTransform that was replaced in a given override
+ # gets re-added in a subsequent override. This is not allowed and ordering
+ # of PTransformOverride objects in 'replacements' is important.
+ for override in replacements:
+ self._check_replacement(override)
+
def run(self, test_runner_api=True):
"""Runs the pipeline. Returns whatever our runner returns after running."""
@@ -188,13 +351,16 @@
Runner-internal implementation detail; no backwards-compatibility guarantees
Args:
- visitor: PipelineVisitor object whose callbacks will be called for each
- node visited. See PipelineVisitor comments.
+ visitor (~apache_beam.pipeline.PipelineVisitor):
+ :class:`~apache_beam.pipeline.PipelineVisitor` object whose callbacks
+ will be called for each node visited. See
+ :class:`~apache_beam.pipeline.PipelineVisitor` comments.
Raises:
- TypeError: if node is specified and is not a PValue.
- pipeline.PipelineError: if node is specified and does not belong to this
- pipeline instance.
+ ~exceptions.TypeError: if node is specified and is not a
+ :class:`~apache_beam.pvalue.PValue`.
+ ~apache_beam.error.PipelineError: if node is specified and does not
+ belong to this pipeline instance.
"""
visited = set()
@@ -204,15 +370,20 @@
"""Applies a custom transform using the pvalueish specified.
Args:
- transform: the PTranform to apply.
- pvalueish: the input for the PTransform (typically a PCollection).
- label: label of the PTransform.
+ transform (~apache_beam.transforms.ptransform.PTransform): the
+ :class:`~apache_beam.transforms.ptransform.PTransform` to apply.
+ pvalueish (~apache_beam.pvalue.PCollection): the input for the
+ :class:`~apache_beam.transforms.ptransform.PTransform` (typically a
+ :class:`~apache_beam.pvalue.PCollection`).
+ label (str): label of the
+ :class:`~apache_beam.transforms.ptransform.PTransform`.
Raises:
- TypeError: if the transform object extracted from the argument list is
- not a PTransform.
- RuntimeError: if the transform object was already applied to this pipeline
- and needs to be cloned in order to apply again.
+ ~exceptions.TypeError: if the transform object extracted from the
+ argument list is not a
+ :class:`~apache_beam.transforms.ptransform.PTransform`.
+ ~exceptions.RuntimeError: if the transform object was already applied to
+ this pipeline and needs to be cloned in order to apply again.
"""
if isinstance(transform, ptransform._NamedPTransform):
return self.apply(transform.transform, pvalueish,
@@ -267,7 +438,7 @@
if type_options is not None and type_options.pipeline_type_check:
transform.type_check_outputs(pvalueish_result)
- for result in ptransform.GetPValues().visit(pvalueish_result):
+ for result in ptransform.get_nested_pvalues(pvalueish_result):
assert isinstance(result, (pvalue.PValue, pvalue.DoOutputsTuple))
# Make sure we set the producer only for a leaf node in the transform DAG.
@@ -314,14 +485,27 @@
self.transforms_stack.pop()
return pvalueish_result
+ def __reduce__(self):
+ # Some transforms contain a reference to their enclosing pipeline,
+ # which in turn reference all other transforms (resulting in quadratic
+ # time/space to pickle each transform individually). As we don't
+ # require pickled pipelines to be executable, break the chain here.
+ return str, ('Pickled pipeline stub.',)
+
def _verify_runner_api_compatible(self):
+ if self._options.view_as(TypeOptions).runtime_type_check:
+ # This option is incompatible with the runner API as it requires
+ # the runner to inspect non-serialized hints on the transform
+ # itself.
+ return False
+
class Visitor(PipelineVisitor): # pylint: disable=used-before-assignment
ok = True # Really a nonlocal.
+ def enter_composite_transform(self, transform_node):
+ self.visit_transform(transform_node)
+
def visit_transform(self, transform_node):
- if transform_node.side_inputs:
- # No side inputs (yet).
- Visitor.ok = False
try:
# Transforms must be picklable.
pickler.loads(pickler.dumps(transform_node.transform,
@@ -340,7 +524,7 @@
def to_runner_api(self):
"""For internal use only; no backwards-compatibility guarantees."""
from apache_beam.runners import pipeline_context
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.api import beam_runner_api_pb2
context = pipeline_context.PipelineContext()
# Mutates context; placing inline would force dependence on
# argument evaluation order.
@@ -363,7 +547,18 @@
p.applied_labels = set([
t.unique_name for t in proto.components.transforms.values()])
for id in proto.components.pcollections:
- context.pcollections.get_by_id(id).pipeline = p
+ pcollection = context.pcollections.get_by_id(id)
+ pcollection.pipeline = p
+
+ # Inject PBegin input where necessary.
+ from apache_beam.io.iobase import Read
+ from apache_beam.transforms.core import Create
+ has_pbegin = [Read, Create]
+ for id in proto.components.transforms:
+ transform = context.transforms.get_by_id(id)
+ if not transform.inputs and transform.transform.__class__ in has_pbegin:
+ transform.inputs = (pvalue.PBegin(p),)
+
return p
@@ -385,7 +580,7 @@
pass
def visit_transform(self, transform_node):
- """Callback for visiting a transform node in the pipeline DAG."""
+ """Callback for visiting a transform leaf node in the pipeline DAG."""
pass
def enter_composite_transform(self, transform_node):
@@ -442,6 +637,20 @@
for side_input in self.side_inputs:
real_producer(side_input.pvalue).refcounts[side_input.pvalue.tag] += 1
+ def replace_output(self, output, tag=None):
+ """Replaces the output defined by the given tag with the given output.
+
+ Args:
+ output: replacement output
+ tag: tag of the output to be replaced.
+ """
+ if isinstance(output, pvalue.DoOutputsTuple):
+ self.replace_output(output[output._main_tag])
+ elif isinstance(output, pvalue.PValue):
+ self.outputs[tag] = output
+ else:
+ raise TypeError("Unexpected output type: %s" % output)
+
def add_output(self, output, tag=None):
if isinstance(output, pvalue.DoOutputsTuple):
self.add_output(output[output._main_tag])
@@ -518,15 +727,19 @@
def named_inputs(self):
# TODO(BEAM-1833): Push names up into the sdk construction.
- return {str(ix): input for ix, input in enumerate(self.inputs)
- if isinstance(input, pvalue.PCollection)}
+ main_inputs = {str(ix): input
+ for ix, input in enumerate(self.inputs)
+ if isinstance(input, pvalue.PCollection)}
+ side_inputs = {'side%s' % ix: si.pvalue
+ for ix, si in enumerate(self.side_inputs)}
+ return dict(main_inputs, **side_inputs)
def named_outputs(self):
return {str(tag): output for tag, output in self.outputs.items()
if isinstance(output, pvalue.PCollection)}
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.api import beam_runner_api_pb2
def transform_to_runner_api(transform, context):
if transform is None:
@@ -536,8 +749,8 @@
return beam_runner_api_pb2.PTransform(
unique_name=self.full_label,
spec=transform_to_runner_api(self.transform, context),
- subtransforms=[context.transforms.get_id(part) for part in self.parts],
- # TODO(BEAM-115): Side inputs.
+ subtransforms=[context.transforms.get_id(part, label=part.full_label)
+ for part in self.parts],
inputs={tag: context.pcollections.get_id(pc)
for tag, pc in self.named_inputs().items()},
outputs={str(tag): context.pcollections.get_id(out)
@@ -547,17 +760,35 @@
@staticmethod
def from_runner_api(proto, context):
+ def is_side_input(tag):
+ # As per named_inputs() above.
+ return tag.startswith('side')
+ main_inputs = [context.pcollections.get_by_id(id)
+ for tag, id in proto.inputs.items()
+ if not is_side_input(tag)]
+ # Ordering is important here.
+ indexed_side_inputs = [(int(tag[4:]), context.pcollections.get_by_id(id))
+ for tag, id in proto.inputs.items()
+ if is_side_input(tag)]
+ side_inputs = [si for _, si in sorted(indexed_side_inputs)]
result = AppliedPTransform(
parent=None,
transform=ptransform.PTransform.from_runner_api(proto.spec, context),
full_label=proto.unique_name,
- inputs=[
- context.pcollections.get_by_id(id) for id in proto.inputs.values()])
+ inputs=main_inputs)
+ if result.transform and result.transform.side_inputs:
+ for si, pcoll in zip(result.transform.side_inputs, side_inputs):
+ si.pvalue = pcoll
+ result.side_inputs = tuple(result.transform.side_inputs)
result.parts = [
context.transforms.get_by_id(id) for id in proto.subtransforms]
result.outputs = {
None if tag == 'None' else tag: context.pcollections.get_by_id(id)
for tag, id in proto.outputs.items()}
+ # This annotation is expected by some runners.
+ if proto.spec.urn == urns.PARDO_TRANSFORM:
+ result.transform.output_tags = set(proto.outputs.keys()).difference(
+ {'None'})
if not result.parts:
for tag, pc in result.outputs.items():
if pc not in result.inputs:
@@ -565,3 +796,37 @@
pc.tag = tag
result.update_input_refcounts()
return result
+
+
+class PTransformOverride(object):
+ """For internal use only; no backwards-compatibility guarantees.
+
+ Gives a matcher and replacements for matching PTransforms.
+
+ TODO: Update this to support cases where input and/our output types are
+ different.
+ """
+ __metaclass__ = abc.ABCMeta
+
+ @abc.abstractmethod
+ def get_matcher(self):
+ """Gives a matcher that will be used to to perform this override.
+
+ Returns:
+ a callable that takes an AppliedPTransform as a parameter and returns a
+ boolean as a result.
+ """
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def get_replacement_transform(self, ptransform):
+ """Provides a runner specific override for a given PTransform.
+
+ Args:
+ ptransform: PTransform to be replaced.
+ Returns:
+ A PTransform that will be the replacement for the PTransform given as an
+ argument.
+ """
+ # Returns a PTransformReplacement
+ raise NotImplementedError
diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index e0775d1..9bbb0d7 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -20,9 +20,7 @@
import logging
import platform
import unittest
-
-# TODO(BEAM-1555): Test is failing on the service, with FakeSource.
-# from nose.plugins.attrib import attr
+from collections import defaultdict
import apache_beam as beam
from apache_beam.io import Read
@@ -30,16 +28,21 @@
from apache_beam.pipeline import Pipeline
from apache_beam.pipeline import PipelineOptions
from apache_beam.pipeline import PipelineVisitor
+from apache_beam.pipeline import PTransformOverride
from apache_beam.pvalue import AsSingleton
+from apache_beam.runners import DirectRunner
from apache_beam.runners.dataflow.native_io.iobase import NativeSource
+from apache_beam.runners.direct.evaluation_context import _ExecutionContext
+from apache_beam.runners.direct.transform_evaluator import _GroupByKeyOnlyEvaluator
+from apache_beam.runners.direct.transform_evaluator import _TransformEvaluator
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import CombineGlobally
from apache_beam.transforms import Create
+from apache_beam.transforms import DoFn
from apache_beam.transforms import FlatMap
from apache_beam.transforms import Map
-from apache_beam.transforms import DoFn
from apache_beam.transforms import ParDo
from apache_beam.transforms import PTransform
from apache_beam.transforms import WindowInto
@@ -47,6 +50,9 @@
from apache_beam.transforms.window import TimestampedValue
from apache_beam.utils.timestamp import MIN_TIMESTAMP
+# TODO(BEAM-1555): Test is failing on the service, with FakeSource.
+# from nose.plugins.attrib import attr
+
class FakeSource(NativeSource):
"""Fake source returning a fixed list of values."""
@@ -75,6 +81,18 @@
return FakeSource._Reader(self._vals)
+class DoubleParDo(beam.PTransform):
+ def expand(self, input):
+ return input | 'Inner' >> beam.Map(lambda a: a * 2)
+
+
+class TripleParDo(beam.PTransform):
+ def expand(self, input):
+ # Keeping labels the same intentionally to make sure that there is no label
+ # conflict due to replacement.
+ return input | 'Inner' >> beam.Map(lambda a: a * 3)
+
+
class PipelineTest(unittest.TestCase):
@staticmethod
@@ -285,6 +303,27 @@
# p = Pipeline('EagerRunner')
# self.assertEqual([1, 4, 9], p | Create([1, 2, 3]) | Map(lambda x: x*x))
+ def test_ptransform_overrides(self):
+
+ def my_par_do_matcher(applied_ptransform):
+ return isinstance(applied_ptransform.transform, DoubleParDo)
+
+ class MyParDoOverride(PTransformOverride):
+
+ def get_matcher(self):
+ return my_par_do_matcher
+
+ def get_replacement_transform(self, ptransform):
+ if isinstance(ptransform, DoubleParDo):
+ return TripleParDo()
+ raise ValueError('Unsupported type of transform: %r', ptransform)
+
+ # Using following private variable for testing.
+ DirectRunner._PTRANSFORM_OVERRIDES.append(MyParDoOverride())
+ with Pipeline() as p:
+ pcoll = p | beam.Create([1, 2, 3]) | 'Multiply' >> DoubleParDo()
+ assert_that(pcoll, equal_to([3, 6, 9]))
+
class DoFnTest(unittest.TestCase):
@@ -445,6 +484,102 @@
p2 = Pipeline.from_runner_api(proto, p.runner, p._options)
p2.run()
+ def test_pickling(self):
+ class MyPTransform(beam.PTransform):
+ pickle_count = [0]
+
+ def expand(self, p):
+ self.p = p
+ return p | beam.Create([None])
+
+ def __reduce__(self):
+ self.pickle_count[0] += 1
+ return str, ()
+
+ p = beam.Pipeline()
+ for k in range(20):
+ p | 'Iter%s' % k >> MyPTransform() # pylint: disable=expression-not-assigned
+ p.to_runner_api()
+ self.assertEqual(MyPTransform.pickle_count[0], 20)
+
+
+class DirectRunnerRetryTests(unittest.TestCase):
+
+ def test_retry_fork_graph(self):
+ pipeline_options = PipelineOptions(['--direct_runner_bundle_retry'])
+ p = beam.Pipeline(options=pipeline_options)
+
+ # TODO(mariagh): Remove the use of globals from the test.
+ global count_b, count_c # pylint: disable=global-variable-undefined
+ count_b, count_c = 0, 0
+
+ def f_b(x):
+ global count_b # pylint: disable=global-variable-undefined
+ count_b += 1
+ raise Exception('exception in f_b')
+
+ def f_c(x):
+ global count_c # pylint: disable=global-variable-undefined
+ count_c += 1
+ raise Exception('exception in f_c')
+
+ names = p | 'CreateNodeA' >> beam.Create(['Ann', 'Joe'])
+
+ fork_b = names | 'SendToB' >> beam.Map(f_b) # pylint: disable=unused-variable
+ fork_c = names | 'SendToC' >> beam.Map(f_c) # pylint: disable=unused-variable
+
+ with self.assertRaises(Exception):
+ p.run().wait_until_finish()
+ assert count_b == count_c == 4
+
+ def test_no_partial_writeouts(self):
+
+ class TestTransformEvaluator(_TransformEvaluator):
+
+ def __init__(self):
+ self._execution_context = _ExecutionContext(None, {})
+
+ def start_bundle(self):
+ self.step_context = self._execution_context.get_step_context()
+
+ def process_element(self, element):
+ k, v = element
+ state = self.step_context.get_keyed_state(k)
+ state.add_state(None, _GroupByKeyOnlyEvaluator.ELEMENTS_TAG, v)
+
+ # Create instance and add key/value, key/value2
+ evaluator = TestTransformEvaluator()
+ evaluator.start_bundle()
+ self.assertIsNone(evaluator.step_context.existing_keyed_state.get('key'))
+ self.assertIsNone(evaluator.step_context.partial_keyed_state.get('key'))
+
+ evaluator.process_element(['key', 'value'])
+ self.assertEqual(
+ evaluator.step_context.existing_keyed_state['key'].state,
+ defaultdict(lambda: defaultdict(list)))
+ self.assertEqual(
+ evaluator.step_context.partial_keyed_state['key'].state,
+ {None: {'elements':['value']}})
+
+ evaluator.process_element(['key', 'value2'])
+ self.assertEqual(
+ evaluator.step_context.existing_keyed_state['key'].state,
+ defaultdict(lambda: defaultdict(list)))
+ self.assertEqual(
+ evaluator.step_context.partial_keyed_state['key'].state,
+ {None: {'elements':['value', 'value2']}})
+
+ # Simulate an exception (redo key/value)
+ evaluator._execution_context.reset()
+ evaluator.start_bundle()
+ evaluator.process_element(['key', 'value'])
+ self.assertEqual(
+ evaluator.step_context.existing_keyed_state['key'].state,
+ defaultdict(lambda: defaultdict(list)))
+ self.assertEqual(
+ evaluator.step_context.partial_keyed_state['key'].state,
+ {None: {'elements':['value']}})
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.DEBUG)
diff --git a/sdks/python/apache_beam/portability/__init__.py b/sdks/python/apache_beam/portability/__init__.py
new file mode 100644
index 0000000..0bce5d6
--- /dev/null
+++ b/sdks/python/apache_beam/portability/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""For internal use only; no backwards-compatibility guarantees."""
diff --git a/sdks/python/apache_beam/portability/api/__init__.py b/sdks/python/apache_beam/portability/api/__init__.py
new file mode 100644
index 0000000..2750859
--- /dev/null
+++ b/sdks/python/apache_beam/portability/api/__init__.py
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""For internal use only; no backwards-compatibility guarantees.
+
+Automatically generated when running setup.py sdist or build[_py].
+"""
diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py
index 7385e82..31922f3 100644
--- a/sdks/python/apache_beam/pvalue.py
+++ b/sdks/python/apache_beam/pvalue.py
@@ -28,8 +28,11 @@
import itertools
+from apache_beam import coders
from apache_beam import typehints
-
+from apache_beam.internal import pickler
+from apache_beam.portability.api import beam_runner_api_pb2
+from apache_beam.utils import urns
__all__ = [
'PCollection',
@@ -128,19 +131,16 @@
return _InvalidUnpickledPCollection, ()
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
- from apache_beam.internal import pickler
return beam_runner_api_pb2.PCollection(
unique_name='%d%s.%s' % (
len(self.producer.full_label), self.producer.full_label, self.tag),
coder_id=pickler.dumps(self.element_type),
- is_bounded=beam_runner_api_pb2.BOUNDED,
+ is_bounded=beam_runner_api_pb2.IsBounded.BOUNDED,
windowing_strategy_id=context.windowing_strategies.get_id(
self.windowing))
@staticmethod
def from_runner_api(proto, context):
- from apache_beam.internal import pickler
# Producer and tag will be filled in later, the key point is that the
# same object is returned for the same pcollection id.
return PCollection(None, element_type=pickler.loads(proto.coder_id))
@@ -289,6 +289,81 @@
def element_type(self):
return typehints.Any
+ # TODO(robertwb): Get rid of _from_runtime_iterable and _view_options
+ # in favor of _side_input_data().
+ def _side_input_data(self):
+ view_options = self._view_options()
+ from_runtime_iterable = type(self)._from_runtime_iterable
+ return SideInputData(
+ urns.ITERABLE_ACCESS,
+ self._window_mapping_fn,
+ lambda iterable: from_runtime_iterable(iterable, view_options),
+ self._input_element_coder())
+
+ def _input_element_coder(self):
+ return coders.WindowedValueCoder(
+ coders.registry.get_coder(self.pvalue.element_type),
+ window_coder=self.pvalue.windowing.windowfn.get_window_coder())
+
+ def to_runner_api(self, context):
+ return self._side_input_data().to_runner_api(context)
+
+ @staticmethod
+ def from_runner_api(proto, context):
+ return _UnpickledSideInput(
+ SideInputData.from_runner_api(proto, context))
+
+
+class _UnpickledSideInput(AsSideInput):
+ def __init__(self, side_input_data):
+ self._data = side_input_data
+ self._window_mapping_fn = side_input_data.window_mapping_fn
+
+ @staticmethod
+ def _from_runtime_iterable(it, options):
+ return options['data'].view_fn(it)
+
+ def _view_options(self):
+ return {
+ 'data': self._data,
+ # For non-fn-api runners.
+ 'window_mapping_fn': self._data.window_mapping_fn,
+ }
+
+ def _side_input_data(self):
+ return self._data
+
+
+class SideInputData(object):
+ """All of the data about a side input except for the bound PCollection."""
+ def __init__(self, access_pattern, window_mapping_fn, view_fn, coder):
+ self.access_pattern = access_pattern
+ self.window_mapping_fn = window_mapping_fn
+ self.view_fn = view_fn
+ self.coder = coder
+
+ def to_runner_api(self, unused_context):
+ return beam_runner_api_pb2.SideInput(
+ access_pattern=beam_runner_api_pb2.FunctionSpec(
+ urn=self.access_pattern),
+ view_fn=beam_runner_api_pb2.SdkFunctionSpec(
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=urns.PICKLED_PYTHON_VIEWFN,
+ payload=pickler.dumps((self.view_fn, self.coder)))),
+ window_mapping_fn=beam_runner_api_pb2.SdkFunctionSpec(
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=urns.PICKLED_WINDOW_MAPPING_FN,
+ payload=pickler.dumps(self.window_mapping_fn))))
+
+ @staticmethod
+ def from_runner_api(proto, unused_context):
+ assert proto.view_fn.spec.urn == urns.PICKLED_PYTHON_VIEWFN
+ assert proto.window_mapping_fn.spec.urn == urns.PICKLED_WINDOW_MAPPING_FN
+ return SideInputData(
+ proto.access_pattern.urn,
+ pickler.loads(proto.window_mapping_fn.spec.payload),
+ *pickler.loads(proto.view_fn.spec.payload))
+
class AsSingleton(AsSideInput):
"""Marker specifying that an entire PCollection is to be used as a side input.
@@ -329,8 +404,9 @@
elif len(head) == 1:
return head[0]
raise ValueError(
- 'PCollection with more than one element accessed as '
- 'a singleton view.')
+ 'PCollection of size %d with more than one element accessed as a '
+ 'singleton view. First two elements encountered are "%s", "%s".' % (
+ len(head), str(head[0]), str(head[1])))
@property
def element_type(self):
@@ -358,6 +434,13 @@
def _from_runtime_iterable(it, options):
return it
+ def _side_input_data(self):
+ return SideInputData(
+ urns.ITERABLE_ACCESS,
+ self._window_mapping_fn,
+ lambda iterable: iterable,
+ self._input_element_coder())
+
@property
def element_type(self):
return typehints.Iterable[self.pvalue.element_type]
@@ -382,6 +465,13 @@
def _from_runtime_iterable(it, options):
return list(it)
+ def _side_input_data(self):
+ return SideInputData(
+ urns.ITERABLE_ACCESS,
+ self._window_mapping_fn,
+ list,
+ self._input_element_coder())
+
class AsDict(AsSideInput):
"""Marker specifying a PCollection to be used as an indexable side input.
@@ -403,6 +493,13 @@
def _from_runtime_iterable(it, options):
return dict(it)
+ def _side_input_data(self):
+ return SideInputData(
+ urns.ITERABLE_ACCESS,
+ self._window_mapping_fn,
+ dict,
+ self._input_element_coder())
+
class EmptySideInput(object):
"""Value indicating when a singleton side input was empty.
diff --git a/sdks/python/apache_beam/pvalue_test.py b/sdks/python/apache_beam/pvalue_test.py
index 4acbc52..48203df 100644
--- a/sdks/python/apache_beam/pvalue_test.py
+++ b/sdks/python/apache_beam/pvalue_test.py
@@ -19,6 +19,7 @@
import unittest
+from apache_beam.pvalue import AsSingleton
from apache_beam.pvalue import PValue
from apache_beam.testing.test_pipeline import TestPipeline
@@ -30,6 +31,13 @@
value = PValue(pipeline)
self.assertEqual(pipeline, value.pipeline)
+ def test_assingleton_multi_element(self):
+ with self.assertRaisesRegexp(
+ ValueError,
+ 'PCollection of size 2 with more than one element accessed as a '
+ 'singleton view. First two elements encountered are \"1\", \"2\".'):
+ AsSingleton._from_runtime_iterable([1, 2], {})
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py
deleted file mode 100644
index bf95208..0000000
--- a/sdks/python/apache_beam/runners/api/__init__.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""For internal use only; no backwards-compatibility guarantees.
-
-Checked in to avoid protoc dependency for Python development.
-
-Regenerate files with::
-
- protoc -I../common/runner-api/src/main/proto/ \
- --python_out=apache_beam/runners/api/ \
- ../common/runner-api/src/main/proto/*.proto
-
- protoc -I../common/{fn,runner}-api/src/main/proto/ \
- --python_out=apache_beam/runners/api/ \
- --grpc_python_out=apache_beam/runners/api/ \
- ../common/fn-api/src/main/proto/*.proto
-"""
diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
deleted file mode 100644
index cb0b72b..0000000
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
+++ /dev/null
@@ -1,2742 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: beam_fn_api.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='beam_fn_api.proto',
- package='org.apache.beam.fn.v1',
- syntax='proto3',
- serialized_pb=_b('\n\x11\x62\x65\x61m_fn_api.proto\x12\x15org.apache.beam.fn.v1\x1a\x19google/protobuf/any.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"t\n\x06Target\x12%\n\x1dprimitive_transform_reference\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x1a\x35\n\x04List\x12-\n\x06target\x18\x01 \x03(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\"&\n\x0bPCollection\x12\x17\n\x0f\x63oder_reference\x18\x01 \x01(\t\"\xcb\x04\n\x12PrimitiveTransform\x12\n\n\x02id\x18\x01 \x01(\t\x12:\n\rfunction_spec\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12\x45\n\x06inputs\x18\x03 \x03(\x0b\x32\x35.org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry\x12G\n\x07outputs\x18\x04 \x03(\x0b\x32\x36.org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry\x12N\n\x0bside_inputs\x18\x05 \x03(\x0b\x32\x39.org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry\x12\x11\n\tstep_name\x18\x06 \x01(\t\x1aQ\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".org.apache.beam.fn.v1.Target.List:\x02\x38\x01\x1aR\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".org.apache.beam.fn.v1.PCollection:\x02\x38\x01\x1aS\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .org.apache.beam.fn.v1.SideInput:\x02\x38\x01\"j\n\x0c\x46unctionSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03urn\x18\x02 \x01(\t\x12\x1d\n\x15\x65nvironment_reference\x18\x03 \x01(\t\x12\"\n\x04\x64\x61ta\x18\x04 \x01(\x0b\x32\x14.google.protobuf.Any\"o\n\tSideInput\x12,\n\x05input\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x34\n\x07view_fn\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\"f\n\x05\x43oder\x12:\n\rfunction_spec\x18\x01 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12!\n\x19\x63omponent_coder_reference\x18\x02 \x03(\t\"]\n\x0eRemoteGrpcPort\x12K\n\x16\x61pi_service_descriptor\x18\x01 \x01(\x0b\x32+.org.apache.beam.fn.v1.ApiServiceDescriptor\"\xe8\x02\n\x12InstructionRequest\x12\x16\n\x0einstruction_id\x18\x01 \x01(\t\x12;\n\x08register\x18\xe8\x07 \x01(\x0b\x32&.org.apache.beam.fn.v1.RegisterRequestH\x00\x12\x46\n\x0eprocess_bundle\x18\xe9\x07 \x01(\x0b\x32+.org.apache.beam.fn.v1.ProcessBundleRequestH\x00\x12W\n\x17process_bundle_progress\x18\xea\x07 \x01(\x0b\x32\x33.org.apache.beam.fn.v1.ProcessBundleProgressRequestH\x00\x12Q\n\x14process_bundle_split\x18\xeb\x07 \x01(\x0b\x32\x30.org.apache.beam.fn.v1.ProcessBundleSplitRequestH\x00\x42\t\n\x07request\"\xfd\x02\n\x13InstructionResponse\x12\x16\n\x0einstruction_id\x18\x01 \x01(\t\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12<\n\x08register\x18\xe8\x07 \x01(\x0b\x32\'.org.apache.beam.fn.v1.RegisterResponseH\x00\x12G\n\x0eprocess_bundle\x18\xe9\x07 \x01(\x0b\x32,.org.apache.beam.fn.v1.ProcessBundleResponseH\x00\x12X\n\x17process_bundle_progress\x18\xea\x07 \x01(\x0b\x32\x34.org.apache.beam.fn.v1.ProcessBundleProgressResponseH\x00\x12R\n\x14process_bundle_split\x18\xeb\x07 \x01(\x0b\x32\x31.org.apache.beam.fn.v1.ProcessBundleSplitResponseH\x00\x42\n\n\x08response\"d\n\x0fRegisterRequest\x12Q\n\x19process_bundle_descriptor\x18\x01 \x03(\x0b\x32..org.apache.beam.fn.v1.ProcessBundleDescriptor\"\x12\n\x10RegisterResponse\"\x9b\x01\n\x17ProcessBundleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x46\n\x13primitive_transform\x18\x02 \x03(\x0b\x32).org.apache.beam.fn.v1.PrimitiveTransform\x12,\n\x06\x63oders\x18\x04 \x03(\x0b\x32\x1c.org.apache.beam.fn.v1.Coder\"|\n\x14ProcessBundleRequest\x12+\n#process_bundle_descriptor_reference\x18\x01 \x01(\t\x12\x37\n\x0c\x63\x61\x63he_tokens\x18\x02 \x03(\x0b\x32!.org.apache.beam.fn.v1.CacheToken\"\x17\n\x15ProcessBundleResponse\"=\n\x1cProcessBundleProgressRequest\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\"G\n\x1dProcessBundleProgressResponse\x12\x15\n\rfinished_work\x18\x01 \x01(\x01\x12\x0f\n\x07\x62\x61\x63klog\x18\x02 \x01(\x01\"L\n\x19ProcessBundleSplitRequest\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\x12\x10\n\x08\x66raction\x18\x02 \x01(\x01\"(\n\x17\x45lementCountRestriction\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\",\n\x1b\x45lementCountSkipRestriction\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\xc8\x01\n\x17PrimitiveTransformSplit\x12%\n\x1dprimitive_transform_reference\x18\x01 \x01(\t\x12\x42\n\x15\x63ompleted_restriction\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12\x42\n\x15remaining_restriction\x18\x03 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\"\\\n\x1aProcessBundleSplitResponse\x12>\n\x06splits\x18\x01 \x03(\x0b\x32..org.apache.beam.fn.v1.PrimitiveTransformSplit\"\xa2\x01\n\x08\x45lements\x12\x32\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32$.org.apache.beam.fn.v1.Elements.Data\x1a\x62\n\x04\x44\x61ta\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\x12-\n\x06target\x18\x02 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xaa\x02\n\x0cStateRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1d\n\x15instruction_reference\x18\x02 \x01(\t\x12\x32\n\tstate_key\x18\x03 \x01(\x0b\x32\x1f.org.apache.beam.fn.v1.StateKey\x12\x36\n\x03get\x18\xe8\x07 \x01(\x0b\x32&.org.apache.beam.fn.v1.StateGetRequestH\x00\x12<\n\x06\x61ppend\x18\xe9\x07 \x01(\x0b\x32).org.apache.beam.fn.v1.StateAppendRequestH\x00\x12:\n\x05\x63lear\x18\xea\x07 \x01(\x0b\x32(.org.apache.beam.fn.v1.StateClearRequestH\x00\x42\t\n\x07request\"\xeb\x01\n\rStateResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x37\n\x03get\x18\xe8\x07 \x01(\x0b\x32\'.org.apache.beam.fn.v1.StateGetResponseH\x00\x12=\n\x06\x61ppend\x18\xe9\x07 \x01(\x0b\x32*.org.apache.beam.fn.v1.StateAppendResponseH\x00\x12;\n\x05\x63lear\x18\xea\x07 \x01(\x0b\x32).org.apache.beam.fn.v1.StateClearResponseH\x00\x42\n\n\x08response\"J\n\nCacheToken\x12-\n\x06target\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\r\n\x05token\x18\x02 \x01(\x0c\"V\n\x08StateKey\x12-\n\x06target\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x0e\n\x06window\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"=\n\x11\x43ontinuableStream\x12\x1a\n\x12\x63ontinuation_token\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"-\n\x0fStateGetRequest\x12\x1a\n\x12\x63ontinuation_token\x18\x01 \x01(\x0c\"L\n\x10StateGetResponse\x12\x38\n\x06stream\x18\x01 \x01(\x0b\x32(.org.apache.beam.fn.v1.ContinuableStream\"\"\n\x12StateAppendRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"\x15\n\x13StateAppendResponse\"\x13\n\x11StateClearRequest\"\x14\n\x12StateClearResponse\"\x9a\x03\n\x08LogEntry\x12:\n\x08severity\x18\x01 \x01(\x0e\x32(.org.apache.beam.fn.v1.LogEntry.Severity\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\r\n\x05trace\x18\x04 \x01(\t\x12\x1d\n\x15instruction_reference\x18\x05 \x01(\t\x12%\n\x1dprimitive_transform_reference\x18\x06 \x01(\t\x12\x14\n\x0clog_location\x18\x07 \x01(\t\x12\x0e\n\x06thread\x18\x08 \x01(\t\x1a<\n\x04List\x12\x34\n\x0blog_entries\x18\x01 \x03(\x0b\x32\x1f.org.apache.beam.fn.v1.LogEntry\"Y\n\x08Severity\x12\t\n\x05TRACE\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\n\x12\x08\n\x04INFO\x10\x14\x12\n\n\x06NOTICE\x10\x1e\x12\x08\n\x04WARN\x10(\x12\t\n\x05\x45RROR\x10\x32\x12\x0c\n\x08\x43RITICAL\x10<\"\x0c\n\nLogControl\"\xa1\x01\n\x14\x41piServiceDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12^\n\x1foauth2_client_credentials_grant\x18\x03 \x01(\x0b\x32\x33.org.apache.beam.fn.v1.OAuth2ClientCredentialsGrantH\x00\x42\x10\n\x0e\x61uthentication\"+\n\x1cOAuth2ClientCredentialsGrant\x12\x0b\n\x03url\x18\x01 \x01(\t\"F\n\x0f\x44ockerContainer\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1a\n\x12registry_reference\x18\x03 \x01(\t2w\n\rBeamFnControl\x12\x66\n\x07\x43ontrol\x12*.org.apache.beam.fn.v1.InstructionResponse\x1a).org.apache.beam.fn.v1.InstructionRequest\"\x00(\x01\x30\x01\x32\\\n\nBeamFnData\x12N\n\x04\x44\x61ta\x12\x1f.org.apache.beam.fn.v1.Elements\x1a\x1f.org.apache.beam.fn.v1.Elements\"\x00(\x01\x30\x01\x32g\n\x0b\x42\x65\x61mFnState\x12X\n\x05State\x12#.org.apache.beam.fn.v1.StateRequest\x1a$.org.apache.beam.fn.v1.StateResponse\"\x00(\x01\x30\x01\x32i\n\rBeamFnLogging\x12X\n\x07Logging\x12$.org.apache.beam.fn.v1.LogEntry.List\x1a!.org.apache.beam.fn.v1.LogControl\"\x00(\x01\x30\x01\x42\"\n\x15org.apache.beam.fn.v1B\tBeamFnApib\x06proto3')
- ,
- dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-
-
-_LOGENTRY_SEVERITY = _descriptor.EnumDescriptor(
- name='Severity',
- full_name='org.apache.beam.fn.v1.LogEntry.Severity',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='TRACE', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DEBUG', index=1, number=10,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='INFO', index=2, number=20,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='NOTICE', index=3, number=30,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='WARN', index=4, number=40,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='ERROR', index=5, number=50,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='CRITICAL', index=6, number=60,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=4529,
- serialized_end=4618,
-)
-_sym_db.RegisterEnumDescriptor(_LOGENTRY_SEVERITY)
-
-
-_TARGET_LIST = _descriptor.Descriptor(
- name='List',
- full_name='org.apache.beam.fn.v1.Target.List',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='target', full_name='org.apache.beam.fn.v1.Target.List.target', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=167,
- serialized_end=220,
-)
-
-_TARGET = _descriptor.Descriptor(
- name='Target',
- full_name='org.apache.beam.fn.v1.Target',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.Target.primitive_transform_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='name', full_name='org.apache.beam.fn.v1.Target.name', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_TARGET_LIST, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=104,
- serialized_end=220,
-)
-
-
-_PCOLLECTION = _descriptor.Descriptor(
- name='PCollection',
- full_name='org.apache.beam.fn.v1.PCollection',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='coder_reference', full_name='org.apache.beam.fn.v1.PCollection.coder_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=222,
- serialized_end=260,
-)
-
-
-_PRIMITIVETRANSFORM_INPUTSENTRY = _descriptor.Descriptor(
- name='InputsEntry',
- full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=600,
- serialized_end=681,
-)
-
-_PRIMITIVETRANSFORM_OUTPUTSENTRY = _descriptor.Descriptor(
- name='OutputsEntry',
- full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=683,
- serialized_end=765,
-)
-
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY = _descriptor.Descriptor(
- name='SideInputsEntry',
- full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=767,
- serialized_end=850,
-)
-
-_PRIMITIVETRANSFORM = _descriptor.Descriptor(
- name='PrimitiveTransform',
- full_name='org.apache.beam.fn.v1.PrimitiveTransform',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.PrimitiveTransform.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='function_spec', full_name='org.apache.beam.fn.v1.PrimitiveTransform.function_spec', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='inputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.inputs', index=2,
- number=3, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='outputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.outputs', index=3,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='side_inputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.side_inputs', index=4,
- number=5, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='step_name', full_name='org.apache.beam.fn.v1.PrimitiveTransform.step_name', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_PRIMITIVETRANSFORM_INPUTSENTRY, _PRIMITIVETRANSFORM_OUTPUTSENTRY, _PRIMITIVETRANSFORM_SIDEINPUTSENTRY, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=263,
- serialized_end=850,
-)
-
-
-_FUNCTIONSPEC = _descriptor.Descriptor(
- name='FunctionSpec',
- full_name='org.apache.beam.fn.v1.FunctionSpec',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.FunctionSpec.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='urn', full_name='org.apache.beam.fn.v1.FunctionSpec.urn', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='environment_reference', full_name='org.apache.beam.fn.v1.FunctionSpec.environment_reference', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='data', full_name='org.apache.beam.fn.v1.FunctionSpec.data', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=852,
- serialized_end=958,
-)
-
-
-_SIDEINPUT = _descriptor.Descriptor(
- name='SideInput',
- full_name='org.apache.beam.fn.v1.SideInput',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='input', full_name='org.apache.beam.fn.v1.SideInput.input', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='view_fn', full_name='org.apache.beam.fn.v1.SideInput.view_fn', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=960,
- serialized_end=1071,
-)
-
-
-_CODER = _descriptor.Descriptor(
- name='Coder',
- full_name='org.apache.beam.fn.v1.Coder',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='function_spec', full_name='org.apache.beam.fn.v1.Coder.function_spec', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='component_coder_reference', full_name='org.apache.beam.fn.v1.Coder.component_coder_reference', index=1,
- number=2, type=9, cpp_type=9, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1073,
- serialized_end=1175,
-)
-
-
-_REMOTEGRPCPORT = _descriptor.Descriptor(
- name='RemoteGrpcPort',
- full_name='org.apache.beam.fn.v1.RemoteGrpcPort',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='api_service_descriptor', full_name='org.apache.beam.fn.v1.RemoteGrpcPort.api_service_descriptor', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1177,
- serialized_end=1270,
-)
-
-
-_INSTRUCTIONREQUEST = _descriptor.Descriptor(
- name='InstructionRequest',
- full_name='org.apache.beam.fn.v1.InstructionRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='instruction_id', full_name='org.apache.beam.fn.v1.InstructionRequest.instruction_id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='register', full_name='org.apache.beam.fn.v1.InstructionRequest.register', index=1,
- number=1000, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle', index=2,
- number=1001, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle_progress', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle_progress', index=3,
- number=1002, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle_split', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle_split', index=4,
- number=1003, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='request', full_name='org.apache.beam.fn.v1.InstructionRequest.request',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=1273,
- serialized_end=1633,
-)
-
-
-_INSTRUCTIONRESPONSE = _descriptor.Descriptor(
- name='InstructionResponse',
- full_name='org.apache.beam.fn.v1.InstructionResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='instruction_id', full_name='org.apache.beam.fn.v1.InstructionResponse.instruction_id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='error', full_name='org.apache.beam.fn.v1.InstructionResponse.error', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='register', full_name='org.apache.beam.fn.v1.InstructionResponse.register', index=2,
- number=1000, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle', index=3,
- number=1001, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle_progress', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle_progress', index=4,
- number=1002, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='process_bundle_split', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle_split', index=5,
- number=1003, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='response', full_name='org.apache.beam.fn.v1.InstructionResponse.response',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=1636,
- serialized_end=2017,
-)
-
-
-_REGISTERREQUEST = _descriptor.Descriptor(
- name='RegisterRequest',
- full_name='org.apache.beam.fn.v1.RegisterRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='process_bundle_descriptor', full_name='org.apache.beam.fn.v1.RegisterRequest.process_bundle_descriptor', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2019,
- serialized_end=2119,
-)
-
-
-_REGISTERRESPONSE = _descriptor.Descriptor(
- name='RegisterResponse',
- full_name='org.apache.beam.fn.v1.RegisterResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2121,
- serialized_end=2139,
-)
-
-
-_PROCESSBUNDLEDESCRIPTOR = _descriptor.Descriptor(
- name='ProcessBundleDescriptor',
- full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='primitive_transform', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.primitive_transform', index=1,
- number=2, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='coders', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.coders', index=2,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2142,
- serialized_end=2297,
-)
-
-
-_PROCESSBUNDLEREQUEST = _descriptor.Descriptor(
- name='ProcessBundleRequest',
- full_name='org.apache.beam.fn.v1.ProcessBundleRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='process_bundle_descriptor_reference', full_name='org.apache.beam.fn.v1.ProcessBundleRequest.process_bundle_descriptor_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='cache_tokens', full_name='org.apache.beam.fn.v1.ProcessBundleRequest.cache_tokens', index=1,
- number=2, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2299,
- serialized_end=2423,
-)
-
-
-_PROCESSBUNDLERESPONSE = _descriptor.Descriptor(
- name='ProcessBundleResponse',
- full_name='org.apache.beam.fn.v1.ProcessBundleResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2425,
- serialized_end=2448,
-)
-
-
-_PROCESSBUNDLEPROGRESSREQUEST = _descriptor.Descriptor(
- name='ProcessBundleProgressRequest',
- full_name='org.apache.beam.fn.v1.ProcessBundleProgressRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='instruction_reference', full_name='org.apache.beam.fn.v1.ProcessBundleProgressRequest.instruction_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2450,
- serialized_end=2511,
-)
-
-
-_PROCESSBUNDLEPROGRESSRESPONSE = _descriptor.Descriptor(
- name='ProcessBundleProgressResponse',
- full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='finished_work', full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse.finished_work', index=0,
- number=1, type=1, cpp_type=5, label=1,
- has_default_value=False, default_value=float(0),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='backlog', full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse.backlog', index=1,
- number=2, type=1, cpp_type=5, label=1,
- has_default_value=False, default_value=float(0),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2513,
- serialized_end=2584,
-)
-
-
-_PROCESSBUNDLESPLITREQUEST = _descriptor.Descriptor(
- name='ProcessBundleSplitRequest',
- full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='instruction_reference', full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest.instruction_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='fraction', full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest.fraction', index=1,
- number=2, type=1, cpp_type=5, label=1,
- has_default_value=False, default_value=float(0),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2586,
- serialized_end=2662,
-)
-
-
-_ELEMENTCOUNTRESTRICTION = _descriptor.Descriptor(
- name='ElementCountRestriction',
- full_name='org.apache.beam.fn.v1.ElementCountRestriction',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='count', full_name='org.apache.beam.fn.v1.ElementCountRestriction.count', index=0,
- number=1, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2664,
- serialized_end=2704,
-)
-
-
-_ELEMENTCOUNTSKIPRESTRICTION = _descriptor.Descriptor(
- name='ElementCountSkipRestriction',
- full_name='org.apache.beam.fn.v1.ElementCountSkipRestriction',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='count', full_name='org.apache.beam.fn.v1.ElementCountSkipRestriction.count', index=0,
- number=1, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2706,
- serialized_end=2750,
-)
-
-
-_PRIMITIVETRANSFORMSPLIT = _descriptor.Descriptor(
- name='PrimitiveTransformSplit',
- full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.primitive_transform_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='completed_restriction', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.completed_restriction', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='remaining_restriction', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.remaining_restriction', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2753,
- serialized_end=2953,
-)
-
-
-_PROCESSBUNDLESPLITRESPONSE = _descriptor.Descriptor(
- name='ProcessBundleSplitResponse',
- full_name='org.apache.beam.fn.v1.ProcessBundleSplitResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='splits', full_name='org.apache.beam.fn.v1.ProcessBundleSplitResponse.splits', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2955,
- serialized_end=3047,
-)
-
-
-_ELEMENTS_DATA = _descriptor.Descriptor(
- name='Data',
- full_name='org.apache.beam.fn.v1.Elements.Data',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='instruction_reference', full_name='org.apache.beam.fn.v1.Elements.Data.instruction_reference', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='target', full_name='org.apache.beam.fn.v1.Elements.Data.target', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='data', full_name='org.apache.beam.fn.v1.Elements.Data.data', index=2,
- number=3, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3114,
- serialized_end=3212,
-)
-
-_ELEMENTS = _descriptor.Descriptor(
- name='Elements',
- full_name='org.apache.beam.fn.v1.Elements',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='data', full_name='org.apache.beam.fn.v1.Elements.data', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_ELEMENTS_DATA, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3050,
- serialized_end=3212,
-)
-
-
-_STATEREQUEST = _descriptor.Descriptor(
- name='StateRequest',
- full_name='org.apache.beam.fn.v1.StateRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.StateRequest.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='instruction_reference', full_name='org.apache.beam.fn.v1.StateRequest.instruction_reference', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='state_key', full_name='org.apache.beam.fn.v1.StateRequest.state_key', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='get', full_name='org.apache.beam.fn.v1.StateRequest.get', index=3,
- number=1000, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='append', full_name='org.apache.beam.fn.v1.StateRequest.append', index=4,
- number=1001, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='clear', full_name='org.apache.beam.fn.v1.StateRequest.clear', index=5,
- number=1002, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='request', full_name='org.apache.beam.fn.v1.StateRequest.request',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=3215,
- serialized_end=3513,
-)
-
-
-_STATERESPONSE = _descriptor.Descriptor(
- name='StateResponse',
- full_name='org.apache.beam.fn.v1.StateResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.StateResponse.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='error', full_name='org.apache.beam.fn.v1.StateResponse.error', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='get', full_name='org.apache.beam.fn.v1.StateResponse.get', index=2,
- number=1000, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='append', full_name='org.apache.beam.fn.v1.StateResponse.append', index=3,
- number=1001, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='clear', full_name='org.apache.beam.fn.v1.StateResponse.clear', index=4,
- number=1002, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='response', full_name='org.apache.beam.fn.v1.StateResponse.response',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=3516,
- serialized_end=3751,
-)
-
-
-_CACHETOKEN = _descriptor.Descriptor(
- name='CacheToken',
- full_name='org.apache.beam.fn.v1.CacheToken',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='target', full_name='org.apache.beam.fn.v1.CacheToken.target', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='token', full_name='org.apache.beam.fn.v1.CacheToken.token', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3753,
- serialized_end=3827,
-)
-
-
-_STATEKEY = _descriptor.Descriptor(
- name='StateKey',
- full_name='org.apache.beam.fn.v1.StateKey',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='target', full_name='org.apache.beam.fn.v1.StateKey.target', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='window', full_name='org.apache.beam.fn.v1.StateKey.window', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.fn.v1.StateKey.key', index=2,
- number=3, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3829,
- serialized_end=3915,
-)
-
-
-_CONTINUABLESTREAM = _descriptor.Descriptor(
- name='ContinuableStream',
- full_name='org.apache.beam.fn.v1.ContinuableStream',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='continuation_token', full_name='org.apache.beam.fn.v1.ContinuableStream.continuation_token', index=0,
- number=1, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='data', full_name='org.apache.beam.fn.v1.ContinuableStream.data', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3917,
- serialized_end=3978,
-)
-
-
-_STATEGETREQUEST = _descriptor.Descriptor(
- name='StateGetRequest',
- full_name='org.apache.beam.fn.v1.StateGetRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='continuation_token', full_name='org.apache.beam.fn.v1.StateGetRequest.continuation_token', index=0,
- number=1, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3980,
- serialized_end=4025,
-)
-
-
-_STATEGETRESPONSE = _descriptor.Descriptor(
- name='StateGetResponse',
- full_name='org.apache.beam.fn.v1.StateGetResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='stream', full_name='org.apache.beam.fn.v1.StateGetResponse.stream', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4027,
- serialized_end=4103,
-)
-
-
-_STATEAPPENDREQUEST = _descriptor.Descriptor(
- name='StateAppendRequest',
- full_name='org.apache.beam.fn.v1.StateAppendRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='data', full_name='org.apache.beam.fn.v1.StateAppendRequest.data', index=0,
- number=1, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4105,
- serialized_end=4139,
-)
-
-
-_STATEAPPENDRESPONSE = _descriptor.Descriptor(
- name='StateAppendResponse',
- full_name='org.apache.beam.fn.v1.StateAppendResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4141,
- serialized_end=4162,
-)
-
-
-_STATECLEARREQUEST = _descriptor.Descriptor(
- name='StateClearRequest',
- full_name='org.apache.beam.fn.v1.StateClearRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4164,
- serialized_end=4183,
-)
-
-
-_STATECLEARRESPONSE = _descriptor.Descriptor(
- name='StateClearResponse',
- full_name='org.apache.beam.fn.v1.StateClearResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4185,
- serialized_end=4205,
-)
-
-
-_LOGENTRY_LIST = _descriptor.Descriptor(
- name='List',
- full_name='org.apache.beam.fn.v1.LogEntry.List',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='log_entries', full_name='org.apache.beam.fn.v1.LogEntry.List.log_entries', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4467,
- serialized_end=4527,
-)
-
-_LOGENTRY = _descriptor.Descriptor(
- name='LogEntry',
- full_name='org.apache.beam.fn.v1.LogEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='severity', full_name='org.apache.beam.fn.v1.LogEntry.severity', index=0,
- number=1, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='timestamp', full_name='org.apache.beam.fn.v1.LogEntry.timestamp', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='message', full_name='org.apache.beam.fn.v1.LogEntry.message', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='trace', full_name='org.apache.beam.fn.v1.LogEntry.trace', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='instruction_reference', full_name='org.apache.beam.fn.v1.LogEntry.instruction_reference', index=4,
- number=5, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.LogEntry.primitive_transform_reference', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='log_location', full_name='org.apache.beam.fn.v1.LogEntry.log_location', index=6,
- number=7, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='thread', full_name='org.apache.beam.fn.v1.LogEntry.thread', index=7,
- number=8, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_LOGENTRY_LIST, ],
- enum_types=[
- _LOGENTRY_SEVERITY,
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4208,
- serialized_end=4618,
-)
-
-
-_LOGCONTROL = _descriptor.Descriptor(
- name='LogControl',
- full_name='org.apache.beam.fn.v1.LogControl',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4620,
- serialized_end=4632,
-)
-
-
-_APISERVICEDESCRIPTOR = _descriptor.Descriptor(
- name='ApiServiceDescriptor',
- full_name='org.apache.beam.fn.v1.ApiServiceDescriptor',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='url', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.url', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='oauth2_client_credentials_grant', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.oauth2_client_credentials_grant', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='authentication', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.authentication',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=4635,
- serialized_end=4796,
-)
-
-
-_OAUTH2CLIENTCREDENTIALSGRANT = _descriptor.Descriptor(
- name='OAuth2ClientCredentialsGrant',
- full_name='org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='url', full_name='org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant.url', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4798,
- serialized_end=4841,
-)
-
-
-_DOCKERCONTAINER = _descriptor.Descriptor(
- name='DockerContainer',
- full_name='org.apache.beam.fn.v1.DockerContainer',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.fn.v1.DockerContainer.id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='uri', full_name='org.apache.beam.fn.v1.DockerContainer.uri', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='registry_reference', full_name='org.apache.beam.fn.v1.DockerContainer.registry_reference', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4843,
- serialized_end=4913,
-)
-
-_TARGET_LIST.fields_by_name['target'].message_type = _TARGET
-_TARGET_LIST.containing_type = _TARGET
-_PRIMITIVETRANSFORM_INPUTSENTRY.fields_by_name['value'].message_type = _TARGET_LIST
-_PRIMITIVETRANSFORM_INPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.fields_by_name['value'].message_type = _PCOLLECTION
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_PRIMITIVETRANSFORM.fields_by_name['inputs'].message_type = _PRIMITIVETRANSFORM_INPUTSENTRY
-_PRIMITIVETRANSFORM.fields_by_name['outputs'].message_type = _PRIMITIVETRANSFORM_OUTPUTSENTRY
-_PRIMITIVETRANSFORM.fields_by_name['side_inputs'].message_type = _PRIMITIVETRANSFORM_SIDEINPUTSENTRY
-_FUNCTIONSPEC.fields_by_name['data'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_SIDEINPUT.fields_by_name['input'].message_type = _TARGET
-_SIDEINPUT.fields_by_name['view_fn'].message_type = _FUNCTIONSPEC
-_CODER.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_REMOTEGRPCPORT.fields_by_name['api_service_descriptor'].message_type = _APISERVICEDESCRIPTOR
-_INSTRUCTIONREQUEST.fields_by_name['register'].message_type = _REGISTERREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle'].message_type = _PROCESSBUNDLEREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'].message_type = _PROCESSBUNDLEPROGRESSREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'].message_type = _PROCESSBUNDLESPLITREQUEST
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
- _INSTRUCTIONREQUEST.fields_by_name['register'])
-_INSTRUCTIONREQUEST.fields_by_name['register'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
- _INSTRUCTIONREQUEST.fields_by_name['process_bundle'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
- _INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
- _INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONRESPONSE.fields_by_name['register'].message_type = _REGISTERRESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle'].message_type = _PROCESSBUNDLERESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'].message_type = _PROCESSBUNDLEPROGRESSRESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'].message_type = _PROCESSBUNDLESPLITRESPONSE
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
- _INSTRUCTIONRESPONSE.fields_by_name['register'])
-_INSTRUCTIONRESPONSE.fields_by_name['register'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
- _INSTRUCTIONRESPONSE.fields_by_name['process_bundle'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
- _INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
- _INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_REGISTERREQUEST.fields_by_name['process_bundle_descriptor'].message_type = _PROCESSBUNDLEDESCRIPTOR
-_PROCESSBUNDLEDESCRIPTOR.fields_by_name['primitive_transform'].message_type = _PRIMITIVETRANSFORM
-_PROCESSBUNDLEDESCRIPTOR.fields_by_name['coders'].message_type = _CODER
-_PROCESSBUNDLEREQUEST.fields_by_name['cache_tokens'].message_type = _CACHETOKEN
-_PRIMITIVETRANSFORMSPLIT.fields_by_name['completed_restriction'].message_type = _FUNCTIONSPEC
-_PRIMITIVETRANSFORMSPLIT.fields_by_name['remaining_restriction'].message_type = _FUNCTIONSPEC
-_PROCESSBUNDLESPLITRESPONSE.fields_by_name['splits'].message_type = _PRIMITIVETRANSFORMSPLIT
-_ELEMENTS_DATA.fields_by_name['target'].message_type = _TARGET
-_ELEMENTS_DATA.containing_type = _ELEMENTS
-_ELEMENTS.fields_by_name['data'].message_type = _ELEMENTS_DATA
-_STATEREQUEST.fields_by_name['state_key'].message_type = _STATEKEY
-_STATEREQUEST.fields_by_name['get'].message_type = _STATEGETREQUEST
-_STATEREQUEST.fields_by_name['append'].message_type = _STATEAPPENDREQUEST
-_STATEREQUEST.fields_by_name['clear'].message_type = _STATECLEARREQUEST
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
- _STATEREQUEST.fields_by_name['get'])
-_STATEREQUEST.fields_by_name['get'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
- _STATEREQUEST.fields_by_name['append'])
-_STATEREQUEST.fields_by_name['append'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
- _STATEREQUEST.fields_by_name['clear'])
-_STATEREQUEST.fields_by_name['clear'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATERESPONSE.fields_by_name['get'].message_type = _STATEGETRESPONSE
-_STATERESPONSE.fields_by_name['append'].message_type = _STATEAPPENDRESPONSE
-_STATERESPONSE.fields_by_name['clear'].message_type = _STATECLEARRESPONSE
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
- _STATERESPONSE.fields_by_name['get'])
-_STATERESPONSE.fields_by_name['get'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
- _STATERESPONSE.fields_by_name['append'])
-_STATERESPONSE.fields_by_name['append'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
- _STATERESPONSE.fields_by_name['clear'])
-_STATERESPONSE.fields_by_name['clear'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_CACHETOKEN.fields_by_name['target'].message_type = _TARGET
-_STATEKEY.fields_by_name['target'].message_type = _TARGET
-_STATEGETRESPONSE.fields_by_name['stream'].message_type = _CONTINUABLESTREAM
-_LOGENTRY_LIST.fields_by_name['log_entries'].message_type = _LOGENTRY
-_LOGENTRY_LIST.containing_type = _LOGENTRY
-_LOGENTRY.fields_by_name['severity'].enum_type = _LOGENTRY_SEVERITY
-_LOGENTRY.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
-_LOGENTRY_SEVERITY.containing_type = _LOGENTRY
-_APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'].message_type = _OAUTH2CLIENTCREDENTIALSGRANT
-_APISERVICEDESCRIPTOR.oneofs_by_name['authentication'].fields.append(
- _APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'])
-_APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'].containing_oneof = _APISERVICEDESCRIPTOR.oneofs_by_name['authentication']
-DESCRIPTOR.message_types_by_name['Target'] = _TARGET
-DESCRIPTOR.message_types_by_name['PCollection'] = _PCOLLECTION
-DESCRIPTOR.message_types_by_name['PrimitiveTransform'] = _PRIMITIVETRANSFORM
-DESCRIPTOR.message_types_by_name['FunctionSpec'] = _FUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['SideInput'] = _SIDEINPUT
-DESCRIPTOR.message_types_by_name['Coder'] = _CODER
-DESCRIPTOR.message_types_by_name['RemoteGrpcPort'] = _REMOTEGRPCPORT
-DESCRIPTOR.message_types_by_name['InstructionRequest'] = _INSTRUCTIONREQUEST
-DESCRIPTOR.message_types_by_name['InstructionResponse'] = _INSTRUCTIONRESPONSE
-DESCRIPTOR.message_types_by_name['RegisterRequest'] = _REGISTERREQUEST
-DESCRIPTOR.message_types_by_name['RegisterResponse'] = _REGISTERRESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleDescriptor'] = _PROCESSBUNDLEDESCRIPTOR
-DESCRIPTOR.message_types_by_name['ProcessBundleRequest'] = _PROCESSBUNDLEREQUEST
-DESCRIPTOR.message_types_by_name['ProcessBundleResponse'] = _PROCESSBUNDLERESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleProgressRequest'] = _PROCESSBUNDLEPROGRESSREQUEST
-DESCRIPTOR.message_types_by_name['ProcessBundleProgressResponse'] = _PROCESSBUNDLEPROGRESSRESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleSplitRequest'] = _PROCESSBUNDLESPLITREQUEST
-DESCRIPTOR.message_types_by_name['ElementCountRestriction'] = _ELEMENTCOUNTRESTRICTION
-DESCRIPTOR.message_types_by_name['ElementCountSkipRestriction'] = _ELEMENTCOUNTSKIPRESTRICTION
-DESCRIPTOR.message_types_by_name['PrimitiveTransformSplit'] = _PRIMITIVETRANSFORMSPLIT
-DESCRIPTOR.message_types_by_name['ProcessBundleSplitResponse'] = _PROCESSBUNDLESPLITRESPONSE
-DESCRIPTOR.message_types_by_name['Elements'] = _ELEMENTS
-DESCRIPTOR.message_types_by_name['StateRequest'] = _STATEREQUEST
-DESCRIPTOR.message_types_by_name['StateResponse'] = _STATERESPONSE
-DESCRIPTOR.message_types_by_name['CacheToken'] = _CACHETOKEN
-DESCRIPTOR.message_types_by_name['StateKey'] = _STATEKEY
-DESCRIPTOR.message_types_by_name['ContinuableStream'] = _CONTINUABLESTREAM
-DESCRIPTOR.message_types_by_name['StateGetRequest'] = _STATEGETREQUEST
-DESCRIPTOR.message_types_by_name['StateGetResponse'] = _STATEGETRESPONSE
-DESCRIPTOR.message_types_by_name['StateAppendRequest'] = _STATEAPPENDREQUEST
-DESCRIPTOR.message_types_by_name['StateAppendResponse'] = _STATEAPPENDRESPONSE
-DESCRIPTOR.message_types_by_name['StateClearRequest'] = _STATECLEARREQUEST
-DESCRIPTOR.message_types_by_name['StateClearResponse'] = _STATECLEARRESPONSE
-DESCRIPTOR.message_types_by_name['LogEntry'] = _LOGENTRY
-DESCRIPTOR.message_types_by_name['LogControl'] = _LOGCONTROL
-DESCRIPTOR.message_types_by_name['ApiServiceDescriptor'] = _APISERVICEDESCRIPTOR
-DESCRIPTOR.message_types_by_name['OAuth2ClientCredentialsGrant'] = _OAUTH2CLIENTCREDENTIALSGRANT
-DESCRIPTOR.message_types_by_name['DockerContainer'] = _DOCKERCONTAINER
-
-Target = _reflection.GeneratedProtocolMessageType('Target', (_message.Message,), dict(
-
- List = _reflection.GeneratedProtocolMessageType('List', (_message.Message,), dict(
- DESCRIPTOR = _TARGET_LIST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Target.List)
- ))
- ,
- DESCRIPTOR = _TARGET,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Target)
- ))
-_sym_db.RegisterMessage(Target)
-_sym_db.RegisterMessage(Target.List)
-
-PCollection = _reflection.GeneratedProtocolMessageType('PCollection', (_message.Message,), dict(
- DESCRIPTOR = _PCOLLECTION,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PCollection)
- ))
-_sym_db.RegisterMessage(PCollection)
-
-PrimitiveTransform = _reflection.GeneratedProtocolMessageType('PrimitiveTransform', (_message.Message,), dict(
-
- InputsEntry = _reflection.GeneratedProtocolMessageType('InputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PRIMITIVETRANSFORM_INPUTSENTRY,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry)
- ))
- ,
-
- OutputsEntry = _reflection.GeneratedProtocolMessageType('OutputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PRIMITIVETRANSFORM_OUTPUTSENTRY,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry)
- ))
- ,
-
- SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PRIMITIVETRANSFORM_SIDEINPUTSENTRY,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry)
- ))
- ,
- DESCRIPTOR = _PRIMITIVETRANSFORM,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform)
- ))
-_sym_db.RegisterMessage(PrimitiveTransform)
-_sym_db.RegisterMessage(PrimitiveTransform.InputsEntry)
-_sym_db.RegisterMessage(PrimitiveTransform.OutputsEntry)
-_sym_db.RegisterMessage(PrimitiveTransform.SideInputsEntry)
-
-FunctionSpec = _reflection.GeneratedProtocolMessageType('FunctionSpec', (_message.Message,), dict(
- DESCRIPTOR = _FUNCTIONSPEC,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.FunctionSpec)
- ))
-_sym_db.RegisterMessage(FunctionSpec)
-
-SideInput = _reflection.GeneratedProtocolMessageType('SideInput', (_message.Message,), dict(
- DESCRIPTOR = _SIDEINPUT,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.SideInput)
- ))
-_sym_db.RegisterMessage(SideInput)
-
-Coder = _reflection.GeneratedProtocolMessageType('Coder', (_message.Message,), dict(
- DESCRIPTOR = _CODER,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Coder)
- ))
-_sym_db.RegisterMessage(Coder)
-
-RemoteGrpcPort = _reflection.GeneratedProtocolMessageType('RemoteGrpcPort', (_message.Message,), dict(
- DESCRIPTOR = _REMOTEGRPCPORT,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RemoteGrpcPort)
- ))
-_sym_db.RegisterMessage(RemoteGrpcPort)
-
-InstructionRequest = _reflection.GeneratedProtocolMessageType('InstructionRequest', (_message.Message,), dict(
- DESCRIPTOR = _INSTRUCTIONREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.InstructionRequest)
- ))
-_sym_db.RegisterMessage(InstructionRequest)
-
-InstructionResponse = _reflection.GeneratedProtocolMessageType('InstructionResponse', (_message.Message,), dict(
- DESCRIPTOR = _INSTRUCTIONRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.InstructionResponse)
- ))
-_sym_db.RegisterMessage(InstructionResponse)
-
-RegisterRequest = _reflection.GeneratedProtocolMessageType('RegisterRequest', (_message.Message,), dict(
- DESCRIPTOR = _REGISTERREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RegisterRequest)
- ))
-_sym_db.RegisterMessage(RegisterRequest)
-
-RegisterResponse = _reflection.GeneratedProtocolMessageType('RegisterResponse', (_message.Message,), dict(
- DESCRIPTOR = _REGISTERRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RegisterResponse)
- ))
-_sym_db.RegisterMessage(RegisterResponse)
-
-ProcessBundleDescriptor = _reflection.GeneratedProtocolMessageType('ProcessBundleDescriptor', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLEDESCRIPTOR,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleDescriptor)
- ))
-_sym_db.RegisterMessage(ProcessBundleDescriptor)
-
-ProcessBundleRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleRequest', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLEREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleRequest)
- ))
-_sym_db.RegisterMessage(ProcessBundleRequest)
-
-ProcessBundleResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleResponse', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLERESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleResponse)
- ))
-_sym_db.RegisterMessage(ProcessBundleResponse)
-
-ProcessBundleProgressRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleProgressRequest', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLEPROGRESSREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleProgressRequest)
- ))
-_sym_db.RegisterMessage(ProcessBundleProgressRequest)
-
-ProcessBundleProgressResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleProgressResponse', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLEPROGRESSRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleProgressResponse)
- ))
-_sym_db.RegisterMessage(ProcessBundleProgressResponse)
-
-ProcessBundleSplitRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleSplitRequest', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLESPLITREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleSplitRequest)
- ))
-_sym_db.RegisterMessage(ProcessBundleSplitRequest)
-
-ElementCountRestriction = _reflection.GeneratedProtocolMessageType('ElementCountRestriction', (_message.Message,), dict(
- DESCRIPTOR = _ELEMENTCOUNTRESTRICTION,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ElementCountRestriction)
- ))
-_sym_db.RegisterMessage(ElementCountRestriction)
-
-ElementCountSkipRestriction = _reflection.GeneratedProtocolMessageType('ElementCountSkipRestriction', (_message.Message,), dict(
- DESCRIPTOR = _ELEMENTCOUNTSKIPRESTRICTION,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ElementCountSkipRestriction)
- ))
-_sym_db.RegisterMessage(ElementCountSkipRestriction)
-
-PrimitiveTransformSplit = _reflection.GeneratedProtocolMessageType('PrimitiveTransformSplit', (_message.Message,), dict(
- DESCRIPTOR = _PRIMITIVETRANSFORMSPLIT,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransformSplit)
- ))
-_sym_db.RegisterMessage(PrimitiveTransformSplit)
-
-ProcessBundleSplitResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleSplitResponse', (_message.Message,), dict(
- DESCRIPTOR = _PROCESSBUNDLESPLITRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleSplitResponse)
- ))
-_sym_db.RegisterMessage(ProcessBundleSplitResponse)
-
-Elements = _reflection.GeneratedProtocolMessageType('Elements', (_message.Message,), dict(
-
- Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dict(
- DESCRIPTOR = _ELEMENTS_DATA,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Elements.Data)
- ))
- ,
- DESCRIPTOR = _ELEMENTS,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Elements)
- ))
-_sym_db.RegisterMessage(Elements)
-_sym_db.RegisterMessage(Elements.Data)
-
-StateRequest = _reflection.GeneratedProtocolMessageType('StateRequest', (_message.Message,), dict(
- DESCRIPTOR = _STATEREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateRequest)
- ))
-_sym_db.RegisterMessage(StateRequest)
-
-StateResponse = _reflection.GeneratedProtocolMessageType('StateResponse', (_message.Message,), dict(
- DESCRIPTOR = _STATERESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateResponse)
- ))
-_sym_db.RegisterMessage(StateResponse)
-
-CacheToken = _reflection.GeneratedProtocolMessageType('CacheToken', (_message.Message,), dict(
- DESCRIPTOR = _CACHETOKEN,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.CacheToken)
- ))
-_sym_db.RegisterMessage(CacheToken)
-
-StateKey = _reflection.GeneratedProtocolMessageType('StateKey', (_message.Message,), dict(
- DESCRIPTOR = _STATEKEY,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateKey)
- ))
-_sym_db.RegisterMessage(StateKey)
-
-ContinuableStream = _reflection.GeneratedProtocolMessageType('ContinuableStream', (_message.Message,), dict(
- DESCRIPTOR = _CONTINUABLESTREAM,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ContinuableStream)
- ))
-_sym_db.RegisterMessage(ContinuableStream)
-
-StateGetRequest = _reflection.GeneratedProtocolMessageType('StateGetRequest', (_message.Message,), dict(
- DESCRIPTOR = _STATEGETREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateGetRequest)
- ))
-_sym_db.RegisterMessage(StateGetRequest)
-
-StateGetResponse = _reflection.GeneratedProtocolMessageType('StateGetResponse', (_message.Message,), dict(
- DESCRIPTOR = _STATEGETRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateGetResponse)
- ))
-_sym_db.RegisterMessage(StateGetResponse)
-
-StateAppendRequest = _reflection.GeneratedProtocolMessageType('StateAppendRequest', (_message.Message,), dict(
- DESCRIPTOR = _STATEAPPENDREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateAppendRequest)
- ))
-_sym_db.RegisterMessage(StateAppendRequest)
-
-StateAppendResponse = _reflection.GeneratedProtocolMessageType('StateAppendResponse', (_message.Message,), dict(
- DESCRIPTOR = _STATEAPPENDRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateAppendResponse)
- ))
-_sym_db.RegisterMessage(StateAppendResponse)
-
-StateClearRequest = _reflection.GeneratedProtocolMessageType('StateClearRequest', (_message.Message,), dict(
- DESCRIPTOR = _STATECLEARREQUEST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateClearRequest)
- ))
-_sym_db.RegisterMessage(StateClearRequest)
-
-StateClearResponse = _reflection.GeneratedProtocolMessageType('StateClearResponse', (_message.Message,), dict(
- DESCRIPTOR = _STATECLEARRESPONSE,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateClearResponse)
- ))
-_sym_db.RegisterMessage(StateClearResponse)
-
-LogEntry = _reflection.GeneratedProtocolMessageType('LogEntry', (_message.Message,), dict(
-
- List = _reflection.GeneratedProtocolMessageType('List', (_message.Message,), dict(
- DESCRIPTOR = _LOGENTRY_LIST,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogEntry.List)
- ))
- ,
- DESCRIPTOR = _LOGENTRY,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogEntry)
- ))
-_sym_db.RegisterMessage(LogEntry)
-_sym_db.RegisterMessage(LogEntry.List)
-
-LogControl = _reflection.GeneratedProtocolMessageType('LogControl', (_message.Message,), dict(
- DESCRIPTOR = _LOGCONTROL,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogControl)
- ))
-_sym_db.RegisterMessage(LogControl)
-
-ApiServiceDescriptor = _reflection.GeneratedProtocolMessageType('ApiServiceDescriptor', (_message.Message,), dict(
- DESCRIPTOR = _APISERVICEDESCRIPTOR,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ApiServiceDescriptor)
- ))
-_sym_db.RegisterMessage(ApiServiceDescriptor)
-
-OAuth2ClientCredentialsGrant = _reflection.GeneratedProtocolMessageType('OAuth2ClientCredentialsGrant', (_message.Message,), dict(
- DESCRIPTOR = _OAUTH2CLIENTCREDENTIALSGRANT,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant)
- ))
-_sym_db.RegisterMessage(OAuth2ClientCredentialsGrant)
-
-DockerContainer = _reflection.GeneratedProtocolMessageType('DockerContainer', (_message.Message,), dict(
- DESCRIPTOR = _DOCKERCONTAINER,
- __module__ = 'beam_fn_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.DockerContainer)
- ))
-_sym_db.RegisterMessage(DockerContainer)
-
-
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\025org.apache.beam.fn.v1B\tBeamFnApi'))
-_PRIMITIVETRANSFORM_INPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_INPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_OUTPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-try:
- # THESE ELEMENTS WILL BE DEPRECATED.
- # Please use the generated *_pb2_grpc.py files instead.
- import grpc
- from grpc.framework.common import cardinality
- from grpc.framework.interfaces.face import utilities as face_utilities
- from grpc.beta import implementations as beta_implementations
- from grpc.beta import interfaces as beta_interfaces
-
-
- class BeamFnControlStub(object):
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Control = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnControl/Control',
- request_serializer=InstructionResponse.SerializeToString,
- response_deserializer=InstructionRequest.FromString,
- )
-
-
- class BeamFnControlServicer(object):
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
-
- def Control(self, request_iterator, context):
- """Instructions sent by the runner to the SDK requesting different types
- of work.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
- def add_BeamFnControlServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Control': grpc.stream_stream_rpc_method_handler(
- servicer.Control,
- request_deserializer=InstructionResponse.FromString,
- response_serializer=InstructionRequest.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnControl', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
- class BeamFnDataStub(object):
- """Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Data = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnData/Data',
- request_serializer=Elements.SerializeToString,
- response_deserializer=Elements.FromString,
- )
-
-
- class BeamFnDataServicer(object):
- """Stable
- """
-
- def Data(self, request_iterator, context):
- """Used to send data between harnesses.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
- def add_BeamFnDataServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Data': grpc.stream_stream_rpc_method_handler(
- servicer.Data,
- request_deserializer=Elements.FromString,
- response_serializer=Elements.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnData', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
- class BeamFnStateStub(object):
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.State = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnState/State',
- request_serializer=StateRequest.SerializeToString,
- response_deserializer=StateResponse.FromString,
- )
-
-
- class BeamFnStateServicer(object):
-
- def State(self, request_iterator, context):
- """Used to get/append/clear state stored by the runner on behalf of the SDK.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
- def add_BeamFnStateServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'State': grpc.stream_stream_rpc_method_handler(
- servicer.State,
- request_deserializer=StateRequest.FromString,
- response_serializer=StateResponse.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnState', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
- class BeamFnLoggingStub(object):
- """Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Logging = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnLogging/Logging',
- request_serializer=LogEntry.List.SerializeToString,
- response_deserializer=LogControl.FromString,
- )
-
-
- class BeamFnLoggingServicer(object):
- """Stable
- """
-
- def Logging(self, request_iterator, context):
- """Allows for the SDK to emit log entries which the runner can
- associate with the active job.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
- def add_BeamFnLoggingServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Logging': grpc.stream_stream_rpc_method_handler(
- servicer.Logging,
- request_deserializer=LogEntry.List.FromString,
- response_serializer=LogControl.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnLogging', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
- class BetaBeamFnControlServicer(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
- def Control(self, request_iterator, context):
- """Instructions sent by the runner to the SDK requesting different types
- of work.
- """
- context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
- class BetaBeamFnControlStub(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
- def Control(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
- """Instructions sent by the runner to the SDK requesting different types
- of work.
- """
- raise NotImplementedError()
-
-
- def beta_create_BeamFnControl_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionResponse.FromString,
- }
- response_serializers = {
- ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionRequest.SerializeToString,
- }
- method_implementations = {
- ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): face_utilities.stream_stream_inline(servicer.Control),
- }
- server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
- return beta_implementations.server(method_implementations, options=server_options)
-
-
- def beta_create_BeamFnControl_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_serializers = {
- ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionResponse.SerializeToString,
- }
- response_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionRequest.FromString,
- }
- cardinalities = {
- 'Control': cardinality.Cardinality.STREAM_STREAM,
- }
- stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
- return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnControl', cardinalities, options=stub_options)
-
-
- class BetaBeamFnDataServicer(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """Stable
- """
- def Data(self, request_iterator, context):
- """Used to send data between harnesses.
- """
- context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
- class BetaBeamFnDataStub(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """Stable
- """
- def Data(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
- """Used to send data between harnesses.
- """
- raise NotImplementedError()
-
-
- def beta_create_BeamFnData_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.FromString,
- }
- response_serializers = {
- ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.SerializeToString,
- }
- method_implementations = {
- ('org.apache.beam.fn.v1.BeamFnData', 'Data'): face_utilities.stream_stream_inline(servicer.Data),
- }
- server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
- return beta_implementations.server(method_implementations, options=server_options)
-
-
- def beta_create_BeamFnData_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_serializers = {
- ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.SerializeToString,
- }
- response_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.FromString,
- }
- cardinalities = {
- 'Data': cardinality.Cardinality.STREAM_STREAM,
- }
- stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
- return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnData', cardinalities, options=stub_options)
-
-
- class BetaBeamFnStateServicer(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- def State(self, request_iterator, context):
- """Used to get/append/clear state stored by the runner on behalf of the SDK.
- """
- context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
- class BetaBeamFnStateStub(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- def State(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
- """Used to get/append/clear state stored by the runner on behalf of the SDK.
- """
- raise NotImplementedError()
-
-
- def beta_create_BeamFnState_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateRequest.FromString,
- }
- response_serializers = {
- ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateResponse.SerializeToString,
- }
- method_implementations = {
- ('org.apache.beam.fn.v1.BeamFnState', 'State'): face_utilities.stream_stream_inline(servicer.State),
- }
- server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
- return beta_implementations.server(method_implementations, options=server_options)
-
-
- def beta_create_BeamFnState_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_serializers = {
- ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateRequest.SerializeToString,
- }
- response_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateResponse.FromString,
- }
- cardinalities = {
- 'State': cardinality.Cardinality.STREAM_STREAM,
- }
- stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
- return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnState', cardinalities, options=stub_options)
-
-
- class BetaBeamFnLoggingServicer(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """Stable
- """
- def Logging(self, request_iterator, context):
- """Allows for the SDK to emit log entries which the runner can
- associate with the active job.
- """
- context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
- class BetaBeamFnLoggingStub(object):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This class was generated
- only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
- """Stable
- """
- def Logging(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
- """Allows for the SDK to emit log entries which the runner can
- associate with the active job.
- """
- raise NotImplementedError()
-
-
- def beta_create_BeamFnLogging_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogEntry.List.FromString,
- }
- response_serializers = {
- ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogControl.SerializeToString,
- }
- method_implementations = {
- ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): face_utilities.stream_stream_inline(servicer.Logging),
- }
- server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
- return beta_implementations.server(method_implementations, options=server_options)
-
-
- def beta_create_BeamFnLogging_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
- """The Beta API is deprecated for 0.15.0 and later.
-
- It is recommended to use the GA API (classes and functions in this
- file not marked beta) for all further purposes. This function was
- generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
- request_serializers = {
- ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogEntry.List.SerializeToString,
- }
- response_deserializers = {
- ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogControl.FromString,
- }
- cardinalities = {
- 'Logging': cardinality.Cardinality.STREAM_STREAM,
- }
- stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
- return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnLogging', cardinalities, options=stub_options)
-except ImportError:
- pass
-# @@protoc_insertion_point(module_scope)
diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
deleted file mode 100644
index 08d7dad..0000000
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-import grpc
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
-
-import beam_fn_api_pb2 as beam__fn__api__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-class BeamFnControlStub(object):
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Control = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnControl/Control',
- request_serializer=beam__fn__api__pb2.InstructionResponse.SerializeToString,
- response_deserializer=beam__fn__api__pb2.InstructionRequest.FromString,
- )
-
-
-class BeamFnControlServicer(object):
- """
- Control Plane API
-
- Progress reporting and splitting still need further vetting. Also, this may change
- with the addition of new types of instructions/responses related to metrics.
-
- An API that describes the work that a SDK harness is meant to do.
- Stable
- """
-
- def Control(self, request_iterator, context):
- """Instructions sent by the runner to the SDK requesting different types
- of work.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnControlServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Control': grpc.stream_stream_rpc_method_handler(
- servicer.Control,
- request_deserializer=beam__fn__api__pb2.InstructionResponse.FromString,
- response_serializer=beam__fn__api__pb2.InstructionRequest.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnControl', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnDataStub(object):
- """Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Data = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnData/Data',
- request_serializer=beam__fn__api__pb2.Elements.SerializeToString,
- response_deserializer=beam__fn__api__pb2.Elements.FromString,
- )
-
-
-class BeamFnDataServicer(object):
- """Stable
- """
-
- def Data(self, request_iterator, context):
- """Used to send data between harnesses.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnDataServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Data': grpc.stream_stream_rpc_method_handler(
- servicer.Data,
- request_deserializer=beam__fn__api__pb2.Elements.FromString,
- response_serializer=beam__fn__api__pb2.Elements.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnData', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnStateStub(object):
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.State = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnState/State',
- request_serializer=beam__fn__api__pb2.StateRequest.SerializeToString,
- response_deserializer=beam__fn__api__pb2.StateResponse.FromString,
- )
-
-
-class BeamFnStateServicer(object):
-
- def State(self, request_iterator, context):
- """Used to get/append/clear state stored by the runner on behalf of the SDK.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnStateServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'State': grpc.stream_stream_rpc_method_handler(
- servicer.State,
- request_deserializer=beam__fn__api__pb2.StateRequest.FromString,
- response_serializer=beam__fn__api__pb2.StateResponse.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnState', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnLoggingStub(object):
- """Stable
- """
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Logging = channel.stream_stream(
- '/org.apache.beam.fn.v1.BeamFnLogging/Logging',
- request_serializer=beam__fn__api__pb2.LogEntry.List.SerializeToString,
- response_deserializer=beam__fn__api__pb2.LogControl.FromString,
- )
-
-
-class BeamFnLoggingServicer(object):
- """Stable
- """
-
- def Logging(self, request_iterator, context):
- """Allows for the SDK to emit log entries which the runner can
- associate with the active job.
- """
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnLoggingServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Logging': grpc.stream_stream_rpc_method_handler(
- servicer.Logging,
- request_deserializer=beam__fn__api__pb2.LogEntry.List.FromString,
- response_serializer=beam__fn__api__pb2.LogControl.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'org.apache.beam.fn.v1.BeamFnLogging', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
diff --git a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
deleted file mode 100644
index e8793b6..0000000
--- a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
+++ /dev/null
@@ -1,2872 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: beam_runner_api.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
-from google.protobuf.internal import enum_type_wrapper
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='beam_runner_api.proto',
- package='org.apache.beam.runner_api.v1',
- syntax='proto3',
- serialized_pb=_b('\n\x15\x62\x65\x61m_runner_api.proto\x12\x1dorg.apache.beam.runner_api.v1\x1a\x19google/protobuf/any.proto\"\x8d\x07\n\nComponents\x12M\n\ntransforms\x18\x01 \x03(\x0b\x32\x39.org.apache.beam.runner_api.v1.Components.TransformsEntry\x12Q\n\x0cpcollections\x18\x02 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.Components.PcollectionsEntry\x12`\n\x14windowing_strategies\x18\x03 \x03(\x0b\x32\x42.org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry\x12\x45\n\x06\x63oders\x18\x04 \x03(\x0b\x32\x35.org.apache.beam.runner_api.v1.Components.CodersEntry\x12Q\n\x0c\x65nvironments\x18\x05 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.Components.EnvironmentsEntry\x1a\\\n\x0fTransformsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).org.apache.beam.runner_api.v1.PTransform:\x02\x38\x01\x1a_\n\x11PcollectionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.PCollection:\x02\x38\x01\x1al\n\x18WindowingStrategiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowingStrategy:\x02\x38\x01\x1aS\n\x0b\x43odersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.org.apache.beam.runner_api.v1.Coder:\x02\x38\x01\x1a_\n\x11\x45nvironmentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.Environment:\x02\x38\x01\"\xe2\x06\n\x15MessageWithComponents\x12=\n\ncomponents\x18\x01 \x01(\x0b\x32).org.apache.beam.runner_api.v1.Components\x12\x35\n\x05\x63oder\x18\x02 \x01(\x0b\x32$.org.apache.beam.runner_api.v1.CoderH\x00\x12H\n\x0f\x63ombine_payload\x18\x03 \x01(\x0b\x32-.org.apache.beam.runner_api.v1.CombinePayloadH\x00\x12K\n\x11sdk_function_spec\x18\x04 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpecH\x00\x12\x45\n\x0epar_do_payload\x18\x06 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.ParDoPayloadH\x00\x12?\n\nptransform\x18\x07 \x01(\x0b\x32).org.apache.beam.runner_api.v1.PTransformH\x00\x12\x41\n\x0bpcollection\x18\x08 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.PCollectionH\x00\x12\x42\n\x0cread_payload\x18\t \x01(\x0b\x32*.org.apache.beam.runner_api.v1.ReadPayloadH\x00\x12>\n\nside_input\x18\x0b \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInputH\x00\x12O\n\x13window_into_payload\x18\x0c \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowIntoPayloadH\x00\x12N\n\x12windowing_strategy\x18\r \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowingStrategyH\x00\x12\x44\n\rfunction_spec\x18\x0e \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpecH\x00\x42\x06\n\x04root\"\xa7\x01\n\x08Pipeline\x12=\n\ncomponents\x18\x01 \x01(\x0b\x32).org.apache.beam.runner_api.v1.Components\x12\x1a\n\x12root_transform_ids\x18\x02 \x03(\t\x12@\n\x0c\x64isplay_data\x18\x03 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\"\xa4\x03\n\nPTransform\x12\x13\n\x0bunique_name\x18\x05 \x01(\t\x12\x39\n\x04spec\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12\x15\n\rsubtransforms\x18\x02 \x03(\t\x12\x45\n\x06inputs\x18\x03 \x03(\x0b\x32\x35.org.apache.beam.runner_api.v1.PTransform.InputsEntry\x12G\n\x07outputs\x18\x04 \x03(\x0b\x32\x36.org.apache.beam.runner_api.v1.PTransform.OutputsEntry\x12@\n\x0c\x64isplay_data\x18\x06 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\x1a-\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd3\x01\n\x0bPCollection\x12\x13\n\x0bunique_name\x18\x01 \x01(\t\x12\x10\n\x08\x63oder_id\x18\x02 \x01(\t\x12<\n\nis_bounded\x18\x03 \x01(\x0e\x32(.org.apache.beam.runner_api.v1.IsBounded\x12\x1d\n\x15windowing_strategy_id\x18\x04 \x01(\t\x12@\n\x0c\x64isplay_data\x18\x05 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\"\x98\x05\n\x0cParDoPayload\x12=\n\x05\x64o_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12<\n\nparameters\x18\x02 \x03(\x0b\x32(.org.apache.beam.runner_api.v1.Parameter\x12P\n\x0bside_inputs\x18\x03 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry\x12P\n\x0bstate_specs\x18\x04 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry\x12P\n\x0btimer_specs\x18\x05 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry\x1a[\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInput:\x02\x38\x01\x1a[\n\x0fStateSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.StateSpec:\x02\x38\x01\x1a[\n\x0fTimerSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.TimerSpec:\x02\x38\x01\"\x8b\x01\n\tParameter\x12;\n\x04type\x18\x01 \x01(\x0e\x32-.org.apache.beam.runner_api.v1.Parameter.Type\"A\n\x04Type\x12\n\n\x06WINDOW\x10\x00\x12\x14\n\x10PIPELINE_OPTIONS\x10\x01\x12\x17\n\x13RESTRICTION_TRACKER\x10\x02\"\x0b\n\tStateSpec\"\x0b\n\tTimerSpec\"\x8b\x01\n\x0bReadPayload\x12>\n\x06source\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12<\n\nis_bounded\x18\x02 \x01(\x0e\x32(.org.apache.beam.runner_api.v1.IsBounded\"V\n\x11WindowIntoPayload\x12\x41\n\twindow_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\"\xe1\x02\n\x0e\x43ombinePayload\x12\x42\n\ncombine_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12\x1c\n\x14\x61\x63\x63umulator_coder_id\x18\x02 \x01(\t\x12<\n\nparameters\x18\x03 \x03(\x0b\x32(.org.apache.beam.runner_api.v1.Parameter\x12R\n\x0bside_inputs\x18\x04 \x03(\x0b\x32=.org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry\x1a[\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInput:\x02\x38\x01\"b\n\x05\x43oder\x12<\n\x04spec\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12\x1b\n\x13\x63omponent_coder_ids\x18\x02 \x03(\t\"\xda\x03\n\x11WindowingStrategy\x12\x41\n\twindow_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12@\n\x0cmerge_status\x18\x02 \x01(\x0e\x32*.org.apache.beam.runner_api.v1.MergeStatus\x12\x17\n\x0fwindow_coder_id\x18\x03 \x01(\t\x12\x37\n\x07trigger\x18\x04 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12J\n\x11\x61\x63\x63umulation_mode\x18\x05 \x01(\x0e\x32/.org.apache.beam.runner_api.v1.AccumulationMode\x12>\n\x0boutput_time\x18\x06 \x01(\x0e\x32).org.apache.beam.runner_api.v1.OutputTime\x12H\n\x10\x63losing_behavior\x18\x07 \x01(\x0e\x32..org.apache.beam.runner_api.v1.ClosingBehavior\x12\x18\n\x10\x61llowed_lateness\x18\x08 \x01(\x03\"\xad\r\n\x07Trigger\x12\x44\n\tafter_all\x18\x01 \x01(\x0b\x32/.org.apache.beam.runner_api.v1.Trigger.AfterAllH\x00\x12\x44\n\tafter_any\x18\x02 \x01(\x0b\x32/.org.apache.beam.runner_api.v1.Trigger.AfterAnyH\x00\x12\x46\n\nafter_each\x18\x03 \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.Trigger.AfterEachH\x00\x12V\n\x13\x61\x66ter_end_of_window\x18\x04 \x01(\x0b\x32\x37.org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindowH\x00\x12[\n\x15\x61\x66ter_processing_time\x18\x05 \x01(\x0b\x32:.org.apache.beam.runner_api.v1.Trigger.AfterProcessingTimeH\x00\x12t\n\"after_synchronized_processing_time\x18\x06 \x01(\x0b\x32\x46.org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTimeH\x00\x12?\n\x06\x61lways\x18\x0c \x01(\x0b\x32-.org.apache.beam.runner_api.v1.Trigger.AlwaysH\x00\x12\x41\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\x0b\x32..org.apache.beam.runner_api.v1.Trigger.DefaultH\x00\x12L\n\relement_count\x18\x08 \x01(\x0b\x32\x33.org.apache.beam.runner_api.v1.Trigger.ElementCountH\x00\x12=\n\x05never\x18\t \x01(\x0b\x32,.org.apache.beam.runner_api.v1.Trigger.NeverH\x00\x12\x46\n\nor_finally\x18\n \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.Trigger.OrFinallyH\x00\x12?\n\x06repeat\x18\x0b \x01(\x0b\x32-.org.apache.beam.runner_api.v1.Trigger.RepeatH\x00\x1aG\n\x08\x41\x66terAll\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1aG\n\x08\x41\x66terAny\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1aH\n\tAfterEach\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x8f\x01\n\x10\x41\x66terEndOfWindow\x12=\n\rearly_firings\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12<\n\x0clate_firings\x18\x02 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x66\n\x13\x41\x66terProcessingTime\x12O\n\x14timestamp_transforms\x18\x01 \x03(\x0b\x32\x31.org.apache.beam.runner_api.v1.TimestampTransform\x1a!\n\x1f\x41\x66terSynchronizedProcessingTime\x1a\t\n\x07\x44\x65\x66\x61ult\x1a%\n\x0c\x45lementCount\x12\x15\n\relement_count\x18\x01 \x01(\x05\x1a\x07\n\x05Never\x1a\x08\n\x06\x41lways\x1az\n\tOrFinally\x12\x34\n\x04main\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12\x37\n\x07\x66inally\x18\x02 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x44\n\x06Repeat\x12:\n\nsubtrigger\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.TriggerB\t\n\x07trigger\"\x8e\x02\n\x12TimestampTransform\x12H\n\x05\x64\x65lay\x18\x01 \x01(\x0b\x32\x37.org.apache.beam.runner_api.v1.TimestampTransform.DelayH\x00\x12M\n\x08\x61lign_to\x18\x02 \x01(\x0b\x32\x39.org.apache.beam.runner_api.v1.TimestampTransform.AlignToH\x00\x1a\x1d\n\x05\x44\x65lay\x12\x14\n\x0c\x64\x65lay_millis\x18\x01 \x01(\x03\x1a)\n\x07\x41lignTo\x12\x0e\n\x06period\x18\x03 \x01(\x03\x12\x0e\n\x06offset\x18\x04 \x01(\x03\x42\x15\n\x13timestamp_transform\"\xdc\x01\n\tSideInput\x12\x43\n\x0e\x61\x63\x63\x65ss_pattern\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12?\n\x07view_fn\x18\x02 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12I\n\x11window_mapping_fn\x18\x03 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\"\x1a\n\x0b\x45nvironment\x12\x0b\n\x03url\x18\x01 \x01(\t\"d\n\x0fSdkFunctionSpec\x12\x39\n\x04spec\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\"D\n\x0c\x46unctionSpec\x12\x0b\n\x03urn\x18\x01 \x01(\t\x12\'\n\tparameter\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\"\xf7\x03\n\x0b\x44isplayData\x12>\n\x05items\x18\x01 \x03(\x0b\x32/.org.apache.beam.runner_api.v1.DisplayData.Item\x1a\x46\n\nIdentifier\x12\x14\n\x0ctransform_id\x18\x01 \x01(\t\x12\x15\n\rtransform_urn\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t\x1a\xf9\x01\n\x04Item\x12\x41\n\x02id\x18\x01 \x01(\x0b\x32\x35.org.apache.beam.runner_api.v1.DisplayData.Identifier\x12=\n\x04type\x18\x02 \x01(\x0e\x32/.org.apache.beam.runner_api.v1.DisplayData.Type\x12#\n\x05value\x18\x03 \x01(\x0b\x32\x14.google.protobuf.Any\x12)\n\x0bshort_value\x18\x04 \x01(\x0b\x32\x14.google.protobuf.Any\x12\r\n\x05label\x18\x05 \x01(\t\x12\x10\n\x08link_url\x18\x06 \x01(\t\"d\n\x04Type\x12\n\n\x06STRING\x10\x00\x12\x0b\n\x07INTEGER\x10\x01\x12\t\n\x05\x46LOAT\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\r\n\tTIMESTAMP\x10\x04\x12\x0c\n\x08\x44URATION\x10\x05\x12\x0e\n\nJAVA_CLASS\x10\x06*\'\n\tIsBounded\x12\x0b\n\x07\x42OUNDED\x10\x00\x12\r\n\tUNBOUNDED\x10\x01*C\n\x0bMergeStatus\x12\x0f\n\x0bNON_MERGING\x10\x00\x12\x0f\n\x0bNEEDS_MERGE\x10\x01\x12\x12\n\x0e\x41LREADY_MERGED\x10\x02*4\n\x10\x41\x63\x63umulationMode\x12\x0e\n\nDISCARDING\x10\x00\x12\x10\n\x0c\x41\x43\x43UMULATING\x10\x01*8\n\x0f\x43losingBehavior\x12\x0f\n\x0b\x45MIT_ALWAYS\x10\x00\x12\x14\n\x10\x45MIT_IF_NONEMPTY\x10\x01*I\n\nOutputTime\x12\x11\n\rEND_OF_WINDOW\x10\x00\x12\x12\n\x0eLATEST_IN_PANE\x10\x01\x12\x14\n\x10\x45\x41RLIEST_IN_PANE\x10\x02*S\n\nTimeDomain\x12\x0e\n\nEVENT_TIME\x10\x00\x12\x13\n\x0fPROCESSING_TIME\x10\x01\x12 \n\x1cSYNCHRONIZED_PROCESSING_TIME\x10\x02\x42\x31\n$org.apache.beam.sdk.common.runner.v1B\tRunnerApib\x06proto3')
- ,
- dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,])
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-_ISBOUNDED = _descriptor.EnumDescriptor(
- name='IsBounded',
- full_name='org.apache.beam.runner_api.v1.IsBounded',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='BOUNDED', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='UNBOUNDED', index=1, number=1,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7583,
- serialized_end=7622,
-)
-_sym_db.RegisterEnumDescriptor(_ISBOUNDED)
-
-IsBounded = enum_type_wrapper.EnumTypeWrapper(_ISBOUNDED)
-_MERGESTATUS = _descriptor.EnumDescriptor(
- name='MergeStatus',
- full_name='org.apache.beam.runner_api.v1.MergeStatus',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='NON_MERGING', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='NEEDS_MERGE', index=1, number=1,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='ALREADY_MERGED', index=2, number=2,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7624,
- serialized_end=7691,
-)
-_sym_db.RegisterEnumDescriptor(_MERGESTATUS)
-
-MergeStatus = enum_type_wrapper.EnumTypeWrapper(_MERGESTATUS)
-_ACCUMULATIONMODE = _descriptor.EnumDescriptor(
- name='AccumulationMode',
- full_name='org.apache.beam.runner_api.v1.AccumulationMode',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='DISCARDING', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='ACCUMULATING', index=1, number=1,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7693,
- serialized_end=7745,
-)
-_sym_db.RegisterEnumDescriptor(_ACCUMULATIONMODE)
-
-AccumulationMode = enum_type_wrapper.EnumTypeWrapper(_ACCUMULATIONMODE)
-_CLOSINGBEHAVIOR = _descriptor.EnumDescriptor(
- name='ClosingBehavior',
- full_name='org.apache.beam.runner_api.v1.ClosingBehavior',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='EMIT_ALWAYS', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='EMIT_IF_NONEMPTY', index=1, number=1,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7747,
- serialized_end=7803,
-)
-_sym_db.RegisterEnumDescriptor(_CLOSINGBEHAVIOR)
-
-ClosingBehavior = enum_type_wrapper.EnumTypeWrapper(_CLOSINGBEHAVIOR)
-_OUTPUTTIME = _descriptor.EnumDescriptor(
- name='OutputTime',
- full_name='org.apache.beam.runner_api.v1.OutputTime',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='END_OF_WINDOW', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='LATEST_IN_PANE', index=1, number=1,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='EARLIEST_IN_PANE', index=2, number=2,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7805,
- serialized_end=7878,
-)
-_sym_db.RegisterEnumDescriptor(_OUTPUTTIME)
-
-OutputTime = enum_type_wrapper.EnumTypeWrapper(_OUTPUTTIME)
-_TIMEDOMAIN = _descriptor.EnumDescriptor(
- name='TimeDomain',
- full_name='org.apache.beam.runner_api.v1.TimeDomain',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='EVENT_TIME', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='PROCESSING_TIME', index=1, number=1,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='SYNCHRONIZED_PROCESSING_TIME', index=2, number=2,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7880,
- serialized_end=7963,
-)
-_sym_db.RegisterEnumDescriptor(_TIMEDOMAIN)
-
-TimeDomain = enum_type_wrapper.EnumTypeWrapper(_TIMEDOMAIN)
-BOUNDED = 0
-UNBOUNDED = 1
-NON_MERGING = 0
-NEEDS_MERGE = 1
-ALREADY_MERGED = 2
-DISCARDING = 0
-ACCUMULATING = 1
-EMIT_ALWAYS = 0
-EMIT_IF_NONEMPTY = 1
-END_OF_WINDOW = 0
-LATEST_IN_PANE = 1
-EARLIEST_IN_PANE = 2
-EVENT_TIME = 0
-PROCESSING_TIME = 1
-SYNCHRONIZED_PROCESSING_TIME = 2
-
-
-_PARAMETER_TYPE = _descriptor.EnumDescriptor(
- name='Type',
- full_name='org.apache.beam.runner_api.v1.Parameter.Type',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='WINDOW', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='PIPELINE_OPTIONS', index=1, number=1,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='RESTRICTION_TRACKER', index=2, number=2,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=3413,
- serialized_end=3478,
-)
-_sym_db.RegisterEnumDescriptor(_PARAMETER_TYPE)
-
-_DISPLAYDATA_TYPE = _descriptor.EnumDescriptor(
- name='Type',
- full_name='org.apache.beam.runner_api.v1.DisplayData.Type',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='STRING', index=0, number=0,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='INTEGER', index=1, number=1,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='FLOAT', index=2, number=2,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='BOOLEAN', index=3, number=3,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='TIMESTAMP', index=4, number=4,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DURATION', index=5, number=5,
- options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='JAVA_CLASS', index=6, number=6,
- options=None,
- type=None),
- ],
- containing_type=None,
- options=None,
- serialized_start=7481,
- serialized_end=7581,
-)
-_sym_db.RegisterEnumDescriptor(_DISPLAYDATA_TYPE)
-
-
-_COMPONENTS_TRANSFORMSENTRY = _descriptor.Descriptor(
- name='TransformsEntry',
- full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=512,
- serialized_end=604,
-)
-
-_COMPONENTS_PCOLLECTIONSENTRY = _descriptor.Descriptor(
- name='PcollectionsEntry',
- full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=606,
- serialized_end=701,
-)
-
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY = _descriptor.Descriptor(
- name='WindowingStrategiesEntry',
- full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=703,
- serialized_end=811,
-)
-
-_COMPONENTS_CODERSENTRY = _descriptor.Descriptor(
- name='CodersEntry',
- full_name='org.apache.beam.runner_api.v1.Components.CodersEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.Components.CodersEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.Components.CodersEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=813,
- serialized_end=896,
-)
-
-_COMPONENTS_ENVIRONMENTSENTRY = _descriptor.Descriptor(
- name='EnvironmentsEntry',
- full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=898,
- serialized_end=993,
-)
-
-_COMPONENTS = _descriptor.Descriptor(
- name='Components',
- full_name='org.apache.beam.runner_api.v1.Components',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='transforms', full_name='org.apache.beam.runner_api.v1.Components.transforms', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='pcollections', full_name='org.apache.beam.runner_api.v1.Components.pcollections', index=1,
- number=2, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='windowing_strategies', full_name='org.apache.beam.runner_api.v1.Components.windowing_strategies', index=2,
- number=3, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='coders', full_name='org.apache.beam.runner_api.v1.Components.coders', index=3,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='environments', full_name='org.apache.beam.runner_api.v1.Components.environments', index=4,
- number=5, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_COMPONENTS_TRANSFORMSENTRY, _COMPONENTS_PCOLLECTIONSENTRY, _COMPONENTS_WINDOWINGSTRATEGIESENTRY, _COMPONENTS_CODERSENTRY, _COMPONENTS_ENVIRONMENTSENTRY, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=84,
- serialized_end=993,
-)
-
-
-_MESSAGEWITHCOMPONENTS = _descriptor.Descriptor(
- name='MessageWithComponents',
- full_name='org.apache.beam.runner_api.v1.MessageWithComponents',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='components', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.components', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='coder', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.coder', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='combine_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.combine_payload', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='sdk_function_spec', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.sdk_function_spec', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='par_do_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.par_do_payload', index=4,
- number=6, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='ptransform', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.ptransform', index=5,
- number=7, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='pcollection', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.pcollection', index=6,
- number=8, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='read_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.read_payload', index=7,
- number=9, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='side_input', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.side_input', index=8,
- number=11, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='window_into_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.window_into_payload', index=9,
- number=12, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='windowing_strategy', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.windowing_strategy', index=10,
- number=13, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='function_spec', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.function_spec', index=11,
- number=14, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='root', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.root',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=996,
- serialized_end=1862,
-)
-
-
-_PIPELINE = _descriptor.Descriptor(
- name='Pipeline',
- full_name='org.apache.beam.runner_api.v1.Pipeline',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='components', full_name='org.apache.beam.runner_api.v1.Pipeline.components', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='root_transform_ids', full_name='org.apache.beam.runner_api.v1.Pipeline.root_transform_ids', index=1,
- number=2, type=9, cpp_type=9, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='display_data', full_name='org.apache.beam.runner_api.v1.Pipeline.display_data', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1865,
- serialized_end=2032,
-)
-
-
-_PTRANSFORM_INPUTSENTRY = _descriptor.Descriptor(
- name='InputsEntry',
- full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry.value', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2362,
- serialized_end=2407,
-)
-
-_PTRANSFORM_OUTPUTSENTRY = _descriptor.Descriptor(
- name='OutputsEntry',
- full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry.value', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2409,
- serialized_end=2455,
-)
-
-_PTRANSFORM = _descriptor.Descriptor(
- name='PTransform',
- full_name='org.apache.beam.runner_api.v1.PTransform',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='unique_name', full_name='org.apache.beam.runner_api.v1.PTransform.unique_name', index=0,
- number=5, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='spec', full_name='org.apache.beam.runner_api.v1.PTransform.spec', index=1,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='subtransforms', full_name='org.apache.beam.runner_api.v1.PTransform.subtransforms', index=2,
- number=2, type=9, cpp_type=9, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='inputs', full_name='org.apache.beam.runner_api.v1.PTransform.inputs', index=3,
- number=3, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='outputs', full_name='org.apache.beam.runner_api.v1.PTransform.outputs', index=4,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='display_data', full_name='org.apache.beam.runner_api.v1.PTransform.display_data', index=5,
- number=6, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_PTRANSFORM_INPUTSENTRY, _PTRANSFORM_OUTPUTSENTRY, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2035,
- serialized_end=2455,
-)
-
-
-_PCOLLECTION = _descriptor.Descriptor(
- name='PCollection',
- full_name='org.apache.beam.runner_api.v1.PCollection',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='unique_name', full_name='org.apache.beam.runner_api.v1.PCollection.unique_name', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='coder_id', full_name='org.apache.beam.runner_api.v1.PCollection.coder_id', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='is_bounded', full_name='org.apache.beam.runner_api.v1.PCollection.is_bounded', index=2,
- number=3, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='windowing_strategy_id', full_name='org.apache.beam.runner_api.v1.PCollection.windowing_strategy_id', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='display_data', full_name='org.apache.beam.runner_api.v1.PCollection.display_data', index=4,
- number=5, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2458,
- serialized_end=2669,
-)
-
-
-_PARDOPAYLOAD_SIDEINPUTSENTRY = _descriptor.Descriptor(
- name='SideInputsEntry',
- full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3059,
- serialized_end=3150,
-)
-
-_PARDOPAYLOAD_STATESPECSENTRY = _descriptor.Descriptor(
- name='StateSpecsEntry',
- full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3152,
- serialized_end=3243,
-)
-
-_PARDOPAYLOAD_TIMERSPECSENTRY = _descriptor.Descriptor(
- name='TimerSpecsEntry',
- full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3245,
- serialized_end=3336,
-)
-
-_PARDOPAYLOAD = _descriptor.Descriptor(
- name='ParDoPayload',
- full_name='org.apache.beam.runner_api.v1.ParDoPayload',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='do_fn', full_name='org.apache.beam.runner_api.v1.ParDoPayload.do_fn', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='parameters', full_name='org.apache.beam.runner_api.v1.ParDoPayload.parameters', index=1,
- number=2, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='side_inputs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.side_inputs', index=2,
- number=3, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='state_specs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.state_specs', index=3,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='timer_specs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.timer_specs', index=4,
- number=5, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_PARDOPAYLOAD_SIDEINPUTSENTRY, _PARDOPAYLOAD_STATESPECSENTRY, _PARDOPAYLOAD_TIMERSPECSENTRY, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=2672,
- serialized_end=3336,
-)
-
-
-_PARAMETER = _descriptor.Descriptor(
- name='Parameter',
- full_name='org.apache.beam.runner_api.v1.Parameter',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='type', full_name='org.apache.beam.runner_api.v1.Parameter.type', index=0,
- number=1, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- _PARAMETER_TYPE,
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3339,
- serialized_end=3478,
-)
-
-
-_STATESPEC = _descriptor.Descriptor(
- name='StateSpec',
- full_name='org.apache.beam.runner_api.v1.StateSpec',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3480,
- serialized_end=3491,
-)
-
-
-_TIMERSPEC = _descriptor.Descriptor(
- name='TimerSpec',
- full_name='org.apache.beam.runner_api.v1.TimerSpec',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3493,
- serialized_end=3504,
-)
-
-
-_READPAYLOAD = _descriptor.Descriptor(
- name='ReadPayload',
- full_name='org.apache.beam.runner_api.v1.ReadPayload',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='source', full_name='org.apache.beam.runner_api.v1.ReadPayload.source', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='is_bounded', full_name='org.apache.beam.runner_api.v1.ReadPayload.is_bounded', index=1,
- number=2, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3507,
- serialized_end=3646,
-)
-
-
-_WINDOWINTOPAYLOAD = _descriptor.Descriptor(
- name='WindowIntoPayload',
- full_name='org.apache.beam.runner_api.v1.WindowIntoPayload',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='window_fn', full_name='org.apache.beam.runner_api.v1.WindowIntoPayload.window_fn', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3648,
- serialized_end=3734,
-)
-
-
-_COMBINEPAYLOAD_SIDEINPUTSENTRY = _descriptor.Descriptor(
- name='SideInputsEntry',
- full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry.value', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3059,
- serialized_end=3150,
-)
-
-_COMBINEPAYLOAD = _descriptor.Descriptor(
- name='CombinePayload',
- full_name='org.apache.beam.runner_api.v1.CombinePayload',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='combine_fn', full_name='org.apache.beam.runner_api.v1.CombinePayload.combine_fn', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='accumulator_coder_id', full_name='org.apache.beam.runner_api.v1.CombinePayload.accumulator_coder_id', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='parameters', full_name='org.apache.beam.runner_api.v1.CombinePayload.parameters', index=2,
- number=3, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='side_inputs', full_name='org.apache.beam.runner_api.v1.CombinePayload.side_inputs', index=3,
- number=4, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_COMBINEPAYLOAD_SIDEINPUTSENTRY, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=3737,
- serialized_end=4090,
-)
-
-
-_CODER = _descriptor.Descriptor(
- name='Coder',
- full_name='org.apache.beam.runner_api.v1.Coder',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='spec', full_name='org.apache.beam.runner_api.v1.Coder.spec', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='component_coder_ids', full_name='org.apache.beam.runner_api.v1.Coder.component_coder_ids', index=1,
- number=2, type=9, cpp_type=9, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4092,
- serialized_end=4190,
-)
-
-
-_WINDOWINGSTRATEGY = _descriptor.Descriptor(
- name='WindowingStrategy',
- full_name='org.apache.beam.runner_api.v1.WindowingStrategy',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='window_fn', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.window_fn', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='merge_status', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.merge_status', index=1,
- number=2, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='window_coder_id', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.window_coder_id', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='trigger', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.trigger', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='accumulation_mode', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.accumulation_mode', index=4,
- number=5, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='output_time', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.output_time', index=5,
- number=6, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='closing_behavior', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.closing_behavior', index=6,
- number=7, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='allowed_lateness', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.allowed_lateness', index=7,
- number=8, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=4193,
- serialized_end=4667,
-)
-
-
-_TRIGGER_AFTERALL = _descriptor.Descriptor(
- name='AfterAll',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterAll',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterAll.subtriggers', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=5602,
- serialized_end=5673,
-)
-
-_TRIGGER_AFTERANY = _descriptor.Descriptor(
- name='AfterAny',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterAny',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterAny.subtriggers', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=5675,
- serialized_end=5746,
-)
-
-_TRIGGER_AFTEREACH = _descriptor.Descriptor(
- name='AfterEach',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterEach',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEach.subtriggers', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=5748,
- serialized_end=5820,
-)
-
-_TRIGGER_AFTERENDOFWINDOW = _descriptor.Descriptor(
- name='AfterEndOfWindow',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='early_firings', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow.early_firings', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='late_firings', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow.late_firings', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=5823,
- serialized_end=5966,
-)
-
-_TRIGGER_AFTERPROCESSINGTIME = _descriptor.Descriptor(
- name='AfterProcessingTime',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='timestamp_transforms', full_name='org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime.timestamp_transforms', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=5968,
- serialized_end=6070,
-)
-
-_TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME = _descriptor.Descriptor(
- name='AfterSynchronizedProcessingTime',
- full_name='org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTime',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6072,
- serialized_end=6105,
-)
-
-_TRIGGER_DEFAULT = _descriptor.Descriptor(
- name='Default',
- full_name='org.apache.beam.runner_api.v1.Trigger.Default',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6107,
- serialized_end=6116,
-)
-
-_TRIGGER_ELEMENTCOUNT = _descriptor.Descriptor(
- name='ElementCount',
- full_name='org.apache.beam.runner_api.v1.Trigger.ElementCount',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='element_count', full_name='org.apache.beam.runner_api.v1.Trigger.ElementCount.element_count', index=0,
- number=1, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6118,
- serialized_end=6155,
-)
-
-_TRIGGER_NEVER = _descriptor.Descriptor(
- name='Never',
- full_name='org.apache.beam.runner_api.v1.Trigger.Never',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6157,
- serialized_end=6164,
-)
-
-_TRIGGER_ALWAYS = _descriptor.Descriptor(
- name='Always',
- full_name='org.apache.beam.runner_api.v1.Trigger.Always',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6166,
- serialized_end=6174,
-)
-
-_TRIGGER_ORFINALLY = _descriptor.Descriptor(
- name='OrFinally',
- full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='main', full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally.main', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='finally', full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally.finally', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6176,
- serialized_end=6298,
-)
-
-_TRIGGER_REPEAT = _descriptor.Descriptor(
- name='Repeat',
- full_name='org.apache.beam.runner_api.v1.Trigger.Repeat',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='subtrigger', full_name='org.apache.beam.runner_api.v1.Trigger.Repeat.subtrigger', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6300,
- serialized_end=6368,
-)
-
-_TRIGGER = _descriptor.Descriptor(
- name='Trigger',
- full_name='org.apache.beam.runner_api.v1.Trigger',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='after_all', full_name='org.apache.beam.runner_api.v1.Trigger.after_all', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='after_any', full_name='org.apache.beam.runner_api.v1.Trigger.after_any', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='after_each', full_name='org.apache.beam.runner_api.v1.Trigger.after_each', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='after_end_of_window', full_name='org.apache.beam.runner_api.v1.Trigger.after_end_of_window', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='after_processing_time', full_name='org.apache.beam.runner_api.v1.Trigger.after_processing_time', index=4,
- number=5, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='after_synchronized_processing_time', full_name='org.apache.beam.runner_api.v1.Trigger.after_synchronized_processing_time', index=5,
- number=6, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='always', full_name='org.apache.beam.runner_api.v1.Trigger.always', index=6,
- number=12, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='default', full_name='org.apache.beam.runner_api.v1.Trigger.default', index=7,
- number=7, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='element_count', full_name='org.apache.beam.runner_api.v1.Trigger.element_count', index=8,
- number=8, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='never', full_name='org.apache.beam.runner_api.v1.Trigger.never', index=9,
- number=9, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='or_finally', full_name='org.apache.beam.runner_api.v1.Trigger.or_finally', index=10,
- number=10, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='repeat', full_name='org.apache.beam.runner_api.v1.Trigger.repeat', index=11,
- number=11, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_TRIGGER_AFTERALL, _TRIGGER_AFTERANY, _TRIGGER_AFTEREACH, _TRIGGER_AFTERENDOFWINDOW, _TRIGGER_AFTERPROCESSINGTIME, _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME, _TRIGGER_DEFAULT, _TRIGGER_ELEMENTCOUNT, _TRIGGER_NEVER, _TRIGGER_ALWAYS, _TRIGGER_ORFINALLY, _TRIGGER_REPEAT, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='trigger', full_name='org.apache.beam.runner_api.v1.Trigger.trigger',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=4670,
- serialized_end=6379,
-)
-
-
-_TIMESTAMPTRANSFORM_DELAY = _descriptor.Descriptor(
- name='Delay',
- full_name='org.apache.beam.runner_api.v1.TimestampTransform.Delay',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='delay_millis', full_name='org.apache.beam.runner_api.v1.TimestampTransform.Delay.delay_millis', index=0,
- number=1, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6557,
- serialized_end=6586,
-)
-
-_TIMESTAMPTRANSFORM_ALIGNTO = _descriptor.Descriptor(
- name='AlignTo',
- full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='period', full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo.period', index=0,
- number=3, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='offset', full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo.offset', index=1,
- number=4, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6588,
- serialized_end=6629,
-)
-
-_TIMESTAMPTRANSFORM = _descriptor.Descriptor(
- name='TimestampTransform',
- full_name='org.apache.beam.runner_api.v1.TimestampTransform',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='delay', full_name='org.apache.beam.runner_api.v1.TimestampTransform.delay', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='align_to', full_name='org.apache.beam.runner_api.v1.TimestampTransform.align_to', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_TIMESTAMPTRANSFORM_DELAY, _TIMESTAMPTRANSFORM_ALIGNTO, ],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- _descriptor.OneofDescriptor(
- name='timestamp_transform', full_name='org.apache.beam.runner_api.v1.TimestampTransform.timestamp_transform',
- index=0, containing_type=None, fields=[]),
- ],
- serialized_start=6382,
- serialized_end=6652,
-)
-
-
-_SIDEINPUT = _descriptor.Descriptor(
- name='SideInput',
- full_name='org.apache.beam.runner_api.v1.SideInput',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='access_pattern', full_name='org.apache.beam.runner_api.v1.SideInput.access_pattern', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='view_fn', full_name='org.apache.beam.runner_api.v1.SideInput.view_fn', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='window_mapping_fn', full_name='org.apache.beam.runner_api.v1.SideInput.window_mapping_fn', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6655,
- serialized_end=6875,
-)
-
-
-_ENVIRONMENT = _descriptor.Descriptor(
- name='Environment',
- full_name='org.apache.beam.runner_api.v1.Environment',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='url', full_name='org.apache.beam.runner_api.v1.Environment.url', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6877,
- serialized_end=6903,
-)
-
-
-_SDKFUNCTIONSPEC = _descriptor.Descriptor(
- name='SdkFunctionSpec',
- full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='spec', full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec.spec', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='environment_id', full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec.environment_id', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=6905,
- serialized_end=7005,
-)
-
-
-_FUNCTIONSPEC = _descriptor.Descriptor(
- name='FunctionSpec',
- full_name='org.apache.beam.runner_api.v1.FunctionSpec',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='urn', full_name='org.apache.beam.runner_api.v1.FunctionSpec.urn', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='parameter', full_name='org.apache.beam.runner_api.v1.FunctionSpec.parameter', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=7007,
- serialized_end=7075,
-)
-
-
-_DISPLAYDATA_IDENTIFIER = _descriptor.Descriptor(
- name='Identifier',
- full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='transform_id', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.transform_id', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='transform_urn', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.transform_urn', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.key', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=7157,
- serialized_end=7227,
-)
-
-_DISPLAYDATA_ITEM = _descriptor.Descriptor(
- name='Item',
- full_name='org.apache.beam.runner_api.v1.DisplayData.Item',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='id', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.id', index=0,
- number=1, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='type', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.type', index=1,
- number=2, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='value', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.value', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='short_value', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.short_value', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='label', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.label', index=4,
- number=5, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- _descriptor.FieldDescriptor(
- name='link_url', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.link_url', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=7230,
- serialized_end=7479,
-)
-
-_DISPLAYDATA = _descriptor.Descriptor(
- name='DisplayData',
- full_name='org.apache.beam.runner_api.v1.DisplayData',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='items', full_name='org.apache.beam.runner_api.v1.DisplayData.items', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None),
- ],
- extensions=[
- ],
- nested_types=[_DISPLAYDATA_IDENTIFIER, _DISPLAYDATA_ITEM, ],
- enum_types=[
- _DISPLAYDATA_TYPE,
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=7078,
- serialized_end=7581,
-)
-
-_COMPONENTS_TRANSFORMSENTRY.fields_by_name['value'].message_type = _PTRANSFORM
-_COMPONENTS_TRANSFORMSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_PCOLLECTIONSENTRY.fields_by_name['value'].message_type = _PCOLLECTION
-_COMPONENTS_PCOLLECTIONSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.fields_by_name['value'].message_type = _WINDOWINGSTRATEGY
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.containing_type = _COMPONENTS
-_COMPONENTS_CODERSENTRY.fields_by_name['value'].message_type = _CODER
-_COMPONENTS_CODERSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_ENVIRONMENTSENTRY.fields_by_name['value'].message_type = _ENVIRONMENT
-_COMPONENTS_ENVIRONMENTSENTRY.containing_type = _COMPONENTS
-_COMPONENTS.fields_by_name['transforms'].message_type = _COMPONENTS_TRANSFORMSENTRY
-_COMPONENTS.fields_by_name['pcollections'].message_type = _COMPONENTS_PCOLLECTIONSENTRY
-_COMPONENTS.fields_by_name['windowing_strategies'].message_type = _COMPONENTS_WINDOWINGSTRATEGIESENTRY
-_COMPONENTS.fields_by_name['coders'].message_type = _COMPONENTS_CODERSENTRY
-_COMPONENTS.fields_by_name['environments'].message_type = _COMPONENTS_ENVIRONMENTSENTRY
-_MESSAGEWITHCOMPONENTS.fields_by_name['components'].message_type = _COMPONENTS
-_MESSAGEWITHCOMPONENTS.fields_by_name['coder'].message_type = _CODER
-_MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'].message_type = _COMBINEPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'].message_type = _SDKFUNCTIONSPEC
-_MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'].message_type = _PARDOPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'].message_type = _PTRANSFORM
-_MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'].message_type = _PCOLLECTION
-_MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'].message_type = _READPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['side_input'].message_type = _SIDEINPUT
-_MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'].message_type = _WINDOWINTOPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'].message_type = _WINDOWINGSTRATEGY
-_MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['coder'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['coder'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['side_input'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['side_input'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
- _MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_PIPELINE.fields_by_name['components'].message_type = _COMPONENTS
-_PIPELINE.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PTRANSFORM_INPUTSENTRY.containing_type = _PTRANSFORM
-_PTRANSFORM_OUTPUTSENTRY.containing_type = _PTRANSFORM
-_PTRANSFORM.fields_by_name['spec'].message_type = _FUNCTIONSPEC
-_PTRANSFORM.fields_by_name['inputs'].message_type = _PTRANSFORM_INPUTSENTRY
-_PTRANSFORM.fields_by_name['outputs'].message_type = _PTRANSFORM_OUTPUTSENTRY
-_PTRANSFORM.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PCOLLECTION.fields_by_name['is_bounded'].enum_type = _ISBOUNDED
-_PCOLLECTION.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PARDOPAYLOAD_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_PARDOPAYLOAD_SIDEINPUTSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD_STATESPECSENTRY.fields_by_name['value'].message_type = _STATESPEC
-_PARDOPAYLOAD_STATESPECSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD_TIMERSPECSENTRY.fields_by_name['value'].message_type = _TIMERSPEC
-_PARDOPAYLOAD_TIMERSPECSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD.fields_by_name['do_fn'].message_type = _SDKFUNCTIONSPEC
-_PARDOPAYLOAD.fields_by_name['parameters'].message_type = _PARAMETER
-_PARDOPAYLOAD.fields_by_name['side_inputs'].message_type = _PARDOPAYLOAD_SIDEINPUTSENTRY
-_PARDOPAYLOAD.fields_by_name['state_specs'].message_type = _PARDOPAYLOAD_STATESPECSENTRY
-_PARDOPAYLOAD.fields_by_name['timer_specs'].message_type = _PARDOPAYLOAD_TIMERSPECSENTRY
-_PARAMETER.fields_by_name['type'].enum_type = _PARAMETER_TYPE
-_PARAMETER_TYPE.containing_type = _PARAMETER
-_READPAYLOAD.fields_by_name['source'].message_type = _SDKFUNCTIONSPEC
-_READPAYLOAD.fields_by_name['is_bounded'].enum_type = _ISBOUNDED
-_WINDOWINTOPAYLOAD.fields_by_name['window_fn'].message_type = _SDKFUNCTIONSPEC
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.containing_type = _COMBINEPAYLOAD
-_COMBINEPAYLOAD.fields_by_name['combine_fn'].message_type = _SDKFUNCTIONSPEC
-_COMBINEPAYLOAD.fields_by_name['parameters'].message_type = _PARAMETER
-_COMBINEPAYLOAD.fields_by_name['side_inputs'].message_type = _COMBINEPAYLOAD_SIDEINPUTSENTRY
-_CODER.fields_by_name['spec'].message_type = _SDKFUNCTIONSPEC
-_WINDOWINGSTRATEGY.fields_by_name['window_fn'].message_type = _SDKFUNCTIONSPEC
-_WINDOWINGSTRATEGY.fields_by_name['merge_status'].enum_type = _MERGESTATUS
-_WINDOWINGSTRATEGY.fields_by_name['trigger'].message_type = _TRIGGER
-_WINDOWINGSTRATEGY.fields_by_name['accumulation_mode'].enum_type = _ACCUMULATIONMODE
-_WINDOWINGSTRATEGY.fields_by_name['output_time'].enum_type = _OUTPUTTIME
-_WINDOWINGSTRATEGY.fields_by_name['closing_behavior'].enum_type = _CLOSINGBEHAVIOR
-_TRIGGER_AFTERALL.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTERALL.containing_type = _TRIGGER
-_TRIGGER_AFTERANY.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTERANY.containing_type = _TRIGGER
-_TRIGGER_AFTEREACH.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTEREACH.containing_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.fields_by_name['early_firings'].message_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.fields_by_name['late_firings'].message_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.containing_type = _TRIGGER
-_TRIGGER_AFTERPROCESSINGTIME.fields_by_name['timestamp_transforms'].message_type = _TIMESTAMPTRANSFORM
-_TRIGGER_AFTERPROCESSINGTIME.containing_type = _TRIGGER
-_TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME.containing_type = _TRIGGER
-_TRIGGER_DEFAULT.containing_type = _TRIGGER
-_TRIGGER_ELEMENTCOUNT.containing_type = _TRIGGER
-_TRIGGER_NEVER.containing_type = _TRIGGER
-_TRIGGER_ALWAYS.containing_type = _TRIGGER
-_TRIGGER_ORFINALLY.fields_by_name['main'].message_type = _TRIGGER
-_TRIGGER_ORFINALLY.fields_by_name['finally'].message_type = _TRIGGER
-_TRIGGER_ORFINALLY.containing_type = _TRIGGER
-_TRIGGER_REPEAT.fields_by_name['subtrigger'].message_type = _TRIGGER
-_TRIGGER_REPEAT.containing_type = _TRIGGER
-_TRIGGER.fields_by_name['after_all'].message_type = _TRIGGER_AFTERALL
-_TRIGGER.fields_by_name['after_any'].message_type = _TRIGGER_AFTERANY
-_TRIGGER.fields_by_name['after_each'].message_type = _TRIGGER_AFTEREACH
-_TRIGGER.fields_by_name['after_end_of_window'].message_type = _TRIGGER_AFTERENDOFWINDOW
-_TRIGGER.fields_by_name['after_processing_time'].message_type = _TRIGGER_AFTERPROCESSINGTIME
-_TRIGGER.fields_by_name['after_synchronized_processing_time'].message_type = _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME
-_TRIGGER.fields_by_name['always'].message_type = _TRIGGER_ALWAYS
-_TRIGGER.fields_by_name['default'].message_type = _TRIGGER_DEFAULT
-_TRIGGER.fields_by_name['element_count'].message_type = _TRIGGER_ELEMENTCOUNT
-_TRIGGER.fields_by_name['never'].message_type = _TRIGGER_NEVER
-_TRIGGER.fields_by_name['or_finally'].message_type = _TRIGGER_ORFINALLY
-_TRIGGER.fields_by_name['repeat'].message_type = _TRIGGER_REPEAT
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_all'])
-_TRIGGER.fields_by_name['after_all'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_any'])
-_TRIGGER.fields_by_name['after_any'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_each'])
-_TRIGGER.fields_by_name['after_each'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_end_of_window'])
-_TRIGGER.fields_by_name['after_end_of_window'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_processing_time'])
-_TRIGGER.fields_by_name['after_processing_time'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['after_synchronized_processing_time'])
-_TRIGGER.fields_by_name['after_synchronized_processing_time'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['always'])
-_TRIGGER.fields_by_name['always'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['default'])
-_TRIGGER.fields_by_name['default'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['element_count'])
-_TRIGGER.fields_by_name['element_count'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['never'])
-_TRIGGER.fields_by_name['never'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['or_finally'])
-_TRIGGER.fields_by_name['or_finally'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
- _TRIGGER.fields_by_name['repeat'])
-_TRIGGER.fields_by_name['repeat'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TIMESTAMPTRANSFORM_DELAY.containing_type = _TIMESTAMPTRANSFORM
-_TIMESTAMPTRANSFORM_ALIGNTO.containing_type = _TIMESTAMPTRANSFORM
-_TIMESTAMPTRANSFORM.fields_by_name['delay'].message_type = _TIMESTAMPTRANSFORM_DELAY
-_TIMESTAMPTRANSFORM.fields_by_name['align_to'].message_type = _TIMESTAMPTRANSFORM_ALIGNTO
-_TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform'].fields.append(
- _TIMESTAMPTRANSFORM.fields_by_name['delay'])
-_TIMESTAMPTRANSFORM.fields_by_name['delay'].containing_oneof = _TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform']
-_TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform'].fields.append(
- _TIMESTAMPTRANSFORM.fields_by_name['align_to'])
-_TIMESTAMPTRANSFORM.fields_by_name['align_to'].containing_oneof = _TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform']
-_SIDEINPUT.fields_by_name['access_pattern'].message_type = _FUNCTIONSPEC
-_SIDEINPUT.fields_by_name['view_fn'].message_type = _SDKFUNCTIONSPEC
-_SIDEINPUT.fields_by_name['window_mapping_fn'].message_type = _SDKFUNCTIONSPEC
-_SDKFUNCTIONSPEC.fields_by_name['spec'].message_type = _FUNCTIONSPEC
-_FUNCTIONSPEC.fields_by_name['parameter'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_IDENTIFIER.containing_type = _DISPLAYDATA
-_DISPLAYDATA_ITEM.fields_by_name['id'].message_type = _DISPLAYDATA_IDENTIFIER
-_DISPLAYDATA_ITEM.fields_by_name['type'].enum_type = _DISPLAYDATA_TYPE
-_DISPLAYDATA_ITEM.fields_by_name['value'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_ITEM.fields_by_name['short_value'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_ITEM.containing_type = _DISPLAYDATA
-_DISPLAYDATA.fields_by_name['items'].message_type = _DISPLAYDATA_ITEM
-_DISPLAYDATA_TYPE.containing_type = _DISPLAYDATA
-DESCRIPTOR.message_types_by_name['Components'] = _COMPONENTS
-DESCRIPTOR.message_types_by_name['MessageWithComponents'] = _MESSAGEWITHCOMPONENTS
-DESCRIPTOR.message_types_by_name['Pipeline'] = _PIPELINE
-DESCRIPTOR.message_types_by_name['PTransform'] = _PTRANSFORM
-DESCRIPTOR.message_types_by_name['PCollection'] = _PCOLLECTION
-DESCRIPTOR.message_types_by_name['ParDoPayload'] = _PARDOPAYLOAD
-DESCRIPTOR.message_types_by_name['Parameter'] = _PARAMETER
-DESCRIPTOR.message_types_by_name['StateSpec'] = _STATESPEC
-DESCRIPTOR.message_types_by_name['TimerSpec'] = _TIMERSPEC
-DESCRIPTOR.message_types_by_name['ReadPayload'] = _READPAYLOAD
-DESCRIPTOR.message_types_by_name['WindowIntoPayload'] = _WINDOWINTOPAYLOAD
-DESCRIPTOR.message_types_by_name['CombinePayload'] = _COMBINEPAYLOAD
-DESCRIPTOR.message_types_by_name['Coder'] = _CODER
-DESCRIPTOR.message_types_by_name['WindowingStrategy'] = _WINDOWINGSTRATEGY
-DESCRIPTOR.message_types_by_name['Trigger'] = _TRIGGER
-DESCRIPTOR.message_types_by_name['TimestampTransform'] = _TIMESTAMPTRANSFORM
-DESCRIPTOR.message_types_by_name['SideInput'] = _SIDEINPUT
-DESCRIPTOR.message_types_by_name['Environment'] = _ENVIRONMENT
-DESCRIPTOR.message_types_by_name['SdkFunctionSpec'] = _SDKFUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['FunctionSpec'] = _FUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['DisplayData'] = _DISPLAYDATA
-DESCRIPTOR.enum_types_by_name['IsBounded'] = _ISBOUNDED
-DESCRIPTOR.enum_types_by_name['MergeStatus'] = _MERGESTATUS
-DESCRIPTOR.enum_types_by_name['AccumulationMode'] = _ACCUMULATIONMODE
-DESCRIPTOR.enum_types_by_name['ClosingBehavior'] = _CLOSINGBEHAVIOR
-DESCRIPTOR.enum_types_by_name['OutputTime'] = _OUTPUTTIME
-DESCRIPTOR.enum_types_by_name['TimeDomain'] = _TIMEDOMAIN
-
-Components = _reflection.GeneratedProtocolMessageType('Components', (_message.Message,), dict(
-
- TransformsEntry = _reflection.GeneratedProtocolMessageType('TransformsEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMPONENTS_TRANSFORMSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.TransformsEntry)
- ))
- ,
-
- PcollectionsEntry = _reflection.GeneratedProtocolMessageType('PcollectionsEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMPONENTS_PCOLLECTIONSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.PcollectionsEntry)
- ))
- ,
-
- WindowingStrategiesEntry = _reflection.GeneratedProtocolMessageType('WindowingStrategiesEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMPONENTS_WINDOWINGSTRATEGIESENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry)
- ))
- ,
-
- CodersEntry = _reflection.GeneratedProtocolMessageType('CodersEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMPONENTS_CODERSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.CodersEntry)
- ))
- ,
-
- EnvironmentsEntry = _reflection.GeneratedProtocolMessageType('EnvironmentsEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMPONENTS_ENVIRONMENTSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.EnvironmentsEntry)
- ))
- ,
- DESCRIPTOR = _COMPONENTS,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components)
- ))
-_sym_db.RegisterMessage(Components)
-_sym_db.RegisterMessage(Components.TransformsEntry)
-_sym_db.RegisterMessage(Components.PcollectionsEntry)
-_sym_db.RegisterMessage(Components.WindowingStrategiesEntry)
-_sym_db.RegisterMessage(Components.CodersEntry)
-_sym_db.RegisterMessage(Components.EnvironmentsEntry)
-
-MessageWithComponents = _reflection.GeneratedProtocolMessageType('MessageWithComponents', (_message.Message,), dict(
- DESCRIPTOR = _MESSAGEWITHCOMPONENTS,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.MessageWithComponents)
- ))
-_sym_db.RegisterMessage(MessageWithComponents)
-
-Pipeline = _reflection.GeneratedProtocolMessageType('Pipeline', (_message.Message,), dict(
- DESCRIPTOR = _PIPELINE,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Pipeline)
- ))
-_sym_db.RegisterMessage(Pipeline)
-
-PTransform = _reflection.GeneratedProtocolMessageType('PTransform', (_message.Message,), dict(
-
- InputsEntry = _reflection.GeneratedProtocolMessageType('InputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PTRANSFORM_INPUTSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform.InputsEntry)
- ))
- ,
-
- OutputsEntry = _reflection.GeneratedProtocolMessageType('OutputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PTRANSFORM_OUTPUTSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform.OutputsEntry)
- ))
- ,
- DESCRIPTOR = _PTRANSFORM,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform)
- ))
-_sym_db.RegisterMessage(PTransform)
-_sym_db.RegisterMessage(PTransform.InputsEntry)
-_sym_db.RegisterMessage(PTransform.OutputsEntry)
-
-PCollection = _reflection.GeneratedProtocolMessageType('PCollection', (_message.Message,), dict(
- DESCRIPTOR = _PCOLLECTION,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PCollection)
- ))
-_sym_db.RegisterMessage(PCollection)
-
-ParDoPayload = _reflection.GeneratedProtocolMessageType('ParDoPayload', (_message.Message,), dict(
-
- SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PARDOPAYLOAD_SIDEINPUTSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry)
- ))
- ,
-
- StateSpecsEntry = _reflection.GeneratedProtocolMessageType('StateSpecsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PARDOPAYLOAD_STATESPECSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry)
- ))
- ,
-
- TimerSpecsEntry = _reflection.GeneratedProtocolMessageType('TimerSpecsEntry', (_message.Message,), dict(
- DESCRIPTOR = _PARDOPAYLOAD_TIMERSPECSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry)
- ))
- ,
- DESCRIPTOR = _PARDOPAYLOAD,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload)
- ))
-_sym_db.RegisterMessage(ParDoPayload)
-_sym_db.RegisterMessage(ParDoPayload.SideInputsEntry)
-_sym_db.RegisterMessage(ParDoPayload.StateSpecsEntry)
-_sym_db.RegisterMessage(ParDoPayload.TimerSpecsEntry)
-
-Parameter = _reflection.GeneratedProtocolMessageType('Parameter', (_message.Message,), dict(
- DESCRIPTOR = _PARAMETER,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Parameter)
- ))
-_sym_db.RegisterMessage(Parameter)
-
-StateSpec = _reflection.GeneratedProtocolMessageType('StateSpec', (_message.Message,), dict(
- DESCRIPTOR = _STATESPEC,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.StateSpec)
- ))
-_sym_db.RegisterMessage(StateSpec)
-
-TimerSpec = _reflection.GeneratedProtocolMessageType('TimerSpec', (_message.Message,), dict(
- DESCRIPTOR = _TIMERSPEC,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimerSpec)
- ))
-_sym_db.RegisterMessage(TimerSpec)
-
-ReadPayload = _reflection.GeneratedProtocolMessageType('ReadPayload', (_message.Message,), dict(
- DESCRIPTOR = _READPAYLOAD,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ReadPayload)
- ))
-_sym_db.RegisterMessage(ReadPayload)
-
-WindowIntoPayload = _reflection.GeneratedProtocolMessageType('WindowIntoPayload', (_message.Message,), dict(
- DESCRIPTOR = _WINDOWINTOPAYLOAD,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.WindowIntoPayload)
- ))
-_sym_db.RegisterMessage(WindowIntoPayload)
-
-CombinePayload = _reflection.GeneratedProtocolMessageType('CombinePayload', (_message.Message,), dict(
-
- SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _COMBINEPAYLOAD_SIDEINPUTSENTRY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry)
- ))
- ,
- DESCRIPTOR = _COMBINEPAYLOAD,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.CombinePayload)
- ))
-_sym_db.RegisterMessage(CombinePayload)
-_sym_db.RegisterMessage(CombinePayload.SideInputsEntry)
-
-Coder = _reflection.GeneratedProtocolMessageType('Coder', (_message.Message,), dict(
- DESCRIPTOR = _CODER,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Coder)
- ))
-_sym_db.RegisterMessage(Coder)
-
-WindowingStrategy = _reflection.GeneratedProtocolMessageType('WindowingStrategy', (_message.Message,), dict(
- DESCRIPTOR = _WINDOWINGSTRATEGY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.WindowingStrategy)
- ))
-_sym_db.RegisterMessage(WindowingStrategy)
-
-Trigger = _reflection.GeneratedProtocolMessageType('Trigger', (_message.Message,), dict(
-
- AfterAll = _reflection.GeneratedProtocolMessageType('AfterAll', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTERALL,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterAll)
- ))
- ,
-
- AfterAny = _reflection.GeneratedProtocolMessageType('AfterAny', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTERANY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterAny)
- ))
- ,
-
- AfterEach = _reflection.GeneratedProtocolMessageType('AfterEach', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTEREACH,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterEach)
- ))
- ,
-
- AfterEndOfWindow = _reflection.GeneratedProtocolMessageType('AfterEndOfWindow', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTERENDOFWINDOW,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow)
- ))
- ,
-
- AfterProcessingTime = _reflection.GeneratedProtocolMessageType('AfterProcessingTime', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTERPROCESSINGTIME,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime)
- ))
- ,
-
- AfterSynchronizedProcessingTime = _reflection.GeneratedProtocolMessageType('AfterSynchronizedProcessingTime', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTime)
- ))
- ,
-
- Default = _reflection.GeneratedProtocolMessageType('Default', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_DEFAULT,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Default)
- ))
- ,
-
- ElementCount = _reflection.GeneratedProtocolMessageType('ElementCount', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_ELEMENTCOUNT,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.ElementCount)
- ))
- ,
-
- Never = _reflection.GeneratedProtocolMessageType('Never', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_NEVER,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Never)
- ))
- ,
-
- Always = _reflection.GeneratedProtocolMessageType('Always', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_ALWAYS,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Always)
- ))
- ,
-
- OrFinally = _reflection.GeneratedProtocolMessageType('OrFinally', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_ORFINALLY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.OrFinally)
- ))
- ,
-
- Repeat = _reflection.GeneratedProtocolMessageType('Repeat', (_message.Message,), dict(
- DESCRIPTOR = _TRIGGER_REPEAT,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Repeat)
- ))
- ,
- DESCRIPTOR = _TRIGGER,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger)
- ))
-_sym_db.RegisterMessage(Trigger)
-_sym_db.RegisterMessage(Trigger.AfterAll)
-_sym_db.RegisterMessage(Trigger.AfterAny)
-_sym_db.RegisterMessage(Trigger.AfterEach)
-_sym_db.RegisterMessage(Trigger.AfterEndOfWindow)
-_sym_db.RegisterMessage(Trigger.AfterProcessingTime)
-_sym_db.RegisterMessage(Trigger.AfterSynchronizedProcessingTime)
-_sym_db.RegisterMessage(Trigger.Default)
-_sym_db.RegisterMessage(Trigger.ElementCount)
-_sym_db.RegisterMessage(Trigger.Never)
-_sym_db.RegisterMessage(Trigger.Always)
-_sym_db.RegisterMessage(Trigger.OrFinally)
-_sym_db.RegisterMessage(Trigger.Repeat)
-
-TimestampTransform = _reflection.GeneratedProtocolMessageType('TimestampTransform', (_message.Message,), dict(
-
- Delay = _reflection.GeneratedProtocolMessageType('Delay', (_message.Message,), dict(
- DESCRIPTOR = _TIMESTAMPTRANSFORM_DELAY,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform.Delay)
- ))
- ,
-
- AlignTo = _reflection.GeneratedProtocolMessageType('AlignTo', (_message.Message,), dict(
- DESCRIPTOR = _TIMESTAMPTRANSFORM_ALIGNTO,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform.AlignTo)
- ))
- ,
- DESCRIPTOR = _TIMESTAMPTRANSFORM,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform)
- ))
-_sym_db.RegisterMessage(TimestampTransform)
-_sym_db.RegisterMessage(TimestampTransform.Delay)
-_sym_db.RegisterMessage(TimestampTransform.AlignTo)
-
-SideInput = _reflection.GeneratedProtocolMessageType('SideInput', (_message.Message,), dict(
- DESCRIPTOR = _SIDEINPUT,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.SideInput)
- ))
-_sym_db.RegisterMessage(SideInput)
-
-Environment = _reflection.GeneratedProtocolMessageType('Environment', (_message.Message,), dict(
- DESCRIPTOR = _ENVIRONMENT,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Environment)
- ))
-_sym_db.RegisterMessage(Environment)
-
-SdkFunctionSpec = _reflection.GeneratedProtocolMessageType('SdkFunctionSpec', (_message.Message,), dict(
- DESCRIPTOR = _SDKFUNCTIONSPEC,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.SdkFunctionSpec)
- ))
-_sym_db.RegisterMessage(SdkFunctionSpec)
-
-FunctionSpec = _reflection.GeneratedProtocolMessageType('FunctionSpec', (_message.Message,), dict(
- DESCRIPTOR = _FUNCTIONSPEC,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.FunctionSpec)
- ))
-_sym_db.RegisterMessage(FunctionSpec)
-
-DisplayData = _reflection.GeneratedProtocolMessageType('DisplayData', (_message.Message,), dict(
-
- Identifier = _reflection.GeneratedProtocolMessageType('Identifier', (_message.Message,), dict(
- DESCRIPTOR = _DISPLAYDATA_IDENTIFIER,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData.Identifier)
- ))
- ,
-
- Item = _reflection.GeneratedProtocolMessageType('Item', (_message.Message,), dict(
- DESCRIPTOR = _DISPLAYDATA_ITEM,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData.Item)
- ))
- ,
- DESCRIPTOR = _DISPLAYDATA,
- __module__ = 'beam_runner_api_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData)
- ))
-_sym_db.RegisterMessage(DisplayData)
-_sym_db.RegisterMessage(DisplayData.Identifier)
-_sym_db.RegisterMessage(DisplayData.Item)
-
-
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n$org.apache.beam.sdk.common.runner.v1B\tRunnerApi'))
-_COMPONENTS_TRANSFORMSENTRY.has_options = True
-_COMPONENTS_TRANSFORMSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_PCOLLECTIONSENTRY.has_options = True
-_COMPONENTS_PCOLLECTIONSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.has_options = True
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_CODERSENTRY.has_options = True
-_COMPONENTS_CODERSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_ENVIRONMENTSENTRY.has_options = True
-_COMPONENTS_ENVIRONMENTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PTRANSFORM_INPUTSENTRY.has_options = True
-_PTRANSFORM_INPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PTRANSFORM_OUTPUTSENTRY.has_options = True
-_PTRANSFORM_OUTPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_SIDEINPUTSENTRY.has_options = True
-_PARDOPAYLOAD_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_STATESPECSENTRY.has_options = True
-_PARDOPAYLOAD_STATESPECSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_TIMERSPECSENTRY.has_options = True
-_PARDOPAYLOAD_TIMERSPECSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.has_options = True
-_COMBINEPAYLOAD_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-# @@protoc_insertion_point(module_scope)
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 8453569..64abe41 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -29,9 +29,9 @@
from apache_beam.metrics.execution import ScopedMetricsContainer
from apache_beam.pvalue import TaggedOutput
from apache_beam.transforms import core
+from apache_beam.transforms.window import GlobalWindow
from apache_beam.transforms.window import TimestampedValue
from apache_beam.transforms.window import WindowFn
-from apache_beam.transforms.window import GlobalWindow
from apache_beam.utils.windowed_value import WindowedValue
@@ -248,14 +248,14 @@
elif d == core.DoFn.SideInputParam:
# If no more args are present then the value must be passed via kwarg
try:
- args_with_placeholders.append(remaining_args_iter.next())
+ args_with_placeholders.append(next(remaining_args_iter))
except StopIteration:
if a not in input_kwargs:
raise ValueError("Value for sideinput %s not provided" % a)
else:
# If no more args are present then the value must be passed via kwarg
try:
- args_with_placeholders.append(remaining_args_iter.next())
+ args_with_placeholders.append(next(remaining_args_iter))
except StopIteration:
pass
args_with_placeholders.extend(list(remaining_args_iter))
diff --git a/sdks/python/apache_beam/runners/common_test.py b/sdks/python/apache_beam/runners/common_test.py
index 62a6955..e0f628c7 100644
--- a/sdks/python/apache_beam/runners/common_test.py
+++ b/sdks/python/apache_beam/runners/common_test.py
@@ -17,8 +17,8 @@
import unittest
-from apache_beam.transforms.core import DoFn
from apache_beam.runners.common import DoFnSignature
+from apache_beam.transforms.core import DoFn
class DoFnSignatureTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
index 24916fd..c7eb88e 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
@@ -21,8 +21,8 @@
service.
"""
-from collections import defaultdict
import numbers
+from collections import defaultdict
from apache_beam.metrics.cells import DistributionData
from apache_beam.metrics.cells import DistributionResult
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 0ecd22a..6253c80 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -25,26 +25,31 @@
import threading
import time
import traceback
+import urllib
+from collections import defaultdict
-from apache_beam import error
+import apache_beam as beam
from apache_beam import coders
+from apache_beam import error
from apache_beam import pvalue
from apache_beam.internal import pickler
from apache_beam.internal.gcp import json_value
+from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import TestOptions
from apache_beam.pvalue import AsSideInput
from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics
from apache_beam.runners.dataflow.internal import names
from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api
from apache_beam.runners.dataflow.internal.names import PropertyNames
from apache_beam.runners.dataflow.internal.names import TransformNames
-from apache_beam.runners.runner import PValueCache
from apache_beam.runners.runner import PipelineResult
from apache_beam.runners.runner import PipelineRunner
from apache_beam.runners.runner import PipelineState
+from apache_beam.runners.runner import PValueCache
from apache_beam.transforms.display import DisplayData
from apache_beam.typehints import typehints
-from apache_beam.options.pipeline_options import StandardOptions
-
+from apache_beam.utils.plugin import BeamPlugin
__all__ = ['DataflowRunner']
@@ -59,11 +64,20 @@
if blocking is set to False.
"""
- # Environment version information. It is passed to the service during a
- # a job submission and is used by the service to establish what features
- # are expected by the workers.
- BATCH_ENVIRONMENT_MAJOR_VERSION = '5'
- STREAMING_ENVIRONMENT_MAJOR_VERSION = '0'
+ # A list of PTransformOverride objects to be applied before running a pipeline
+ # using DataflowRunner.
+ # Currently this only works for overrides where the input and output types do
+ # not change.
+ # For internal SDK use only. This should not be updated by Beam pipeline
+ # authors.
+
+ # Imported here to avoid circular dependencies.
+ # TODO: Remove the apache_beam.pipeline dependency in CreatePTransformOverride
+ from apache_beam.runners.dataflow.ptransform_overrides import CreatePTransformOverride
+
+ _PTRANSFORM_OVERRIDES = [
+ CreatePTransformOverride(),
+ ]
def __init__(self, cache=None):
# Cache of CloudWorkflowStep protos generated while the runner
@@ -76,8 +90,18 @@
return 's%s' % self._unique_step_id
@staticmethod
- def poll_for_job_completion(runner, result):
- """Polls for the specified job to finish running (successfully or not)."""
+ def poll_for_job_completion(runner, result, duration):
+ """Polls for the specified job to finish running (successfully or not).
+
+ Updates the result with the new job information before returning.
+
+ Args:
+ runner: DataflowRunner instance to use for polling job state.
+ result: DataflowPipelineResult instance used for job information.
+ duration (int): The time to wait (in milliseconds) for job to finish.
+ If it is set to :data:`None`, it will wait indefinitely until the job
+ is finished.
+ """
last_message_time = None
last_message_hash = None
@@ -98,6 +122,10 @@
return 1
return 0
+ if duration:
+ start_secs = time.time()
+ duration_secs = duration / 1000
+
job_id = result.job_id()
while True:
response = runner.dataflow_client.get_job(job_id)
@@ -150,6 +178,13 @@
if not page_token:
break
+ if duration:
+ passed_secs = time.time() - start_secs
+ if passed_secs > duration_secs:
+ logging.warning('Timing out on waiting for job %s after %d seconds',
+ job_id, passed_secs)
+ break
+
result._job = response
runner.last_error_msg = last_error_msg
@@ -177,18 +212,31 @@
if not input_type:
input_type = typehints.Any
- if not isinstance(input_type, typehints.TupleHint.TupleConstraint):
- if isinstance(input_type, typehints.AnyTypeConstraint):
+ def coerce_to_kv_type(element_type):
+ if isinstance(element_type, typehints.TupleHint.TupleConstraint):
+ if len(element_type.tuple_types) == 2:
+ return element_type
+ else:
+ raise ValueError(
+ "Tuple input to GroupByKey must be have two components. "
+ "Found %s for %s" % (element_type, pcoll))
+ elif isinstance(input_type, typehints.AnyTypeConstraint):
# `Any` type needs to be replaced with a KV[Any, Any] to
# force a KV coder as the main output coder for the pcollection
# preceding a GroupByKey.
- pcoll.element_type = typehints.KV[typehints.Any, typehints.Any]
+ return typehints.KV[typehints.Any, typehints.Any]
+ elif isinstance(element_type, typehints.UnionConstraint):
+ union_types = [
+ coerce_to_kv_type(t) for t in element_type.union_types]
+ return typehints.KV[
+ typehints.Union[tuple(t.tuple_types[0] for t in union_types)],
+ typehints.Union[tuple(t.tuple_types[1] for t in union_types)]]
else:
- # TODO: Handle other valid types,
- # e.g. Union[KV[str, int], KV[str, float]]
+ # TODO: Possibly handle other valid types.
raise ValueError(
"Input to GroupByKey must be of Tuple or Any type. "
- "Found %s for %s" % (input_type, pcoll))
+ "Found %s for %s" % (element_type, pcoll))
+ pcoll.element_type = coerce_to_kv_type(input_type)
return GroupByKeyInputVisitor()
@@ -213,7 +261,6 @@
return FlattenInputVisitor()
- # TODO(mariagh): Make this method take pipepline_options
def run(self, pipeline):
"""Remotely executes entire pipeline or parts reachable from node."""
# Import here to avoid adding the dependency for local running scenarios.
@@ -224,7 +271,21 @@
raise ImportError(
'Google Cloud Dataflow runner not available, '
'please install apache_beam[gcp]')
- self.job = apiclient.Job(pipeline._options)
+
+ # Snapshot the pipeline in a portable proto before mutating it
+ proto_pipeline = pipeline.to_runner_api()
+
+ # Performing configured PTransform overrides.
+ pipeline.replace_all(DataflowRunner._PTRANSFORM_OVERRIDES)
+
+ # Add setup_options for all the BeamPlugin imports
+ setup_options = pipeline._options.view_as(SetupOptions)
+ plugins = BeamPlugin.get_all_plugin_paths()
+ if setup_options.beam_plugins is not None:
+ plugins = list(set(plugins + setup_options.beam_plugins))
+ setup_options.beam_plugins = plugins
+
+ self.job = apiclient.Job(pipeline._options, proto_pipeline)
# Dataflow runner requires a KV type for GBK inputs, hence we enforce that
# here.
@@ -237,17 +298,19 @@
# The superclass's run will trigger a traversal of all reachable nodes.
super(DataflowRunner, self).run(pipeline)
- standard_options = pipeline._options.view_as(StandardOptions)
- if standard_options.streaming:
- job_version = DataflowRunner.STREAMING_ENVIRONMENT_MAJOR_VERSION
- else:
- job_version = DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION
+ test_options = pipeline._options.view_as(TestOptions)
+ # If it is a dry run, return without submitting the job.
+ if test_options.dry_run:
+ return None
# Get a Dataflow API client and set its options
self.dataflow_client = apiclient.DataflowApplicationClient(
- pipeline._options, job_version)
+ pipeline._options)
- # Create the job
+ # Create the job description and send a request to the service. The result
+ # can be None if there is no need to send a request to the service (e.g.
+ # template creation). If a request was sent and failed then the call will
+ # raise an exception.
result = DataflowPipelineResult(
self.dataflow_client.create_job(self.job), self)
@@ -358,6 +421,26 @@
PropertyNames.OUTPUT_NAME: PropertyNames.OUT}])
return step
+ def run_Impulse(self, transform_node):
+ standard_options = (
+ transform_node.outputs[None].pipeline._options.view_as(StandardOptions))
+ if standard_options.streaming:
+ step = self._add_step(
+ TransformNames.READ, transform_node.full_label, transform_node)
+ step.add_property(PropertyNames.FORMAT, 'pubsub')
+ step.add_property(PropertyNames.PUBSUB_SUBSCRIPTION, '_starting_signal/')
+
+ step.encoding = self._get_encoded_output_coder(transform_node)
+ step.add_property(
+ PropertyNames.OUTPUT_INFO,
+ [{PropertyNames.USER_NAME: (
+ '%s.%s' % (
+ transform_node.full_label, PropertyNames.OUT)),
+ PropertyNames.ENCODING: step.encoding,
+ PropertyNames.OUTPUT_NAME: PropertyNames.OUT}])
+ else:
+ ValueError('Impulse source for batch pipelines has not been defined.')
+
def run_Flatten(self, transform_node):
step = self._add_step(TransformNames.FLATTEN,
transform_node.full_label, transform_node)
@@ -377,6 +460,26 @@
PropertyNames.ENCODING: step.encoding,
PropertyNames.OUTPUT_NAME: PropertyNames.OUT}])
+ def apply_WriteToBigQuery(self, transform, pcoll):
+ # Make sure this is the WriteToBigQuery class that we expected
+ if not isinstance(transform, beam.io.WriteToBigQuery):
+ return self.apply_PTransform(transform, pcoll)
+ standard_options = pcoll.pipeline._options.view_as(StandardOptions)
+ if standard_options.streaming:
+ if (transform.write_disposition ==
+ beam.io.BigQueryDisposition.WRITE_TRUNCATE):
+ raise RuntimeError('Can not use write truncation mode in streaming')
+ return self.apply_PTransform(transform, pcoll)
+ else:
+ return pcoll | 'WriteToBigQuery' >> beam.io.Write(
+ beam.io.BigQuerySink(
+ transform.table_reference.tableId,
+ transform.table_reference.datasetId,
+ transform.table_reference.projectId,
+ transform.schema,
+ transform.create_disposition,
+ transform.write_disposition))
+
def apply_GroupByKey(self, transform, pcoll):
# Infer coder of parent.
#
@@ -416,7 +519,9 @@
PropertyNames.OUTPUT_NAME: PropertyNames.OUT}])
windowing = transform_node.transform.get_windowing(
transform_node.inputs)
- step.add_property(PropertyNames.SERIALIZED_FN, pickler.dumps(windowing))
+ step.add_property(
+ PropertyNames.SERIALIZED_FN,
+ self.serialize_windowing_strategy(windowing))
def run_ParDo(self, transform_node):
transform = transform_node.transform
@@ -427,11 +532,24 @@
si_dict = {}
# We must call self._cache.get_pvalue exactly once due to refcounting.
si_labels = {}
+ full_label_counts = defaultdict(int)
lookup_label = lambda side_pval: si_labels[side_pval]
for side_pval in transform_node.side_inputs:
assert isinstance(side_pval, AsSideInput)
- si_label = 'SideInput-' + self._get_unique_step_name()
- si_full_label = '%s/%s' % (transform_node.full_label, si_label)
+ step_number = self._get_unique_step_name()
+ si_label = 'SideInput-' + step_number
+ pcollection_label = '%s.%s' % (
+ side_pval.pvalue.producer.full_label.split('/')[-1],
+ side_pval.pvalue.tag if side_pval.pvalue.tag else 'out')
+ si_full_label = '%s/%s(%s.%s)' % (transform_node.full_label,
+ side_pval.__class__.__name__,
+ pcollection_label,
+ full_label_counts[pcollection_label])
+
+ # Count the number of times the same PCollection is a side input
+ # to the same ParDo.
+ full_label_counts[pcollection_label] += 1
+
self._add_singleton_step(
si_label, si_full_label, side_pval.pvalue.tag,
self._cache.get_pvalue(side_pval.pvalue))
@@ -442,10 +560,12 @@
si_labels[side_pval] = si_label
# Now create the step for the ParDo transform being handled.
+ transform_name = transform_node.full_label.rsplit('/', 1)[-1]
step = self._add_step(
TransformNames.DO,
transform_node.full_label + (
- '/Do' if transform_node.side_inputs else ''),
+ '/{}'.format(transform_name)
+ if transform_node.side_inputs else ''),
transform_node,
transform_node.transform.output_tags)
fn_data = self._pardo_fn_data(transform_node, lookup_label)
@@ -516,8 +636,8 @@
PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag)})
# Note that the accumulator must not have a WindowedValue encoding, while
# the output of this step does in fact have a WindowedValue encoding.
- accumulator_encoding = self._get_encoded_output_coder(transform_node,
- window_value=False)
+ accumulator_encoding = self._get_cloud_encoding(
+ transform_node.transform.fn.get_accumulator_coder())
output_encoding = self._get_encoded_output_coder(transform_node)
step.encoding = output_encoding
@@ -595,12 +715,15 @@
standard_options = (
transform_node.inputs[0].pipeline.options.view_as(StandardOptions))
if not standard_options.streaming:
- raise ValueError('PubSubSource is currently available for use only in '
- 'streaming pipelines.')
- step.add_property(PropertyNames.PUBSUB_TOPIC, transform.source.topic)
- if transform.source.subscription:
+ raise ValueError('PubSubPayloadSource is currently available for use '
+ 'only in streaming pipelines.')
+ # Only one of topic or subscription should be set.
+ if transform.source.full_subscription:
step.add_property(PropertyNames.PUBSUB_SUBSCRIPTION,
- transform.source.topic)
+ transform.source.full_subscription)
+ elif transform.source.full_topic:
+ step.add_property(PropertyNames.PUBSUB_TOPIC,
+ transform.source.full_topic)
if transform.source.id_label:
step.add_property(PropertyNames.PUBSUB_ID_LABEL,
transform.source.id_label)
@@ -618,7 +741,12 @@
# step should be the type of value outputted by each step. Read steps
# automatically wrap output values in a WindowedValue wrapper, if necessary.
# This is also necessary for proper encoding for size estimation.
- coder = coders.WindowedValueCoder(transform._infer_output_coder()) # pylint: disable=protected-access
+ # Using a GlobalWindowCoder as a place holder instead of the default
+ # PickleCoder because GlobalWindowCoder is known coder.
+ # TODO(robertwb): Query the collection for the windowfn to extract the
+ # correct coder.
+ coder = coders.WindowedValueCoder(transform._infer_output_coder(),
+ coders.coders.GlobalWindowCoder()) # pylint: disable=protected-access
step.encoding = self._get_cloud_encoding(coder)
step.add_property(
@@ -677,9 +805,9 @@
standard_options = (
transform_node.inputs[0].pipeline.options.view_as(StandardOptions))
if not standard_options.streaming:
- raise ValueError('PubSubSink is currently available for use only in '
- 'streaming pipelines.')
- step.add_property(PropertyNames.PUBSUB_TOPIC, transform.sink.topic)
+ raise ValueError('PubSubPayloadSink is currently available for use '
+ 'only in streaming pipelines.')
+ step.add_property(PropertyNames.PUBSUB_TOPIC, transform.sink.full_topic)
else:
raise ValueError(
'Sink %r has unexpected format %s.' % (
@@ -687,8 +815,12 @@
step.add_property(PropertyNames.FORMAT, transform.sink.format)
# Wrap coder in WindowedValueCoder: this is necessary for proper encoding
- # for size estimation.
- coder = coders.WindowedValueCoder(transform.sink.coder)
+ # for size estimation. Using a GlobalWindowCoder as a place holder instead
+ # of the default PickleCoder because GlobalWindowCoder is known coder.
+ # TODO(robertwb): Query the collection for the windowfn to extract the
+ # correct coder.
+ coder = coders.WindowedValueCoder(transform.sink.coder,
+ coders.coders.GlobalWindowCoder())
step.encoding = self._get_cloud_encoding(coder)
step.add_property(PropertyNames.ENCODING, step.encoding)
step.add_property(
@@ -697,16 +829,62 @@
PropertyNames.STEP_NAME: input_step.proto.name,
PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag)})
+ @classmethod
+ def serialize_windowing_strategy(cls, windowing):
+ from apache_beam.runners import pipeline_context
+ from apache_beam.portability.api import beam_runner_api_pb2
+ context = pipeline_context.PipelineContext()
+ windowing_proto = windowing.to_runner_api(context)
+ return cls.byte_array_to_json_string(
+ beam_runner_api_pb2.MessageWithComponents(
+ components=context.to_runner_api(),
+ windowing_strategy=windowing_proto).SerializeToString())
+
+ @classmethod
+ def deserialize_windowing_strategy(cls, serialized_data):
+ # Imported here to avoid circular dependencies.
+ # pylint: disable=wrong-import-order, wrong-import-position
+ from apache_beam.runners import pipeline_context
+ from apache_beam.portability.api import beam_runner_api_pb2
+ from apache_beam.transforms.core import Windowing
+ proto = beam_runner_api_pb2.MessageWithComponents()
+ proto.ParseFromString(cls.json_string_to_byte_array(serialized_data))
+ return Windowing.from_runner_api(
+ proto.windowing_strategy,
+ pipeline_context.PipelineContext(proto.components))
+
+ @staticmethod
+ def byte_array_to_json_string(raw_bytes):
+ """Implements org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString."""
+ return urllib.quote(raw_bytes)
+
+ @staticmethod
+ def json_string_to_byte_array(encoded_string):
+ """Implements org.apache.beam.sdk.util.StringUtils.jsonStringToByteArray."""
+ return urllib.unquote(encoded_string)
+
class DataflowPipelineResult(PipelineResult):
"""Represents the state of a pipeline run on the Dataflow service."""
def __init__(self, job, runner):
- """Job is a Job message from the Dataflow API."""
+ """Initialize a new DataflowPipelineResult instance.
+
+ Args:
+ job: Job message from the Dataflow API. Could be :data:`None` if a job
+ request was not sent to Dataflow service (e.g. template jobs).
+ runner: DataflowRunner instance.
+ """
self._job = job
self._runner = runner
self.metric_results = None
+ def _update_job(self):
+ # We need the job id to be able to update job information. There is no need
+ # to update the job if we are in a known terminal state.
+ if self.has_job and not self._is_in_terminal_state():
+ self._job = self._runner.dataflow_client.get_job(self.job_id())
+
def job_id(self):
return self._job.id
@@ -727,7 +905,12 @@
if not self.has_job:
return PipelineState.UNKNOWN
+ self._update_job()
+
values_enum = dataflow_api.Job.CurrentStateValueValuesEnum
+
+ # TODO: Move this table to a another location.
+ # Ordered by the enum values.
api_jobstate_map = {
values_enum.JOB_STATE_UNKNOWN: PipelineState.UNKNOWN,
values_enum.JOB_STATE_STOPPED: PipelineState.STOPPED,
@@ -738,6 +921,8 @@
values_enum.JOB_STATE_UPDATED: PipelineState.UPDATED,
values_enum.JOB_STATE_DRAINING: PipelineState.DRAINING,
values_enum.JOB_STATE_DRAINED: PipelineState.DRAINED,
+ values_enum.JOB_STATE_PENDING: PipelineState.PENDING,
+ values_enum.JOB_STATE_CANCELLING: PipelineState.CANCELLING,
}
return (api_jobstate_map[self._job.currentState] if self._job.currentState
@@ -747,21 +932,20 @@
if not self.has_job:
return True
- return self.state in [
- PipelineState.STOPPED, PipelineState.DONE, PipelineState.FAILED,
- PipelineState.CANCELLED, PipelineState.DRAINED]
+ values_enum = dataflow_api.Job.CurrentStateValueValuesEnum
+ return self._job.currentState in [
+ values_enum.JOB_STATE_STOPPED, values_enum.JOB_STATE_DONE,
+ values_enum.JOB_STATE_FAILED, values_enum.JOB_STATE_CANCELLED,
+ values_enum.JOB_STATE_DRAINED]
def wait_until_finish(self, duration=None):
if not self._is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
- if duration:
- raise NotImplementedError(
- 'DataflowRunner does not support duration argument.')
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
- args=(self._runner, self))
+ args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
@@ -770,14 +954,42 @@
thread.start()
while thread.isAlive():
time.sleep(5.0)
- if self.state != PipelineState.DONE:
- # TODO(BEAM-1290): Consider converting this to an error log based on the
- # resolution of the issue.
+
+ # TODO: Merge the termination code in poll_for_job_completion and
+ # _is_in_terminal_state.
+ terminated = (str(self._job.currentState) != 'JOB_STATE_RUNNING')
+ assert duration or terminated, (
+ 'Job did not reach to a terminal state after waiting indefinitely.')
+
+ if terminated and self.state != PipelineState.DONE:
+ # TODO(BEAM-1290): Consider converting this to an error log based on
+ # theresolution of the issue.
raise DataflowRuntimeException(
'Dataflow pipeline failed. State: %s, Error:\n%s' %
(self.state, getattr(self._runner, 'last_error_msg', None)), self)
return self.state
+ def cancel(self):
+ if not self.has_job:
+ raise IOError('Failed to get the Dataflow job id.')
+
+ self._update_job()
+
+ if self._is_in_terminal_state():
+ logging.warning(
+ 'Cancel failed because job %s is already terminated in state %s.',
+ self.job_id(), self.state)
+ else:
+ if not self._runner.dataflow_client.modify_job_state(
+ self.job_id(), 'JOB_STATE_CANCELLED'):
+ cancel_failed_message = (
+ 'Failed to cancel job %s, please go to the Developers Console to '
+ 'cancel it manually.') % self.job_id()
+ logging.error(cancel_failed_message)
+ raise DataflowRuntimeException(cancel_failed_message, self)
+
+ return self.state
+
def __str__(self):
return '<%s %s %s>' % (
self.__class__.__name__,
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index ff4b51d..2d529e11 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -25,19 +25,22 @@
import apache_beam as beam
import apache_beam.transforms as ptransform
-
from apache_beam.options.pipeline_options import PipelineOptions
-from apache_beam.pipeline import Pipeline, AppliedPTransform
+from apache_beam.pipeline import AppliedPTransform
+from apache_beam.pipeline import Pipeline
from apache_beam.pvalue import PCollection
-from apache_beam.runners import create_runner
from apache_beam.runners import DataflowRunner
from apache_beam.runners import TestDataflowRunner
+from apache_beam.runners import create_runner
from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult
from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException
from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api
+from apache_beam.runners.runner import PipelineState
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms import window
+from apache_beam.transforms.core import Windowing
from apache_beam.transforms.core import _GroupByKeyOnly
+from apache_beam.transforms.display import DisplayDataItem
from apache_beam.typehints import typehints
# Protect against environments where apitools library is not available.
@@ -57,7 +60,8 @@
'--project=test-project',
'--staging_location=ignored',
'--temp_location=/dev/null',
- '--no_auth=True']
+ '--no_auth=True',
+ '--dry_run=True']
@mock.patch('time.sleep', return_value=None)
def test_wait_until_finish(self, patched_time_sleep):
@@ -65,13 +69,17 @@
class MockDataflowRunner(object):
- def __init__(self, final_state):
+ def __init__(self, states):
self.dataflow_client = mock.MagicMock()
self.job = mock.MagicMock()
self.job.currentState = values_enum.JOB_STATE_UNKNOWN
+ self._states = states
+ self._next_state_index = 0
def get_job_side_effect(*args, **kwargs):
- self.job.currentState = final_state
+ self.job.currentState = self._states[self._next_state_index]
+ if self._next_state_index < (len(self._states) - 1):
+ self._next_state_index += 1
return mock.DEFAULT
self.dataflow_client.get_job = mock.MagicMock(
@@ -81,14 +89,76 @@
with self.assertRaisesRegexp(
DataflowRuntimeException, 'Dataflow pipeline failed. State: FAILED'):
- failed_runner = MockDataflowRunner(values_enum.JOB_STATE_FAILED)
+ failed_runner = MockDataflowRunner([values_enum.JOB_STATE_FAILED])
failed_result = DataflowPipelineResult(failed_runner.job, failed_runner)
failed_result.wait_until_finish()
- succeeded_runner = MockDataflowRunner(values_enum.JOB_STATE_DONE)
+ succeeded_runner = MockDataflowRunner([values_enum.JOB_STATE_DONE])
succeeded_result = DataflowPipelineResult(
succeeded_runner.job, succeeded_runner)
- succeeded_result.wait_until_finish()
+ result = succeeded_result.wait_until_finish()
+ self.assertEqual(result, PipelineState.DONE)
+
+ # Time array has duplicate items, because some logging implementations also
+ # call time.
+ with mock.patch('time.time', mock.MagicMock(side_effect=[1, 1, 2, 2, 3])):
+ duration_succeeded_runner = MockDataflowRunner(
+ [values_enum.JOB_STATE_RUNNING, values_enum.JOB_STATE_DONE])
+ duration_succeeded_result = DataflowPipelineResult(
+ duration_succeeded_runner.job, duration_succeeded_runner)
+ result = duration_succeeded_result.wait_until_finish(5000)
+ self.assertEqual(result, PipelineState.DONE)
+
+ with mock.patch('time.time', mock.MagicMock(side_effect=[1, 9, 9, 20, 20])):
+ duration_timedout_runner = MockDataflowRunner(
+ [values_enum.JOB_STATE_RUNNING])
+ duration_timedout_result = DataflowPipelineResult(
+ duration_timedout_runner.job, duration_timedout_runner)
+ result = duration_timedout_result.wait_until_finish(5000)
+ self.assertEqual(result, PipelineState.RUNNING)
+
+ with mock.patch('time.time', mock.MagicMock(side_effect=[1, 1, 2, 2, 3])):
+ with self.assertRaisesRegexp(
+ DataflowRuntimeException,
+ 'Dataflow pipeline failed. State: CANCELLED'):
+ duration_failed_runner = MockDataflowRunner(
+ [values_enum.JOB_STATE_CANCELLED])
+ duration_failed_result = DataflowPipelineResult(
+ duration_failed_runner.job, duration_failed_runner)
+ duration_failed_result.wait_until_finish(5000)
+
+ @mock.patch('time.sleep', return_value=None)
+ def test_cancel(self, patched_time_sleep):
+ values_enum = dataflow_api.Job.CurrentStateValueValuesEnum
+
+ class MockDataflowRunner(object):
+
+ def __init__(self, state, cancel_result):
+ self.dataflow_client = mock.MagicMock()
+ self.job = mock.MagicMock()
+ self.job.currentState = state
+
+ self.dataflow_client.get_job = mock.MagicMock(return_value=self.job)
+ self.dataflow_client.modify_job_state = mock.MagicMock(
+ return_value=cancel_result)
+ self.dataflow_client.list_messages = mock.MagicMock(
+ return_value=([], None))
+
+ with self.assertRaisesRegexp(
+ DataflowRuntimeException, 'Failed to cancel job'):
+ failed_runner = MockDataflowRunner(values_enum.JOB_STATE_RUNNING, False)
+ failed_result = DataflowPipelineResult(failed_runner.job, failed_runner)
+ failed_result.cancel()
+
+ succeeded_runner = MockDataflowRunner(values_enum.JOB_STATE_RUNNING, True)
+ succeeded_result = DataflowPipelineResult(
+ succeeded_runner.job, succeeded_runner)
+ succeeded_result.cancel()
+
+ terminal_runner = MockDataflowRunner(values_enum.JOB_STATE_DONE, False)
+ terminal_result = DataflowPipelineResult(
+ terminal_runner.job, terminal_runner)
+ terminal_result.cancel()
def test_create_runner(self):
self.assertTrue(
@@ -106,8 +176,22 @@
(p | ptransform.Create([1, 2, 3]) # pylint: disable=expression-not-assigned
| 'Do' >> ptransform.FlatMap(lambda x: [(x, x)])
| ptransform.GroupByKey())
- remote_runner.job = apiclient.Job(p._options)
- super(DataflowRunner, remote_runner).run(p)
+ p.run()
+
+ def test_streaming_create_translation(self):
+ remote_runner = DataflowRunner()
+ self.default_properties.append("--streaming")
+ p = Pipeline(remote_runner, PipelineOptions(self.default_properties))
+ p | ptransform.Create([1]) # pylint: disable=expression-not-assigned
+ p.run()
+ job_dict = json.loads(str(remote_runner.job))
+ self.assertEqual(len(job_dict[u'steps']), 2)
+
+ self.assertEqual(job_dict[u'steps'][0][u'kind'], u'ParallelRead')
+ self.assertEqual(
+ job_dict[u'steps'][0][u'properties'][u'pubsub_subscription'],
+ '_starting_signal/')
+ self.assertEqual(job_dict[u'steps'][1][u'kind'], u'ParallelDo')
def test_remote_runner_display_data(self):
remote_runner = DataflowRunner()
@@ -140,8 +224,7 @@
(p | ptransform.Create([1, 2, 3, 4, 5])
| 'Do' >> SpecialParDo(SpecialDoFn(), now))
- remote_runner.job = apiclient.Job(p._options)
- super(DataflowRunner, remote_runner).run(p)
+ p.run()
job_dict = json.loads(str(remote_runner.job))
steps = [step
for step in job_dict['steps']
@@ -240,6 +323,37 @@
for _ in range(num_inputs):
self.assertEqual(inputs[0].element_type, output_type)
+ def test_gbk_then_flatten_input_visitor(self):
+ p = TestPipeline(
+ runner=DataflowRunner(),
+ options=PipelineOptions(self.default_properties))
+ none_str_pc = p | 'c1' >> beam.Create({None: 'a'})
+ none_int_pc = p | 'c2' >> beam.Create({None: 3})
+ flat = (none_str_pc, none_int_pc) | beam.Flatten()
+ _ = flat | beam.GroupByKey()
+
+ # This may change if type inference changes, but we assert it here
+ # to make sure the check below is not vacuous.
+ self.assertNotIsInstance(flat.element_type, typehints.TupleConstraint)
+
+ p.visit(DataflowRunner.group_by_key_input_visitor())
+ p.visit(DataflowRunner.flatten_input_visitor())
+
+ # The dataflow runner requires gbk input to be tuples *and* flatten
+ # inputs to be equal to their outputs. Assert both hold.
+ self.assertIsInstance(flat.element_type, typehints.TupleConstraint)
+ self.assertEqual(flat.element_type, none_str_pc.element_type)
+ self.assertEqual(flat.element_type, none_int_pc.element_type)
+
+ def test_serialize_windowing_strategy(self):
+ # This just tests the basic path; more complete tests
+ # are in window_test.py.
+ strategy = Windowing(window.FixedWindows(10))
+ self.assertEqual(
+ strategy,
+ DataflowRunner.deserialize_windowing_strategy(
+ DataflowRunner.serialize_windowing_strategy(strategy)))
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index bfdd5e4..3aa563d 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -26,8 +26,8 @@
import os
import re
import time
-from StringIO import StringIO
from datetime import datetime
+from StringIO import StringIO
from apitools.base.py import encoding
from apitools.base.py import exceptions
@@ -36,18 +36,24 @@
from apache_beam.internal.gcp.json_value import to_json_value
from apache_beam.io.filesystems import FileSystems
from apache_beam.io.gcp.internal.clients import storage
+from apache_beam.options.pipeline_options import DebugOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import WorkerOptions
from apache_beam.runners.dataflow.internal import dependency
+from apache_beam.runners.dataflow.internal import names
from apache_beam.runners.dataflow.internal.clients import dataflow
-from apache_beam.runners.dataflow.internal.dependency import get_required_container_version
from apache_beam.runners.dataflow.internal.dependency import get_sdk_name_and_version
from apache_beam.runners.dataflow.internal.names import PropertyNames
from apache_beam.transforms import cy_combiners
from apache_beam.transforms.display import DisplayData
from apache_beam.utils import retry
-from apache_beam.options.pipeline_options import DebugOptions
-from apache_beam.options.pipeline_options import GoogleCloudOptions
-from apache_beam.options.pipeline_options import StandardOptions
-from apache_beam.options.pipeline_options import WorkerOptions
+
+# Environment version information. It is passed to the service during a
+# a job submission and is used by the service to establish what features
+# are expected by the workers.
+_LEGACY_ENVIRONMENT_MAJOR_VERSION = '6'
+_FNAPI_ENVIRONMENT_MAJOR_VERSION = '1'
class Step(object):
@@ -113,11 +119,12 @@
class Environment(object):
"""Wrapper for a dataflow Environment protobuf."""
- def __init__(self, packages, options, environment_version):
+ def __init__(self, packages, options, environment_version, pipeline_url):
self.standard_options = options.view_as(StandardOptions)
self.google_cloud_options = options.view_as(GoogleCloudOptions)
self.worker_options = options.view_as(WorkerOptions)
self.debug_options = options.view_as(DebugOptions)
+ self.pipeline_url = pipeline_url
self.proto = dataflow.Environment()
self.proto.clusterManagerApiService = GoogleCloudOptions.COMPUTE_API_SERVICE
self.proto.dataset = '{}/cloud_dataflow'.format(
@@ -145,15 +152,26 @@
# Version information.
self.proto.version = dataflow.Environment.VersionValue()
if self.standard_options.streaming:
- job_type = 'PYTHON_STREAMING'
+ job_type = 'FNAPI_STREAMING'
else:
- job_type = 'PYTHON_BATCH'
+ if _use_fnapi(options):
+ job_type = 'FNAPI_BATCH'
+ else:
+ job_type = 'PYTHON_BATCH'
self.proto.version.additionalProperties.extend([
dataflow.Environment.VersionValue.AdditionalProperty(
key='job_type',
value=to_json_value(job_type)),
dataflow.Environment.VersionValue.AdditionalProperty(
key='major', value=to_json_value(environment_version))])
+ # TODO: Use enumerated type instead of strings for job types.
+ if job_type.startswith('FNAPI_'):
+ runner_harness_override = (
+ dependency.get_runner_harness_container_image())
+ if runner_harness_override:
+ self.debug_options.experiments = self.debug_options.experiments or []
+ self.debug_options.experiments.append(
+ 'runner_harness_container_image=' + runner_harness_override)
# Experiments
if self.debug_options.experiments:
for experiment in self.debug_options.experiments:
@@ -172,10 +190,18 @@
pool = dataflow.WorkerPool(
kind='local' if self.local else 'harness',
packages=package_descriptors,
+ # https://issues.apache.org/jira/browse/BEAM-3116
+ # metadata=dataflow.WorkerPool.MetadataValue(),
taskrunnerSettings=dataflow.TaskRunnerSettings(
parallelWorkerSettings=dataflow.WorkerSettings(
baseUrl=GoogleCloudOptions.DATAFLOW_ENDPOINT,
servicePath=self.google_cloud_options.dataflow_endpoint)))
+
+ # https://issues.apache.org/jira/browse/BEAM-3116
+ # pool.metadata.additionalProperties.append(
+ # dataflow.WorkerPool.MetadataValue.AdditionalProperty(
+ # key=names.STAGED_PIPELINE_URL_METADATA_FIELD, value=pipeline_url))
+
pool.autoscalingSettings = dataflow.AutoscalingSettings()
# Set worker pool options received through command line.
if self.worker_options.num_workers:
@@ -205,11 +231,8 @@
pool.workerHarnessContainerImage = (
self.worker_options.worker_harness_container_image)
else:
- # Default to using the worker harness container image for the current SDK
- # version.
pool.workerHarnessContainerImage = (
- 'dataflow.gcr.io/v1beta3/python:%s' %
- get_required_container_version())
+ dependency.get_default_container_image_for_current_sdk(job_type))
if self.worker_options.use_public_ips is not None:
if self.worker_options.use_public_ips:
pool.ipConfiguration = (
@@ -310,8 +333,9 @@
job_name = Job._build_default_job_name(getpass.getuser())
return job_name
- def __init__(self, options):
+ def __init__(self, options, proto_pipeline):
self.options = options
+ self.proto_pipeline = proto_pipeline
self.google_cloud_options = options.view_as(GoogleCloudOptions)
if not self.google_cloud_options.job_name:
self.google_cloud_options.job_name = self.default_job_name(
@@ -350,6 +374,17 @@
self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_STREAMING
else:
self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_BATCH
+
+ # Labels.
+ if self.google_cloud_options.labels:
+ self.proto.labels = dataflow.Job.LabelsValue()
+ for label in self.google_cloud_options.labels:
+ parts = label.split('=', 1)
+ key = parts[0]
+ value = parts[1] if len(parts) > 1 else ''
+ self.proto.labels.additionalProperties.append(
+ dataflow.Job.LabelsValue.AdditionalProperty(key=key, value=value))
+
self.base64_str_re = re.compile(r'^[A-Za-z0-9+/]*=*$')
self.coder_str_re = re.compile(r'^([A-Za-z]+\$)([A-Za-z0-9+/]*=*)$')
@@ -364,11 +399,16 @@
class DataflowApplicationClient(object):
"""A Dataflow API client used by application code to create and query jobs."""
- def __init__(self, options, environment_version):
+ def __init__(self, options):
"""Initializes a Dataflow API client object."""
self.standard_options = options.view_as(StandardOptions)
self.google_cloud_options = options.view_as(GoogleCloudOptions)
- self.environment_version = environment_version
+
+ if _use_fnapi(options):
+ self.environment_version = _FNAPI_ENVIRONMENT_MAJOR_VERSION
+ else:
+ self.environment_version = _LEGACY_ENVIRONMENT_MAJOR_VERSION
+
if self.google_cloud_options.no_auth:
credentials = None
else:
@@ -446,9 +486,19 @@
def create_job_description(self, job):
"""Creates a job described by the workflow proto."""
+
+ # Stage the pipeline for the runner harness
+ self.stage_file(job.google_cloud_options.staging_location,
+ names.STAGED_PIPELINE_FILENAME,
+ StringIO(job.proto_pipeline.SerializeToString()))
+
+ # Stage other resources for the SDK harness
resources = dependency.stage_job_resources(
job.options, file_copy=self._gcs_file_copy)
+
job.proto.environment = Environment(
+ pipeline_url=FileSystems.join(job.google_cloud_options.staging_location,
+ names.STAGED_PIPELINE_FILENAME),
packages=resources, options=job.options,
environment_version=self.environment_version).proto
logging.debug('JOB: %s', job)
@@ -489,8 +539,11 @@
logging.info('Created job with id: [%s]', response.id)
logging.info(
'To access the Dataflow monitoring console, please navigate to '
- 'https://console.developers.google.com/project/%s/dataflow/job/%s',
- self.google_cloud_options.project, response.id)
+ 'https://console.cloud.google.com/dataflow/jobsDetail'
+ '/locations/%s/jobs/%s?project=%s',
+ self.google_cloud_options.region,
+ response.id,
+ self.google_cloud_options.project)
return response
@@ -696,10 +749,6 @@
metric_update_proto.integer = to_split_int(value)
-def translate_scalar(accumulator, metric_update):
- metric_update.scalar = to_json_value(accumulator.value, with_type=True)
-
-
def translate_mean(accumulator, metric_update):
if accumulator.count:
metric_update.meanSum = to_json_value(accumulator.sum, with_type=True)
@@ -710,21 +759,52 @@
metric_update.kind = None
+def _use_fnapi(pipeline_options):
+ standard_options = pipeline_options.view_as(StandardOptions)
+ debug_options = pipeline_options.view_as(DebugOptions)
+
+ return standard_options.streaming or (
+ debug_options.experiments and 'beam_fn_api' in debug_options.experiments)
+
+
# To enable a counter on the service, add it to this dictionary.
-metric_translations = {
- cy_combiners.CountCombineFn: ('sum', translate_scalar),
- cy_combiners.SumInt64Fn: ('sum', translate_scalar),
- cy_combiners.MinInt64Fn: ('min', translate_scalar),
- cy_combiners.MaxInt64Fn: ('max', translate_scalar),
- cy_combiners.MeanInt64Fn: ('mean', translate_mean),
- cy_combiners.SumFloatFn: ('sum', translate_scalar),
- cy_combiners.MinFloatFn: ('min', translate_scalar),
- cy_combiners.MaxFloatFn: ('max', translate_scalar),
- cy_combiners.MeanFloatFn: ('mean', translate_mean),
- cy_combiners.AllCombineFn: ('and', translate_scalar),
- cy_combiners.AnyCombineFn: ('or', translate_scalar),
+structured_counter_translations = {
+ cy_combiners.CountCombineFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.SumInt64Fn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MinInt64Fn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MIN,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MaxInt64Fn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MAX,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MeanInt64Fn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MEAN,
+ MetricUpdateTranslators.translate_scalar_mean_int),
+ cy_combiners.SumFloatFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MinFloatFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MIN,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MaxFloatFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MAX,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MeanFloatFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.MEAN,
+ MetricUpdateTranslators.translate_scalar_mean_float),
+ cy_combiners.AllCombineFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.AND,
+ MetricUpdateTranslators.translate_boolean),
+ cy_combiners.AnyCombineFn: (
+ dataflow.CounterMetadata.KindValueValuesEnum.OR,
+ MetricUpdateTranslators.translate_boolean),
}
+
counter_translations = {
cy_combiners.CountCombineFn: (
dataflow.NameAndKind.KindValueValuesEnum.SUM,
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 67cf77f..f8a4471 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -17,12 +17,12 @@
"""Unit tests for the apiclient module."""
import unittest
-from mock import Mock
+import mock
+import pkg_resources
from apache_beam.metrics.cells import DistributionData
from apache_beam.options.pipeline_options import PipelineOptions
-
-from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
+from apache_beam.runners.dataflow.internal import dependency
from apache_beam.runners.dataflow.internal.clients import dataflow
# Protect against environments where apitools library is not available.
@@ -33,6 +33,8 @@
apiclient = None
# pylint: enable=wrong-import-order, wrong-import-position
+FAKE_PIPELINE_URL = "gs://invalid-bucket/anywhere"
+
@unittest.skipIf(apiclient is None, 'GCP dependencies are not installed')
class UtilTest(unittest.TestCase):
@@ -40,9 +42,7 @@
@unittest.skip("Enable once BEAM-1080 is fixed.")
def test_create_application_client(self):
pipeline_options = PipelineOptions()
- apiclient.DataflowApplicationClient(
- pipeline_options,
- DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
+ apiclient.DataflowApplicationClient(pipeline_options)
def test_set_network(self):
pipeline_options = PipelineOptions(
@@ -50,7 +50,8 @@
'--temp_location', 'gs://any-location/temp'])
env = apiclient.Environment([], #packages
pipeline_options,
- '2.0.0') #any environment version
+ '2.0.0', #any environment version
+ FAKE_PIPELINE_URL)
self.assertEqual(env.proto.workerPools[0].network,
'anetworkname')
@@ -61,7 +62,8 @@
env = apiclient.Environment([], #packages
pipeline_options,
- '2.0.0') #any environment version
+ '2.0.0', #any environment version
+ FAKE_PIPELINE_URL)
self.assertEqual(env.proto.workerPools[0].subnetwork,
'/regions/MY/subnetworks/SUBNETWORK')
@@ -106,7 +108,7 @@
def test_translate_means(self):
metric_update = dataflow.CounterUpdate()
- accumulator = Mock()
+ accumulator = mock.Mock()
accumulator.sum = 16
accumulator.count = 2
apiclient.MetricUpdateTranslators.translate_scalar_mean_int(accumulator,
@@ -122,6 +124,121 @@
self.assertEqual(
metric_update.floatingPointMean.count.lowBits, accumulator.count)
+ def test_default_ip_configuration(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp'])
+ env = apiclient.Environment([],
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL)
+ self.assertEqual(env.proto.workerPools[0].ipConfiguration, None)
+
+ def test_public_ip_configuration(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp',
+ '--use_public_ips'])
+ env = apiclient.Environment([],
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL)
+ self.assertEqual(
+ env.proto.workerPools[0].ipConfiguration,
+ dataflow.WorkerPool.IpConfigurationValueValuesEnum.WORKER_IP_PUBLIC)
+
+ def test_private_ip_configuration(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp',
+ '--no_use_public_ips'])
+ env = apiclient.Environment([],
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL)
+ self.assertEqual(
+ env.proto.workerPools[0].ipConfiguration,
+ dataflow.WorkerPool.IpConfigurationValueValuesEnum.WORKER_IP_PRIVATE)
+
+ def test_harness_override_present_in_dataflow_distributions(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp', '--streaming'])
+ override = ''.join(
+ ['runner_harness_container_image=',
+ dependency.DATAFLOW_CONTAINER_IMAGE_REPOSITORY,
+ '/harness:2.2.0'])
+ distribution = pkg_resources.Distribution(version='2.2.0')
+ with mock.patch(
+ 'apache_beam.runners.dataflow.internal.dependency.pkg_resources'
+ '.get_distribution',
+ mock.MagicMock(return_value=distribution)):
+ env = apiclient.Environment([], #packages
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL) #any environment version
+ self.assertIn(override, env.proto.experiments)
+
+ @mock.patch('apache_beam.runners.dataflow.internal.dependency.'
+ 'beam_version.__version__', '2.2.0')
+ def test_harness_override_present_in_beam_releases(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp', '--streaming'])
+ override = ''.join(
+ ['runner_harness_container_image=',
+ dependency.DATAFLOW_CONTAINER_IMAGE_REPOSITORY,
+ '/harness:2.2.0'])
+ with mock.patch(
+ 'apache_beam.runners.dataflow.internal.dependency.pkg_resources'
+ '.get_distribution',
+ mock.Mock(side_effect=pkg_resources.DistributionNotFound())):
+ env = apiclient.Environment([], #packages
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL) #any environment version
+ self.assertIn(override, env.proto.experiments)
+
+ @mock.patch('apache_beam.runners.dataflow.internal.dependency.'
+ 'beam_version.__version__', '2.2.0-dev')
+ def test_harness_override_absent_in_unreleased_sdk(self):
+ pipeline_options = PipelineOptions(
+ ['--temp_location', 'gs://any-location/temp', '--streaming'])
+ with mock.patch(
+ 'apache_beam.runners.dataflow.internal.dependency.pkg_resources'
+ '.get_distribution',
+ mock.Mock(side_effect=pkg_resources.DistributionNotFound())):
+ env = apiclient.Environment([], #packages
+ pipeline_options,
+ '2.0.0',
+ FAKE_PIPELINE_URL) #any environment version
+ if env.proto.experiments:
+ for experiment in env.proto.experiments:
+ self.assertNotIn('runner_harness_container_image=', experiment)
+
+ def test_labels(self):
+ pipeline_options = PipelineOptions(
+ ['--project', 'test_project', '--job_name', 'test_job_name',
+ '--temp_location', 'gs://test-location/temp'])
+ job = apiclient.Job(pipeline_options, FAKE_PIPELINE_URL)
+ self.assertIsNone(job.proto.labels)
+
+ pipeline_options = PipelineOptions(
+ ['--project', 'test_project', '--job_name', 'test_job_name',
+ '--temp_location', 'gs://test-location/temp',
+ '--label', 'key1=value1',
+ '--label', 'key2',
+ '--label', 'key3=value3',
+ '--labels', 'key4=value4',
+ '--labels', 'key5'])
+ job = apiclient.Job(pipeline_options, FAKE_PIPELINE_URL)
+ self.assertEqual(5, len(job.proto.labels.additionalProperties))
+ self.assertEqual('key1', job.proto.labels.additionalProperties[0].key)
+ self.assertEqual('value1', job.proto.labels.additionalProperties[0].value)
+ self.assertEqual('key2', job.proto.labels.additionalProperties[1].key)
+ self.assertEqual('', job.proto.labels.additionalProperties[1].value)
+ self.assertEqual('key3', job.proto.labels.additionalProperties[2].key)
+ self.assertEqual('value3', job.proto.labels.additionalProperties[2].value)
+ self.assertEqual('key4', job.proto.labels.additionalProperties[3].key)
+ self.assertEqual('value4', job.proto.labels.additionalProperties[3].value)
+ self.assertEqual('key5', job.proto.labels.additionalProperties[4].key)
+ self.assertEqual('', job.proto.labels.additionalProperties[4].value)
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
index f280217..61d0273 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -29,7 +29,7 @@
BASE_URL = u'https://dataflow.googleapis.com/'
_PACKAGE = u'dataflow'
- _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/userinfo.email']
+ _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/compute', u'https://www.googleapis.com/auth/compute.readonly', u'https://www.googleapis.com/auth/userinfo.email']
_VERSION = u'v1b3'
_CLIENT_ID = '1042881264118.apps.googleusercontent.com'
_CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b'
@@ -56,9 +56,11 @@
self.projects_jobs_messages = self.ProjectsJobsMessagesService(self)
self.projects_jobs_workItems = self.ProjectsJobsWorkItemsService(self)
self.projects_jobs = self.ProjectsJobsService(self)
+ self.projects_locations_jobs_debug = self.ProjectsLocationsJobsDebugService(self)
self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self)
self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self)
self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
+ self.projects_locations_templates = self.ProjectsLocationsTemplatesService(self)
self.projects_locations = self.ProjectsLocationsService(self)
self.projects_templates = self.ProjectsTemplatesService(self)
self.projects = self.ProjectsService(self)
@@ -235,6 +237,18 @@
def __init__(self, client):
super(DataflowV1b3.ProjectsJobsService, self).__init__(client)
self._method_configs = {
+ 'Aggregated': base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.jobs.aggregated',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'filter', u'location', u'pageSize', u'pageToken', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/jobs:aggregated',
+ request_field='',
+ request_type_name=u'DataflowProjectsJobsAggregatedRequest',
+ response_type_name=u'ListJobsResponse',
+ supports_download=False,
+ ),
'Create': base_api.ApiMethodInfo(
http_method=u'POST',
method_id=u'dataflow.projects.jobs.create',
@@ -300,6 +314,19 @@
self._upload_configs = {
}
+ def Aggregated(self, request, global_params=None):
+ """List the jobs of a project across all regions.
+
+ Args:
+ request: (DataflowProjectsJobsAggregatedRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListJobsResponse) The response message.
+ """
+ config = self.GetMethodConfig('Aggregated')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
def Create(self, request, global_params=None):
"""Creates a Cloud Dataflow job.
@@ -340,7 +367,7 @@
config, request, global_params=global_params)
def List(self, request, global_params=None):
- """List the jobs of a project.
+ """List the jobs of a project in a given region.
Args:
request: (DataflowProjectsJobsListRequest) input message
@@ -365,6 +392,69 @@
return self._RunMethod(
config, request, global_params=global_params)
+ class ProjectsLocationsJobsDebugService(base_api.BaseApiService):
+ """Service class for the projects_locations_jobs_debug resource."""
+
+ _NAME = u'projects_locations_jobs_debug'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsJobsDebugService, self).__init__(client)
+ self._method_configs = {
+ 'GetConfig': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.jobs.debug.getConfig',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/debug/getConfig',
+ request_field=u'getDebugConfigRequest',
+ request_type_name=u'DataflowProjectsLocationsJobsDebugGetConfigRequest',
+ response_type_name=u'GetDebugConfigResponse',
+ supports_download=False,
+ ),
+ 'SendCapture': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.jobs.debug.sendCapture',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/debug/sendCapture',
+ request_field=u'sendDebugCaptureRequest',
+ request_type_name=u'DataflowProjectsLocationsJobsDebugSendCaptureRequest',
+ response_type_name=u'SendDebugCaptureResponse',
+ supports_download=False,
+ ),
+ }
+
+ self._upload_configs = {
+ }
+
+ def GetConfig(self, request, global_params=None):
+ """Get encoded debug configuration for component. Not cacheable.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsDebugGetConfigRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (GetDebugConfigResponse) The response message.
+ """
+ config = self.GetMethodConfig('GetConfig')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ def SendCapture(self, request, global_params=None):
+ """Send encoded debug capture data for component.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsDebugSendCaptureRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (SendDebugCaptureResponse) The response message.
+ """
+ config = self.GetMethodConfig('SendCapture')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
class ProjectsLocationsJobsMessagesService(base_api.BaseApiService):
"""Service class for the projects_locations_jobs_messages resource."""
@@ -579,7 +669,7 @@
config, request, global_params=global_params)
def List(self, request, global_params=None):
- """List the jobs of a project.
+ """List the jobs of a project in a given region.
Args:
request: (DataflowProjectsLocationsJobsListRequest) input message
@@ -604,6 +694,94 @@
return self._RunMethod(
config, request, global_params=global_params)
+ class ProjectsLocationsTemplatesService(base_api.BaseApiService):
+ """Service class for the projects_locations_templates resource."""
+
+ _NAME = u'projects_locations_templates'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsTemplatesService, self).__init__(client)
+ self._method_configs = {
+ 'Create': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.templates.create',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates',
+ request_field=u'createJobFromTemplateRequest',
+ request_type_name=u'DataflowProjectsLocationsTemplatesCreateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ ),
+ 'Get': base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.templates.get',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[u'gcsPath', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates:get',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsTemplatesGetRequest',
+ response_type_name=u'GetTemplateResponse',
+ supports_download=False,
+ ),
+ 'Launch': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.templates.launch',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[u'gcsPath', u'validateOnly'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates:launch',
+ request_field=u'launchTemplateParameters',
+ request_type_name=u'DataflowProjectsLocationsTemplatesLaunchRequest',
+ response_type_name=u'LaunchTemplateResponse',
+ supports_download=False,
+ ),
+ }
+
+ self._upload_configs = {
+ }
+
+ def Create(self, request, global_params=None):
+ """Creates a Cloud Dataflow job from a template.
+
+ Args:
+ request: (DataflowProjectsLocationsTemplatesCreateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Create')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ def Get(self, request, global_params=None):
+ """Get the template associated with a template.
+
+ Args:
+ request: (DataflowProjectsLocationsTemplatesGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (GetTemplateResponse) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ def Launch(self, request, global_params=None):
+ """Launch a template.
+
+ Args:
+ request: (DataflowProjectsLocationsTemplatesLaunchRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (LaunchTemplateResponse) The response message.
+ """
+ config = self.GetMethodConfig('Launch')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
class ProjectsLocationsService(base_api.BaseApiService):
"""Service class for the projects_locations resource."""
@@ -612,11 +790,36 @@
def __init__(self, client):
super(DataflowV1b3.ProjectsLocationsService, self).__init__(client)
self._method_configs = {
+ 'WorkerMessages': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.workerMessages',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/WorkerMessages',
+ request_field=u'sendWorkerMessagesRequest',
+ request_type_name=u'DataflowProjectsLocationsWorkerMessagesRequest',
+ response_type_name=u'SendWorkerMessagesResponse',
+ supports_download=False,
+ ),
}
self._upload_configs = {
}
+ def WorkerMessages(self, request, global_params=None):
+ """Send a worker_message to the service.
+
+ Args:
+ request: (DataflowProjectsLocationsWorkerMessagesRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (SendWorkerMessagesResponse) The response message.
+ """
+ config = self.GetMethodConfig('WorkerMessages')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
class ProjectsTemplatesService(base_api.BaseApiService):
"""Service class for the projects_templates resource."""
@@ -637,6 +840,30 @@
response_type_name=u'Job',
supports_download=False,
),
+ 'Get': base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.templates.get',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'gcsPath', u'location', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/templates:get',
+ request_field='',
+ request_type_name=u'DataflowProjectsTemplatesGetRequest',
+ response_type_name=u'GetTemplateResponse',
+ supports_download=False,
+ ),
+ 'Launch': base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.templates.launch',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'gcsPath', u'location', u'validateOnly'],
+ relative_path=u'v1b3/projects/{projectId}/templates:launch',
+ request_field=u'launchTemplateParameters',
+ request_type_name=u'DataflowProjectsTemplatesLaunchRequest',
+ response_type_name=u'LaunchTemplateResponse',
+ supports_download=False,
+ ),
}
self._upload_configs = {
@@ -655,6 +882,32 @@
return self._RunMethod(
config, request, global_params=global_params)
+ def Get(self, request, global_params=None):
+ """Get the template associated with a template.
+
+ Args:
+ request: (DataflowProjectsTemplatesGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (GetTemplateResponse) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ def Launch(self, request, global_params=None):
+ """Launch a template.
+
+ Args:
+ request: (DataflowProjectsTemplatesLaunchRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (LaunchTemplateResponse) The response message.
+ """
+ config = self.GetMethodConfig('Launch')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
class ProjectsService(base_api.BaseApiService):
"""Service class for the projects resource."""
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
index eb88bce..b0d4e44 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -26,7 +26,6 @@
from apitools.base.py import encoding
from apitools.base.py import extra_types
-
package = 'dataflow'
@@ -242,7 +241,6 @@
outputs: The outputs from the computation.
stateFamilies: The state family values.
systemStageName: The system stage name.
- userStageName: The user stage name.
"""
computationId = _messages.StringField(1)
@@ -251,7 +249,6 @@
outputs = _messages.MessageField('StreamLocation', 4, repeated=True)
stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True)
systemStageName = _messages.StringField(6)
- userStageName = _messages.StringField(7)
class ConcatPosition(_messages.Message):
@@ -522,6 +519,64 @@
vmInstance = _messages.StringField(2)
+class DataflowProjectsJobsAggregatedRequest(_messages.Message):
+ """A DataflowProjectsJobsAggregatedRequest object.
+
+ Enums:
+ FilterValueValuesEnum: The kind of filter to use.
+ ViewValueValuesEnum: Level of information requested in response. Default
+ is `JOB_VIEW_SUMMARY`.
+
+ Fields:
+ filter: The kind of filter to use.
+ location: The location that contains this job.
+ pageSize: If there are many jobs, limit response to at most this many. The
+ actual number of jobs returned will be the lesser of max_responses and
+ an unspecified server-defined limit.
+ pageToken: Set this to the 'next_page_token' field of a previous response
+ to request additional results in a long list.
+ projectId: The project which owns the jobs.
+ view: Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+ """
+
+ class FilterValueValuesEnum(_messages.Enum):
+ """The kind of filter to use.
+
+ Values:
+ UNKNOWN: <no description>
+ ALL: <no description>
+ TERMINATED: <no description>
+ ACTIVE: <no description>
+ """
+ UNKNOWN = 0
+ ALL = 1
+ TERMINATED = 2
+ ACTIVE = 3
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ JOB_VIEW_DESCRIPTION: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+ JOB_VIEW_DESCRIPTION = 3
+
+ filter = _messages.EnumField('FilterValueValuesEnum', 1)
+ location = _messages.StringField(2)
+ pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(4)
+ projectId = _messages.StringField(5, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 6)
+
+
class DataflowProjectsJobsCreateRequest(_messages.Message):
"""A DataflowProjectsJobsCreateRequest object.
@@ -3013,13 +3068,14 @@
"""
-
class RuntimeEnvironment(_messages.Message):
"""The environment values to set at runtime.
Fields:
bypassTempDirValidation: Whether to bypass the safety checks for the job's
temporary directory. Use with caution.
+ machineType: The machine type to use for the job. Defaults to the value
+ from the template if not specified.
maxWorkers: The maximum number of Google Compute Engine instances to be
made available to your pipeline during execution, from 1 to 1000.
serviceAccountEmail: The email address of the service account to run the
@@ -3032,10 +3088,11 @@
"""
bypassTempDirValidation = _messages.BooleanField(1)
- maxWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32)
- serviceAccountEmail = _messages.StringField(3)
- tempLocation = _messages.StringField(4)
- zone = _messages.StringField(5)
+ machineType = _messages.StringField(2)
+ maxWorkers = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ serviceAccountEmail = _messages.StringField(4)
+ tempLocation = _messages.StringField(5)
+ zone = _messages.StringField(6)
class SendDebugCaptureRequest(_messages.Message):
@@ -3702,7 +3759,7 @@
user-facing error message is needed, put the localized message in the error
details or localize it in the client. The optional error details may contain
arbitrary information about the error. There is a predefined set of error
- detail types in the package `google.rpc` which can be used for common error
+ detail types in the package `google.rpc` that can be used for common error
conditions. # Language mapping The `Status` message is the logical
representation of the error model, but it is not necessarily the actual wire
format. When the `Status` message is exposed in different client libraries
@@ -3715,8 +3772,8 @@
If a service needs to return partial errors to the client, it may embed
the `Status` in the normal response to indicate the partial errors. -
Workflow errors. A typical workflow has multiple steps. Each step may
- have a `Status` message for error reporting purpose. - Batch operations. If
- a client uses batch request and batch response, the `Status` message
+ have a `Status` message for error reporting. - Batch operations. If a
+ client uses batch request and batch response, the `Status` message
should be used directly inside batch response, one for each error sub-
response. - Asynchronous operations. If an API call embeds asynchronous
operation results in its response, the status of those operations should
@@ -3729,7 +3786,7 @@
Fields:
code: The status code, which should be an enum value of google.rpc.Code.
- details: A list of messages that carry the error details. There will be a
+ details: A list of messages that carry the error details. There is a
common set of message types for APIs to use.
message: A developer-facing error message, which should be in English. Any
user-facing error message should be localized and sent in the
@@ -4096,19 +4153,14 @@
"""Metadata describing a template.
Fields:
- bypassTempDirValidation: If true, will bypass the validation that the temp
- directory is writable. This should only be used with templates for
- pipelines that are guaranteed not to need to write to the temp
- directory, which is subject to change based on the optimizer.
description: Optional. A description of the template.
name: Required. The name of the template.
parameters: The parameters for the template.
"""
- bypassTempDirValidation = _messages.BooleanField(1)
- description = _messages.StringField(2)
- name = _messages.StringField(3)
- parameters = _messages.MessageField('ParameterMetadata', 4, repeated=True)
+ description = _messages.StringField(1)
+ name = _messages.StringField(2)
+ parameters = _messages.MessageField('ParameterMetadata', 3, repeated=True)
class TopologyConfig(_messages.Message):
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py
index 4dda47a..805473a 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py
@@ -17,7 +17,6 @@
from hamcrest.core.base_matcher import BaseMatcher
-
IGNORED = object()
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py
index 3163c9b2..15bb9ef 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
import unittest
-import hamcrest as hc
-import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow
+import hamcrest as hc
+
+import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow
from apache_beam.internal.gcp.json_value import to_json_value
from apache_beam.runners.dataflow.internal.clients.dataflow import message_matchers
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 892d9f9..fba2df2 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -1,5 +1,3 @@
-
-#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -61,27 +59,46 @@
import sys
import tempfile
+import pkg_resources
+
from apache_beam import version as beam_version
from apache_beam.internal import pickler
from apache_beam.io.filesystems import FileSystems
-from apache_beam.runners.dataflow.internal import names
-from apache_beam.utils import processes
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.runners.dataflow.internal import names
+from apache_beam.utils import processes
+# All constants are for internal use only; no backwards-compatibility
+# guarantees.
+# In a released version BEAM_CONTAINER_VERSION and BEAM_FNAPI_CONTAINER_VERSION
+# should match each other, and should be in the same format as the SDK version
+# (i.e. MAJOR.MINOR.PATCH). For non-released (dev) versions, read below.
# Update this version to the next version whenever there is a change that will
-# require changes to the execution environment.
-BEAM_CONTAINER_VERSION = '2.0.0'
+# require changes to legacy Dataflow worker execution environment.
+# This should be in the beam-[version]-[date] format, date is optional.
+BEAM_CONTAINER_VERSION = 'beam-2.2.0-20170928'
+# Update this version to the next version whenever there is a change that
+# requires changes to SDK harness container or SDK harness launcher.
+# This should be in the beam-[version]-[date] format, date is optional.
+BEAM_FNAPI_CONTAINER_VERSION = 'beam-2.1.0-20170621'
# Standard file names used for staging files.
WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'
REQUIREMENTS_FILE = 'requirements.txt'
EXTRA_PACKAGES_FILE = 'extra_packages.txt'
+# Package names for different distributions
GOOGLE_PACKAGE_NAME = 'google-cloud-dataflow'
BEAM_PACKAGE_NAME = 'apache-beam'
+# SDK identifiers for different distributions
+GOOGLE_SDK_NAME = 'Google Cloud Dataflow SDK for Python'
+BEAM_SDK_NAME = 'Apache Beam SDK for Python'
+
+DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'dataflow.gcr.io/v1beta3'
+
def _dependency_file_copy(from_path, to_path):
"""Copies a local file to a GCS file or vice versa."""
@@ -162,10 +179,11 @@
for package in extra_packages:
if not (os.path.basename(package).endswith('.tar') or
os.path.basename(package).endswith('.tar.gz') or
- os.path.basename(package).endswith('.whl')):
+ os.path.basename(package).endswith('.whl') or
+ os.path.basename(package).endswith('.zip')):
raise RuntimeError(
'The --extra_package option expects a full path ending with '
- '".tar" or ".tar.gz" instead of %s' % package)
+ '".tar", ".tar.gz", ".whl" or ".zip" instead of %s' % package)
if os.path.basename(package).endswith('.whl'):
logging.warning(
'The .whl package "%s" is provided in --extra_package. '
@@ -180,7 +198,12 @@
staging_temp_dir = tempfile.mkdtemp(dir=temp_dir)
logging.info('Downloading extra package: %s locally before staging',
package)
- _dependency_file_copy(package, staging_temp_dir)
+ if os.path.isfile(staging_temp_dir):
+ local_file_path = staging_temp_dir
+ else:
+ _, last_component = FileSystems.split(package)
+ local_file_path = FileSystems.join(staging_temp_dir, last_component)
+ _dependency_file_copy(package, local_file_path)
else:
raise RuntimeError(
'The file %s cannot be found. It was specified in the '
@@ -468,56 +491,106 @@
'type of location: %s' % sdk_remote_location)
-def get_required_container_version():
+def get_runner_harness_container_image():
"""For internal use only; no backwards-compatibility guarantees.
- Returns the Google Cloud Dataflow container version for remote execution.
+ Returns:
+ str: Runner harness container image that shall be used by default
+ for current SDK version or None if the runner harness container image
+ bundled with the service shall be used.
+ """
+ try:
+ version = pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME).version
+ # Pin runner harness for Dataflow releases.
+ return (DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/' + 'harness' + ':' +
+ version)
+ except pkg_resources.DistributionNotFound:
+ # Pin runner harness for BEAM releases.
+ if 'dev' not in beam_version.__version__:
+ return (DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/' + 'harness' + ':' +
+ beam_version.__version__)
+ # Don't pin runner harness for BEAM head so that we can notice
+ # potential incompatibility between runner and sdk harnesses.
+ return None
+
+
+def get_default_container_image_for_current_sdk(job_type):
+ """For internal use only; no backwards-compatibility guarantees.
+
+ Args:
+ job_type (str): BEAM job type.
+
+ Returns:
+ str: Google Cloud Dataflow container image for remote execution.
+ """
+ # TODO(tvalentyn): Use enumerated type instead of strings for job types.
+ if job_type == 'FNAPI_BATCH' or job_type == 'FNAPI_STREAMING':
+ image_name = 'dataflow.gcr.io/v1beta3/python-fnapi'
+ else:
+ image_name = 'dataflow.gcr.io/v1beta3/python'
+ image_tag = _get_required_container_version(job_type)
+ return image_name + ':' + image_tag
+
+
+def _get_required_container_version(job_type=None):
+ """For internal use only; no backwards-compatibility guarantees.
+
+ Args:
+ job_type (str, optional): BEAM job type. Defaults to None.
+
+ Returns:
+ str: The tag of worker container images in GCR that corresponds to
+ current version of the SDK.
"""
# TODO(silviuc): Handle apache-beam versions when we have official releases.
- import pkg_resources as pkg
try:
- version = pkg.get_distribution(GOOGLE_PACKAGE_NAME).version
+ version = pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME).version
# We drop any pre/post parts of the version and we keep only the X.Y.Z
# format. For instance the 0.3.0rc2 SDK version translates into 0.3.0.
- container_version = '%s.%s.%s' % pkg.parse_version(version)._version.release
+ container_version = (
+ '%s.%s.%s' % pkg_resources.parse_version(version)._version.release)
# We do, however, keep the ".dev" suffix if it is present.
if re.match(r'.*\.dev[0-9]*$', version):
container_version += '.dev'
return container_version
- except pkg.DistributionNotFound:
+ except pkg_resources.DistributionNotFound:
# This case covers Apache Beam end-to-end testing scenarios. All these tests
# will run with a special container version.
- return BEAM_CONTAINER_VERSION
+ if job_type == 'FNAPI_BATCH' or job_type == 'FNAPI_STREAMING':
+ return BEAM_FNAPI_CONTAINER_VERSION
+ else:
+ return BEAM_CONTAINER_VERSION
def get_sdk_name_and_version():
"""For internal use only; no backwards-compatibility guarantees.
Returns name and version of SDK reported to Google Cloud Dataflow."""
- # TODO(ccy): Make this check cleaner.
- container_version = get_required_container_version()
- if container_version == BEAM_CONTAINER_VERSION:
- return ('Apache Beam SDK for Python', beam_version.__version__)
- return ('Google Cloud Dataflow SDK for Python', container_version)
+ container_version = _get_required_container_version()
+ try:
+ pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME)
+ return (GOOGLE_SDK_NAME, container_version)
+ except pkg_resources.DistributionNotFound:
+ return (BEAM_SDK_NAME, beam_version.__version__)
def get_sdk_package_name():
"""For internal use only; no backwards-compatibility guarantees.
Returns the PyPI package name to be staged to Google Cloud Dataflow."""
- container_version = get_required_container_version()
- if container_version == BEAM_CONTAINER_VERSION:
+ sdk_name, _ = get_sdk_name_and_version()
+ if sdk_name == GOOGLE_SDK_NAME:
+ return GOOGLE_PACKAGE_NAME
+ else:
return BEAM_PACKAGE_NAME
- return GOOGLE_PACKAGE_NAME
def _download_pypi_sdk_package(temp_dir):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = get_sdk_package_name()
- import pkg_resources as pkg
try:
- version = pkg.get_distribution(package_name).version
- except pkg.DistributionNotFound:
+ version = pkg_resources.get_distribution(package_name).version
+ except pkg_resources.DistributionNotFound:
raise RuntimeError('Please set --sdk_location command-line option '
'or install a valid {} distribution.'
.format(package_name))
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
index 5eac7d6..f0e59bc 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
@@ -24,13 +24,22 @@
import unittest
from apache_beam.io.filesystems import FileSystems
-from apache_beam.runners.dataflow.internal import dependency
-from apache_beam.runners.dataflow.internal import names
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.runners.dataflow.internal import dependency
+from apache_beam.runners.dataflow.internal import names
+
+# Protect against environments where GCS library is not available.
+# pylint: disable=wrong-import-order, wrong-import-position
+try:
+ from apitools.base.py.exceptions import HttpError
+except ImportError:
+ HttpError = None
+# pylint: enable=wrong-import-order, wrong-import-position
+@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
class SetupTest(unittest.TestCase):
def update_options(self, options):
@@ -369,7 +378,9 @@
if from_path.startswith('gs://'):
gcs_copied_files.append(from_path)
_, from_name = os.path.split(from_path)
- self.create_temp_file(os.path.join(to_path, from_name), 'nothing')
+ if os.path.isdir(to_path):
+ to_path = os.path.join(to_path, from_name)
+ self.create_temp_file(to_path, 'nothing')
logging.info('Fake copied GCS file: %s to %s', from_path, to_path)
elif to_path.startswith('gs://'):
logging.info('Faking file_copy(%s, %s)', from_path, to_path)
@@ -416,8 +427,9 @@
dependency.stage_job_resources(options)
self.assertEqual(
cm.exception.message,
- 'The --extra_package option expects a full path ending with ".tar" or '
- '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz'))
+ 'The --extra_package option expects a full path ending with '
+ '".tar", ".tar.gz", ".whl" or ".zip" '
+ 'instead of %s' % os.path.join(source_dir, 'abc.tgz'))
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py
index be67224..559b445 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/names.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py
@@ -21,6 +21,8 @@
# Standard file names used for staging files.
PICKLED_MAIN_SESSION_FILE = 'pickled_main_session'
DATAFLOW_SDK_TARBALL_FILE = 'dataflow_python_sdk.tar'
+STAGED_PIPELINE_FILENAME = "pipeline.pb"
+STAGED_PIPELINE_URL_METADATA_FIELD = "pipeline_url"
# String constants related to sources framework
SOURCE_FORMAT = 'custom_source'
diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
index c1f4238..2f2316f 100644
--- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
+++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
@@ -23,6 +23,7 @@
import logging
from apache_beam import pvalue
+from apache_beam.io import iobase
from apache_beam.transforms import ptransform
from apache_beam.transforms.display import HasDisplayData
@@ -42,7 +43,7 @@
'compression_type']
-class NativeSource(HasDisplayData):
+class NativeSource(iobase.SourceBase):
"""A source implemented by Dataflow service.
This class is to be only inherited by sources natively implemented by Cloud
@@ -55,6 +56,9 @@
"""Returns a NativeSourceReader instance associated with this source."""
raise NotImplementedError
+ def is_bounded(self):
+ return True
+
def __repr__(self):
return '<{name} {vals}>'.format(
name=self.__class__.__name__,
diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py
index 7610baf..01fd35f 100644
--- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py
@@ -20,18 +20,20 @@
import unittest
-from apache_beam import error, pvalue
-from apache_beam.runners.dataflow.native_io.iobase import (
- _dict_printable_fields,
- _NativeWrite,
- ConcatPosition,
- DynamicSplitRequest,
- DynamicSplitResultWithPosition,
- NativeSink,
- NativeSource,
- ReaderPosition,
- ReaderProgress
-)
+from apache_beam import Create
+from apache_beam import error
+from apache_beam import pvalue
+from apache_beam.runners.dataflow.native_io.iobase import ConcatPosition
+from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitRequest
+from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitResultWithPosition
+from apache_beam.runners.dataflow.native_io.iobase import NativeSink
+from apache_beam.runners.dataflow.native_io.iobase import NativeSinkWriter
+from apache_beam.runners.dataflow.native_io.iobase import NativeSource
+from apache_beam.runners.dataflow.native_io.iobase import ReaderPosition
+from apache_beam.runners.dataflow.native_io.iobase import ReaderProgress
+from apache_beam.runners.dataflow.native_io.iobase import _dict_printable_fields
+from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite
+from apache_beam.testing.test_pipeline import TestPipeline
class TestHelperFunctions(unittest.TestCase):
@@ -154,6 +156,39 @@
fake_sink = FakeSink()
self.assertEqual(fake_sink.__repr__(), "<FakeSink ['validate=False']>")
+ def test_on_direct_runner(self):
+ class FakeSink(NativeSink):
+ """A fake sink outputing a number of elements."""
+
+ def __init__(self):
+ self.written_values = []
+ self.writer_instance = FakeSinkWriter(self.written_values)
+
+ def writer(self):
+ return self.writer_instance
+
+ class FakeSinkWriter(NativeSinkWriter):
+ """A fake sink writer for testing."""
+
+ def __init__(self, written_values):
+ self.written_values = written_values
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *unused_args):
+ pass
+
+ def Write(self, value):
+ self.written_values.append(value)
+
+ p = TestPipeline()
+ sink = FakeSink()
+ p | Create(['a', 'b', 'c']) | _NativeWrite(sink) # pylint: disable=expression-not-assigned
+ p.run()
+
+ self.assertEqual(['a', 'b', 'c'], sink.written_values)
+
class Test_NativeWrite(unittest.TestCase):
diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py
new file mode 100644
index 0000000..a54ee77
--- /dev/null
+++ b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Create transform for streaming."""
+
+from apache_beam import DoFn
+from apache_beam import ParDo
+from apache_beam import PTransform
+from apache_beam import Windowing
+from apache_beam import pvalue
+from apache_beam.transforms.window import GlobalWindows
+
+
+class StreamingCreate(PTransform):
+ """A specialized implementation for ``Create`` transform in streaming mode.
+
+ Note: There is no unbounded source API in python to wrap the Create source,
+ so we map this to composite of Impulse primitive and an SDF.
+ """
+
+ def __init__(self, values, coder):
+ self.coder = coder
+ self.encoded_values = map(coder.encode, values)
+
+ class DecodeAndEmitDoFn(DoFn):
+ """A DoFn which stores encoded versions of elements.
+
+ It also stores a Coder to decode and emit those elements.
+ TODO: BEAM-2422 - Make this a SplittableDoFn.
+ """
+
+ def __init__(self, encoded_values, coder):
+ self.encoded_values = encoded_values
+ self.coder = coder
+
+ def process(self, unused_element):
+ for encoded_value in self.encoded_values:
+ yield self.coder.decode(encoded_value)
+
+ class Impulse(PTransform):
+ """The Dataflow specific override for the impulse primitive."""
+
+ def expand(self, pbegin):
+ assert isinstance(pbegin, pvalue.PBegin), (
+ 'Input to Impulse transform must be a PBegin but found %s' % pbegin)
+ return pvalue.PCollection(pbegin.pipeline)
+
+ def get_windowing(self, inputs):
+ return Windowing(GlobalWindows())
+
+ def infer_output_type(self, unused_input_type):
+ return bytes
+
+ def expand(self, pbegin):
+ return (pbegin
+ | 'Impulse' >> self.Impulse()
+ | 'Decode Values' >> ParDo(
+ self.DecodeAndEmitDoFn(self.encoded_values, self.coder)))
diff --git a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py
new file mode 100644
index 0000000..680a4b7
--- /dev/null
+++ b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Ptransform overrides for DataflowRunner."""
+
+from apache_beam.coders import typecoders
+from apache_beam.pipeline import PTransformOverride
+
+
+class CreatePTransformOverride(PTransformOverride):
+ """A ``PTransformOverride`` for ``Create`` in streaming mode."""
+
+ def get_matcher(self):
+ return self.is_streaming_create
+
+ @staticmethod
+ def is_streaming_create(applied_ptransform):
+ # Imported here to avoid circular dependencies.
+ # pylint: disable=wrong-import-order, wrong-import-position
+ from apache_beam import Create
+ from apache_beam.options.pipeline_options import StandardOptions
+
+ if isinstance(applied_ptransform.transform, Create):
+ standard_options = (applied_ptransform
+ .outputs[None]
+ .pipeline._options
+ .view_as(StandardOptions))
+ return standard_options.streaming
+ else:
+ return False
+
+ def get_replacement_transform(self, ptransform):
+ # Imported here to avoid circular dependencies.
+ # pylint: disable=wrong-import-order, wrong-import-position
+ from apache_beam.runners.dataflow.native_io.streaming_create import \
+ StreamingCreate
+ coder = typecoders.registry.get_coder(ptransform.get_output_type())
+ return StreamingCreate(ptransform.value, coder)
diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
index 7927219..82eb76b 100644
--- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
@@ -24,8 +24,8 @@
import unittest
import apache_beam as beam
-from apache_beam.pipeline import Pipeline
from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.pipeline import Pipeline
from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
# Protect against environments where apitools library is not available.
@@ -87,7 +87,8 @@
'--temp_location=/dev/null',
'--template_location=/bad/path',
'--no_auth=True']))
- remote_runner.job = apiclient.Job(pipeline._options)
+ remote_runner.job = apiclient.Job(pipeline._options,
+ pipeline.to_runner_api())
with self.assertRaises(IOError):
pipeline.run().wait_until_finish()
diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
index b339882..b2330c0 100644
--- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
@@ -16,12 +16,13 @@
#
"""Wrapper of Beam runners that's built for running and verifying e2e tests."""
+from __future__ import print_function
from apache_beam.internal import pickler
-from apache_beam.options.pipeline_options import TestOptions, GoogleCloudOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import TestOptions
from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
-
__all__ = ['TestDataflowRunner']
@@ -38,12 +39,13 @@
self.result = super(TestDataflowRunner, self).run(pipeline)
if self.result.has_job:
project = pipeline._options.view_as(GoogleCloudOptions).project
+ region_id = pipeline._options.view_as(GoogleCloudOptions).region
job_id = self.result.job_id()
# TODO(markflyhigh)(BEAM-1890): Use print since Nose dosen't show logs
# in some cases.
print (
- 'Found: https://console.cloud.google.com/dataflow/job/%s?project=%s' %
- (job_id, project))
+ 'Found: https://console.cloud.google.com/dataflow/jobsDetail'
+ '/locations/%s/jobs/%s?project=%s' % (region_id, job_id, project))
self.result.wait_until_finish()
if on_success_matcher:
diff --git a/sdks/python/apache_beam/runners/direct/bundle_factory.py b/sdks/python/apache_beam/runners/direct/bundle_factory.py
index ed00b03..0182b4c 100644
--- a/sdks/python/apache_beam/runners/direct/bundle_factory.py
+++ b/sdks/python/apache_beam/runners/direct/bundle_factory.py
@@ -108,7 +108,7 @@
self._initial_windowed_value.windows)
def __init__(self, pcollection, stacked=True):
- assert isinstance(pcollection, pvalue.PCollection)
+ assert isinstance(pcollection, (pvalue.PBegin, pvalue.PCollection))
self._pcollection = pcollection
self._elements = []
self._stacked = stacked
diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py
index 97d1ee8..4efaa27 100644
--- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py
+++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py
@@ -21,8 +21,8 @@
import unittest
from apache_beam import pvalue
-from apache_beam.io import iobase
from apache_beam.io import Read
+from apache_beam.io import iobase
from apache_beam.pipeline import Pipeline
from apache_beam.pvalue import AsList
from apache_beam.runners.direct import DirectRunner
diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics.py b/sdks/python/apache_beam/runners/direct/direct_metrics.py
index 9d23487..aa35fb7 100644
--- a/sdks/python/apache_beam/runners/direct/direct_metrics.py
+++ b/sdks/python/apache_beam/runners/direct/direct_metrics.py
@@ -20,8 +20,8 @@
responding to queries of current metrics, but also of keeping the common
state consistent.
"""
-from collections import defaultdict
import threading
+from collections import defaultdict
from apache_beam.metrics.cells import CounterAggregator
from apache_beam.metrics.cells import DistributionAggregator
diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py
index 256b91f..f361786 100644
--- a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py
+++ b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py
@@ -19,12 +19,12 @@
import hamcrest as hc
-from apache_beam.metrics.metricbase import MetricName
-from apache_beam.metrics.execution import MetricUpdates
-from apache_beam.metrics.execution import MetricResult
-from apache_beam.metrics.execution import MetricKey
from apache_beam.metrics.cells import DistributionData
from apache_beam.metrics.cells import DistributionResult
+from apache_beam.metrics.execution import MetricKey
+from apache_beam.metrics.execution import MetricResult
+from apache_beam.metrics.execution import MetricUpdates
+from apache_beam.metrics.metricbase import MetricName
from apache_beam.runners.direct.direct_metrics import DirectMetrics
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py
index ecf5114..794a96b 100644
--- a/sdks/python/apache_beam/runners/direct/direct_runner.py
+++ b/sdks/python/apache_beam/runners/direct/direct_runner.py
@@ -26,22 +26,77 @@
import collections
import logging
+from google.protobuf import wrappers_pb2
+
+import apache_beam as beam
+from apache_beam import typehints
from apache_beam.metrics.execution import MetricsEnvironment
+from apache_beam.options.pipeline_options import DirectOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.pvalue import PCollection
from apache_beam.runners.direct.bundle_factory import BundleFactory
from apache_beam.runners.runner import PipelineResult
from apache_beam.runners.runner import PipelineRunner
from apache_beam.runners.runner import PipelineState
from apache_beam.runners.runner import PValueCache
-from apache_beam.options.pipeline_options import DirectOptions
-from apache_beam.options.value_provider import RuntimeValueProvider
-
+from apache_beam.transforms.core import _GroupAlsoByWindow
+from apache_beam.transforms.core import _GroupByKeyOnly
+from apache_beam.transforms.ptransform import PTransform
__all__ = ['DirectRunner']
+# Type variables.
+K = typehints.TypeVariable('K')
+V = typehints.TypeVariable('V')
+
+
+@typehints.with_input_types(typehints.KV[K, V])
+@typehints.with_output_types(typehints.KV[K, typehints.Iterable[V]])
+class _StreamingGroupByKeyOnly(_GroupByKeyOnly):
+ """Streaming GroupByKeyOnly placeholder for overriding in DirectRunner."""
+ urn = "direct_runner:streaming_gbko:v0.1"
+
+ # These are needed due to apply overloads.
+ def to_runner_api_parameter(self, unused_context):
+ return _StreamingGroupByKeyOnly.urn, None
+
+ @PTransform.register_urn(urn, None)
+ def from_runner_api_parameter(unused_payload, unused_context):
+ return _StreamingGroupByKeyOnly()
+
+
+@typehints.with_input_types(typehints.KV[K, typehints.Iterable[V]])
+@typehints.with_output_types(typehints.KV[K, typehints.Iterable[V]])
+class _StreamingGroupAlsoByWindow(_GroupAlsoByWindow):
+ """Streaming GroupAlsoByWindow placeholder for overriding in DirectRunner."""
+ urn = "direct_runner:streaming_gabw:v0.1"
+
+ # These are needed due to apply overloads.
+ def to_runner_api_parameter(self, context):
+ return (
+ _StreamingGroupAlsoByWindow.urn,
+ wrappers_pb2.BytesValue(value=context.windowing_strategies.get_id(
+ self.windowing)))
+
+ @PTransform.register_urn(urn, wrappers_pb2.BytesValue)
+ def from_runner_api_parameter(payload, context):
+ return _StreamingGroupAlsoByWindow(
+ context.windowing_strategies.get_by_id(payload.value))
+
+
class DirectRunner(PipelineRunner):
"""Executes a single pipeline on the local machine."""
+ # A list of PTransformOverride objects to be applied before running a pipeline
+ # using DirectRunner.
+ # Currently this only works for overrides where the input and output types do
+ # not change.
+ # For internal SDK use only. This should not be updated by Beam pipeline
+ # authors.
+ _PTRANSFORM_OVERRIDES = []
+
def __init__(self):
self._cache = None
@@ -56,9 +111,84 @@
except NotImplementedError:
return transform.expand(pcoll)
+ def apply__GroupByKeyOnly(self, transform, pcoll):
+ if (transform.__class__ == _GroupByKeyOnly and
+ pcoll.pipeline._options.view_as(StandardOptions).streaming):
+ # Use specialized streaming implementation, if requested.
+ type_hints = transform.get_type_hints()
+ return pcoll | (_StreamingGroupByKeyOnly()
+ .with_input_types(*type_hints.input_types[0])
+ .with_output_types(*type_hints.output_types[0]))
+ return transform.expand(pcoll)
+
+ def apply__GroupAlsoByWindow(self, transform, pcoll):
+ if (transform.__class__ == _GroupAlsoByWindow and
+ pcoll.pipeline._options.view_as(StandardOptions).streaming):
+ # Use specialized streaming implementation, if requested.
+ type_hints = transform.get_type_hints()
+ return pcoll | (_StreamingGroupAlsoByWindow(transform.windowing)
+ .with_input_types(*type_hints.input_types[0])
+ .with_output_types(*type_hints.output_types[0]))
+ return transform.expand(pcoll)
+
+ def apply_ReadStringsFromPubSub(self, transform, pcoll):
+ try:
+ from google.cloud import pubsub as unused_pubsub
+ except ImportError:
+ raise ImportError('Google Cloud PubSub not available, please install '
+ 'apache_beam[gcp]')
+ # Execute this as a native transform.
+ output = PCollection(pcoll.pipeline)
+ output.element_type = unicode
+ return output
+
+ def apply_WriteStringsToPubSub(self, transform, pcoll):
+ try:
+ from google.cloud import pubsub
+ except ImportError:
+ raise ImportError('Google Cloud PubSub not available, please install '
+ 'apache_beam[gcp]')
+ project = transform._sink.project
+ topic_name = transform._sink.topic_name
+
+ class DirectWriteToPubSub(beam.DoFn):
+ _topic = None
+
+ def __init__(self, project, topic_name):
+ self.project = project
+ self.topic_name = topic_name
+
+ def start_bundle(self):
+ if self._topic is None:
+ self._topic = pubsub.Client(project=self.project).topic(
+ self.topic_name)
+ self._buffer = []
+
+ def process(self, elem):
+ self._buffer.append(elem.encode('utf-8'))
+ if len(self._buffer) >= 100:
+ self._flush()
+
+ def finish_bundle(self):
+ self._flush()
+
+ def _flush(self):
+ if self._buffer:
+ with self._topic.batch() as batch:
+ for datum in self._buffer:
+ batch.publish(datum)
+ self._buffer = []
+
+ output = pcoll | beam.ParDo(DirectWriteToPubSub(project, topic_name))
+ output.element_type = unicode
+ return output
+
def run(self, pipeline):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
+ # Performing configured PTransform overrides.
+ pipeline.replace_all(DirectRunner._PTRANSFORM_OVERRIDES)
+
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
@@ -152,6 +282,16 @@
self._executor = executor
self._evaluation_context = evaluation_context
+ def __del__(self):
+ if self._state == PipelineState.RUNNING:
+ logging.warning(
+ 'The DirectPipelineResult is being garbage-collected while the '
+ 'DirectRunner is still running the corresponding pipeline. This may '
+ 'lead to incomplete execution of the pipeline if the main thread '
+ 'exits before pipeline completion. Consider using '
+ 'result.wait_until_finish() to wait for completion of pipeline '
+ 'execution.')
+
def _is_in_terminal_state(self):
return self._state is not PipelineState.RUNNING
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner_test.py b/sdks/python/apache_beam/runners/direct/direct_runner_test.py
new file mode 100644
index 0000000..1c8b785
--- /dev/null
+++ b/sdks/python/apache_beam/runners/direct/direct_runner_test.py
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import threading
+import unittest
+
+import apache_beam as beam
+from apache_beam.testing import test_pipeline
+
+
+class DirectPipelineResultTest(unittest.TestCase):
+
+ def test_waiting_on_result_stops_executor_threads(self):
+ pre_test_threads = set(t.ident for t in threading.enumerate())
+
+ pipeline = test_pipeline.TestPipeline()
+ _ = (pipeline | beam.Create([{'foo': 'bar'}]))
+ result = pipeline.run()
+ result.wait_until_finish()
+
+ post_test_threads = set(t.ident for t in threading.enumerate())
+ new_threads = post_test_threads - pre_test_threads
+ self.assertEqual(len(new_threads), 0)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/apache_beam/runners/direct/evaluation_context.py b/sdks/python/apache_beam/runners/direct/evaluation_context.py
index 68d99d3..abb2dc4 100644
--- a/sdks/python/apache_beam/runners/direct/evaluation_context.py
+++ b/sdks/python/apache_beam/runners/direct/evaluation_context.py
@@ -22,27 +22,30 @@
import collections
import threading
-from apache_beam.transforms import sideinputs
from apache_beam.runners.direct.clock import Clock
-from apache_beam.runners.direct.watermark_manager import WatermarkManager
-from apache_beam.runners.direct.executor import TransformExecutor
from apache_beam.runners.direct.direct_metrics import DirectMetrics
+from apache_beam.runners.direct.executor import TransformExecutor
+from apache_beam.runners.direct.watermark_manager import WatermarkManager
+from apache_beam.transforms import sideinputs
+from apache_beam.transforms.trigger import InMemoryUnmergedState
from apache_beam.utils import counters
class _ExecutionContext(object):
- def __init__(self, watermarks, existing_state):
- self._watermarks = watermarks
- self._existing_state = existing_state
+ def __init__(self, watermarks, keyed_states):
+ self.watermarks = watermarks
+ self.keyed_states = keyed_states
- @property
- def watermarks(self):
- return self._watermarks
+ self._step_context = None
- @property
- def existing_state(self):
- return self._existing_state
+ def get_step_context(self):
+ if not self._step_context:
+ self._step_context = DirectStepContext(self.keyed_states)
+ return self._step_context
+
+ def reset(self):
+ self._step_context = None
class _SideInputView(object):
@@ -145,11 +148,11 @@
self._pcollection_to_views = collections.defaultdict(list)
for view in views:
self._pcollection_to_views[view.pvalue].append(view)
-
- # AppliedPTransform -> Evaluator specific state objects
- self._application_state_interals = {}
+ self._transform_keyed_states = self._initialize_keyed_states(
+ root_transforms, value_to_consumers)
self._watermark_manager = WatermarkManager(
- Clock(), root_transforms, value_to_consumers)
+ Clock(), root_transforms, value_to_consumers,
+ self._transform_keyed_states)
self._side_inputs_container = _SideInputsContainer(views)
self._pending_unblocked_tasks = []
self._counter_factory = counters.CounterFactory()
@@ -158,6 +161,15 @@
self._lock = threading.Lock()
+ def _initialize_keyed_states(self, root_transforms, value_to_consumers):
+ transform_keyed_states = {}
+ for transform in root_transforms:
+ transform_keyed_states[transform] = {}
+ for consumers in value_to_consumers.values():
+ for consumer in consumers:
+ transform_keyed_states[consumer] = {}
+ return transform_keyed_states
+
def use_pvalue_cache(self, cache):
assert not self._cache
self._cache = cache
@@ -199,11 +211,12 @@
the committed bundles contained within the handled result.
"""
with self._lock:
- committed_bundles = self._commit_bundles(
- result.uncommitted_output_bundles)
+ committed_bundles, unprocessed_bundles = self._commit_bundles(
+ result.uncommitted_output_bundles,
+ result.unprocessed_bundles)
self._watermark_manager.update_watermarks(
completed_bundle, result.transform, completed_timers,
- committed_bundles, result.watermark_hold)
+ committed_bundles, unprocessed_bundles, result.keyed_watermark_holds)
self._metrics.commit_logical(completed_bundle,
result.logical_metric_updates)
@@ -231,7 +244,10 @@
counter.name, counter.combine_fn)
merged_counter.accumulator.merge([counter.accumulator])
- self._application_state_interals[result.transform] = result.state
+ # Commit partial GBK states
+ existing_keyed_state = self._transform_keyed_states[result.transform]
+ for k, v in result.partial_keyed_state.iteritems():
+ existing_keyed_state[k] = v
return committed_bundles
def get_aggregator_values(self, aggregator_or_name):
@@ -244,19 +260,22 @@
executor_service.submit(task)
self._pending_unblocked_tasks = []
- def _commit_bundles(self, uncommitted_bundles):
+ def _commit_bundles(self, uncommitted_bundles, unprocessed_bundles):
"""Commits bundles and returns a immutable set of committed bundles."""
for in_progress_bundle in uncommitted_bundles:
producing_applied_ptransform = in_progress_bundle.pcollection.producer
watermarks = self._watermark_manager.get_watermarks(
producing_applied_ptransform)
in_progress_bundle.commit(watermarks.synchronized_processing_output_time)
- return tuple(uncommitted_bundles)
+
+ for unprocessed_bundle in unprocessed_bundles:
+ unprocessed_bundle.commit(None)
+ return tuple(uncommitted_bundles), tuple(unprocessed_bundles)
def get_execution_context(self, applied_ptransform):
return _ExecutionContext(
self._watermark_manager.get_watermarks(applied_ptransform),
- self._application_state_interals.get(applied_ptransform))
+ self._transform_keyed_states[applied_ptransform])
def create_bundle(self, output_pcollection):
"""Create an uncommitted bundle for the specified PCollection."""
@@ -296,3 +315,28 @@
assert isinstance(task, TransformExecutor)
return self._side_inputs_container.get_value_or_schedule_after_output(
side_input, task)
+
+
+class DirectUnmergedState(InMemoryUnmergedState):
+ """UnmergedState implementation for the DirectRunner."""
+
+ def __init__(self):
+ super(DirectUnmergedState, self).__init__(defensive_copy=False)
+
+
+class DirectStepContext(object):
+ """Context for the currently-executing step."""
+
+ def __init__(self, existing_keyed_state):
+ self.existing_keyed_state = existing_keyed_state
+ # In order to avoid partial writes of a bundle, every time
+ # existing_keyed_state is accessed, a copy of the state is made
+ # to be transferred to the bundle state once the bundle is committed.
+ self.partial_keyed_state = {}
+
+ def get_keyed_state(self, key):
+ if not self.existing_keyed_state.get(key):
+ self.existing_keyed_state[key] = DirectUnmergedState()
+ if not self.partial_keyed_state.get(key):
+ self.partial_keyed_state[key] = self.existing_keyed_state[key].copy()
+ return self.partial_keyed_state[key]
diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py
index 86db291..51fe908 100644
--- a/sdks/python/apache_beam/runners/direct/executor.py
+++ b/sdks/python/apache_beam/runners/direct/executor.py
@@ -20,14 +20,17 @@
from __future__ import absolute_import
import collections
+import itertools
import logging
import Queue
import sys
import threading
+import traceback
from weakref import WeakValueDictionary
from apache_beam.metrics.execution import MetricsContainer
from apache_beam.metrics.execution import ScopedMetricsContainer
+from apache_beam.options.pipeline_options import DirectOptions
class _ExecutorService(object):
@@ -55,6 +58,9 @@
self._default_name = 'ExecutorServiceWorker-' + str(index)
self._update_name()
self.shutdown_requested = False
+
+ # Stop worker thread when main thread exits.
+ self.daemon = True
self.start()
def _update_name(self, task=None):
@@ -75,7 +81,6 @@
return None
def run(self):
-
while not self.shutdown_requested:
task = self._get_task_or_none()
if task:
@@ -221,22 +226,30 @@
or for a source transform.
"""
- def __init__(self, evaluation_context, all_updates, timers=None):
+ def __init__(self, evaluation_context, all_updates, timer_firings=None):
self._evaluation_context = evaluation_context
self._all_updates = all_updates
- self._timers = timers
+ self._timer_firings = timer_firings or []
- def handle_result(self, input_committed_bundle, transform_result):
+ def handle_result(self, transform_executor, input_committed_bundle,
+ transform_result):
output_committed_bundles = self._evaluation_context.handle_result(
- input_committed_bundle, self._timers, transform_result)
+ input_committed_bundle, self._timer_firings, transform_result)
for output_committed_bundle in output_committed_bundles:
self._all_updates.offer(_ExecutorServiceParallelExecutor._ExecutorUpdate(
- output_committed_bundle, None))
+ transform_executor,
+ committed_bundle=output_committed_bundle))
+ for unprocessed_bundle in transform_result.unprocessed_bundles:
+ self._all_updates.offer(
+ _ExecutorServiceParallelExecutor._ExecutorUpdate(
+ transform_executor,
+ unprocessed_bundle=unprocessed_bundle))
return output_committed_bundles
- def handle_exception(self, exception):
+ def handle_exception(self, transform_executor, exception):
self._all_updates.offer(
- _ExecutorServiceParallelExecutor._ExecutorUpdate(None, exception))
+ _ExecutorServiceParallelExecutor._ExecutorUpdate(
+ transform_executor, exception=exception))
class TransformExecutor(_ExecutorService.CallableTask):
@@ -249,26 +262,37 @@
completion callback.
"""
+ _MAX_RETRY_PER_BUNDLE = 4
+
def __init__(self, transform_evaluator_registry, evaluation_context,
- input_bundle, applied_transform, completion_callback,
- transform_evaluation_state):
+ input_bundle, fired_timers, applied_ptransform,
+ completion_callback, transform_evaluation_state):
self._transform_evaluator_registry = transform_evaluator_registry
self._evaluation_context = evaluation_context
self._input_bundle = input_bundle
- self._applied_transform = applied_transform
+ self._fired_timers = fired_timers
+ self._applied_ptransform = applied_ptransform
self._completion_callback = completion_callback
self._transform_evaluation_state = transform_evaluation_state
self._side_input_values = {}
self.blocked = False
self._call_count = 0
+ self._retry_count = 0
+ # Switch to turn on/off the retry of bundles.
+ pipeline_options = self._evaluation_context.pipeline_options
+ # TODO(mariagh): Remove once "bundle retry" is no longer experimental.
+ if not pipeline_options.view_as(DirectOptions).direct_runner_bundle_retry:
+ self._max_retries_per_bundle = 1
+ else:
+ self._max_retries_per_bundle = TransformExecutor._MAX_RETRY_PER_BUNDLE
def call(self):
self._call_count += 1
- assert self._call_count <= (1 + len(self._applied_transform.side_inputs))
- metrics_container = MetricsContainer(self._applied_transform.full_label)
+ assert self._call_count <= (1 + len(self._applied_ptransform.side_inputs))
+ metrics_container = MetricsContainer(self._applied_ptransform.full_label)
scoped_metrics_container = ScopedMetricsContainer(metrics_container)
- for side_input in self._applied_transform.side_inputs:
+ for side_input in self._applied_ptransform.side_inputs:
if side_input not in self._side_input_values:
has_result, value = (
self._evaluation_context.get_value_or_schedule_after_output(
@@ -278,43 +302,67 @@
# available.
return
self._side_input_values[side_input] = value
-
side_input_values = [self._side_input_values[side_input]
- for side_input in self._applied_transform.side_inputs]
+ for side_input in self._applied_ptransform.side_inputs]
- try:
- evaluator = self._transform_evaluator_registry.for_application(
- self._applied_transform, self._input_bundle,
- side_input_values, scoped_metrics_container)
+ while self._retry_count < self._max_retries_per_bundle:
+ try:
+ self.attempt_call(metrics_container,
+ scoped_metrics_container,
+ side_input_values)
+ break
+ except Exception as e:
+ self._retry_count += 1
+ logging.error(
+ 'Exception at bundle %r, due to an exception.\n %s',
+ self._input_bundle, traceback.format_exc())
+ if self._retry_count == self._max_retries_per_bundle:
+ logging.error('Giving up after %s attempts.',
+ self._max_retries_per_bundle)
+ if self._retry_count == 1:
+ logging.info(
+ 'Use the experimental flag --direct_runner_bundle_retry'
+ ' to retry failed bundles (up to %d times).',
+ TransformExecutor._MAX_RETRY_PER_BUNDLE)
+ self._completion_callback.handle_exception(self, e)
- if self._input_bundle:
- for value in self._input_bundle.get_elements_iterable():
- evaluator.process_element(value)
+ self._evaluation_context.metrics().commit_physical(
+ self._input_bundle,
+ metrics_container.get_cumulative())
+ self._transform_evaluation_state.complete(self)
- with scoped_metrics_container:
- result = evaluator.finish_bundle()
- result.logical_metric_updates = metrics_container.get_cumulative()
+ def attempt_call(self, metrics_container,
+ scoped_metrics_container,
+ side_input_values):
+ evaluator = self._transform_evaluator_registry.get_evaluator(
+ self._applied_ptransform, self._input_bundle,
+ side_input_values, scoped_metrics_container)
- if self._evaluation_context.has_cache:
- for uncommitted_bundle in result.uncommitted_output_bundles:
+ if self._fired_timers:
+ for timer_firing in self._fired_timers:
+ evaluator.process_timer_wrapper(timer_firing)
+
+ if self._input_bundle:
+ for value in self._input_bundle.get_elements_iterable():
+ evaluator.process_element(value)
+
+ with scoped_metrics_container:
+ result = evaluator.finish_bundle()
+ result.logical_metric_updates = metrics_container.get_cumulative()
+
+ if self._evaluation_context.has_cache:
+ for uncommitted_bundle in result.uncommitted_output_bundles:
+ self._evaluation_context.append_to_cache(
+ self._applied_ptransform, uncommitted_bundle.tag,
+ uncommitted_bundle.get_elements_iterable())
+ undeclared_tag_values = result.undeclared_tag_values
+ if undeclared_tag_values:
+ for tag, value in undeclared_tag_values.iteritems():
self._evaluation_context.append_to_cache(
- self._applied_transform, uncommitted_bundle.tag,
- uncommitted_bundle.get_elements_iterable())
- undeclared_tag_values = result.undeclared_tag_values
- if undeclared_tag_values:
- for tag, value in undeclared_tag_values.iteritems():
- self._evaluation_context.append_to_cache(
- self._applied_transform, tag, value)
+ self._applied_ptransform, tag, value)
- self._completion_callback.handle_result(self._input_bundle, result)
- return result
- except Exception as e: # pylint: disable=broad-except
- self._completion_callback.handle_exception(e)
- finally:
- self._evaluation_context.metrics().commit_physical(
- self._input_bundle,
- metrics_container.get_cumulative())
- self._transform_evaluation_state.complete(self)
+ self._completion_callback.handle_result(self, self._input_bundle, result)
+ return result
class Executor(object):
@@ -353,6 +401,15 @@
def start(self, roots):
self.root_nodes = frozenset(roots)
+ self.all_nodes = frozenset(
+ itertools.chain(
+ roots,
+ *itertools.chain(self.value_to_consumers.values())))
+ self.node_to_pending_bundles = {}
+ for root_node in self.root_nodes:
+ provider = (self.transform_evaluator_registry
+ .get_root_bundle_provider(root_node))
+ self.node_to_pending_bundles[root_node] = provider.get_root_bundles()
self.executor_service.submit(
_ExecutorServiceParallelExecutor._MonitorTask(self))
@@ -364,31 +421,36 @@
raise t, v, tb
finally:
self.executor_service.shutdown()
+ self.executor_service.await_completion()
def schedule_consumers(self, committed_bundle):
if committed_bundle.pcollection in self.value_to_consumers:
consumers = self.value_to_consumers[committed_bundle.pcollection]
for applied_ptransform in consumers:
- self.schedule_consumption(applied_ptransform, committed_bundle,
+ self.schedule_consumption(applied_ptransform, committed_bundle, [],
self.default_completion_callback)
- def schedule_consumption(self, consumer_applied_transform, committed_bundle,
- on_complete):
+ def schedule_unprocessed_bundle(self, applied_ptransform,
+ unprocessed_bundle):
+ self.node_to_pending_bundles[applied_ptransform].append(unprocessed_bundle)
+
+ def schedule_consumption(self, consumer_applied_ptransform, committed_bundle,
+ fired_timers, on_complete):
"""Schedules evaluation of the given bundle with the transform."""
- assert all([consumer_applied_transform, on_complete])
- assert committed_bundle or consumer_applied_transform in self.root_nodes
- if (committed_bundle
- and self.transform_evaluator_registry.should_execute_serially(
- consumer_applied_transform)):
+ assert consumer_applied_ptransform
+ assert committed_bundle
+ assert on_complete
+ if self.transform_evaluator_registry.should_execute_serially(
+ consumer_applied_ptransform):
transform_executor_service = self.transform_executor_services.serial(
- consumer_applied_transform)
+ consumer_applied_ptransform)
else:
transform_executor_service = self.transform_executor_services.parallel()
transform_executor = TransformExecutor(
self.transform_evaluator_registry, self.evaluation_context,
- committed_bundle, consumer_applied_transform, on_complete,
- transform_executor_service)
+ committed_bundle, fired_timers, consumer_applied_ptransform,
+ on_complete, transform_executor_service)
transform_executor_service.schedule(transform_executor)
class _TypedUpdateQueue(object):
@@ -407,9 +469,17 @@
return None
def take(self):
- item = self._queue.get()
- self._queue.task_done()
- return item
+ # The implementation of Queue.Queue.get() does not propagate
+ # KeyboardInterrupts when a timeout is not used. We therefore use a
+ # one-second timeout in the following loop to allow KeyboardInterrupts
+ # to be correctly propagated.
+ while True:
+ try:
+ item = self._queue.get(timeout=1)
+ self._queue.task_done()
+ return item
+ except Queue.Empty:
+ pass
def offer(self, item):
assert isinstance(item, self._item_type)
@@ -418,10 +488,16 @@
class _ExecutorUpdate(object):
"""An internal status update on the state of the executor."""
- def __init__(self, produced_bundle=None, exception=None):
+ def __init__(self, transform_executor, committed_bundle=None,
+ unprocessed_bundle=None, exception=None):
+ self.transform_executor = transform_executor
# Exactly one of them should be not-None
- assert bool(produced_bundle) != bool(exception)
- self.committed_bundle = produced_bundle
+ assert sum([
+ bool(committed_bundle),
+ bool(unprocessed_bundle),
+ bool(exception)]) == 1
+ self.committed_bundle = committed_bundle
+ self.unprocessed_bundle = unprocessed_bundle
self.exception = exception
self.exc_info = sys.exc_info()
if self.exc_info[1] is not exception:
@@ -456,9 +532,13 @@
while update:
if update.committed_bundle:
self._executor.schedule_consumers(update.committed_bundle)
+ elif update.unprocessed_bundle:
+ self._executor.schedule_unprocessed_bundle(
+ update.transform_executor._applied_ptransform,
+ update.unprocessed_bundle)
else:
assert update.exception
- logging.warning('A task failed with exception.\n %s',
+ logging.warning('A task failed with exception: %s',
update.exception)
self._executor.visible_updates.offer(
_ExecutorServiceParallelExecutor._VisibleExecutorUpdate(
@@ -517,19 +597,21 @@
Returns:
True if timers fired.
"""
- fired_timers = self._executor.evaluation_context.extract_fired_timers()
- for applied_ptransform in fired_timers:
+ transform_fired_timers = (
+ self._executor.evaluation_context.extract_fired_timers())
+ for applied_ptransform, fired_timers in transform_fired_timers:
# Use an empty committed bundle. just to trigger.
empty_bundle = (
self._executor.evaluation_context.create_empty_committed_bundle(
applied_ptransform.inputs[0]))
timer_completion_callback = _CompletionCallback(
self._executor.evaluation_context, self._executor.all_updates,
- applied_ptransform)
+ timer_firings=fired_timers)
self._executor.schedule_consumption(
- applied_ptransform, empty_bundle, timer_completion_callback)
- return bool(fired_timers)
+ applied_ptransform, empty_bundle, fired_timers,
+ timer_completion_callback)
+ return bool(transform_fired_timers)
def _is_executing(self):
"""Returns True if there is at least one non-blocked TransformExecutor."""
@@ -564,10 +646,14 @@
# additional work.
return
- # All current TransformExecutors are blocked; add more work from the
- # roots.
- for applied_transform in self._executor.root_nodes:
- if not self._executor.evaluation_context.is_done(applied_transform):
- self._executor.schedule_consumption(
- applied_transform, None,
- self._executor.default_completion_callback)
+ # All current TransformExecutors are blocked; add more work from any
+ # pending bundles.
+ for applied_ptransform in self._executor.all_nodes:
+ if not self._executor.evaluation_context.is_done(applied_ptransform):
+ pending_bundles = self._executor.node_to_pending_bundles.get(
+ applied_ptransform, [])
+ for bundle in pending_bundles:
+ self._executor.schedule_consumption(
+ applied_ptransform, bundle, [],
+ self._executor.default_completion_callback)
+ self._executor.node_to_pending_bundles[applied_ptransform] = []
diff --git a/sdks/python/apache_beam/runners/direct/helper_transforms.py b/sdks/python/apache_beam/runners/direct/helper_transforms.py
index 374cd4e..26b0701 100644
--- a/sdks/python/apache_beam/runners/direct/helper_transforms.py
+++ b/sdks/python/apache_beam/runners/direct/helper_transforms.py
@@ -20,8 +20,8 @@
import apache_beam as beam
from apache_beam import typehints
-from apache_beam.utils.windowed_value import WindowedValue
from apache_beam.internal.util import ArgumentPlaceholder
+from apache_beam.utils.windowed_value import WindowedValue
class LiftedCombinePerKey(beam.PTransform):
diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
index b1cb626..2f3ac4f 100644
--- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
@@ -20,24 +20,39 @@
from __future__ import absolute_import
import collections
+import random
+import time
+import apache_beam.io as io
from apache_beam import coders
from apache_beam import pvalue
from apache_beam.internal import pickler
-import apache_beam.io as io
+from apache_beam.options.pipeline_options import TypeOptions
from apache_beam.runners.common import DoFnRunner
from apache_beam.runners.common import DoFnState
-from apache_beam.runners.direct.watermark_manager import WatermarkManager
-from apache_beam.runners.direct.transform_result import TransformResult
from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access
+from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow
+from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly
+from apache_beam.runners.direct.util import KeyedWorkItem
+from apache_beam.runners.direct.util import TransformResult
+from apache_beam.runners.direct.watermark_manager import WatermarkManager
+from apache_beam.testing.test_stream import ElementEvent
+from apache_beam.testing.test_stream import ProcessingTimeEvent
+from apache_beam.testing.test_stream import TestStream
+from apache_beam.testing.test_stream import WatermarkEvent
from apache_beam.transforms import core
+from apache_beam.transforms.trigger import TimeDomain
+from apache_beam.transforms.trigger import _CombiningValueStateTag
+from apache_beam.transforms.trigger import _ListStateTag
+from apache_beam.transforms.trigger import create_trigger_driver
from apache_beam.transforms.window import GlobalWindows
from apache_beam.transforms.window import WindowedValue
from apache_beam.typehints.typecheck import OutputCheckWrapperDoFn
from apache_beam.typehints.typecheck import TypeCheckError
from apache_beam.typehints.typecheck import TypeCheckWrapperDoFn
from apache_beam.utils import counters
-from apache_beam.options.pipeline_options import TypeOptions
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import Timestamp
class TransformEvaluatorRegistry(object):
@@ -51,13 +66,21 @@
self._evaluation_context = evaluation_context
self._evaluators = {
io.Read: _BoundedReadEvaluator,
+ io.ReadStringsFromPubSub: _PubSubReadEvaluator,
core.Flatten: _FlattenEvaluator,
core.ParDo: _ParDoEvaluator,
core._GroupByKeyOnly: _GroupByKeyOnlyEvaluator,
+ _StreamingGroupByKeyOnly: _StreamingGroupByKeyOnlyEvaluator,
+ _StreamingGroupAlsoByWindow: _StreamingGroupAlsoByWindowEvaluator,
_NativeWrite: _NativeWriteEvaluator,
+ TestStream: _TestStreamEvaluator,
+ }
+ self._root_bundle_providers = {
+ core.PTransform: DefaultRootBundleProvider,
+ TestStream: _TestStreamRootBundleProvider,
}
- def for_application(
+ def get_evaluator(
self, applied_ptransform, input_committed_bundle,
side_inputs, scoped_metrics_container):
"""Returns a TransformEvaluator suitable for processing given inputs."""
@@ -79,6 +102,18 @@
input_committed_bundle, side_inputs,
scoped_metrics_container)
+ def get_root_bundle_provider(self, applied_ptransform):
+ provider_cls = None
+ for cls in applied_ptransform.transform.__class__.mro():
+ provider_cls = self._root_bundle_providers.get(cls)
+ if provider_cls:
+ break
+ if not provider_cls:
+ raise NotImplementedError(
+ 'Root provider for [%s] not implemented in runner %s' % (
+ type(applied_ptransform.transform), self))
+ return provider_cls(self._evaluation_context, applied_ptransform)
+
def should_execute_serially(self, applied_ptransform):
"""Returns True if this applied_ptransform should run one bundle at a time.
@@ -99,7 +134,48 @@
True if executor should execute applied_ptransform serially.
"""
return isinstance(applied_ptransform.transform,
- (core._GroupByKeyOnly, _NativeWrite))
+ (core._GroupByKeyOnly,
+ _StreamingGroupByKeyOnly,
+ _StreamingGroupAlsoByWindow,
+ _NativeWrite))
+
+
+class RootBundleProvider(object):
+ """Provides bundles for the initial execution of a root transform."""
+
+ def __init__(self, evaluation_context, applied_ptransform):
+ self._evaluation_context = evaluation_context
+ self._applied_ptransform = applied_ptransform
+
+ def get_root_bundles(self):
+ raise NotImplementedError
+
+
+class DefaultRootBundleProvider(RootBundleProvider):
+ """Provides an empty bundle by default for root transforms."""
+
+ def get_root_bundles(self):
+ input_node = pvalue.PBegin(self._applied_ptransform.transform.pipeline)
+ empty_bundle = (
+ self._evaluation_context.create_empty_committed_bundle(input_node))
+ return [empty_bundle]
+
+
+class _TestStreamRootBundleProvider(RootBundleProvider):
+ """Provides an initial bundle for the TestStream evaluator."""
+
+ def get_root_bundles(self):
+ test_stream = self._applied_ptransform.transform
+ bundles = []
+ if len(test_stream.events) > 0:
+ bundle = self._evaluation_context.create_bundle(
+ pvalue.PBegin(self._applied_ptransform.transform.pipeline))
+ # Explicitly set timestamp to MIN_TIMESTAMP to ensure that we hold the
+ # watermark.
+ bundle.add(GlobalWindows.windowed_value(0, timestamp=MIN_TIMESTAMP))
+ bundle.commit(None)
+ bundles.append(bundle)
+ return bundles
class _TransformEvaluator(object):
@@ -161,6 +237,27 @@
"""Starts a new bundle."""
pass
+ def process_timer_wrapper(self, timer_firing):
+ """Process timer by clearing and then calling process_timer().
+
+ This method is called with any timer firing and clears the delivered
+ timer from the keyed state and then calls process_timer(). The default
+ process_timer() implementation emits a KeyedWorkItem for the particular
+ timer and passes it to process_element(). Evaluator subclasses which
+ desire different timer delivery semantics can override process_timer().
+ """
+ state = self.step_context.get_keyed_state(timer_firing.encoded_key)
+ state.clear_timer(
+ timer_firing.window, timer_firing.name, timer_firing.time_domain)
+ self.process_timer(timer_firing)
+
+ def process_timer(self, timer_firing):
+ """Default process_timer() impl. generating KeyedWorkItem element."""
+ self.process_element(
+ GlobalWindows.windowed_value(
+ KeyedWorkItem(timer_firing.encoded_key,
+ timer_firings=[timer_firing])))
+
def process_element(self, element):
"""Processes a new element as part of the current bundle."""
raise NotImplementedError('%s do not process elements.', type(self))
@@ -178,7 +275,6 @@
def __init__(self, evaluation_context, applied_ptransform,
input_committed_bundle, side_inputs, scoped_metrics_container):
- assert not input_committed_bundle
assert not side_inputs
self._source = applied_ptransform.transform.source
self._source.pipeline_options = evaluation_context.pipeline_options
@@ -206,8 +302,148 @@
with self._source.reader() as reader:
bundles = _read_values_to_bundles(reader)
+ return TransformResult(self, bundles, [], None, None)
+
+
+class _TestStreamEvaluator(_TransformEvaluator):
+ """TransformEvaluator for the TestStream transform."""
+
+ def __init__(self, evaluation_context, applied_ptransform,
+ input_committed_bundle, side_inputs, scoped_metrics_container):
+ assert not side_inputs
+ self.test_stream = applied_ptransform.transform
+ super(_TestStreamEvaluator, self).__init__(
+ evaluation_context, applied_ptransform, input_committed_bundle,
+ side_inputs, scoped_metrics_container)
+
+ def start_bundle(self):
+ self.current_index = -1
+ self.watermark = MIN_TIMESTAMP
+ self.bundles = []
+
+ def process_element(self, element):
+ index = element.value
+ self.watermark = element.timestamp
+ assert isinstance(index, int)
+ assert 0 <= index <= len(self.test_stream.events)
+ self.current_index = index
+ event = self.test_stream.events[self.current_index]
+ if isinstance(event, ElementEvent):
+ assert len(self._outputs) == 1
+ output_pcollection = list(self._outputs)[0]
+ bundle = self._evaluation_context.create_bundle(output_pcollection)
+ for tv in event.timestamped_values:
+ bundle.output(
+ GlobalWindows.windowed_value(tv.value, timestamp=tv.timestamp))
+ self.bundles.append(bundle)
+ elif isinstance(event, WatermarkEvent):
+ assert event.new_watermark >= self.watermark
+ self.watermark = event.new_watermark
+ elif isinstance(event, ProcessingTimeEvent):
+ # TODO(ccy): advance processing time in the context's mock clock.
+ pass
+ else:
+ raise ValueError('Invalid TestStream event: %s.' % event)
+
+ def finish_bundle(self):
+ unprocessed_bundles = []
+ hold = None
+ if self.current_index < len(self.test_stream.events) - 1:
+ unprocessed_bundle = self._evaluation_context.create_bundle(
+ pvalue.PBegin(self._applied_ptransform.transform.pipeline))
+ unprocessed_bundle.add(GlobalWindows.windowed_value(
+ self.current_index + 1, timestamp=self.watermark))
+ unprocessed_bundles.append(unprocessed_bundle)
+ hold = self.watermark
+
return TransformResult(
- self._applied_ptransform, bundles, None, None, None, None)
+ self, self.bundles, unprocessed_bundles, None, {None: hold})
+
+
+class _PubSubSubscriptionWrapper(object):
+ """Wrapper for garbage-collecting temporary PubSub subscriptions."""
+
+ def __init__(self, subscription, should_cleanup):
+ self.subscription = subscription
+ self.should_cleanup = should_cleanup
+
+ def __del__(self):
+ if self.should_cleanup:
+ self.subscription.delete()
+
+
+class _PubSubReadEvaluator(_TransformEvaluator):
+ """TransformEvaluator for PubSub read."""
+
+ _subscription_cache = {}
+
+ def __init__(self, evaluation_context, applied_ptransform,
+ input_committed_bundle, side_inputs, scoped_metrics_container):
+ assert not side_inputs
+ super(_PubSubReadEvaluator, self).__init__(
+ evaluation_context, applied_ptransform, input_committed_bundle,
+ side_inputs, scoped_metrics_container)
+
+ source = self._applied_ptransform.transform._source
+ self._subscription = _PubSubReadEvaluator.get_subscription(
+ self._applied_ptransform, source.project, source.topic_name,
+ source.subscription_name)
+
+ @classmethod
+ def get_subscription(cls, transform, project, topic, subscription_name):
+ if transform not in cls._subscription_cache:
+ from google.cloud import pubsub
+ should_create = not subscription_name
+ if should_create:
+ subscription_name = 'beam_%d_%x' % (
+ int(time.time()), random.randrange(1 << 32))
+ cls._subscription_cache[transform] = _PubSubSubscriptionWrapper(
+ pubsub.Client(project=project).topic(topic).subscription(
+ subscription_name),
+ should_create)
+ if should_create:
+ cls._subscription_cache[transform].subscription.create()
+ return cls._subscription_cache[transform].subscription
+
+ def start_bundle(self):
+ pass
+
+ def process_element(self, element):
+ pass
+
+ def _read_from_pubsub(self):
+ from google.cloud import pubsub
+ # Because of the AutoAck, we are not able to reread messages if this
+ # evaluator fails with an exception before emitting a bundle. However,
+ # the DirectRunner currently doesn't retry work items anyway, so the
+ # pipeline would enter an inconsistent state on any error.
+ with pubsub.subscription.AutoAck(
+ self._subscription, return_immediately=True,
+ max_messages=10) as results:
+ return [message.data for unused_ack_id, message in results.items()]
+
+ def finish_bundle(self):
+ data = self._read_from_pubsub()
+ if data:
+ output_pcollection = list(self._outputs)[0]
+ bundle = self._evaluation_context.create_bundle(output_pcollection)
+ # TODO(ccy): we currently do not use the PubSub message timestamp or
+ # respect the PubSub source's id_label field.
+ now = Timestamp.of(time.time())
+ for message_data in data:
+ bundle.output(GlobalWindows.windowed_value(message_data, timestamp=now))
+ bundles = [bundle]
+ else:
+ bundles = []
+ if self._applied_ptransform.inputs:
+ input_pvalue = self._applied_ptransform.inputs[0]
+ else:
+ input_pvalue = pvalue.PBegin(self._applied_ptransform.transform.pipeline)
+ unprocessed_bundle = self._evaluation_context.create_bundle(
+ input_pvalue)
+
+ return TransformResult(self, bundles, [unprocessed_bundle], None,
+ {None: Timestamp.of(time.time())})
class _FlattenEvaluator(_TransformEvaluator):
@@ -230,8 +466,7 @@
def finish_bundle(self):
bundles = [self.bundle]
- return TransformResult(
- self._applied_ptransform, bundles, None, None, None, None)
+ return TransformResult(self, bundles, [], None, None)
class _TaggedReceivers(dict):
@@ -320,7 +555,7 @@
bundles = self._tagged_receivers.values()
result_counters = self._counter_factory.get_counters()
return TransformResult(
- self._applied_ptransform, bundles, None, None, result_counters, None,
+ self, bundles, [], result_counters, None,
self._tagged_receivers.undeclared_in_memory_tag_values)
@@ -328,13 +563,8 @@
"""TransformEvaluator for _GroupByKeyOnly transform."""
MAX_ELEMENT_PER_BUNDLE = None
-
- class _GroupByKeyOnlyEvaluatorState(object):
-
- def __init__(self):
- # output: {} key -> [values]
- self.output = collections.defaultdict(list)
- self.completed = False
+ ELEMENTS_TAG = _ListStateTag('elements')
+ COMPLETION_TAG = _CombiningValueStateTag('completed', any)
def __init__(self, evaluation_context, applied_ptransform,
input_committed_bundle, side_inputs, scoped_metrics_container):
@@ -343,15 +573,103 @@
evaluation_context, applied_ptransform, input_committed_bundle,
side_inputs, scoped_metrics_container)
- @property
def _is_final_bundle(self):
return (self._execution_context.watermarks.input_watermark
== WatermarkManager.WATERMARK_POS_INF)
def start_bundle(self):
- self.state = (self._execution_context.existing_state
- if self._execution_context.existing_state
- else _GroupByKeyOnlyEvaluator._GroupByKeyOnlyEvaluatorState())
+ self.step_context = self._execution_context.get_step_context()
+ self.global_state = self.step_context.get_keyed_state(None)
+
+ assert len(self._outputs) == 1
+ self.output_pcollection = list(self._outputs)[0]
+
+ # The output type of a GroupByKey will be KV[Any, Any] or more specific.
+ # TODO(BEAM-2717): Infer coders earlier.
+ kv_type_hint = (
+ self._applied_ptransform.outputs[None].element_type
+ or
+ self._applied_ptransform.transform.get_type_hints().input_types[0][0])
+ self.key_coder = coders.registry.get_coder(kv_type_hint.tuple_types[0])
+
+ def process_timer(self, timer_firing):
+ # We do not need to emit a KeyedWorkItem to process_element().
+ pass
+
+ def process_element(self, element):
+ assert not self.global_state.get_state(
+ None, _GroupByKeyOnlyEvaluator.COMPLETION_TAG)
+ if (isinstance(element, WindowedValue)
+ and isinstance(element.value, collections.Iterable)
+ and len(element.value) == 2):
+ k, v = element.value
+ encoded_k = self.key_coder.encode(k)
+ state = self.step_context.get_keyed_state(encoded_k)
+ state.add_state(None, _GroupByKeyOnlyEvaluator.ELEMENTS_TAG, v)
+ else:
+ raise TypeCheckError('Input to _GroupByKeyOnly must be a PCollection of '
+ 'windowed key-value pairs. Instead received: %r.'
+ % element)
+
+ def finish_bundle(self):
+ if self._is_final_bundle():
+ if self.global_state.get_state(
+ None, _GroupByKeyOnlyEvaluator.COMPLETION_TAG):
+ # Ignore empty bundles after emitting output. (This may happen because
+ # empty bundles do not affect input watermarks.)
+ bundles = []
+ else:
+ gbk_result = []
+ # TODO(ccy): perhaps we can clean this up to not use this
+ # internal attribute of the DirectStepContext.
+ for encoded_k in self.step_context.existing_keyed_state:
+ # Ignore global state.
+ if encoded_k is None:
+ continue
+ k = self.key_coder.decode(encoded_k)
+ state = self.step_context.get_keyed_state(encoded_k)
+ vs = state.get_state(None, _GroupByKeyOnlyEvaluator.ELEMENTS_TAG)
+ gbk_result.append(GlobalWindows.windowed_value((k, vs)))
+
+ def len_element_fn(element):
+ _, v = element.value
+ return len(v)
+
+ bundles = self._split_list_into_bundles(
+ self.output_pcollection, gbk_result,
+ _GroupByKeyOnlyEvaluator.MAX_ELEMENT_PER_BUNDLE, len_element_fn)
+
+ self.global_state.add_state(
+ None, _GroupByKeyOnlyEvaluator.COMPLETION_TAG, True)
+ hold = WatermarkManager.WATERMARK_POS_INF
+ else:
+ bundles = []
+ hold = WatermarkManager.WATERMARK_NEG_INF
+ self.global_state.set_timer(
+ None, '', TimeDomain.WATERMARK, WatermarkManager.WATERMARK_POS_INF)
+
+ return TransformResult(self, bundles, [], None, {None: hold})
+
+
+class _StreamingGroupByKeyOnlyEvaluator(_TransformEvaluator):
+ """TransformEvaluator for _StreamingGroupByKeyOnly transform.
+
+ The _GroupByKeyOnlyEvaluator buffers elements until its input watermark goes
+ to infinity, which is suitable for batch mode execution. During streaming
+ mode execution, we emit each bundle as it comes to the next transform.
+ """
+
+ MAX_ELEMENT_PER_BUNDLE = None
+
+ def __init__(self, evaluation_context, applied_ptransform,
+ input_committed_bundle, side_inputs, scoped_metrics_container):
+ assert not side_inputs
+ super(_StreamingGroupByKeyOnlyEvaluator, self).__init__(
+ evaluation_context, applied_ptransform, input_committed_bundle,
+ side_inputs, scoped_metrics_container)
+
+ def start_bundle(self):
+ self.gbk_items = collections.defaultdict(list)
assert len(self._outputs) == 1
self.output_pcollection = list(self._outputs)[0]
@@ -362,52 +680,95 @@
self.key_coder = coders.registry.get_coder(kv_type_hint[0].tuple_types[0])
def process_element(self, element):
- assert not self.state.completed
if (isinstance(element, WindowedValue)
and isinstance(element.value, collections.Iterable)
and len(element.value) == 2):
k, v = element.value
- self.state.output[self.key_coder.encode(k)].append(v)
+ self.gbk_items[self.key_coder.encode(k)].append(v)
else:
raise TypeCheckError('Input to _GroupByKeyOnly must be a PCollection of '
'windowed key-value pairs. Instead received: %r.'
% element)
def finish_bundle(self):
- if self._is_final_bundle:
- if self.state.completed:
- # Ignore empty bundles after emitting output. (This may happen because
- # empty bundles do not affect input watermarks.)
- bundles = []
- else:
- gbk_result = (
- map(GlobalWindows.windowed_value, (
- (self.key_coder.decode(k), v)
- for k, v in self.state.output.iteritems())))
+ bundles = []
+ bundle = None
+ for encoded_k, vs in self.gbk_items.iteritems():
+ if not bundle:
+ bundle = self._evaluation_context.create_bundle(
+ self.output_pcollection)
+ bundles.append(bundle)
+ kwi = KeyedWorkItem(encoded_k, elements=vs)
+ bundle.add(GlobalWindows.windowed_value(kwi))
- def len_element_fn(element):
- _, v = element.value
- return len(v)
+ return TransformResult(self, bundles, [], None, None)
- bundles = self._split_list_into_bundles(
- self.output_pcollection, gbk_result,
- _GroupByKeyOnlyEvaluator.MAX_ELEMENT_PER_BUNDLE, len_element_fn)
- self.state.completed = True
- state = self.state
- hold = WatermarkManager.WATERMARK_POS_INF
- else:
- bundles = []
- state = self.state
- hold = WatermarkManager.WATERMARK_NEG_INF
+class _StreamingGroupAlsoByWindowEvaluator(_TransformEvaluator):
+ """TransformEvaluator for the _StreamingGroupAlsoByWindow transform.
- return TransformResult(
- self._applied_ptransform, bundles, state, None, None, hold)
+ This evaluator is only used in streaming mode. In batch mode, the
+ GroupAlsoByWindow operation is evaluated as a normal DoFn, as defined
+ in transforms/core.py.
+ """
+
+ def __init__(self, evaluation_context, applied_ptransform,
+ input_committed_bundle, side_inputs, scoped_metrics_container):
+ assert not side_inputs
+ super(_StreamingGroupAlsoByWindowEvaluator, self).__init__(
+ evaluation_context, applied_ptransform, input_committed_bundle,
+ side_inputs, scoped_metrics_container)
+
+ def start_bundle(self):
+ assert len(self._outputs) == 1
+ self.output_pcollection = list(self._outputs)[0]
+ self.step_context = self._execution_context.get_step_context()
+ self.driver = create_trigger_driver(
+ self._applied_ptransform.transform.windowing)
+ self.gabw_items = []
+ self.keyed_holds = {}
+
+ # The input type of a GroupAlsoByWindow will be KV[Any, Iter[Any]] or more
+ # specific.
+ kv_type_hint = (
+ self._applied_ptransform.transform.get_type_hints().input_types[0])
+ self.key_coder = coders.registry.get_coder(kv_type_hint[0].tuple_types[0])
+
+ def process_element(self, element):
+ kwi = element.value
+ assert isinstance(kwi, KeyedWorkItem), kwi
+ encoded_k, timer_firings, vs = (
+ kwi.encoded_key, kwi.timer_firings, kwi.elements)
+ k = self.key_coder.decode(encoded_k)
+ state = self.step_context.get_keyed_state(encoded_k)
+
+ for timer_firing in timer_firings:
+ for wvalue in self.driver.process_timer(
+ timer_firing.window, timer_firing.name, timer_firing.time_domain,
+ timer_firing.timestamp, state):
+ self.gabw_items.append(wvalue.with_value((k, wvalue.value)))
+ if vs:
+ for wvalue in self.driver.process_elements(state, vs, MIN_TIMESTAMP):
+ self.gabw_items.append(wvalue.with_value((k, wvalue.value)))
+
+ self.keyed_holds[encoded_k] = state.get_earliest_hold()
+
+ def finish_bundle(self):
+ bundles = []
+ if self.gabw_items:
+ bundle = self._evaluation_context.create_bundle(self.output_pcollection)
+ for item in self.gabw_items:
+ bundle.add(item)
+ bundles.append(bundle)
+
+ return TransformResult(self, bundles, [], None, self.keyed_holds)
class _NativeWriteEvaluator(_TransformEvaluator):
"""TransformEvaluator for _NativeWrite transform."""
+ ELEMENTS_TAG = _ListStateTag('elements')
+
def __init__(self, evaluation_context, applied_ptransform,
input_committed_bundle, side_inputs, scoped_metrics_container):
assert not side_inputs
@@ -429,12 +790,16 @@
== WatermarkManager.WATERMARK_POS_INF)
def start_bundle(self):
- # state: [values]
- self.state = (self._execution_context.existing_state
- if self._execution_context.existing_state else [])
+ self.step_context = self._execution_context.get_step_context()
+ self.global_state = self.step_context.get_keyed_state(None)
+
+ def process_timer(self, timer_firing):
+ # We do not need to emit a KeyedWorkItem to process_element().
+ pass
def process_element(self, element):
- self.state.append(element)
+ self.global_state.add_state(
+ None, _NativeWriteEvaluator.ELEMENTS_TAG, element)
def finish_bundle(self):
# finish_bundle will append incoming bundles in memory until all the bundles
@@ -444,19 +809,20 @@
# ignored and would not generate additional output files.
# TODO(altay): Do not wait until the last bundle to write in a single shard.
if self._is_final_bundle:
+ elements = self.global_state.get_state(
+ None, _NativeWriteEvaluator.ELEMENTS_TAG)
if self._has_already_produced_output:
# Ignore empty bundles that arrive after the output is produced.
- assert self.state == []
+ assert elements == []
else:
self._sink.pipeline_options = self._evaluation_context.pipeline_options
with self._sink.writer() as writer:
- for v in self.state:
+ for v in elements:
writer.Write(v.value)
- state = None
hold = WatermarkManager.WATERMARK_POS_INF
else:
- state = self.state
hold = WatermarkManager.WATERMARK_NEG_INF
+ self.global_state.set_timer(
+ None, '', TimeDomain.WATERMARK, WatermarkManager.WATERMARK_POS_INF)
- return TransformResult(
- self._applied_ptransform, [], state, None, None, hold)
+ return TransformResult(self, [], [], None, {None: hold})
diff --git a/sdks/python/apache_beam/runners/direct/transform_result.py b/sdks/python/apache_beam/runners/direct/transform_result.py
deleted file mode 100644
index febdd20..0000000
--- a/sdks/python/apache_beam/runners/direct/transform_result.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""The result of evaluating an AppliedPTransform with a TransformEvaluator."""
-
-from __future__ import absolute_import
-
-
-class TransformResult(object):
- """For internal use only; no backwards-compatibility guarantees.
-
- The result of evaluating an AppliedPTransform with a TransformEvaluator."""
-
- def __init__(self, applied_ptransform, uncommitted_output_bundles, state,
- timer_update, counters, watermark_hold,
- undeclared_tag_values=None):
- self.transform = applied_ptransform
- self.uncommitted_output_bundles = uncommitted_output_bundles
- self.state = state
- # TODO: timer update is currently unused.
- self.timer_update = timer_update
- self.counters = counters
- self.watermark_hold = watermark_hold
- # Only used when caching (materializing) all values is requested.
- self.undeclared_tag_values = undeclared_tag_values
- # Populated by the TransformExecutor.
- self.logical_metric_updates = None
diff --git a/sdks/python/apache_beam/runners/direct/util.py b/sdks/python/apache_beam/runners/direct/util.py
new file mode 100644
index 0000000..96a6ee2
--- /dev/null
+++ b/sdks/python/apache_beam/runners/direct/util.py
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utility classes used by the DirectRunner.
+
+For internal use only. No backwards compatibility guarantees.
+"""
+
+from __future__ import absolute_import
+
+
+class TransformResult(object):
+ """Result of evaluating an AppliedPTransform with a TransformEvaluator."""
+
+ def __init__(self, transform_evaluator, uncommitted_output_bundles,
+ unprocessed_bundles, counters, keyed_watermark_holds,
+ undeclared_tag_values=None):
+ self.transform = transform_evaluator._applied_ptransform
+ self.uncommitted_output_bundles = uncommitted_output_bundles
+ self.unprocessed_bundles = unprocessed_bundles
+ self.counters = counters
+ # Mapping of key -> earliest hold timestamp or None. Keys should be
+ # strings or None.
+ #
+ # For each key, we receive as its corresponding value the earliest
+ # watermark hold for that key (the key can be None for global state), past
+ # which the output watermark for the currently-executing step will not
+ # advance. If the value is None or utils.timestamp.MAX_TIMESTAMP, the
+ # watermark hold will be removed.
+ self.keyed_watermark_holds = keyed_watermark_holds or {}
+ # Only used when caching (materializing) all values is requested.
+ self.undeclared_tag_values = undeclared_tag_values
+ # Populated by the TransformExecutor.
+ self.logical_metric_updates = None
+
+ step_context = transform_evaluator._execution_context.get_step_context()
+ self.partial_keyed_state = step_context.partial_keyed_state
+
+
+class TimerFiring(object):
+ """A single instance of a fired timer."""
+
+ def __init__(self, encoded_key, window, name, time_domain, timestamp):
+ self.encoded_key = encoded_key
+ self.window = window
+ self.name = name
+ self.time_domain = time_domain
+ self.timestamp = timestamp
+
+
+class KeyedWorkItem(object):
+ """A keyed item that can either be a timer firing or a list of elements."""
+ def __init__(self, encoded_key, timer_firings=None, elements=None):
+ self.encoded_key = encoded_key
+ self.timer_firings = timer_firings or []
+ self.elements = elements or []
diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py
index 3a13539..935998d 100644
--- a/sdks/python/apache_beam/runners/direct/watermark_manager.py
+++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py
@@ -23,8 +23,10 @@
from apache_beam import pipeline
from apache_beam import pvalue
+from apache_beam.runners.direct.util import TimerFiring
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import TIME_GRANULARITY
class WatermarkManager(object):
@@ -35,21 +37,23 @@
WATERMARK_POS_INF = MAX_TIMESTAMP
WATERMARK_NEG_INF = MIN_TIMESTAMP
- def __init__(self, clock, root_transforms, value_to_consumers):
+ def __init__(self, clock, root_transforms, value_to_consumers,
+ transform_keyed_states):
self._clock = clock # processing time clock
- self._value_to_consumers = value_to_consumers
self._root_transforms = root_transforms
+ self._value_to_consumers = value_to_consumers
+ self._transform_keyed_states = transform_keyed_states
# AppliedPTransform -> TransformWatermarks
self._transform_to_watermarks = {}
for root_transform in root_transforms:
self._transform_to_watermarks[root_transform] = _TransformWatermarks(
- self._clock)
+ self._clock, transform_keyed_states[root_transform], root_transform)
for consumers in value_to_consumers.values():
for consumer in consumers:
self._transform_to_watermarks[consumer] = _TransformWatermarks(
- self._clock)
+ self._clock, transform_keyed_states[consumer], consumer)
for consumers in value_to_consumers.values():
for consumer in consumers:
@@ -89,16 +93,19 @@
return self._transform_to_watermarks[applied_ptransform]
def update_watermarks(self, completed_committed_bundle, applied_ptransform,
- timer_update, outputs, earliest_hold):
+ completed_timers, outputs, unprocessed_bundles,
+ keyed_earliest_holds):
assert isinstance(applied_ptransform, pipeline.AppliedPTransform)
self._update_pending(
- completed_committed_bundle, applied_ptransform, timer_update, outputs)
+ completed_committed_bundle, applied_ptransform, completed_timers,
+ outputs, unprocessed_bundles)
tw = self.get_watermarks(applied_ptransform)
- tw.hold(earliest_hold)
+ tw.hold(keyed_earliest_holds)
self._refresh_watermarks(applied_ptransform)
def _update_pending(self, input_committed_bundle, applied_ptransform,
- timer_update, output_committed_bundles):
+ completed_timers, output_committed_bundles,
+ unprocessed_bundles):
"""Updated list of pending bundles for the given AppliedPTransform."""
# Update pending elements. Filter out empty bundles. They do not impact
@@ -112,7 +119,10 @@
consumer_tw.add_pending(output)
completed_tw = self._transform_to_watermarks[applied_ptransform]
- completed_tw.update_timers(timer_update)
+ completed_tw.update_timers(completed_timers)
+
+ for unprocessed_bundle in unprocessed_bundles:
+ completed_tw.add_pending(unprocessed_bundle)
assert input_committed_bundle or applied_ptransform in self._root_transforms
if input_committed_bundle and input_committed_bundle.has_elements():
@@ -136,33 +146,36 @@
def extract_fired_timers(self):
all_timers = []
for applied_ptransform, tw in self._transform_to_watermarks.iteritems():
- if tw.extract_fired_timers():
- all_timers.append(applied_ptransform)
+ fired_timers = tw.extract_fired_timers()
+ if fired_timers:
+ all_timers.append((applied_ptransform, fired_timers))
return all_timers
class _TransformWatermarks(object):
- """Tracks input and output watermarks for aan AppliedPTransform."""
+ """Tracks input and output watermarks for an AppliedPTransform."""
- def __init__(self, clock):
+ def __init__(self, clock, keyed_states, transform):
self._clock = clock
+ self._keyed_states = keyed_states
self._input_transform_watermarks = []
self._input_watermark = WatermarkManager.WATERMARK_NEG_INF
self._output_watermark = WatermarkManager.WATERMARK_NEG_INF
- self._earliest_hold = WatermarkManager.WATERMARK_POS_INF
+ self._keyed_earliest_holds = {}
self._pending = set() # Scheduled bundles targeted for this transform.
- self._fired_timers = False
+ self._fired_timers = set()
self._lock = threading.Lock()
+ self._label = str(transform)
+
def update_input_transform_watermarks(self, input_transform_watermarks):
with self._lock:
self._input_transform_watermarks = input_transform_watermarks
- def update_timers(self, timer_update):
+ def update_timers(self, completed_timers):
with self._lock:
- if timer_update:
- assert self._fired_timers
- self._fired_timers = False
+ for timer_firing in completed_timers:
+ self._fired_timers.remove(timer_firing)
@property
def input_watermark(self):
@@ -174,11 +187,13 @@
with self._lock:
return self._output_watermark
- def hold(self, value):
+ def hold(self, keyed_earliest_holds):
with self._lock:
- if value is None:
- value = WatermarkManager.WATERMARK_POS_INF
- self._earliest_hold = value
+ for key, hold_value in keyed_earliest_holds.iteritems():
+ self._keyed_earliest_holds[key] = hold_value
+ if (hold_value is None or
+ hold_value == WatermarkManager.WATERMARK_POS_INF):
+ del self._keyed_earliest_holds[key]
def add_pending(self, pending):
with self._lock:
@@ -193,9 +208,22 @@
def refresh(self):
with self._lock:
- pending_holder = (WatermarkManager.WATERMARK_NEG_INF
- if self._pending else
- WatermarkManager.WATERMARK_POS_INF)
+ min_pending_timestamp = WatermarkManager.WATERMARK_POS_INF
+ has_pending_elements = False
+ for input_bundle in self._pending:
+ # TODO(ccy): we can have the Bundle class keep track of the minimum
+ # timestamp so we don't have to do an iteration here.
+ for wv in input_bundle.get_elements_iterable():
+ has_pending_elements = True
+ if wv.timestamp < min_pending_timestamp:
+ min_pending_timestamp = wv.timestamp
+
+ # If there is a pending element with a certain timestamp, we can at most
+ # advance our watermark to the maximum timestamp less than that
+ # timestamp.
+ pending_holder = WatermarkManager.WATERMARK_POS_INF
+ if has_pending_elements:
+ pending_holder = min_pending_timestamp - TIME_GRANULARITY
input_watermarks = [
tw.output_watermark for tw in self._input_transform_watermarks]
@@ -204,7 +232,11 @@
self._input_watermark = max(self._input_watermark,
min(pending_holder, producer_watermark))
- new_output_watermark = min(self._input_watermark, self._earliest_hold)
+ earliest_hold = WatermarkManager.WATERMARK_POS_INF
+ for hold in self._keyed_earliest_holds.values():
+ if hold < earliest_hold:
+ earliest_hold = hold
+ new_output_watermark = min(self._input_watermark, earliest_hold)
advanced = new_output_watermark > self._output_watermark
self._output_watermark = new_output_watermark
@@ -219,8 +251,12 @@
if self._fired_timers:
return False
- should_fire = (
- self._earliest_hold < WatermarkManager.WATERMARK_POS_INF and
- self._input_watermark == WatermarkManager.WATERMARK_POS_INF)
- self._fired_timers = should_fire
- return should_fire
+ fired_timers = []
+ for encoded_key, state in self._keyed_states.iteritems():
+ timers = state.get_timers(watermark=self._input_watermark)
+ for expired in timers:
+ window, (name, time_domain, timestamp) = expired
+ fired_timers.append(
+ TimerFiring(encoded_key, window, name, time_domain, timestamp))
+ self._fired_timers.update(fired_timers)
+ return fired_timers
diff --git a/sdks/python/apache_beam/runners/experimental/__init__.py b/sdks/python/apache_beam/runners/experimental/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/sdks/python/apache_beam/runners/experimental/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/__init__.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/__init__.py
new file mode 100644
index 0000000..5d14030
--- /dev/null
+++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/__init__.py
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This is the experimental direct runner for testing the job api that
+sends a runner API proto over the API and then runs it on the other side.
+"""
+
+from apache_beam.runners.experimental.python_rpc_direct.python_rpc_direct_runner import PythonRPCDirectRunner
diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py
new file mode 100644
index 0000000..84bed427
--- /dev/null
+++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py
@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A runner implementation that submits a job for remote execution.
+"""
+
+import logging
+import random
+import string
+
+import grpc
+
+from apache_beam.portability.api import beam_job_api_pb2
+from apache_beam.runners.job import utils as job_utils
+from apache_beam.runners.job.manager import DockerRPCManager
+from apache_beam.runners.runner import PipelineResult
+from apache_beam.runners.runner import PipelineRunner
+
+__all__ = ['PythonRPCDirectRunner']
+
+
+class PythonRPCDirectRunner(PipelineRunner):
+ """Executes a single pipeline on the local machine inside a container."""
+
+ # A list of PTransformOverride objects to be applied before running a pipeline
+ # using DirectRunner.
+ # Currently this only works for overrides where the input and output types do
+ # not change.
+ # For internal SDK use only. This should not be updated by Beam pipeline
+ # authors.
+ _PTRANSFORM_OVERRIDES = []
+
+ def __init__(self):
+ self._cache = None
+
+ def run(self, pipeline):
+ """Remotely executes entire pipeline or parts reachable from node."""
+
+ # Performing configured PTransform overrides.
+ pipeline.replace_all(PythonRPCDirectRunner._PTRANSFORM_OVERRIDES)
+
+ # Start the RPC co-process
+ manager = DockerRPCManager()
+
+ # Submit the job to the RPC co-process
+ jobName = ('Job-' +
+ ''.join(random.choice(string.ascii_uppercase) for _ in range(6)))
+ options = {k: v for k, v in pipeline._options.get_all_options().iteritems()
+ if v is not None}
+
+ try:
+ response = manager.service.run(beam_job_api_pb2.SubmitJobRequest(
+ pipeline=pipeline.to_runner_api(),
+ pipelineOptions=job_utils.dict_to_struct(options),
+ jobName=jobName))
+
+ logging.info('Submitted a job with id: %s', response.jobId)
+
+ # Return the result object that references the manager instance
+ result = PythonRPCDirectPipelineResult(response.jobId, manager)
+ return result
+ except grpc.RpcError:
+ logging.error('Failed to run the job with name: %s', jobName)
+ raise
+
+
+class PythonRPCDirectPipelineResult(PipelineResult):
+ """Represents the state of a pipeline run on the Dataflow service."""
+
+ def __init__(self, job_id, job_manager):
+ self.job_id = job_id
+ self.manager = job_manager
+
+ @property
+ def state(self):
+ return self.manager.service.getState(
+ beam_job_api_pb2.GetJobStateRequest(jobId=self.job_id))
+
+ def wait_until_finish(self):
+ messages_request = beam_job_api_pb2.JobMessagesRequest(jobId=self.job_id)
+ for message in self.manager.service.getMessageStream(messages_request):
+ if message.HasField('stateResponse'):
+ logging.info(
+ 'Current state of job: %s',
+ beam_job_api_pb2.JobState.Enum.Name(
+ message.stateResponse.state))
+ else:
+ logging.info('Message %s', message.messageResponse)
+ logging.info('Job with id: %s in terminal state now.', self.job_id)
+
+ def cancel(self):
+ return self.manager.service.cancel(
+ beam_job_api_pb2.CancelJobRequest(jobId=self.job_id))
+
+ def metrics(self):
+ raise NotImplementedError
diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py
new file mode 100644
index 0000000..4986dc4
--- /dev/null
+++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A runner implementation that submits a job for remote execution.
+"""
+import time
+import uuid
+from concurrent import futures
+
+import grpc
+
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.pipeline import Pipeline
+from apache_beam.portability.api import beam_job_api_pb2
+from apache_beam.portability.api import beam_job_api_pb2_grpc
+from apache_beam.runners.runner import PipelineState
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+class JobService(beam_job_api_pb2_grpc.JobServiceServicer):
+
+ def __init__(self):
+ self.jobs = {}
+
+ def run(self, request, context):
+ job_id = uuid.uuid4().get_hex()
+ pipeline_result = Pipeline.from_runner_api(
+ request.pipeline,
+ 'DirectRunner',
+ PipelineOptions()).run()
+ self.jobs[job_id] = pipeline_result
+ return beam_job_api_pb2.SubmitJobResponse(jobId=job_id)
+
+ def getState(self, request, context):
+ pipeline_result = self.jobs[request.jobId]
+ return beam_job_api_pb2.GetJobStateResponse(
+ state=self._map_state_to_jobState(pipeline_result.state))
+
+ def cancel(self, request, context):
+ pipeline_result = self.jobs[request.jobId]
+ pipeline_result.cancel()
+ return beam_job_api_pb2.CancelJobResponse(
+ state=self._map_state_to_jobState(pipeline_result.state))
+
+ def getMessageStream(self, request, context):
+ pipeline_result = self.jobs[request.jobId]
+ pipeline_result.wait_until_finish()
+ yield beam_job_api_pb2.JobMessagesResponse(
+ stateResponse=beam_job_api_pb2.GetJobStateResponse(
+ state=self._map_state_to_jobState(pipeline_result.state)))
+
+ def getStateStream(self, request, context):
+ context.set_details('Not Implemented for direct runner!')
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ return
+
+ @staticmethod
+ def _map_state_to_jobState(state):
+ if state == PipelineState.UNKNOWN:
+ return beam_job_api_pb2.JobState.UNSPECIFIED
+ elif state == PipelineState.STOPPED:
+ return beam_job_api_pb2.JobState.STOPPED
+ elif state == PipelineState.RUNNING:
+ return beam_job_api_pb2.JobState.RUNNING
+ elif state == PipelineState.DONE:
+ return beam_job_api_pb2.JobState.DONE
+ elif state == PipelineState.FAILED:
+ return beam_job_api_pb2.JobState.FAILED
+ elif state == PipelineState.CANCELLED:
+ return beam_job_api_pb2.JobState.CANCELLED
+ elif state == PipelineState.UPDATED:
+ return beam_job_api_pb2.JobState.UPDATED
+ elif state == PipelineState.DRAINING:
+ return beam_job_api_pb2.JobState.DRAINING
+ elif state == PipelineState.DRAINED:
+ return beam_job_api_pb2.JobState.DRAINED
+ else:
+ raise ValueError('Unknown pipeline state')
+
+
+def serve():
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+ beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(JobService(), server)
+
+ server.add_insecure_port('[::]:50051')
+ server.start()
+
+ try:
+ while True:
+ time.sleep(_ONE_DAY_IN_SECONDS)
+ except KeyboardInterrupt:
+ server.stop(0)
+
+
+if __name__ == '__main__':
+ serve()
diff --git a/sdks/python/apache_beam/runners/job/__init__.py b/sdks/python/apache_beam/runners/job/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/sdks/python/apache_beam/runners/job/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/sdks/python/apache_beam/runners/job/manager.py b/sdks/python/apache_beam/runners/job/manager.py
new file mode 100644
index 0000000..4d88a11
--- /dev/null
+++ b/sdks/python/apache_beam/runners/job/manager.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A object to control to the Job API Co-Process
+"""
+
+import logging
+import subprocess
+import time
+
+import grpc
+
+from apache_beam.portability.api import beam_job_api_pb2_grpc
+
+
+class DockerRPCManager(object):
+ """A native co-process to start a contianer that speaks the JobApi
+ """
+ def __init__(self, run_command=None):
+ # TODO(BEAM-2431): Change this to a docker container from a command.
+ self.process = subprocess.Popen(
+ ['python',
+ '-m',
+ 'apache_beam.runners.experimental.python_rpc_direct.server'])
+
+ self.channel = grpc.insecure_channel('localhost:50051')
+ self.service = beam_job_api_pb2_grpc.JobServiceStub(self.channel)
+
+ # Sleep for 2 seconds for process to start completely
+ # This is just for the co-process and would be removed
+ # once we migrate to docker.
+ time.sleep(2)
+
+ def __del__(self):
+ """Terminate the co-process when the manager is GC'ed
+ """
+ logging.info('Shutting the co-process')
+ self.process.terminate()
diff --git a/sdks/python/apache_beam/runners/job/utils.py b/sdks/python/apache_beam/runners/job/utils.py
new file mode 100644
index 0000000..84c727f
--- /dev/null
+++ b/sdks/python/apache_beam/runners/job/utils.py
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utility functions for efficiently processing with the job API
+"""
+
+import json
+
+from google.protobuf import json_format
+from google.protobuf import struct_pb2
+
+
+def dict_to_struct(dict_obj):
+ return json_format.Parse(json.dumps(dict_obj), struct_pb2.Struct())
+
+
+def struct_to_dict(struct_obj):
+ return json.loads(json_format.MessageToJson(struct_obj))
diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py
index 1c89d06..3506099 100644
--- a/sdks/python/apache_beam/runners/pipeline_context.py
+++ b/sdks/python/apache_beam/runners/pipeline_context.py
@@ -21,10 +21,11 @@
"""
+from apache_beam import coders
from apache_beam import pipeline
from apache_beam import pvalue
-from apache_beam import coders
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.transforms import core
@@ -39,20 +40,21 @@
self._obj_type = obj_type
self._obj_to_id = {}
self._id_to_obj = {}
- self._id_to_proto = proto_map if proto_map else {}
+ self._id_to_proto = dict(proto_map) if proto_map else {}
self._counter = 0
- def _unique_ref(self):
+ def _unique_ref(self, obj=None, label=None):
self._counter += 1
- return "ref_%s_%s" % (self._obj_type.__name__, self._counter)
+ return "ref_%s_%s_%s" % (
+ self._obj_type.__name__, label or type(obj).__name__, self._counter)
def populate_map(self, proto_map):
for id, proto in self._id_to_proto.items():
proto_map[id].CopyFrom(proto)
- def get_id(self, obj):
+ def get_id(self, obj, label=None):
if obj not in self._obj_to_id:
- id = self._unique_ref()
+ id = self._unique_ref(obj, label)
self._id_to_obj[id] = obj
self._obj_to_id[obj] = id
self._id_to_proto[id] = obj.to_runner_api(self._pipeline_context)
@@ -64,6 +66,12 @@
self._id_to_proto[id], self._pipeline_context)
return self._id_to_obj[id]
+ def __getitem__(self, id):
+ return self.get_by_id(id)
+
+ def __contains__(self, id):
+ return id in self._id_to_proto
+
class PipelineContext(object):
"""For internal use only; no backwards-compatibility guarantees.
@@ -79,11 +87,16 @@
# TODO: environment
}
- def __init__(self, context_proto=None):
+ def __init__(self, proto=None):
+ if isinstance(proto, beam_fn_api_pb2.ProcessBundleDescriptor):
+ proto = beam_runner_api_pb2.Components(
+ coders=dict(proto.coders.items()),
+ windowing_strategies=dict(proto.windowing_strategies.items()),
+ environments=dict(proto.environments.items()))
for name, cls in self._COMPONENT_TYPES.items():
setattr(
self, name, _PipelineContextMap(
- self, cls, getattr(context_proto, name, None)))
+ self, cls, getattr(proto, name, None)))
@staticmethod
def from_runner_api(proto):
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
index 2635559..838ce1e 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
@@ -17,27 +17,39 @@
"""A PipelineRunner using the SDK harness.
"""
+import base64
import collections
-import json
+import copy
import logging
import Queue as queue
import threading
-
-import grpc
+import time
from concurrent import futures
-import apache_beam as beam
+import grpc
+
+import apache_beam as beam # pylint: disable=ungrouped-imports
from apache_beam.coders import WindowedValueCoder
+from apache_beam.coders import registry
from apache_beam.coders.coder_impl import create_InputStream
from apache_beam.coders.coder_impl import create_OutputStream
from apache_beam.internal import pickler
from apache_beam.io import iobase
-from apache_beam.transforms.window import GlobalWindows
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.metrics.execution import MetricsEnvironment
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+from apache_beam.portability.api import beam_runner_api_pb2
+from apache_beam.runners import pipeline_context
from apache_beam.runners.portability import maptask_executor_runner
+from apache_beam.runners.runner import PipelineState
+from apache_beam.runners.worker import bundle_processor
from apache_beam.runners.worker import data_plane
from apache_beam.runners.worker import operation_specs
from apache_beam.runners.worker import sdk_worker
+from apache_beam.transforms import trigger
+from apache_beam.transforms.window import GlobalWindows
+from apache_beam.utils import proto_utils
+from apache_beam.utils import urns
# This module is experimental. No backwards-compatibility guarantees.
@@ -53,12 +65,15 @@
self._push_queue = queue.Queue()
self._pull_queue = queue.Queue()
setattr(self, method_name, self.run)
- self._read_thread = threading.Thread(target=self._read)
+ self._read_thread = threading.Thread(
+ name='streaming_rpc_handler_read', target=self._read)
+ self._started = False
def run(self, iterator, context):
self._inputs = iterator
# Note: We only support one client for now.
self._read_thread.start()
+ self._started = True
while True:
to_push = self._push_queue.get()
if to_push is self._DONE:
@@ -80,6 +95,9 @@
def done(self):
self.push(self._DONE)
+ # Can't join a thread before it's started.
+ while not self._started:
+ time.sleep(.01)
self._read_thread.join()
return StreamingRpcHandler()
@@ -108,11 +126,74 @@
beam.transforms.core.Windowing(GlobalWindows())))
+class _GroupingBuffer(object):
+ """Used to accumulate groupded (shuffled) results."""
+ def __init__(self, pre_grouped_coder, post_grouped_coder, windowing):
+ self._key_coder = pre_grouped_coder.key_coder()
+ self._pre_grouped_coder = pre_grouped_coder
+ self._post_grouped_coder = post_grouped_coder
+ self._table = collections.defaultdict(list)
+ self._windowing = windowing
+
+ def append(self, elements_data):
+ input_stream = create_InputStream(elements_data)
+ while input_stream.size() > 0:
+ windowed_key_value = self._pre_grouped_coder.get_impl(
+ ).decode_from_stream(input_stream, True)
+ key = windowed_key_value.value[0]
+ windowed_value = windowed_key_value.with_value(
+ windowed_key_value.value[1])
+ self._table[self._key_coder.encode(key)].append(windowed_value)
+
+ def __iter__(self):
+ output_stream = create_OutputStream()
+ trigger_driver = trigger.create_trigger_driver(self._windowing, True)
+ for encoded_key, windowed_values in self._table.items():
+ key = self._key_coder.decode(encoded_key)
+ for wkvs in trigger_driver.process_entire_key(key, windowed_values):
+ self._post_grouped_coder.get_impl().encode_to_stream(
+ wkvs, output_stream, True)
+ return iter([output_stream.get()])
+
+
+class _WindowGroupingBuffer(object):
+ """Used to partition windowed side inputs."""
+ def __init__(self, side_input_data):
+ # Here's where we would use a different type of partitioning
+ # (e.g. also by key) for a different access pattern.
+ assert side_input_data.access_pattern == urns.ITERABLE_ACCESS
+ self._windowed_value_coder = side_input_data.coder
+ self._window_coder = side_input_data.coder.window_coder
+ self._value_coder = side_input_data.coder.wrapped_value_coder
+ self._values_by_window = collections.defaultdict(list)
+
+ def append(self, elements_data):
+ input_stream = create_InputStream(elements_data)
+ while input_stream.size() > 0:
+ windowed_value = self._windowed_value_coder.get_impl(
+ ).decode_from_stream(input_stream, True)
+ for window in windowed_value.windows:
+ self._values_by_window[window].append(windowed_value.value)
+
+ def items(self):
+ value_coder_impl = self._value_coder.get_impl()
+ for window, values in self._values_by_window.items():
+ encoded_window = self._window_coder.encode(window)
+ output_stream = create_OutputStream()
+ for value in values:
+ value_coder_impl.encode_to_stream(value, output_stream, True)
+ yield encoded_window, output_stream.get()
+
+
class FnApiRunner(maptask_executor_runner.MapTaskExecutorRunner):
- def __init__(self):
+ def __init__(self, use_grpc=False, sdk_harness_factory=None):
super(FnApiRunner, self).__init__()
self._last_uid = -1
+ self._use_grpc = use_grpc
+ if sdk_harness_factory and not use_grpc:
+ raise ValueError('GRPC must be used if a harness factory is provided.')
+ self._sdk_harness_factory = sdk_harness_factory
def has_metrics_support(self):
return False
@@ -121,193 +202,761 @@
self._last_uid += 1
return str(self._last_uid)
- def _map_task_registration(self, map_task, state_handler,
- data_operation_spec):
- input_data = {}
- runner_sinks = {}
- transforms = []
- transform_index_to_id = {}
+ def run(self, pipeline):
+ MetricsEnvironment.set_metrics_supported(self.has_metrics_support())
+ if pipeline._verify_runner_api_compatible():
+ return self.run_via_runner_api(pipeline.to_runner_api())
+ else:
+ return super(FnApiRunner, self).run(pipeline)
- # Maps coders to new coder objects and references.
- coders = {}
+ def run_via_runner_api(self, pipeline_proto):
+ return self.run_stages(*self.create_stages(pipeline_proto))
- def coder_id(coder):
- if coder not in coders:
- coders[coder] = beam_fn_api_pb2.Coder(
- function_spec=sdk_worker.pack_function_spec_data(
- json.dumps(coder.as_cloud_object()),
- sdk_worker.PYTHON_CODER_URN, id=self._next_uid()))
+ def create_stages(self, pipeline_proto):
- return coders[coder].function_spec.id
+ # First define a couple of helpers.
- def output_tags(op):
- return getattr(op, 'output_tags', ['out'])
-
- def as_target(op_input):
- input_op_index, input_output_index = op_input
- input_op = map_task[input_op_index][1]
- return {
- 'ignored_input_tag':
- beam_fn_api_pb2.Target.List(target=[
- beam_fn_api_pb2.Target(
- primitive_transform_reference=transform_index_to_id[
- input_op_index],
- name=output_tags(input_op)[input_output_index])
- ])
- }
-
- def outputs(op):
- return {
- tag: beam_fn_api_pb2.PCollection(coder_reference=coder_id(coder))
- for tag, coder in zip(output_tags(op), op.output_coders)
- }
-
- for op_ix, (stage_name, operation) in enumerate(map_task):
- transform_id = transform_index_to_id[op_ix] = self._next_uid()
- if isinstance(operation, operation_specs.WorkerInMemoryWrite):
- # Write this data back to the runner.
- fn = beam_fn_api_pb2.FunctionSpec(urn=sdk_worker.DATA_OUTPUT_URN,
- id=self._next_uid())
- if data_operation_spec:
- fn.data.Pack(data_operation_spec)
- inputs = as_target(operation.input)
- side_inputs = {}
- runner_sinks[(transform_id, 'out')] = operation
-
- elif isinstance(operation, operation_specs.WorkerRead):
- # A Read is either translated to a direct injection of windowed values
- # into the sdk worker, or an injection of the source object into the
- # sdk worker as data followed by an SDF that reads that source.
- if (isinstance(operation.source.source,
- worker_runner_base.InMemorySource)
- and isinstance(operation.source.source.default_output_coder(),
- WindowedValueCoder)):
- output_stream = create_OutputStream()
- element_coder = (
- operation.source.source.default_output_coder().get_impl())
- # Re-encode the elements in the nested context and
- # concatenate them together
- for element in operation.source.source.read(None):
- element_coder.encode_to_stream(element, output_stream, True)
- target_name = self._next_uid()
- input_data[(transform_id, target_name)] = output_stream.get()
- fn = beam_fn_api_pb2.FunctionSpec(urn=sdk_worker.DATA_INPUT_URN,
- id=self._next_uid())
- if data_operation_spec:
- fn.data.Pack(data_operation_spec)
- inputs = {target_name: beam_fn_api_pb2.Target.List()}
- side_inputs = {}
- else:
- # Read the source object from the runner.
- source_coder = beam.coders.DillCoder()
- input_transform_id = self._next_uid()
- output_stream = create_OutputStream()
- source_coder.get_impl().encode_to_stream(
- GlobalWindows.windowed_value(operation.source),
- output_stream,
- True)
- target_name = self._next_uid()
- input_data[(input_transform_id, target_name)] = output_stream.get()
- input_ptransform = beam_fn_api_pb2.PrimitiveTransform(
- id=input_transform_id,
- function_spec=beam_fn_api_pb2.FunctionSpec(
- urn=sdk_worker.DATA_INPUT_URN,
- id=self._next_uid()),
- # TODO(robertwb): Possible name collision.
- step_name=stage_name + '/inject_source',
- inputs={target_name: beam_fn_api_pb2.Target.List()},
- outputs={
- 'out':
- beam_fn_api_pb2.PCollection(
- coder_reference=coder_id(source_coder))
- })
- if data_operation_spec:
- input_ptransform.function_spec.data.Pack(data_operation_spec)
- transforms.append(input_ptransform)
-
- # Read the elements out of the source.
- fn = sdk_worker.pack_function_spec_data(
- OLDE_SOURCE_SPLITTABLE_DOFN_DATA,
- sdk_worker.PYTHON_DOFN_URN,
- id=self._next_uid())
- inputs = {
- 'ignored_input_tag':
- beam_fn_api_pb2.Target.List(target=[
- beam_fn_api_pb2.Target(
- primitive_transform_reference=input_transform_id,
- name='out')
- ])
- }
- side_inputs = {}
-
- elif isinstance(operation, operation_specs.WorkerDoFn):
- fn = sdk_worker.pack_function_spec_data(
- operation.serialized_fn,
- sdk_worker.PYTHON_DOFN_URN,
- id=self._next_uid())
- inputs = as_target(operation.input)
- # Store the contents of each side input for state access.
- for si in operation.side_inputs:
- assert isinstance(si.source, iobase.BoundedSource)
- element_coder = si.source.default_output_coder()
- view_id = self._next_uid()
- # TODO(robertwb): Actually flesh out the ViewFn API.
- side_inputs[si.tag] = beam_fn_api_pb2.SideInput(
- view_fn=sdk_worker.serialize_and_pack_py_fn(
- element_coder, urn=sdk_worker.PYTHON_ITERABLE_VIEWFN_URN,
- id=view_id))
- # Re-encode the elements in the nested context and
- # concatenate them together
- output_stream = create_OutputStream()
- for element in si.source.read(
- si.source.get_range_tracker(None, None)):
- element_coder.get_impl().encode_to_stream(
- element, output_stream, True)
- elements_data = output_stream.get()
- state_key = beam_fn_api_pb2.StateKey(function_spec_reference=view_id)
- state_handler.Clear(state_key)
- state_handler.Append(
- beam_fn_api_pb2.SimpleStateAppendRequest(
- state_key=state_key, data=[elements_data]))
-
- elif isinstance(operation, operation_specs.WorkerFlatten):
- fn = sdk_worker.pack_function_spec_data(
- operation.serialized_fn,
- sdk_worker.IDENTITY_DOFN_URN,
- id=self._next_uid())
- inputs = {
- 'ignored_input_tag':
- beam_fn_api_pb2.Target.List(target=[
- beam_fn_api_pb2.Target(
- primitive_transform_reference=transform_index_to_id[
- input_op_index],
- name=output_tags(map_task[input_op_index][1])[
- input_output_index])
- for input_op_index, input_output_index in operation.inputs
- ])
- }
- side_inputs = {}
-
+ def union(a, b):
+ # Minimize the number of distinct sets.
+ if not a or a == b:
+ return b
+ elif not b:
+ return a
else:
- raise TypeError(operation)
+ return frozenset.union(a, b)
- ptransform = beam_fn_api_pb2.PrimitiveTransform(
- id=transform_id,
- function_spec=fn,
- step_name=stage_name,
- inputs=inputs,
- side_inputs=side_inputs,
- outputs=outputs(operation))
- transforms.append(ptransform)
+ class Stage(object):
+ """A set of Transforms that can be sent to the worker for processing."""
+ def __init__(self, name, transforms,
+ downstream_side_inputs=None, must_follow=frozenset()):
+ self.name = name
+ self.transforms = transforms
+ self.downstream_side_inputs = downstream_side_inputs
+ self.must_follow = must_follow
+
+ def __repr__(self):
+ must_follow = ', '.join(prev.name for prev in self.must_follow)
+ downstream_side_inputs = ', '.join(
+ str(si) for si in self.downstream_side_inputs)
+ return "%s\n %s\n must follow: %s\n downstream_side_inputs: %s" % (
+ self.name,
+ '\n'.join(["%s:%s" % (transform.unique_name, transform.spec.urn)
+ for transform in self.transforms]),
+ must_follow,
+ downstream_side_inputs)
+
+ def can_fuse(self, consumer):
+ def no_overlap(a, b):
+ return not a.intersection(b)
+ return (
+ not self in consumer.must_follow
+ and not self.is_flatten() and not consumer.is_flatten()
+ and no_overlap(self.downstream_side_inputs, consumer.side_inputs()))
+
+ def fuse(self, other):
+ return Stage(
+ "(%s)+(%s)" % (self.name, other.name),
+ self.transforms + other.transforms,
+ union(self.downstream_side_inputs, other.downstream_side_inputs),
+ union(self.must_follow, other.must_follow))
+
+ def is_flatten(self):
+ return any(transform.spec.urn == urns.FLATTEN_TRANSFORM
+ for transform in self.transforms)
+
+ def side_inputs(self):
+ for transform in self.transforms:
+ if transform.spec.urn == urns.PARDO_TRANSFORM:
+ payload = proto_utils.parse_Bytes(
+ transform.spec.payload, beam_runner_api_pb2.ParDoPayload)
+ for side_input in payload.side_inputs:
+ yield transform.inputs[side_input]
+
+ def has_as_main_input(self, pcoll):
+ for transform in self.transforms:
+ if transform.spec.urn == urns.PARDO_TRANSFORM:
+ payload = proto_utils.parse_Bytes(
+ transform.spec.payload, beam_runner_api_pb2.ParDoPayload)
+ local_side_inputs = payload.side_inputs
+ else:
+ local_side_inputs = {}
+ for local_id, pipeline_id in transform.inputs.items():
+ if pcoll == pipeline_id and local_id not in local_side_inputs:
+ return True
+
+ def deduplicate_read(self):
+ seen_pcolls = set()
+ new_transforms = []
+ for transform in self.transforms:
+ if transform.spec.urn == bundle_processor.DATA_INPUT_URN:
+ pcoll = only_element(transform.outputs.items())[1]
+ if pcoll in seen_pcolls:
+ continue
+ seen_pcolls.add(pcoll)
+ new_transforms.append(transform)
+ self.transforms = new_transforms
+
+ # Now define the "optimization" phases.
+
+ safe_coders = {}
+
+ def expand_gbk(stages):
+ """Transforms each GBK into a write followed by a read.
+ """
+ good_coder_urns = set(beam.coders.Coder._known_urns.keys()) - set([
+ urns.PICKLED_CODER])
+ coders = pipeline_components.coders
+
+ for coder_id, coder_proto in coders.items():
+ if coder_proto.spec.spec.urn == urns.BYTES_CODER:
+ bytes_coder_id = coder_id
+ break
+ else:
+ bytes_coder_id = unique_name(coders, 'bytes_coder')
+ pipeline_components.coders[bytes_coder_id].CopyFrom(
+ beam.coders.BytesCoder().to_runner_api(None))
+
+ coder_substitutions = {}
+
+ def wrap_unknown_coders(coder_id, with_bytes):
+ if (coder_id, with_bytes) not in coder_substitutions:
+ wrapped_coder_id = None
+ coder_proto = coders[coder_id]
+ if coder_proto.spec.spec.urn == urns.LENGTH_PREFIX_CODER:
+ coder_substitutions[coder_id, with_bytes] = (
+ bytes_coder_id if with_bytes else coder_id)
+ elif coder_proto.spec.spec.urn in good_coder_urns:
+ wrapped_components = [wrap_unknown_coders(c, with_bytes)
+ for c in coder_proto.component_coder_ids]
+ if wrapped_components == list(coder_proto.component_coder_ids):
+ # Use as is.
+ coder_substitutions[coder_id, with_bytes] = coder_id
+ else:
+ wrapped_coder_id = unique_name(
+ coders,
+ coder_id + ("_bytes" if with_bytes else "_len_prefix"))
+ coders[wrapped_coder_id].CopyFrom(coder_proto)
+ coders[wrapped_coder_id].component_coder_ids[:] = [
+ wrap_unknown_coders(c, with_bytes)
+ for c in coder_proto.component_coder_ids]
+ coder_substitutions[coder_id, with_bytes] = wrapped_coder_id
+ else:
+ # Not a known coder.
+ if with_bytes:
+ coder_substitutions[coder_id, with_bytes] = bytes_coder_id
+ else:
+ wrapped_coder_id = unique_name(coders, coder_id + "_len_prefix")
+ len_prefix_coder_proto = beam_runner_api_pb2.Coder(
+ spec=beam_runner_api_pb2.SdkFunctionSpec(
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=urns.LENGTH_PREFIX_CODER)),
+ component_coder_ids=[coder_id])
+ coders[wrapped_coder_id].CopyFrom(len_prefix_coder_proto)
+ coder_substitutions[coder_id, with_bytes] = wrapped_coder_id
+ # This operation is idempotent.
+ if wrapped_coder_id:
+ coder_substitutions[wrapped_coder_id, with_bytes] = wrapped_coder_id
+ return coder_substitutions[coder_id, with_bytes]
+
+ def fix_pcoll_coder(pcoll):
+ new_coder_id = wrap_unknown_coders(pcoll.coder_id, False)
+ safe_coders[new_coder_id] = wrap_unknown_coders(pcoll.coder_id, True)
+ pcoll.coder_id = new_coder_id
+
+ for stage in stages:
+ assert len(stage.transforms) == 1
+ transform = stage.transforms[0]
+ if transform.spec.urn == urns.GROUP_BY_KEY_TRANSFORM:
+ for pcoll_id in transform.inputs.values():
+ fix_pcoll_coder(pipeline_components.pcollections[pcoll_id])
+ for pcoll_id in transform.outputs.values():
+ fix_pcoll_coder(pipeline_components.pcollections[pcoll_id])
+
+ # This is used later to correlate the read and write.
+ param = str("group:%s" % stage.name)
+ gbk_write = Stage(
+ transform.unique_name + '/Write',
+ [beam_runner_api_pb2.PTransform(
+ unique_name=transform.unique_name + '/Write',
+ inputs=transform.inputs,
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_OUTPUT_URN,
+ payload=param))],
+ downstream_side_inputs=frozenset(),
+ must_follow=stage.must_follow)
+ yield gbk_write
+
+ yield Stage(
+ transform.unique_name + '/Read',
+ [beam_runner_api_pb2.PTransform(
+ unique_name=transform.unique_name + '/Read',
+ outputs=transform.outputs,
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_INPUT_URN,
+ payload=param))],
+ downstream_side_inputs=frozenset(),
+ must_follow=union(frozenset([gbk_write]), stage.must_follow))
+ else:
+ yield stage
+
+ def sink_flattens(stages):
+ """Sink flattens and remove them from the graph.
+
+ A flatten that cannot be sunk/fused away becomes multiple writes (to the
+ same logical sink) followed by a read.
+ """
+ # TODO(robertwb): Actually attempt to sink rather than always materialize.
+ # TODO(robertwb): Possibly fuse this into one of the stages.
+ pcollections = pipeline_components.pcollections
+ for stage in stages:
+ assert len(stage.transforms) == 1
+ transform = stage.transforms[0]
+ if transform.spec.urn == urns.FLATTEN_TRANSFORM:
+ # This is used later to correlate the read and writes.
+ param = str("materialize:%s" % transform.unique_name)
+ output_pcoll_id, = transform.outputs.values()
+ output_coder_id = pcollections[output_pcoll_id].coder_id
+ flatten_writes = []
+ for local_in, pcoll_in in transform.inputs.items():
+
+ if pcollections[pcoll_in].coder_id != output_coder_id:
+ # Flatten inputs must all be written with the same coder as is
+ # used to read them.
+ pcollections[pcoll_in].coder_id = output_coder_id
+ transcoded_pcollection = (
+ transform.unique_name + '/Transcode/' + local_in + '/out')
+ yield Stage(
+ transform.unique_name + '/Transcode/' + local_in,
+ [beam_runner_api_pb2.PTransform(
+ unique_name=
+ transform.unique_name + '/Transcode/' + local_in,
+ inputs={local_in: pcoll_in},
+ outputs={'out': transcoded_pcollection},
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.IDENTITY_DOFN_URN))],
+ downstream_side_inputs=frozenset(),
+ must_follow=stage.must_follow)
+ pcollections[transcoded_pcollection].CopyFrom(
+ pcollections[pcoll_in])
+ pcollections[transcoded_pcollection].coder_id = output_coder_id
+ else:
+ transcoded_pcollection = pcoll_in
+
+ flatten_write = Stage(
+ transform.unique_name + '/Write/' + local_in,
+ [beam_runner_api_pb2.PTransform(
+ unique_name=transform.unique_name + '/Write/' + local_in,
+ inputs={local_in: transcoded_pcollection},
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_OUTPUT_URN,
+ payload=param))],
+ downstream_side_inputs=frozenset(),
+ must_follow=stage.must_follow)
+ flatten_writes.append(flatten_write)
+ yield flatten_write
+
+ yield Stage(
+ transform.unique_name + '/Read',
+ [beam_runner_api_pb2.PTransform(
+ unique_name=transform.unique_name + '/Read',
+ outputs=transform.outputs,
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_INPUT_URN,
+ payload=param))],
+ downstream_side_inputs=frozenset(),
+ must_follow=union(frozenset(flatten_writes), stage.must_follow))
+
+ else:
+ yield stage
+
+ def annotate_downstream_side_inputs(stages):
+ """Annotate each stage with fusion-prohibiting information.
+
+ Each stage is annotated with the (transitive) set of pcollections that
+ depend on this stage that are also used later in the pipeline as a
+ side input.
+
+ While theoretically this could result in O(n^2) annotations, the size of
+ each set is bounded by the number of side inputs (typically much smaller
+ than the number of total nodes) and the number of *distinct* side-input
+ sets is also generally small (and shared due to the use of union
+ defined above).
+
+ This representation is also amenable to simple recomputation on fusion.
+ """
+ consumers = collections.defaultdict(list)
+ all_side_inputs = set()
+ for stage in stages:
+ for transform in stage.transforms:
+ for input in transform.inputs.values():
+ consumers[input].append(stage)
+ for si in stage.side_inputs():
+ all_side_inputs.add(si)
+ all_side_inputs = frozenset(all_side_inputs)
+
+ downstream_side_inputs_by_stage = {}
+
+ def compute_downstream_side_inputs(stage):
+ if stage not in downstream_side_inputs_by_stage:
+ downstream_side_inputs = frozenset()
+ for transform in stage.transforms:
+ for output in transform.outputs.values():
+ if output in all_side_inputs:
+ downstream_side_inputs = union(
+ downstream_side_inputs, frozenset([output]))
+ for consumer in consumers[output]:
+ downstream_side_inputs = union(
+ downstream_side_inputs,
+ compute_downstream_side_inputs(consumer))
+ downstream_side_inputs_by_stage[stage] = downstream_side_inputs
+ return downstream_side_inputs_by_stage[stage]
+
+ for stage in stages:
+ stage.downstream_side_inputs = compute_downstream_side_inputs(stage)
+ return stages
+
+ def greedily_fuse(stages):
+ """Places transforms sharing an edge in the same stage, whenever possible.
+ """
+ producers_by_pcoll = {}
+ consumers_by_pcoll = collections.defaultdict(list)
+
+ # Used to always reference the correct stage as the producer and
+ # consumer maps are not updated when stages are fused away.
+ replacements = {}
+
+ def replacement(s):
+ old_ss = []
+ while s in replacements:
+ old_ss.append(s)
+ s = replacements[s]
+ for old_s in old_ss[:-1]:
+ replacements[old_s] = s
+ return s
+
+ def fuse(producer, consumer):
+ fused = producer.fuse(consumer)
+ replacements[producer] = fused
+ replacements[consumer] = fused
+
+ # First record the producers and consumers of each PCollection.
+ for stage in stages:
+ for transform in stage.transforms:
+ for input in transform.inputs.values():
+ consumers_by_pcoll[input].append(stage)
+ for output in transform.outputs.values():
+ producers_by_pcoll[output] = stage
+
+ logging.debug('consumers\n%s', consumers_by_pcoll)
+ logging.debug('producers\n%s', producers_by_pcoll)
+
+ # Now try to fuse away all pcollections.
+ for pcoll, producer in producers_by_pcoll.items():
+ pcoll_as_param = str("materialize:%s" % pcoll)
+ write_pcoll = None
+ for consumer in consumers_by_pcoll[pcoll]:
+ producer = replacement(producer)
+ consumer = replacement(consumer)
+ # Update consumer.must_follow set, as it's used in can_fuse.
+ consumer.must_follow = frozenset(
+ replacement(s) for s in consumer.must_follow)
+ if producer.can_fuse(consumer):
+ fuse(producer, consumer)
+ else:
+ # If we can't fuse, do a read + write.
+ if write_pcoll is None:
+ write_pcoll = Stage(
+ pcoll + '/Write',
+ [beam_runner_api_pb2.PTransform(
+ unique_name=pcoll + '/Write',
+ inputs={'in': pcoll},
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_OUTPUT_URN,
+ payload=pcoll_as_param))])
+ fuse(producer, write_pcoll)
+ if consumer.has_as_main_input(pcoll):
+ read_pcoll = Stage(
+ pcoll + '/Read',
+ [beam_runner_api_pb2.PTransform(
+ unique_name=pcoll + '/Read',
+ outputs={'out': pcoll},
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_INPUT_URN,
+ payload=pcoll_as_param))],
+ must_follow=frozenset([write_pcoll]))
+ fuse(read_pcoll, consumer)
+ else:
+ consumer.must_follow = union(
+ consumer.must_follow, frozenset([write_pcoll]))
+
+ # Everything that was originally a stage or a replacement, but wasn't
+ # replaced, should be in the final graph.
+ final_stages = frozenset(stages).union(replacements.values()).difference(
+ replacements.keys())
+
+ for stage in final_stages:
+ # Update all references to their final values before throwing
+ # the replacement data away.
+ stage.must_follow = frozenset(replacement(s) for s in stage.must_follow)
+ # Two reads of the same stage may have been fused. This is unneeded.
+ stage.deduplicate_read()
+ return final_stages
+
+ def sort_stages(stages):
+ """Order stages suitable for sequential execution.
+ """
+ seen = set()
+ ordered = []
+
+ def process(stage):
+ if stage not in seen:
+ seen.add(stage)
+ for prev in stage.must_follow:
+ process(prev)
+ ordered.append(stage)
+ for stage in stages:
+ process(stage)
+ return ordered
+
+ # Now actually apply the operations.
+
+ pipeline_components = copy.deepcopy(pipeline_proto.components)
+
+ # Reify coders.
+ # TODO(BEAM-2717): Remove once Coders are already in proto.
+ coders = pipeline_context.PipelineContext(pipeline_components).coders
+ for pcoll in pipeline_components.pcollections.values():
+ if pcoll.coder_id not in coders:
+ window_coder = coders[
+ pipeline_components.windowing_strategies[
+ pcoll.windowing_strategy_id].window_coder_id]
+ coder = WindowedValueCoder(
+ registry.get_coder(pickler.loads(pcoll.coder_id)),
+ window_coder=window_coder)
+ pcoll.coder_id = coders.get_id(coder)
+ coders.populate_map(pipeline_components.coders)
+
+ known_composites = set([urns.GROUP_BY_KEY_TRANSFORM])
+
+ def leaf_transforms(root_ids):
+ for root_id in root_ids:
+ root = pipeline_proto.components.transforms[root_id]
+ if root.spec.urn in known_composites or not root.subtransforms:
+ yield root_id
+ else:
+ for leaf in leaf_transforms(root.subtransforms):
+ yield leaf
+
+ # Initial set of stages are singleton leaf transforms.
+ stages = [
+ Stage(name, [pipeline_proto.components.transforms[name]])
+ for name in leaf_transforms(pipeline_proto.root_transform_ids)]
+
+ # Apply each phase in order.
+ for phase in [
+ annotate_downstream_side_inputs, expand_gbk, sink_flattens,
+ greedily_fuse, sort_stages]:
+ logging.info('%s %s %s', '=' * 20, phase, '=' * 20)
+ stages = list(phase(stages))
+ logging.debug('Stages: %s', [str(s) for s in stages])
+
+ # Return the (possibly mutated) context and ordered set of stages.
+ return pipeline_components, stages, safe_coders
+
+ def run_stages(self, pipeline_components, stages, safe_coders):
+
+ if self._use_grpc:
+ controller = FnApiRunner.GrpcController(self._sdk_harness_factory)
+ else:
+ controller = FnApiRunner.DirectController()
+ metrics_by_stage = {}
+
+ try:
+ pcoll_buffers = collections.defaultdict(list)
+ for stage in stages:
+ metrics_by_stage[stage.name] = self.run_stage(
+ controller, pipeline_components, stage,
+ pcoll_buffers, safe_coders).process_bundle.metrics
+ finally:
+ controller.close()
+
+ return RunnerResult(PipelineState.DONE, metrics_by_stage)
+
+ def run_stage(
+ self, controller, pipeline_components, stage, pcoll_buffers, safe_coders):
+
+ context = pipeline_context.PipelineContext(pipeline_components)
+ data_operation_spec = controller.data_operation_spec()
+
+ def extract_endpoints(stage):
+ # Returns maps of transform names to PCollection identifiers.
+ # Also mutates IO stages to point to the data data_operation_spec.
+ data_input = {}
+ data_side_input = {}
+ data_output = {}
+ for transform in stage.transforms:
+ if transform.spec.urn in (bundle_processor.DATA_INPUT_URN,
+ bundle_processor.DATA_OUTPUT_URN):
+ pcoll_id = transform.spec.payload
+ if transform.spec.urn == bundle_processor.DATA_INPUT_URN:
+ target = transform.unique_name, only_element(transform.outputs)
+ data_input[target] = pcoll_id
+ elif transform.spec.urn == bundle_processor.DATA_OUTPUT_URN:
+ target = transform.unique_name, only_element(transform.inputs)
+ data_output[target] = pcoll_id
+ else:
+ raise NotImplementedError
+ if data_operation_spec:
+ transform.spec.payload = data_operation_spec.SerializeToString()
+ else:
+ transform.spec.payload = ""
+ elif transform.spec.urn == urns.PARDO_TRANSFORM:
+ payload = proto_utils.parse_Bytes(
+ transform.spec.payload, beam_runner_api_pb2.ParDoPayload)
+ for tag, si in payload.side_inputs.items():
+ data_side_input[transform.unique_name, tag] = (
+ 'materialize:' + transform.inputs[tag],
+ beam.pvalue.SideInputData.from_runner_api(si, None))
+ return data_input, data_side_input, data_output
+
+ logging.info('Running %s', stage.name)
+ logging.debug(' %s', stage)
+ data_input, data_side_input, data_output = extract_endpoints(stage)
process_bundle_descriptor = beam_fn_api_pb2.ProcessBundleDescriptor(
- id=self._next_uid(), coders=coders.values(),
- primitive_transform=transforms)
+ id=self._next_uid(),
+ transforms={transform.unique_name: transform
+ for transform in stage.transforms},
+ pcollections=dict(pipeline_components.pcollections.items()),
+ coders=dict(pipeline_components.coders.items()),
+ windowing_strategies=dict(
+ pipeline_components.windowing_strategies.items()),
+ environments=dict(pipeline_components.environments.items()))
+
+ process_bundle_registration = beam_fn_api_pb2.InstructionRequest(
+ instruction_id=self._next_uid(),
+ register=beam_fn_api_pb2.RegisterRequest(
+ process_bundle_descriptor=[process_bundle_descriptor]))
+
+ process_bundle = beam_fn_api_pb2.InstructionRequest(
+ instruction_id=self._next_uid(),
+ process_bundle=beam_fn_api_pb2.ProcessBundleRequest(
+ process_bundle_descriptor_reference=
+ process_bundle_descriptor.id))
+
+ # Write all the input data to the channel.
+ for (transform_id, name), pcoll_id in data_input.items():
+ data_out = controller.data_plane_handler.output_stream(
+ process_bundle.instruction_id, beam_fn_api_pb2.Target(
+ primitive_transform_reference=transform_id, name=name))
+ for element_data in pcoll_buffers[pcoll_id]:
+ data_out.write(element_data)
+ data_out.close()
+
+ # Store the required side inputs into state.
+ for (transform_id, tag), (pcoll_id, si) in data_side_input.items():
+ elements_by_window = _WindowGroupingBuffer(si)
+ for element_data in pcoll_buffers[pcoll_id]:
+ elements_by_window.append(element_data)
+ for window, elements_data in elements_by_window.items():
+ state_key = beam_fn_api_pb2.StateKey(
+ multimap_side_input=beam_fn_api_pb2.StateKey.MultimapSideInput(
+ ptransform_id=transform_id,
+ side_input_id=tag,
+ window=window))
+ controller.state_handler.blocking_append(
+ state_key, elements_data, process_bundle.instruction_id)
+
+ # Register and start running the bundle.
+ controller.control_handler.push(process_bundle_registration)
+ controller.control_handler.push(process_bundle)
+
+ # Wait for the bundle to finish.
+ while True:
+ result = controller.control_handler.pull()
+ if result and result.instruction_id == process_bundle.instruction_id:
+ if result.error:
+ raise RuntimeError(result.error)
+ break
+
+ # Gather all output data.
+ expected_targets = [
+ beam_fn_api_pb2.Target(primitive_transform_reference=transform_id,
+ name=output_name)
+ for (transform_id, output_name), _ in data_output.items()]
+ for output in controller.data_plane_handler.input_elements(
+ process_bundle.instruction_id, expected_targets):
+ target_tuple = (
+ output.target.primitive_transform_reference, output.target.name)
+ if target_tuple in data_output:
+ pcoll_id = data_output[target_tuple]
+ if pcoll_id.startswith('materialize:'):
+ # Just store the data chunks for replay.
+ pcoll_buffers[pcoll_id].append(output.data)
+ elif pcoll_id.startswith('group:'):
+ # This is a grouping write, create a grouping buffer if needed.
+ if pcoll_id not in pcoll_buffers:
+ original_gbk_transform = pcoll_id.split(':', 1)[1]
+ transform_proto = pipeline_components.transforms[
+ original_gbk_transform]
+ input_pcoll = only_element(transform_proto.inputs.values())
+ output_pcoll = only_element(transform_proto.outputs.values())
+ pre_gbk_coder = context.coders[safe_coders[
+ pipeline_components.pcollections[input_pcoll].coder_id]]
+ post_gbk_coder = context.coders[safe_coders[
+ pipeline_components.pcollections[output_pcoll].coder_id]]
+ windowing_strategy = context.windowing_strategies[
+ pipeline_components
+ .pcollections[output_pcoll].windowing_strategy_id]
+ pcoll_buffers[pcoll_id] = _GroupingBuffer(
+ pre_gbk_coder, post_gbk_coder, windowing_strategy)
+ pcoll_buffers[pcoll_id].append(output.data)
+ else:
+ # These should be the only two identifiers we produce for now,
+ # but special side input writes may go here.
+ raise NotImplementedError(pcoll_id)
+ return result
+
+ # This is the "old" way of executing pipelines.
+ # TODO(robertwb): Remove once runner API supports side inputs.
+
+ def _map_task_registration(self, map_task, state_handler,
+ data_operation_spec):
+ input_data, side_input_data, runner_sinks, process_bundle_descriptor = (
+ self._map_task_to_protos(map_task, data_operation_spec))
+ # Side inputs will be accessed over the state API.
+ for key, elements_data in side_input_data.items():
+ state_key = beam_fn_api_pb2.StateKey.MultimapSideInput(key=key)
+ state_handler.Clear(state_key)
+ state_handler.Append(state_key, [elements_data])
return beam_fn_api_pb2.InstructionRequest(
instruction_id=self._next_uid(),
register=beam_fn_api_pb2.RegisterRequest(
- process_bundle_descriptor=[process_bundle_descriptor
- ])), runner_sinks, input_data
+ process_bundle_descriptor=[process_bundle_descriptor])
+ ), runner_sinks, input_data
+
+ def _map_task_to_protos(self, map_task, data_operation_spec):
+ input_data = {}
+ side_input_data = {}
+ runner_sinks = {}
+
+ context = pipeline_context.PipelineContext()
+ transform_protos = {}
+ used_pcollections = {}
+
+ def uniquify(*names):
+ # An injective mapping from string* to string.
+ return ':'.join("%s:%d" % (name, len(name)) for name in names)
+
+ def pcollection_id(op_ix, out_ix):
+ if (op_ix, out_ix) not in used_pcollections:
+ used_pcollections[op_ix, out_ix] = uniquify(
+ map_task[op_ix][0], 'out', str(out_ix))
+ return used_pcollections[op_ix, out_ix]
+
+ def get_inputs(op):
+ if hasattr(op, 'inputs'):
+ inputs = op.inputs
+ elif hasattr(op, 'input'):
+ inputs = [op.input]
+ else:
+ inputs = []
+ return {'in%s' % ix: pcollection_id(*input)
+ for ix, input in enumerate(inputs)}
+
+ def get_outputs(op_ix):
+ op = map_task[op_ix][1]
+ return {tag: pcollection_id(op_ix, out_ix)
+ for out_ix, tag in enumerate(getattr(op, 'output_tags', ['out']))}
+
+ for op_ix, (stage_name, operation) in enumerate(map_task):
+ transform_id = uniquify(stage_name)
+
+ if isinstance(operation, operation_specs.WorkerInMemoryWrite):
+ # Write this data back to the runner.
+ target_name = only_element(get_inputs(operation).keys())
+ runner_sinks[(transform_id, target_name)] = operation
+ transform_spec = beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_OUTPUT_URN,
+ payload=data_operation_spec.SerializeToString() \
+ if data_operation_spec is not None else None)
+
+ elif isinstance(operation, operation_specs.WorkerRead):
+ # A Read from an in-memory source is done over the data plane.
+ if (isinstance(operation.source.source,
+ maptask_executor_runner.InMemorySource)
+ and isinstance(operation.source.source.default_output_coder(),
+ WindowedValueCoder)):
+ target_name = only_element(get_outputs(op_ix).keys())
+ input_data[(transform_id, target_name)] = self._reencode_elements(
+ operation.source.source.read(None),
+ operation.source.source.default_output_coder())
+ transform_spec = beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.DATA_INPUT_URN,
+ payload=data_operation_spec.SerializeToString() \
+ if data_operation_spec is not None else None)
+
+ else:
+ # Otherwise serialize the source and execute it there.
+ # TODO: Use SDFs with an initial impulse.
+ # The Dataflow runner harness strips the base64 encoding. do the same
+ # here until we get the same thing back that we sent in.
+ source_bytes = base64.b64decode(
+ pickler.dumps(operation.source.source))
+ transform_spec = beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.PYTHON_SOURCE_URN,
+ payload=source_bytes)
+
+ elif isinstance(operation, operation_specs.WorkerDoFn):
+ # Record the contents of each side input for access via the state api.
+ side_input_extras = []
+ for si in operation.side_inputs:
+ assert isinstance(si.source, iobase.BoundedSource)
+ element_coder = si.source.default_output_coder()
+ # TODO(robertwb): Actually flesh out the ViewFn API.
+ side_input_extras.append((si.tag, element_coder))
+ side_input_data[
+ bundle_processor.side_input_tag(transform_id, si.tag)] = (
+ self._reencode_elements(
+ si.source.read(si.source.get_range_tracker(None, None)),
+ element_coder))
+ augmented_serialized_fn = pickler.dumps(
+ (operation.serialized_fn, side_input_extras))
+ transform_spec = beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.PYTHON_DOFN_URN,
+ payload=augmented_serialized_fn)
+
+ elif isinstance(operation, operation_specs.WorkerFlatten):
+ # Flatten is nice and simple.
+ transform_spec = beam_runner_api_pb2.FunctionSpec(
+ urn=bundle_processor.IDENTITY_DOFN_URN)
+
+ else:
+ raise NotImplementedError(operation)
+
+ transform_protos[transform_id] = beam_runner_api_pb2.PTransform(
+ unique_name=stage_name,
+ spec=transform_spec,
+ inputs=get_inputs(operation),
+ outputs=get_outputs(op_ix))
+
+ pcollection_protos = {
+ name: beam_runner_api_pb2.PCollection(
+ unique_name=name,
+ coder_id=context.coders.get_id(
+ map_task[op_id][1].output_coders[out_id]))
+ for (op_id, out_id), name in used_pcollections.items()
+ }
+ # Must follow creation of pcollection_protos to capture used coders.
+ context_proto = context.to_runner_api()
+ process_bundle_descriptor = beam_fn_api_pb2.ProcessBundleDescriptor(
+ id=self._next_uid(),
+ transforms=transform_protos,
+ pcollections=pcollection_protos,
+ coders=dict(context_proto.coders.items()),
+ windowing_strategies=dict(context_proto.windowing_strategies.items()),
+ environments=dict(context_proto.environments.items()))
+ return input_data, side_input_data, runner_sinks, process_bundle_descriptor
def _run_map_task(
self, map_task, control_handler, state_handler, data_plane_handler,
@@ -358,7 +1007,7 @@
sink_op.output_buffer.append(e)
return
- def execute_map_tasks(self, ordered_map_tasks, direct=True):
+ def execute_map_tasks(self, ordered_map_tasks, direct=False):
if direct:
controller = FnApiRunner.DirectController()
else:
@@ -373,36 +1022,65 @@
finally:
controller.close()
- class SimpleState(object): # TODO(robertwb): Inherit from GRPC servicer.
+ @staticmethod
+ def _reencode_elements(elements, element_coder):
+ output_stream = create_OutputStream()
+ for element in elements:
+ element_coder.get_impl().encode_to_stream(element, output_stream, True)
+ return output_stream.get()
+
+ # These classes are used to interact with the worker.
+
+ class StateServicer(beam_fn_api_pb2_grpc.BeamFnStateServicer):
def __init__(self):
- self._all = collections.defaultdict(list)
+ self._lock = threading.Lock()
+ self._state = collections.defaultdict(list)
- def Get(self, state_key):
- return beam_fn_api_pb2.Elements.Data(
- data=''.join(self._all[self._to_key(state_key)]))
+ def blocking_get(self, state_key, instruction_reference=None):
+ with self._lock:
+ return ''.join(self._state[self._to_key(state_key)])
- def Append(self, append_request):
- self._all[self._to_key(append_request.state_key)].extend(
- append_request.data)
+ def blocking_append(self, state_key, data, instruction_reference=None):
+ with self._lock:
+ self._state[self._to_key(state_key)].append(data)
- def Clear(self, state_key):
- try:
- del self._all[self._to_key(state_key)]
- except KeyError:
- pass
+ def blocking_clear(self, state_key, instruction_reference=None):
+ with self._lock:
+ del self._state[self._to_key(state_key)]
@staticmethod
def _to_key(state_key):
- return (state_key.function_spec_reference, state_key.window,
- state_key.key)
+ return state_key.SerializeToString()
+
+ class GrpcStateServicer(
+ StateServicer, beam_fn_api_pb2_grpc.BeamFnStateServicer):
+ def State(self, request_stream, context=None):
+ # Note that this eagerly mutates state, assuming any failures are fatal.
+ # Thus it is safe to ignore instruction_reference.
+ for request in request_stream:
+ if request.get:
+ yield beam_fn_api_pb2.StateResponse(
+ id=request.id,
+ get=beam_fn_api_pb2.StateGetResponse(
+ data=self.blocking_get(request.state_key)))
+ elif request.append:
+ self.blocking_append(request.state_key, request.append.data)
+ yield beam_fn_api_pb2.StateResponse(
+ id=request.id,
+ append=beam_fn_api_pb2.AppendResponse())
+ elif request.clear:
+ self.blocking_clear(request.state_key)
+ yield beam_fn_api_pb2.StateResponse(
+ id=request.id,
+ clear=beam_fn_api_pb2.ClearResponse())
class DirectController(object):
"""An in-memory controller for fn API control, state and data planes."""
def __init__(self):
self._responses = []
- self.state_handler = FnApiRunner.SimpleState()
+ self.state_handler = FnApiRunner.StateServicer()
self.control_handler = self
self.data_plane_handler = data_plane.InMemoryDataChannel()
self.worker = sdk_worker.SdkWorker(
@@ -430,8 +1108,8 @@
class GrpcController(object):
"""An grpc based controller for fn API control, state and data planes."""
- def __init__(self):
- self.state_handler = FnApiRunner.SimpleState()
+ def __init__(self, sdk_harness_factory=None):
+ self.sdk_harness_factory = sdk_harness_factory
self.control_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10))
self.control_port = self.control_server.add_insecure_port('[::]:0')
@@ -440,22 +1118,29 @@
self.data_port = self.data_server.add_insecure_port('[::]:0')
self.control_handler = streaming_rpc_handler(
- beam_fn_api_pb2.BeamFnControlServicer, 'Control')
- beam_fn_api_pb2.add_BeamFnControlServicer_to_server(
+ beam_fn_api_pb2_grpc.BeamFnControlServicer, 'Control')
+ beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server(
self.control_handler, self.control_server)
self.data_plane_handler = data_plane.GrpcServerDataChannel()
- beam_fn_api_pb2.add_BeamFnDataServicer_to_server(
+ beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server(
self.data_plane_handler, self.data_server)
+ # TODO(robertwb): Is sharing the control channel fine? Alternatively,
+ # how should this be plumbed?
+ self.state_handler = FnApiRunner.GrpcStateServicer()
+ beam_fn_api_pb2_grpc.add_BeamFnStateServicer_to_server(
+ self.state_handler, self.control_server)
+
logging.info('starting control server on port %s', self.control_port)
logging.info('starting data server on port %s', self.data_port)
self.data_server.start()
self.control_server.start()
- self.worker = sdk_worker.SdkHarness(
- grpc.insecure_channel('localhost:%s' % self.control_port))
- self.worker_thread = threading.Thread(target=self.worker.run)
+ self.worker = (self.sdk_harness_factory or sdk_worker.SdkHarness)(
+ 'localhost:%s' % self.control_port)
+ self.worker_thread = threading.Thread(
+ name='run_worker', target=self.worker.run)
logging.info('starting worker')
self.worker_thread.start()
@@ -471,3 +1156,26 @@
self.data_plane_handler.close()
self.control_server.stop(5).wait()
self.data_server.stop(5).wait()
+
+
+class RunnerResult(maptask_executor_runner.WorkerRunnerResult):
+ def __init__(self, state, metrics_by_stage):
+ super(RunnerResult, self).__init__(state)
+ self._metrics_by_stage = metrics_by_stage
+
+
+def only_element(iterable):
+ element, = iterable
+ return element
+
+
+def unique_name(existing, prefix):
+ if prefix in existing:
+ counter = 0
+ while True:
+ counter += 1
+ prefix_counter = prefix + "_%s" % counter
+ if prefix_counter not in existing:
+ return prefix_counter
+ else:
+ return prefix
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
index 633602f..ea9ed1a 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
@@ -16,25 +16,151 @@
#
import logging
+import time
import unittest
import apache_beam as beam
from apache_beam.runners.portability import fn_api_runner
-from apache_beam.runners.portability import maptask_executor_runner
+from apache_beam.runners.portability import maptask_executor_runner_test
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+from apache_beam.transforms import window
+
+try:
+ from apache_beam.runners.worker.statesampler import DEFAULT_SAMPLING_PERIOD_MS
+except ImportError:
+ DEFAULT_SAMPLING_PERIOD_MS = 0
-class FnApiRunnerTest(maptask_executor_runner.MapTaskExecutorRunner):
+class FnApiRunnerTest(
+ maptask_executor_runner_test.MapTaskExecutorRunnerTest):
def create_pipeline(self):
- return beam.Pipeline(runner=fn_api_runner.FnApiRunner())
+ return beam.Pipeline(
+ runner=fn_api_runner.FnApiRunner(use_grpc=False))
def test_combine_per_key(self):
- # TODO(robertwb): Implement PGBKCV operation.
+ # TODO(BEAM-1348): Enable once Partial GBK is supported in fn API.
pass
+ def test_combine_per_key(self):
+ # TODO(BEAM-1348): Enable once Partial GBK is supported in fn API.
+ pass
+
+ def test_pardo_side_inputs(self):
+ def cross_product(elem, sides):
+ for side in sides:
+ yield elem, side
+ with self.create_pipeline() as p:
+ main = p | 'main' >> beam.Create(['a', 'b', 'c'])
+ side = p | 'side' >> beam.Create(['x', 'y'])
+ assert_that(main | beam.FlatMap(cross_product, beam.pvalue.AsList(side)),
+ equal_to([('a', 'x'), ('b', 'x'), ('c', 'x'),
+ ('a', 'y'), ('b', 'y'), ('c', 'y')]))
+
+ # Now with some windowing.
+ pcoll = p | beam.Create(range(10)) | beam.Map(
+ lambda t: window.TimestampedValue(t, t))
+ # Intentionally choosing non-aligned windows to highlight the transition.
+ main = pcoll | 'WindowMain' >> beam.WindowInto(window.FixedWindows(5))
+ side = pcoll | 'WindowSide' >> beam.WindowInto(window.FixedWindows(7))
+ res = main | beam.Map(lambda x, s: (x, sorted(s)),
+ beam.pvalue.AsList(side))
+ assert_that(
+ res,
+ equal_to([
+ # The window [0, 5) maps to the window [0, 7).
+ (0, range(7)),
+ (1, range(7)),
+ (2, range(7)),
+ (3, range(7)),
+ (4, range(7)),
+ # The window [5, 10) maps to the window [7, 14).
+ (5, range(7, 10)),
+ (6, range(7, 10)),
+ (7, range(7, 10)),
+ (8, range(7, 10)),
+ (9, range(7, 10))]),
+ label='windowed')
+
+ def test_assert_that(self):
+ # TODO: figure out a way for fn_api_runner to parse and raise the
+ # underlying exception.
+ with self.assertRaisesRegexp(Exception, 'Failed assert'):
+ with self.create_pipeline() as p:
+ assert_that(p | beam.Create(['a', 'b']), equal_to(['a']))
+
+ def test_progress_metrics(self):
+ p = self.create_pipeline()
+ if not isinstance(p.runner, fn_api_runner.FnApiRunner):
+ # This test is inherited by others that may not support the same
+ # internal way of accessing progress metrics.
+ return
+
+ _ = (p
+ | beam.Create([0, 0, 0, 2.1e-3 * DEFAULT_SAMPLING_PERIOD_MS])
+ | beam.Map(time.sleep)
+ | beam.Map(lambda x: ('key', x))
+ | beam.GroupByKey()
+ | 'm_out' >> beam.FlatMap(lambda x: [
+ 1, 2, 3, 4, 5,
+ beam.pvalue.TaggedOutput('once', x),
+ beam.pvalue.TaggedOutput('twice', x),
+ beam.pvalue.TaggedOutput('twice', x)]))
+ res = p.run()
+ res.wait_until_finish()
+ try:
+ self.assertEqual(2, len(res._metrics_by_stage))
+ pregbk_metrics, postgbk_metrics = res._metrics_by_stage.values()
+ if 'Create/Read' not in pregbk_metrics.ptransforms:
+ # The metrics above are actually unordered. Swap.
+ pregbk_metrics, postgbk_metrics = postgbk_metrics, pregbk_metrics
+
+ self.assertEqual(
+ 4,
+ pregbk_metrics.ptransforms['Create/Read']
+ .processed_elements.measured.output_element_counts['None'])
+ self.assertEqual(
+ 4,
+ pregbk_metrics.ptransforms['Map(sleep)']
+ .processed_elements.measured.output_element_counts['None'])
+ self.assertLessEqual(
+ 2e-3 * DEFAULT_SAMPLING_PERIOD_MS,
+ pregbk_metrics.ptransforms['Map(sleep)']
+ .processed_elements.measured.total_time_spent)
+ self.assertEqual(
+ 1,
+ postgbk_metrics.ptransforms['GroupByKey/Read']
+ .processed_elements.measured.output_element_counts['None'])
+
+ # The actual stage name ends up being something like 'm_out/lamdbda...'
+ m_out, = [
+ metrics for name, metrics in postgbk_metrics.ptransforms.items()
+ if name.startswith('m_out')]
+ self.assertEqual(
+ 5,
+ m_out.processed_elements.measured.output_element_counts['None'])
+ self.assertEqual(
+ 1,
+ m_out.processed_elements.measured.output_element_counts['once'])
+ self.assertEqual(
+ 2,
+ m_out.processed_elements.measured.output_element_counts['twice'])
+
+ except:
+ print res._metrics_by_stage
+ raise
+
# Inherits all tests from maptask_executor_runner.MapTaskExecutorRunner
+class FnApiRunnerTestWithGrpc(FnApiRunnerTest):
+
+ def create_pipeline(self):
+ return beam.Pipeline(
+ runner=fn_api_runner.FnApiRunner(use_grpc=True))
+
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
unittest.main()
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
index ddfc4cc..5b580a6 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
@@ -36,13 +36,14 @@
from apache_beam.runners.runner import PipelineState
from apache_beam.runners.worker import operation_specs
from apache_beam.runners.worker import operations
+from apache_beam.typehints import typehints
+from apache_beam.utils import profiler
+from apache_beam.utils.counters import CounterFactory
+
try:
from apache_beam.runners.worker import statesampler
except ImportError:
from apache_beam.runners.worker import statesampler_fake as statesampler
-from apache_beam.typehints import typehints
-from apache_beam.utils import profiler
-from apache_beam.utils.counters import CounterFactory
# This module is experimental. No backwards-compatibility guarantees.
@@ -129,7 +130,7 @@
# Create the CounterFactory and StateSampler for this MapTask.
# TODO(robertwb): Output counters produced here are currently ignored.
counter_factory = CounterFactory()
- state_sampler = statesampler.StateSampler('%s-' % ix, counter_factory)
+ state_sampler = statesampler.StateSampler('%s' % ix, counter_factory)
map_executor = operations.SimpleMapTaskExecutor(
operation_specs.MapTask(
all_operations, 'S%02d' % ix,
@@ -434,7 +435,7 @@
def to_accumulator(v):
return self.combine_fn.add_input(
self.combine_fn.create_accumulator(), v)
- return input | beam.Map(lambda (k, v): (k, to_accumulator(v)))
+ return input | beam.Map(lambda k_v: (k_v[0], to_accumulator(k_v[1])))
class MergeAccumulators(beam.PTransform):
@@ -448,7 +449,11 @@
return beam.pvalue.PCollection(input.pipeline)
else:
merge_accumulators = self.combine_fn.merge_accumulators
- return input | beam.Map(lambda (k, vs): (k, merge_accumulators(vs)))
+
+ def merge_with_existing_key(k_vs):
+ return (k_vs[0], merge_accumulators(k_vs[1]))
+
+ return input | beam.Map(merge_with_existing_key)
class ExtractOutputs(beam.PTransform):
@@ -462,7 +467,7 @@
return beam.pvalue.PCollection(input.pipeline)
else:
extract_output = self.combine_fn.extract_output
- return input | beam.Map(lambda (k, v): (k, extract_output(v)))
+ return input | beam.Map(lambda k_v1: (k_v1[0], extract_output(k_v1[1])))
class WorkerRunnerResult(PipelineResult):
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
index b7ba15a..0f8637f 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
@@ -21,18 +21,16 @@
import unittest
import apache_beam as beam
-
from apache_beam.metrics import Metrics
from apache_beam.metrics.execution import MetricKey
from apache_beam.metrics.execution import MetricsEnvironment
from apache_beam.metrics.metricbase import MetricName
-
from apache_beam.pvalue import AsList
-from apache_beam.testing.util import assert_that
+from apache_beam.runners.portability import maptask_executor_runner
from apache_beam.testing.util import BeamAssertException
+from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms.window import TimestampedValue
-from apache_beam.runners.portability import maptask_executor_runner
class MapTaskExecutorRunnerTest(unittest.TestCase):
@@ -154,7 +152,7 @@
derived = ((pcoll,) | beam.Flatten()
| beam.Map(lambda x: (x, x))
| beam.GroupByKey()
- | 'Unkey' >> beam.Map(lambda (x, _): x))
+ | 'Unkey' >> beam.Map(lambda kv: kv[0]))
assert_that(
pcoll | beam.FlatMap(cross_product, AsList(derived)),
equal_to([('a', 'a'), ('a', 'b'), ('b', 'a'), ('b', 'b')]))
@@ -164,7 +162,7 @@
res = (p
| beam.Create([('a', 1), ('a', 2), ('b', 3)])
| beam.GroupByKey()
- | beam.Map(lambda (k, vs): (k, sorted(vs))))
+ | beam.Map(lambda k_vs: (k_vs[0], sorted(k_vs[1]))))
assert_that(res, equal_to([('a', [1, 2]), ('b', [3])]))
def test_flatten(self):
@@ -201,7 +199,7 @@
| beam.Map(lambda t: TimestampedValue(('k', t), t))
| beam.WindowInto(beam.transforms.window.Sessions(10))
| beam.GroupByKey()
- | beam.Map(lambda (k, vs): (k, sorted(vs))))
+ | beam.Map(lambda k_vs1: (k_vs1[0], sorted(k_vs1[1]))))
assert_that(res, equal_to([('k', [1, 2]), ('k', [100, 101, 102])]))
def test_errors(self):
diff --git a/sdks/python/apache_beam/runners/portability/universal_local_runner.py b/sdks/python/apache_beam/runners/portability/universal_local_runner.py
new file mode 100644
index 0000000..579983c
--- /dev/null
+++ b/sdks/python/apache_beam/runners/portability/universal_local_runner.py
@@ -0,0 +1,409 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import functools
+import logging
+import os
+import Queue as queue
+import socket
+import subprocess
+import sys
+import threading
+import time
+import traceback
+import uuid
+from concurrent import futures
+
+import grpc
+from google.protobuf import text_format
+
+from apache_beam.portability.api import beam_job_api_pb2
+from apache_beam.portability.api import beam_job_api_pb2_grpc
+from apache_beam.portability.api import endpoints_pb2
+from apache_beam.runners import runner
+from apache_beam.runners.portability import fn_api_runner
+
+TERMINAL_STATES = [
+ beam_job_api_pb2.JobState.DONE,
+ beam_job_api_pb2.JobState.STOPPED,
+ beam_job_api_pb2.JobState.FAILED,
+ beam_job_api_pb2.JobState.CANCELLED,
+]
+
+
+class UniversalLocalRunner(runner.PipelineRunner):
+ """A BeamRunner that executes Python pipelines via the Beam Job API.
+
+ By default, this runner executes in process but still uses GRPC to communicate
+ pipeline and worker state. It can also be configured to use inline calls
+ rather than GRPC (for speed) or launch completely separate subprocesses for
+ the runner and worker(s).
+ """
+
+ def __init__(self, use_grpc=True, use_subprocesses=False):
+ if use_subprocesses and not use_grpc:
+ raise ValueError("GRPC must be used with subprocesses")
+ super(UniversalLocalRunner, self).__init__()
+ self._use_grpc = use_grpc
+ self._use_subprocesses = use_subprocesses
+
+ self._job_service = None
+ self._job_service_lock = threading.Lock()
+ self._subprocess = None
+
+ def __del__(self):
+ # Best effort to not leave any dangling processes around.
+ self.cleanup()
+
+ def cleanup(self):
+ if self._subprocess:
+ self._subprocess.kill()
+ time.sleep(0.1)
+ self._subprocess = None
+
+ def _get_job_service(self):
+ with self._job_service_lock:
+ if not self._job_service:
+ if self._use_subprocesses:
+ self._job_service = self._start_local_runner_subprocess_job_service()
+
+ elif self._use_grpc:
+ self._servicer = JobServicer(use_grpc=True)
+ self._job_service = beam_job_api_pb2_grpc.JobServiceStub(
+ grpc.insecure_channel(
+ 'localhost:%d' % self._servicer.start_grpc()))
+
+ else:
+ self._job_service = JobServicer(use_grpc=False)
+
+ return self._job_service
+
+ def _start_local_runner_subprocess_job_service(self):
+ if self._subprocess:
+ # Kill the old one if it exists.
+ self._subprocess.kill()
+ # TODO(robertwb): Consider letting the subprocess pick one and
+ # communicate it back...
+ port = _pick_unused_port()
+ logging.info("Starting server on port %d.", port)
+ self._subprocess = subprocess.Popen([
+ sys.executable,
+ '-m',
+ 'apache_beam.runners.portability.universal_local_runner_main',
+ '-p',
+ str(port),
+ '--worker_command_line',
+ '%s -m apache_beam.runners.worker.sdk_worker_main' % sys.executable
+ ])
+ job_service = beam_job_api_pb2_grpc.JobServiceStub(
+ grpc.insecure_channel('localhost:%d' % port))
+ logging.info("Waiting for server to be ready...")
+ start = time.time()
+ timeout = 30
+ while True:
+ time.sleep(0.1)
+ if self._subprocess.poll() is not None:
+ raise RuntimeError(
+ "Subprocess terminated unexpectedly with exit code %d." %
+ self._subprocess.returncode)
+ elif time.time() - start > timeout:
+ raise RuntimeError(
+ "Pipeline timed out waiting for job service subprocess.")
+ else:
+ try:
+ job_service.GetState(
+ beam_job_api_pb2.GetJobStateRequest(job_id='[fake]'))
+ break
+ except grpc.RpcError as exn:
+ if exn.code != grpc.StatusCode.UNAVAILABLE:
+ # We were able to contact the service for our fake state request.
+ break
+ logging.info("Server ready.")
+ return job_service
+
+ def run(self, pipeline):
+ job_service = self._get_job_service()
+ prepare_response = job_service.Prepare(
+ beam_job_api_pb2.PrepareJobRequest(
+ job_name='job',
+ pipeline=pipeline.to_runner_api()))
+ run_response = job_service.Run(beam_job_api_pb2.RunJobRequest(
+ preparation_id=prepare_response.preparation_id))
+ return PipelineResult(job_service, run_response.job_id)
+
+
+class PipelineResult(runner.PipelineResult):
+ def __init__(self, job_service, job_id):
+ super(PipelineResult, self).__init__(beam_job_api_pb2.JobState.UNSPECIFIED)
+ self._job_service = job_service
+ self._job_id = job_id
+ self._messages = []
+
+ def cancel(self):
+ self._job_service.Cancel()
+
+ @property
+ def state(self):
+ runner_api_state = self._job_service.GetState(
+ beam_job_api_pb2.GetJobStateRequest(job_id=self._job_id)).state
+ self._state = self._runner_api_state_to_pipeline_state(runner_api_state)
+ return self._state
+
+ @staticmethod
+ def _runner_api_state_to_pipeline_state(runner_api_state):
+ return getattr(
+ runner.PipelineState,
+ beam_job_api_pb2.JobState.Enum.Name(runner_api_state))
+
+ @staticmethod
+ def _pipeline_state_to_runner_api_state(pipeline_state):
+ return beam_job_api_pb2.JobState.Enum.Value(pipeline_state)
+
+ def wait_until_finish(self):
+ def read_messages():
+ for message in self._job_service.GetMessageStream(
+ beam_job_api_pb2.JobMessagesRequest(job_id=self._job_id)):
+ self._messages.append(message)
+ threading.Thread(target=read_messages).start()
+
+ for state_response in self._job_service.GetStateStream(
+ beam_job_api_pb2.GetJobStateRequest(job_id=self._job_id)):
+ self._state = self._runner_api_state_to_pipeline_state(
+ state_response.state)
+ if state_response.state in TERMINAL_STATES:
+ break
+ if self._state != runner.PipelineState.DONE:
+ raise RuntimeError(
+ "Pipeline %s failed in state %s." % (self._job_id, self._state))
+
+
+class BeamJob(threading.Thread):
+ """This class handles running and managing a single pipeline.
+
+ The current state of the pipeline is available as self.state.
+ """
+ def __init__(self, job_id, pipeline_options, pipeline_proto,
+ use_grpc=True, sdk_harness_factory=None):
+ super(BeamJob, self).__init__()
+ self._job_id = job_id
+ self._pipeline_options = pipeline_options
+ self._pipeline_proto = pipeline_proto
+ self._use_grpc = use_grpc
+ self._sdk_harness_factory = sdk_harness_factory
+ self._log_queue = queue.Queue()
+ self._state_change_callbacks = [
+ lambda new_state: self._log_queue.put(
+ beam_job_api_pb2.JobMessagesResponse(
+ state_response=
+ beam_job_api_pb2.GetJobStateResponse(state=new_state)))
+ ]
+ self._state = None
+ self.state = beam_job_api_pb2.JobState.STARTING
+ self.daemon = True
+
+ def add_state_change_callback(self, f):
+ self._state_change_callbacks.append(f)
+
+ @property
+ def log_queue(self):
+ return self._log_queue
+
+ @property
+ def state(self):
+ return self._state
+
+ @state.setter
+ def state(self, new_state):
+ for state_change_callback in self._state_change_callbacks:
+ state_change_callback(new_state)
+ self._state = new_state
+
+ def run(self):
+ with JobLogHandler(self._log_queue):
+ try:
+ fn_api_runner.FnApiRunner(
+ use_grpc=self._use_grpc,
+ sdk_harness_factory=self._sdk_harness_factory
+ ).run_via_runner_api(self._pipeline_proto)
+ self.state = beam_job_api_pb2.JobState.DONE
+ except: # pylint: disable=bare-except
+ logging.exception("Error running pipeline.")
+ traceback.print_exc()
+ self.state = beam_job_api_pb2.JobState.FAILED
+
+ def cancel(self):
+ if self.state not in TERMINAL_STATES:
+ self.state = beam_job_api_pb2.JobState.CANCELLING
+ # TODO(robertwb): Actually cancel...
+ self.state = beam_job_api_pb2.JobState.CANCELLED
+
+
+class JobServicer(beam_job_api_pb2_grpc.JobServiceServicer):
+ """Servicer for the Beam Job API.
+
+ Manages one or more pipelines, possibly concurrently.
+ """
+ def __init__(
+ self, worker_command_line=None, use_grpc=True):
+ self._worker_command_line = worker_command_line
+ self._use_grpc = use_grpc or bool(worker_command_line)
+ self._jobs = {}
+
+ def start_grpc(self, port=0):
+ self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=3))
+ port = self._server.add_insecure_port('localhost:%d' % port)
+ beam_job_api_pb2_grpc.add_JobServiceServicer_to_server(self, self._server)
+ self._server.start()
+ return port
+
+ def Prepare(self, request, context=None):
+ # For now, just use the job name as the job id.
+ preparation_id = "%s-%s" % (request.job_name, uuid.uuid4())
+ if self._worker_command_line:
+ sdk_harness_factory = functools.partial(
+ SubprocessSdkWorker, self._worker_command_line)
+ else:
+ sdk_harness_factory = None
+ self._jobs[preparation_id] = BeamJob(
+ preparation_id, request.pipeline_options, request.pipeline,
+ use_grpc=self._use_grpc, sdk_harness_factory=sdk_harness_factory)
+ return beam_job_api_pb2.PrepareJobResponse(preparation_id=preparation_id)
+
+ def Run(self, request, context=None):
+ job_id = request.preparation_id
+ self._jobs[job_id].start()
+ return beam_job_api_pb2.RunJobResponse(job_id=job_id)
+
+ def GetState(self, request, context=None):
+ return beam_job_api_pb2.GetJobStateResponse(
+ state=self._jobs[request.job_id].state)
+
+ def Cancel(self, request, context=None):
+ self._jobs[request.job_id].cancel()
+ return beam_job_api_pb2.CancelJobRequest(
+ state=self._jobs[request.job_id].state)
+
+ def GetStateStream(self, request, context=None):
+ job = self._jobs[request.job_id]
+ state_queue = queue.Queue()
+ job.add_state_change_callback(lambda state: state_queue.put(state))
+ try:
+ current_state = state_queue.get()
+ except queue.Empty:
+ current_state = job.state
+ yield beam_job_api_pb2.GetJobStateResponse(
+ state=current_state)
+ while current_state not in TERMINAL_STATES:
+ current_state = state_queue.get(block=True)
+ yield beam_job_api_pb2.GetJobStateResponse(
+ state=current_state)
+
+ def GetMessageStream(self, request, context=None):
+ job = self._jobs[request.job_id]
+ current_state = job.state
+ while current_state not in TERMINAL_STATES:
+ msg = job.log_queue.get(block=True)
+ yield msg
+ if msg.HasField('state_response'):
+ current_state = msg.state_response.state
+ try:
+ while True:
+ yield job.log_queue.get(block=False)
+ except queue.Empty:
+ pass
+
+
+class SubprocessSdkWorker(object):
+ """Manages a SDK worker implemented as a subprocess communicating over grpc.
+ """
+
+ def __init__(self, worker_command_line, control_address):
+ self._worker_command_line = worker_command_line
+ self._control_address = control_address
+
+ def run(self):
+ control_descriptor = text_format.MessageToString(
+ endpoints_pb2.ApiServiceDescriptor(url=self._control_address))
+ p = subprocess.Popen(
+ self._worker_command_line,
+ shell=True,
+ env=dict(os.environ,
+ CONTROL_API_SERVICE_DESCRIPTOR=control_descriptor))
+ try:
+ p.wait()
+ if p.returncode:
+ raise RuntimeError(
+ "Worker subprocess exited with return code %s" % p.returncode)
+ finally:
+ if p.poll() is None:
+ p.kill()
+
+
+class JobLogHandler(logging.Handler):
+ """Captures logs to be returned via the Beam Job API.
+
+ Enabled via the with statement."""
+
+ # Mapping from logging levels to LogEntry levels.
+ LOG_LEVEL_MAP = {
+ logging.FATAL: beam_job_api_pb2.JobMessage.JOB_MESSAGE_ERROR,
+ logging.ERROR: beam_job_api_pb2.JobMessage.JOB_MESSAGE_ERROR,
+ logging.WARNING: beam_job_api_pb2.JobMessage.JOB_MESSAGE_WARNING,
+ logging.INFO: beam_job_api_pb2.JobMessage.JOB_MESSAGE_BASIC,
+ logging.DEBUG: beam_job_api_pb2.JobMessage.JOB_MESSAGE_DEBUG,
+ }
+
+ def __init__(self, message_queue):
+ super(JobLogHandler, self).__init__()
+ self._message_queue = message_queue
+ self._last_id = 0
+ self._logged_thread = None
+
+ def __enter__(self):
+ # Remember the current thread to demultiplex the logs of concurrently
+ # running pipelines (as Python log handlers are global).
+ self._logged_thread = threading.current_thread()
+ logging.getLogger().addHandler(self)
+
+ def __exit__(self, *args):
+ self._logged_thread = None
+ self.close()
+
+ def _next_id(self):
+ self._last_id += 1
+ return str(self._last_id)
+
+ def emit(self, record):
+ if self._logged_thread is threading.current_thread():
+ self._message_queue.put(beam_job_api_pb2.JobMessagesResponse(
+ message_response=beam_job_api_pb2.JobMessage(
+ message_id=self._next_id(),
+ time=time.strftime(
+ '%Y-%m-%d %H:%M:%S.', time.localtime(record.created)),
+ importance=self.LOG_LEVEL_MAP[record.levelno],
+ message_text=self.format(record))))
+
+
+def _pick_unused_port():
+ """Not perfect, but we have to provide a port to the subprocess."""
+ # TODO(robertwb): Consider letting the subprocess communicate a choice of
+ # port back.
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind(('localhost', 0))
+ _, port = s.getsockname()
+ s.close()
+ return port
diff --git a/sdks/python/apache_beam/runners/portability/universal_local_runner_main.py b/sdks/python/apache_beam/runners/portability/universal_local_runner_main.py
new file mode 100644
index 0000000..9dd3a7e
--- /dev/null
+++ b/sdks/python/apache_beam/runners/portability/universal_local_runner_main.py
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import logging
+import sys
+import time
+
+from apache_beam.runners.portability import universal_local_runner
+
+
+def run(argv):
+ if argv[0] == __file__:
+ argv = argv[1:]
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-p', '--port',
+ type=int,
+ help='port on which to serve the job api')
+ parser.add_argument('--worker_command_line',
+ help='command line for starting up a worker process')
+ options = parser.parse_args(argv)
+ job_servicer = universal_local_runner.JobServicer(options.worker_command_line)
+ port = job_servicer.start_grpc(options.port)
+ while True:
+ logging.info("Listening for jobs at %d", port)
+ time.sleep(300)
+
+
+if __name__ == '__main__':
+ run(sys.argv)
diff --git a/sdks/python/apache_beam/runners/portability/universal_local_runner_test.py b/sdks/python/apache_beam/runners/portability/universal_local_runner_test.py
new file mode 100644
index 0000000..e1104dc
--- /dev/null
+++ b/sdks/python/apache_beam/runners/portability/universal_local_runner_test.py
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.runners.portability import fn_api_runner_test
+from apache_beam.runners.portability import universal_local_runner
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+
+class UniversalLocalRunnerTest(fn_api_runner_test.FnApiRunnerTest):
+
+ _use_grpc = False
+ _use_subprocesses = False
+
+ @classmethod
+ def get_runner(cls):
+ # Don't inherit.
+ if '_runner' not in cls.__dict__:
+ cls._runner = universal_local_runner.UniversalLocalRunner(
+ use_grpc=cls._use_grpc,
+ use_subprocesses=cls._use_subprocesses)
+ return cls._runner
+
+ @classmethod
+ def tearDownClass(cls):
+ cls._runner.cleanup()
+
+ def create_pipeline(self):
+ return beam.Pipeline(self.get_runner())
+
+ def test_assert_that(self):
+ # TODO: figure out a way for runner to parse and raise the
+ # underlying exception.
+ with self.assertRaises(Exception):
+ with self.create_pipeline() as p:
+ assert_that(p | beam.Create(['a', 'b']), equal_to(['a']))
+
+ def test_errors(self):
+ # TODO: figure out a way for runner to parse and raise the
+ # underlying exception.
+ with self.assertRaises(BaseException):
+ with self.create_pipeline() as p:
+ def raise_error(x):
+ raise RuntimeError('x')
+ # pylint: disable=expression-not-assigned
+ (p
+ | beam.Create(['a', 'b'])
+ | 'StageA' >> beam.Map(lambda x: x)
+ | 'StageB' >> beam.Map(lambda x: x)
+ | 'StageC' >> beam.Map(raise_error)
+ | 'StageD' >> beam.Map(lambda x: x))
+
+ # Inherits all tests from fn_api_runner_test.FnApiRunnerTest
+
+
+class UniversalLocalRunnerTestWithGrpc(UniversalLocalRunnerTest):
+ _use_grpc = True
+
+
+@unittest.skip("BEAM-3040")
+class UniversalLocalRunnerTestWithSubprocesses(UniversalLocalRunnerTest):
+ _use_grpc = True
+ _use_subprocesses = True
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py
index af00d8f..bdabd81 100644
--- a/sdks/python/apache_beam/runners/runner.py
+++ b/sdks/python/apache_beam/runners/runner.py
@@ -25,7 +25,6 @@
import shutil
import tempfile
-
__all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult']
@@ -41,7 +40,11 @@
_DATAFLOW_RUNNER_PATH = (
'apache_beam.runners.dataflow.dataflow_runner.')
_TEST_RUNNER_PATH = 'apache_beam.runners.test.'
+_PYTHON_RPC_DIRECT_RUNNER = (
+ 'apache_beam.runners.experimental.python_rpc_direct.'
+ 'python_rpc_direct_runner.')
+_KNOWN_PYTHON_RPC_DIRECT_RUNNER = ('PythonRPCDirectRunner',)
_KNOWN_DIRECT_RUNNERS = ('DirectRunner', 'EagerRunner')
_KNOWN_DATAFLOW_RUNNERS = ('DataflowRunner',)
_KNOWN_TEST_RUNNERS = ('TestDataflowRunner',)
@@ -51,6 +54,8 @@
_DIRECT_RUNNER_PATH))
_RUNNER_MAP.update(_get_runner_map(_KNOWN_DATAFLOW_RUNNERS,
_DATAFLOW_RUNNER_PATH))
+_RUNNER_MAP.update(_get_runner_map(_KNOWN_PYTHON_RPC_DIRECT_RUNNER,
+ _PYTHON_RPC_DIRECT_RUNNER))
_RUNNER_MAP.update(_get_runner_map(_KNOWN_TEST_RUNNERS,
_TEST_RUNNER_PATH))
@@ -242,17 +247,18 @@
self._cache[
self.to_cache_key(transform, tag)] = [value, transform.refcounts[tag]]
- def get_pvalue(self, pvalue):
+ def get_pvalue(self, pvalue, decref=True):
"""Gets the value associated with a PValue from the cache."""
self._ensure_pvalue_has_real_producer(pvalue)
try:
value_with_refcount = self._cache[self.key(pvalue)]
- value_with_refcount[1] -= 1
- logging.debug('PValue computed by %s (tag %s): refcount: %d => %d',
- pvalue.real_producer.full_label, self.key(pvalue)[1],
- value_with_refcount[1] + 1, value_with_refcount[1])
- if value_with_refcount[1] <= 0:
- self.clear_pvalue(pvalue)
+ if decref:
+ value_with_refcount[1] -= 1
+ logging.debug('PValue computed by %s (tag %s): refcount: %d => %d',
+ pvalue.real_producer.full_label, self.key(pvalue)[1],
+ value_with_refcount[1] + 1, value_with_refcount[1])
+ if value_with_refcount[1] <= 0:
+ self.clear_pvalue(pvalue)
return value_with_refcount[0]
except KeyError:
if (pvalue.tag is not None
@@ -263,8 +269,8 @@
else:
raise
- def get_unwindowed_pvalue(self, pvalue):
- return [v.value for v in self.get_pvalue(pvalue)]
+ def get_unwindowed_pvalue(self, pvalue, decref=True):
+ return [v.value for v in self.get_pvalue(pvalue, decref)]
def clear_pvalue(self, pvalue):
"""Removes a PValue from the cache."""
@@ -277,13 +283,14 @@
class PipelineState(object):
- """State of the Pipeline, as returned by PipelineResult.state.
+ """State of the Pipeline, as returned by :attr:`PipelineResult.state`.
This is meant to be the union of all the states any runner can put a
- pipeline in. Currently, it represents the values of the dataflow
+ pipeline in. Currently, it represents the values of the dataflow
API JobState enum.
"""
UNKNOWN = 'UNKNOWN' # not specified
+ STARTING = 'STARTING' # not yet started
STOPPED = 'STOPPED' # paused or not yet started
RUNNING = 'RUNNING' # currently running
DONE = 'DONE' # successfully completed (terminal state)
@@ -292,10 +299,13 @@
UPDATED = 'UPDATED' # replaced by another job (terminal state)
DRAINING = 'DRAINING' # still processing, no longer reading data
DRAINED = 'DRAINED' # draining completed (terminal state)
+ PENDING = 'PENDING' # the job has been created but is not yet running.
+ CANCELLING = 'CANCELLING' # job has been explicitly cancelled and is
+ # in the process of stopping
class PipelineResult(object):
- """A PipelineResult provides access to info about a pipeline."""
+ """A :class:`PipelineResult` provides access to info about a pipeline."""
def __init__(self, state):
self._state = state
@@ -309,15 +319,18 @@
"""Waits until the pipeline finishes and returns the final status.
Args:
- duration: The time to wait (in milliseconds) for job to finish. If it is
- set to None, it will wait indefinitely until the job is finished.
+ duration (int): The time to wait (in milliseconds) for job to finish.
+ If it is set to :data:`None`, it will wait indefinitely until the job
+ is finished.
Raises:
- IOError: If there is a persistent problem getting job information.
- NotImplementedError: If the runner does not support this operation.
+ ~exceptions.IOError: If there is a persistent problem getting job
+ information.
+ ~exceptions.NotImplementedError: If the runner does not support this
+ operation.
Returns:
- The final state of the pipeline, or None on timeout.
+ The final state of the pipeline, or :data:`None` on timeout.
"""
raise NotImplementedError
@@ -325,8 +338,10 @@
"""Cancels the pipeline execution.
Raises:
- IOError: If there is a persistent problem getting job information.
- NotImplementedError: If the runner does not support this operation.
+ ~exceptions.IOError: If there is a persistent problem getting job
+ information.
+ ~exceptions.NotImplementedError: If the runner does not support this
+ operation.
Returns:
The final state of the pipeline.
@@ -334,10 +349,12 @@
raise NotImplementedError
def metrics(self):
- """Returns MetricsResult object to query metrics from the runner.
+ """Returns :class:`~apache_beam.metrics.metric.MetricResults` object to
+ query metrics from the runner.
Raises:
- NotImplementedError: If the runner does not support this operation.
+ ~exceptions.NotImplementedError: If the runner does not support this
+ operation.
"""
raise NotImplementedError
diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py
index fa80b1c..063c8a2 100644
--- a/sdks/python/apache_beam/runners/runner_test.py
+++ b/sdks/python/apache_beam/runners/runner_test.py
@@ -33,12 +33,12 @@
from apache_beam.metrics.execution import MetricKey
from apache_beam.metrics.execution import MetricResult
from apache_beam.metrics.metricbase import MetricName
+from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.pipeline import Pipeline
from apache_beam.runners import DirectRunner
from apache_beam.runners import create_runner
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
-from apache_beam.options.pipeline_options import PipelineOptions
class RunnerTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py
new file mode 100644
index 0000000..689eab7
--- /dev/null
+++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py
@@ -0,0 +1,590 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""SDK harness for executing Python Fns via the Fn API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import base64
+import collections
+import json
+import logging
+
+from google.protobuf import wrappers_pb2
+
+import apache_beam as beam
+from apache_beam.coders import WindowedValueCoder
+from apache_beam.coders import coder_impl
+from apache_beam.internal import pickler
+from apache_beam.io import iobase
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_runner_api_pb2
+from apache_beam.runners import pipeline_context
+from apache_beam.runners.dataflow.native_io import iobase as native_iobase
+from apache_beam.runners.worker import operation_specs
+from apache_beam.runners.worker import operations
+from apache_beam.transforms import sideinputs
+from apache_beam.utils import counters
+from apache_beam.utils import proto_utils
+from apache_beam.utils import urns
+
+# This module is experimental. No backwards-compatibility guarantees.
+
+
+try:
+ from apache_beam.runners.worker import statesampler
+except ImportError:
+ from apache_beam.runners.worker import statesampler_fake as statesampler
+
+
+DATA_INPUT_URN = 'urn:org.apache.beam:source:runner:0.1'
+DATA_OUTPUT_URN = 'urn:org.apache.beam:sink:runner:0.1'
+IDENTITY_DOFN_URN = 'urn:org.apache.beam:dofn:identity:0.1'
+PYTHON_ITERABLE_VIEWFN_URN = 'urn:org.apache.beam:viewfn:iterable:python:0.1'
+PYTHON_CODER_URN = 'urn:org.apache.beam:coder:python:0.1'
+# TODO(vikasrk): Fix this once runner sends appropriate python urns.
+OLD_DATAFLOW_RUNNER_HARNESS_PARDO_URN = 'urn:beam:dofn:javasdk:0.1'
+OLD_DATAFLOW_RUNNER_HARNESS_READ_URN = 'urn:org.apache.beam:source:java:0.1'
+
+
+class RunnerIOOperation(operations.Operation):
+ """Common baseclass for runner harness IO operations."""
+
+ def __init__(self, operation_name, step_name, consumers, counter_factory,
+ state_sampler, windowed_coder, target, data_channel):
+ super(RunnerIOOperation, self).__init__(
+ operation_name, None, counter_factory, state_sampler)
+ self.windowed_coder = windowed_coder
+ self.step_name = step_name
+ # target represents the consumer for the bytes in the data plane for a
+ # DataInputOperation or a producer of these bytes for a DataOutputOperation.
+ self.target = target
+ self.data_channel = data_channel
+ for _, consumer_ops in consumers.items():
+ for consumer in consumer_ops:
+ self.add_receiver(consumer, 0)
+
+
+class DataOutputOperation(RunnerIOOperation):
+ """A sink-like operation that gathers outputs to be sent back to the runner.
+ """
+
+ def set_output_stream(self, output_stream):
+ self.output_stream = output_stream
+
+ def process(self, windowed_value):
+ self.windowed_coder.get_impl().encode_to_stream(
+ windowed_value, self.output_stream, True)
+
+ def finish(self):
+ self.output_stream.close()
+ super(DataOutputOperation, self).finish()
+
+
+class DataInputOperation(RunnerIOOperation):
+ """A source-like operation that gathers input from the runner.
+ """
+
+ def __init__(self, operation_name, step_name, consumers, counter_factory,
+ state_sampler, windowed_coder, input_target, data_channel):
+ super(DataInputOperation, self).__init__(
+ operation_name, step_name, consumers, counter_factory, state_sampler,
+ windowed_coder, target=input_target, data_channel=data_channel)
+ # We must do this manually as we don't have a spec or spec.output_coders.
+ self.receivers = [
+ operations.ConsumerSet(self.counter_factory, self.step_name, 0,
+ next(consumers.itervalues()),
+ self.windowed_coder)]
+
+ def process(self, windowed_value):
+ self.output(windowed_value)
+
+ def process_encoded(self, encoded_windowed_values):
+ input_stream = coder_impl.create_InputStream(encoded_windowed_values)
+ while input_stream.size() > 0:
+ decoded_value = self.windowed_coder.get_impl().decode_from_stream(
+ input_stream, True)
+ self.output(decoded_value)
+
+
+# TODO(robertwb): Revise side input API to not be in terms of native sources.
+# This will enable lookups, but there's an open question as to how to handle
+# custom sources without forcing intermediate materialization. This seems very
+# related to the desire to inject key and window preserving [Splittable]DoFns
+# into the view computation.
+class SideInputSource(native_iobase.NativeSource,
+ native_iobase.NativeSourceReader):
+ """A 'source' for reading side inputs via state API calls.
+ """
+
+ def __init__(self, state_handler, state_key, coder):
+ self._state_handler = state_handler
+ self._state_key = state_key
+ self._coder = coder
+
+ def reader(self):
+ return self
+
+ @property
+ def returns_windowed_values(self):
+ return True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exn_info):
+ pass
+
+ def __iter__(self):
+ # TODO(robertwb): Support pagination.
+ input_stream = coder_impl.create_InputStream(
+ self._state_handler.Get(self._state_key).data)
+ while input_stream.size() > 0:
+ yield self._coder.get_impl().decode_from_stream(input_stream, True)
+
+
+class StateBackedSideInputMap(object):
+ def __init__(self, state_handler, transform_id, tag, side_input_data):
+ self._state_handler = state_handler
+ self._transform_id = transform_id
+ self._tag = tag
+ self._side_input_data = side_input_data
+ self._element_coder = side_input_data.coder.wrapped_value_coder
+ self._target_window_coder = side_input_data.coder.window_coder
+ # TODO(robertwb): Limit the cache size.
+ # TODO(robertwb): Cross-bundle caching respecting cache tokens.
+ self._cache = {}
+
+ def __getitem__(self, window):
+ target_window = self._side_input_data.window_mapping_fn(window)
+ if target_window not in self._cache:
+ state_key = beam_fn_api_pb2.StateKey(
+ multimap_side_input=beam_fn_api_pb2.StateKey.MultimapSideInput(
+ ptransform_id=self._transform_id,
+ side_input_id=self._tag,
+ window=self._target_window_coder.encode(target_window)))
+ element_coder_impl = self._element_coder.get_impl()
+ state_handler = self._state_handler
+
+ class AllElements(object):
+ def __iter__(self):
+ # TODO(robertwb): Support pagination.
+ input_stream = coder_impl.create_InputStream(
+ state_handler.blocking_get(state_key, None))
+ while input_stream.size() > 0:
+ yield element_coder_impl.decode_from_stream(input_stream, True)
+ self._cache[target_window] = self._side_input_data.view_fn(AllElements())
+ return self._cache[target_window]
+
+ def is_globally_windowed(self):
+ return (self._side_input_data.window_mapping_fn
+ == sideinputs._global_window_mapping_fn)
+
+
+def memoize(func):
+ cache = {}
+ missing = object()
+
+ def wrapper(*args):
+ result = cache.get(args, missing)
+ if result is missing:
+ result = cache[args] = func(*args)
+ return result
+ return wrapper
+
+
+def only_element(iterable):
+ element, = iterable
+ return element
+
+
+class BundleProcessor(object):
+ """A class for processing bundles of elements.
+ """
+ def __init__(
+ self, process_bundle_descriptor, state_handler, data_channel_factory):
+ self.process_bundle_descriptor = process_bundle_descriptor
+ self.state_handler = state_handler
+ self.data_channel_factory = data_channel_factory
+ # TODO(robertwb): Figure out the correct prefix to use for output counters
+ # from StateSampler.
+ self.counter_factory = counters.CounterFactory()
+ self.state_sampler = statesampler.StateSampler(
+ 'fnapi-step-%s' % self.process_bundle_descriptor.id,
+ self.counter_factory)
+ self.ops = self.create_execution_tree(self.process_bundle_descriptor)
+
+ def create_execution_tree(self, descriptor):
+
+ transform_factory = BeamTransformFactory(
+ descriptor, self.data_channel_factory, self.counter_factory,
+ self.state_sampler, self.state_handler)
+
+ def is_side_input(transform_proto, tag):
+ if transform_proto.spec.urn == urns.PARDO_TRANSFORM:
+ return tag in proto_utils.parse_Bytes(
+ transform_proto.spec.payload,
+ beam_runner_api_pb2.ParDoPayload).side_inputs
+
+ pcoll_consumers = collections.defaultdict(list)
+ for transform_id, transform_proto in descriptor.transforms.items():
+ for tag, pcoll_id in transform_proto.inputs.items():
+ if not is_side_input(transform_proto, tag):
+ pcoll_consumers[pcoll_id].append(transform_id)
+
+ @memoize
+ def get_operation(transform_id):
+ transform_consumers = {
+ tag: [get_operation(op) for op in pcoll_consumers[pcoll_id]]
+ for tag, pcoll_id
+ in descriptor.transforms[transform_id].outputs.items()
+ }
+ return transform_factory.create_operation(
+ transform_id, transform_consumers)
+
+ # Operations must be started (hence returned) in order.
+ @memoize
+ def topological_height(transform_id):
+ return 1 + max(
+ [0] +
+ [topological_height(consumer)
+ for pcoll in descriptor.transforms[transform_id].outputs.values()
+ for consumer in pcoll_consumers[pcoll]])
+
+ return collections.OrderedDict([
+ (transform_id, get_operation(transform_id))
+ for transform_id in sorted(
+ descriptor.transforms, key=topological_height, reverse=True)])
+
+ def process_bundle(self, instruction_id):
+
+ expected_inputs = []
+ for op in self.ops.values():
+ if isinstance(op, DataOutputOperation):
+ # TODO(robertwb): Is there a better way to pass the instruction id to
+ # the operation?
+ op.set_output_stream(op.data_channel.output_stream(
+ instruction_id, op.target))
+ elif isinstance(op, DataInputOperation):
+ # We must wait until we receive "end of stream" for each of these ops.
+ expected_inputs.append(op)
+
+ try:
+ self.state_sampler.start()
+ # Start all operations.
+ for op in reversed(self.ops.values()):
+ logging.info('start %s', op)
+ op.start()
+
+ # Inject inputs from data plane.
+ for input_op in expected_inputs:
+ for data in input_op.data_channel.input_elements(
+ instruction_id, [input_op.target]):
+ # ignores input name
+ input_op.process_encoded(data.data)
+
+ # Finish all operations.
+ for op in self.ops.values():
+ logging.info('finish %s', op)
+ op.finish()
+ finally:
+ self.state_sampler.stop_if_still_running()
+
+ def metrics(self):
+ return beam_fn_api_pb2.Metrics(
+ # TODO(robertwb): Rename to progress?
+ ptransforms=
+ {transform_id:
+ self._fix_output_tags(transform_id, op.progress_metrics())
+ for transform_id, op in self.ops.items()})
+
+ def _fix_output_tags(self, transform_id, metrics):
+ # Outputs are still referred to by index, not by name, in many Operations.
+ # However, if there is exactly one output, we can fix up the name here.
+ def fix_only_output_tag(actual_output_tag, mapping):
+ if len(mapping) == 1:
+ fake_output_tag, count = only_element(mapping.items())
+ if fake_output_tag != actual_output_tag:
+ del mapping[fake_output_tag]
+ mapping[actual_output_tag] = count
+ actual_output_tags = list(
+ self.process_bundle_descriptor.transforms[transform_id].outputs.keys())
+ if len(actual_output_tags) == 1:
+ fix_only_output_tag(
+ actual_output_tags[0],
+ metrics.processed_elements.measured.output_element_counts)
+ fix_only_output_tag(
+ actual_output_tags[0],
+ metrics.active_elements.measured.output_element_counts)
+ return metrics
+
+
+class BeamTransformFactory(object):
+ """Factory for turning transform_protos into executable operations."""
+ def __init__(self, descriptor, data_channel_factory, counter_factory,
+ state_sampler, state_handler):
+ self.descriptor = descriptor
+ self.data_channel_factory = data_channel_factory
+ self.counter_factory = counter_factory
+ self.state_sampler = state_sampler
+ self.state_handler = state_handler
+ self.context = pipeline_context.PipelineContext(descriptor)
+
+ _known_urns = {}
+
+ @classmethod
+ def register_urn(cls, urn, parameter_type):
+ def wrapper(func):
+ cls._known_urns[urn] = func, parameter_type
+ return func
+ return wrapper
+
+ def create_operation(self, transform_id, consumers):
+ transform_proto = self.descriptor.transforms[transform_id]
+ creator, parameter_type = self._known_urns[transform_proto.spec.urn]
+ payload = proto_utils.parse_Bytes(
+ transform_proto.spec.payload, parameter_type)
+ return creator(self, transform_id, transform_proto, payload, consumers)
+
+ def get_coder(self, coder_id):
+ coder_proto = self.descriptor.coders[coder_id]
+ if coder_proto.spec.spec.urn:
+ return self.context.coders.get_by_id(coder_id)
+ else:
+ # No URN, assume cloud object encoding json bytes.
+ return operation_specs.get_coder_from_spec(
+ json.loads(coder_proto.spec.spec.payload))
+
+ def get_output_coders(self, transform_proto):
+ return {
+ tag: self.get_coder(self.descriptor.pcollections[pcoll_id].coder_id)
+ for tag, pcoll_id in transform_proto.outputs.items()
+ }
+
+ def get_only_output_coder(self, transform_proto):
+ return only_element(self.get_output_coders(transform_proto).values())
+
+ def get_input_coders(self, transform_proto):
+ return {
+ tag: self.get_coder(self.descriptor.pcollections[pcoll_id].coder_id)
+ for tag, pcoll_id in transform_proto.inputs.items()
+ }
+
+ def get_only_input_coder(self, transform_proto):
+ return only_element(self.get_input_coders(transform_proto).values())
+
+ # TODO(robertwb): Update all operations to take these in the constructor.
+ @staticmethod
+ def augment_oldstyle_op(op, step_name, consumers, tag_list=None):
+ op.step_name = step_name
+ for tag, op_consumers in consumers.items():
+ for consumer in op_consumers:
+ op.add_receiver(consumer, tag_list.index(tag) if tag_list else 0)
+ return op
+
+
+@BeamTransformFactory.register_urn(
+ DATA_INPUT_URN, beam_fn_api_pb2.RemoteGrpcPort)
+def create(factory, transform_id, transform_proto, grpc_port, consumers):
+ target = beam_fn_api_pb2.Target(
+ primitive_transform_reference=transform_id,
+ name=only_element(transform_proto.outputs.keys()))
+ return DataInputOperation(
+ transform_proto.unique_name,
+ transform_proto.unique_name,
+ consumers,
+ factory.counter_factory,
+ factory.state_sampler,
+ factory.get_only_output_coder(transform_proto),
+ input_target=target,
+ data_channel=factory.data_channel_factory.create_data_channel(grpc_port))
+
+
+@BeamTransformFactory.register_urn(
+ DATA_OUTPUT_URN, beam_fn_api_pb2.RemoteGrpcPort)
+def create(factory, transform_id, transform_proto, grpc_port, consumers):
+ target = beam_fn_api_pb2.Target(
+ primitive_transform_reference=transform_id,
+ name=only_element(transform_proto.inputs.keys()))
+ return DataOutputOperation(
+ transform_proto.unique_name,
+ transform_proto.unique_name,
+ consumers,
+ factory.counter_factory,
+ factory.state_sampler,
+ # TODO(robertwb): Perhaps this could be distinct from the input coder?
+ factory.get_only_input_coder(transform_proto),
+ target=target,
+ data_channel=factory.data_channel_factory.create_data_channel(grpc_port))
+
+
+@BeamTransformFactory.register_urn(OLD_DATAFLOW_RUNNER_HARNESS_READ_URN, None)
+def create(factory, transform_id, transform_proto, parameter, consumers):
+ # The Dataflow runner harness strips the base64 encoding.
+ source = pickler.loads(base64.b64encode(parameter))
+ spec = operation_specs.WorkerRead(
+ iobase.SourceBundle(1.0, source, None, None),
+ [WindowedValueCoder(source.default_output_coder())])
+ return factory.augment_oldstyle_op(
+ operations.ReadOperation(
+ transform_proto.unique_name,
+ spec,
+ factory.counter_factory,
+ factory.state_sampler),
+ transform_proto.unique_name,
+ consumers)
+
+
+@BeamTransformFactory.register_urn(
+ urns.READ_TRANSFORM, beam_runner_api_pb2.ReadPayload)
+def create(factory, transform_id, transform_proto, parameter, consumers):
+ source = iobase.SourceBase.from_runner_api(parameter.source, factory.context)
+ spec = operation_specs.WorkerRead(
+ iobase.SourceBundle(1.0, source, None, None),
+ [WindowedValueCoder(source.default_output_coder())])
+ return factory.augment_oldstyle_op(
+ operations.ReadOperation(
+ transform_proto.unique_name,
+ spec,
+ factory.counter_factory,
+ factory.state_sampler),
+ transform_proto.unique_name,
+ consumers)
+
+
+@BeamTransformFactory.register_urn(OLD_DATAFLOW_RUNNER_HARNESS_PARDO_URN, None)
+def create(factory, transform_id, transform_proto, serialized_fn, consumers):
+ return _create_pardo_operation(
+ factory, transform_id, transform_proto, consumers, serialized_fn)
+
+
+@BeamTransformFactory.register_urn(
+ urns.PARDO_TRANSFORM, beam_runner_api_pb2.ParDoPayload)
+def create(factory, transform_id, transform_proto, parameter, consumers):
+ assert parameter.do_fn.spec.urn == urns.PICKLED_DO_FN_INFO
+ serialized_fn = parameter.do_fn.spec.payload
+ return _create_pardo_operation(
+ factory, transform_id, transform_proto, consumers,
+ serialized_fn, parameter.side_inputs)
+
+
+def _create_pardo_operation(
+ factory, transform_id, transform_proto, consumers,
+ serialized_fn, side_inputs_proto=None):
+
+ if side_inputs_proto:
+ tagged_side_inputs = [
+ (tag, beam.pvalue.SideInputData.from_runner_api(si, factory.context))
+ for tag, si in side_inputs_proto.items()]
+ tagged_side_inputs.sort(key=lambda tag_si: int(tag_si[0][4:]))
+ side_input_maps = [
+ StateBackedSideInputMap(factory.state_handler, transform_id, tag, si)
+ for tag, si in tagged_side_inputs]
+ else:
+ side_input_maps = []
+
+ output_tags = list(transform_proto.outputs.keys())
+
+ # Hack to match out prefix injected by dataflow runner.
+ def mutate_tag(tag):
+ if 'None' in output_tags:
+ if tag == 'None':
+ return 'out'
+ else:
+ return 'out_' + tag
+ else:
+ return tag
+
+ dofn_data = pickler.loads(serialized_fn)
+ if not dofn_data[-1]:
+ # Windowing not set.
+ side_input_tags = side_inputs_proto or ()
+ pcoll_id, = [pcoll for tag, pcoll in transform_proto.inputs.items()
+ if tag not in side_input_tags]
+ windowing = factory.context.windowing_strategies.get_by_id(
+ factory.descriptor.pcollections[pcoll_id].windowing_strategy_id)
+ serialized_fn = pickler.dumps(dofn_data[:-1] + (windowing,))
+
+ output_coders = factory.get_output_coders(transform_proto)
+ spec = operation_specs.WorkerDoFn(
+ serialized_fn=serialized_fn,
+ output_tags=[mutate_tag(tag) for tag in output_tags],
+ input=None,
+ side_inputs=[], # Obsoleted by side_input_maps.
+ output_coders=[output_coders[tag] for tag in output_tags])
+ return factory.augment_oldstyle_op(
+ operations.DoOperation(
+ transform_proto.unique_name,
+ spec,
+ factory.counter_factory,
+ factory.state_sampler,
+ side_input_maps),
+ transform_proto.unique_name,
+ consumers,
+ output_tags)
+
+
+def _create_simple_pardo_operation(
+ factory, transform_id, transform_proto, consumers, dofn):
+ serialized_fn = pickler.dumps((dofn, (), {}, [], None))
+ return _create_pardo_operation(
+ factory, transform_id, transform_proto, consumers, serialized_fn)
+
+
+@BeamTransformFactory.register_urn(
+ urns.GROUP_ALSO_BY_WINDOW_TRANSFORM, wrappers_pb2.BytesValue)
+def create(factory, transform_id, transform_proto, parameter, consumers):
+ # Perhaps this hack can go away once all apply overloads are gone.
+ from apache_beam.transforms.core import _GroupAlsoByWindowDoFn
+ return _create_simple_pardo_operation(
+ factory, transform_id, transform_proto, consumers,
+ _GroupAlsoByWindowDoFn(
+ factory.context.windowing_strategies.get_by_id(parameter.value)))
+
+
+@BeamTransformFactory.register_urn(
+ urns.WINDOW_INTO_TRANSFORM, beam_runner_api_pb2.WindowingStrategy)
+def create(factory, transform_id, transform_proto, parameter, consumers):
+ class WindowIntoDoFn(beam.DoFn):
+ def __init__(self, windowing):
+ self.windowing = windowing
+
+ def process(self, element, timestamp=beam.DoFn.TimestampParam):
+ new_windows = self.windowing.windowfn.assign(
+ WindowFn.AssignContext(timestamp, element=element))
+ yield WindowedValue(element, timestamp, new_windows)
+ from apache_beam.transforms.core import Windowing
+ from apache_beam.transforms.window import WindowFn, WindowedValue
+ windowing = Windowing.from_runner_api(parameter, factory.context)
+ return _create_simple_pardo_operation(
+ factory, transform_id, transform_proto, consumers,
+ WindowIntoDoFn(windowing))
+
+
+@BeamTransformFactory.register_urn(IDENTITY_DOFN_URN, None)
+def create(factory, transform_id, transform_proto, unused_parameter, consumers):
+ return factory.augment_oldstyle_op(
+ operations.FlattenOperation(
+ transform_proto.unique_name,
+ operation_specs.WorkerFlatten(
+ None, [factory.get_only_output_coder(transform_proto)]),
+ factory.counter_factory,
+ factory.state_sampler),
+ transform_proto.unique_name,
+ consumers)
diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py
index 5edd0b4..f2a3751 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane.py
@@ -25,12 +25,15 @@
import collections
import logging
import Queue as queue
+import sys
import threading
-from apache_beam.coders import coder_impl
-from apache_beam.runners.api import beam_fn_api_pb2
import grpc
+from apache_beam.coders import coder_impl
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+
# This module is experimental. No backwards-compatibility guarantees.
@@ -144,9 +147,12 @@
self._received = collections.defaultdict(queue.Queue)
self._receive_lock = threading.Lock()
self._reads_finished = threading.Event()
+ self._closed = False
+ self._exc_info = None
def close(self):
self._to_send.put(self._WRITES_FINISHED)
+ self._closed = True
def wait(self, timeout=None):
self._reads_finished.wait(timeout)
@@ -159,20 +165,31 @@
received = self._receiving_queue(instruction_id)
done_targets = []
while len(done_targets) < len(expected_targets):
- data = received.get()
- if not data.data and data.target in expected_targets:
- done_targets.append(data.target)
+ try:
+ data = received.get(timeout=1)
+ except queue.Empty:
+ if self._exc_info:
+ raise exc_info[0], exc_info[1], exc_info[2]
else:
- assert data.target not in done_targets
- yield data
+ if not data.data and data.target in expected_targets:
+ done_targets.append(data.target)
+ else:
+ assert data.target not in done_targets
+ yield data
def output_stream(self, instruction_id, target):
+ # TODO: Return an output stream that sends data
+ # to the Runner once a fixed size buffer is full.
+ # Currently we buffer all the data before sending
+ # any messages.
def add_to_send_queue(data):
- self._to_send.put(
- beam_fn_api_pb2.Elements.Data(
- instruction_reference=instruction_id,
- target=target,
- data=data))
+ if data:
+ self._to_send.put(
+ beam_fn_api_pb2.Elements.Data(
+ instruction_reference=instruction_id,
+ target=target,
+ data=data))
+ # End of stream marker.
self._to_send.put(
beam_fn_api_pb2.Elements.Data(
instruction_reference=instruction_id,
@@ -202,9 +219,11 @@
for elements in elements_iterator:
for data in elements.data:
self._receiving_queue(data.instruction_reference).put(data)
- except: # pylint: disable=broad-except
- logging.exception('Failed to read inputs in the data plane')
- raise
+ except: # pylint: disable=bare-except
+ if not self._closed:
+ logging.exception('Failed to read inputs in the data plane')
+ self._exc_info = sys.exc_info()
+ raise
finally:
self._reads_finished.set()
@@ -225,7 +244,7 @@
class GrpcServerDataChannel(
- beam_fn_api_pb2.BeamFnDataServicer, _GrpcDataChannel):
+ beam_fn_api_pb2_grpc.BeamFnDataServicer, _GrpcDataChannel):
"""A DataChannel wrapping the server side of a BeamFnData connection."""
def Data(self, elements_iterator, context):
@@ -240,8 +259,8 @@
__metaclass__ = abc.ABCMeta
@abc.abstractmethod
- def create_data_channel(self, function_spec):
- """Returns a ``DataChannel`` from the given function_spec."""
+ def create_data_channel(self, remote_grpc_port):
+ """Returns a ``DataChannel`` from the given RemoteGrpcPort."""
raise NotImplementedError(type(self))
@abc.abstractmethod
@@ -259,15 +278,19 @@
def __init__(self):
self._data_channel_cache = {}
- def create_data_channel(self, function_spec):
- remote_grpc_port = beam_fn_api_pb2.RemoteGrpcPort()
- function_spec.data.Unpack(remote_grpc_port)
+ def create_data_channel(self, remote_grpc_port):
url = remote_grpc_port.api_service_descriptor.url
if url not in self._data_channel_cache:
logging.info('Creating channel for %s', url)
- grpc_channel = grpc.insecure_channel(url)
+ grpc_channel = grpc.insecure_channel(
+ url,
+ # Options to have no limits (-1) on the size of the messages
+ # received or sent over the data plane. The actual buffer size is
+ # controlled in a layer above.
+ options=[("grpc.max_receive_message_length", -1),
+ ("grpc.max_send_message_length", -1)])
self._data_channel_cache[url] = GrpcClientDataChannel(
- beam_fn_api_pb2.BeamFnDataStub(grpc_channel))
+ beam_fn_api_pb2_grpc.BeamFnDataStub(grpc_channel))
return self._data_channel_cache[url]
def close(self):
@@ -283,7 +306,7 @@
def __init__(self, in_memory_data_channel):
self._in_memory_data_channel = in_memory_data_channel
- def create_data_channel(self, unused_function_spec):
+ def create_data_channel(self, unused_remote_grpc_port):
return self._in_memory_data_channel
def close(self):
diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py
index 7340789..07ba8fd 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane_test.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py
@@ -25,11 +25,12 @@
import sys
import threading
import unittest
-
-import grpc
from concurrent import futures
-from apache_beam.runners.api import beam_fn_api_pb2
+import grpc
+
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
from apache_beam.runners.worker import data_plane
@@ -62,12 +63,12 @@
data_channel_service = data_plane.GrpcServerDataChannel()
server = grpc.server(futures.ThreadPoolExecutor(max_workers=2))
- beam_fn_api_pb2.add_BeamFnDataServicer_to_server(
+ beam_fn_api_pb2_grpc.add_BeamFnDataServicer_to_server(
data_channel_service, server)
test_port = server.add_insecure_port('[::]:0')
server.start()
- data_channel_stub = beam_fn_api_pb2.BeamFnDataStub(
+ data_channel_stub = beam_fn_api_pb2_grpc.BeamFnDataStub(
grpc.insecure_channel('localhost:%s' % test_port))
data_channel_client = data_plane.GrpcClientDataChannel(data_channel_stub)
diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py
index 59ffbf4..6d8a1d9 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler.py
@@ -21,9 +21,11 @@
import Queue as queue
import threading
-from apache_beam.runners.api import beam_fn_api_pb2
import grpc
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+
# This module is experimental. No backwards-compatibility guarantees.
@@ -37,17 +39,18 @@
# Mapping from logging levels to LogEntry levels.
LOG_LEVEL_MAP = {
- logging.FATAL: beam_fn_api_pb2.LogEntry.CRITICAL,
- logging.ERROR: beam_fn_api_pb2.LogEntry.ERROR,
- logging.WARNING: beam_fn_api_pb2.LogEntry.WARN,
- logging.INFO: beam_fn_api_pb2.LogEntry.INFO,
- logging.DEBUG: beam_fn_api_pb2.LogEntry.DEBUG
+ logging.FATAL: beam_fn_api_pb2.LogEntry.Severity.CRITICAL,
+ logging.ERROR: beam_fn_api_pb2.LogEntry.Severity.ERROR,
+ logging.WARNING: beam_fn_api_pb2.LogEntry.Severity.WARN,
+ logging.INFO: beam_fn_api_pb2.LogEntry.Severity.INFO,
+ logging.DEBUG: beam_fn_api_pb2.LogEntry.Severity.DEBUG
}
def __init__(self, log_service_descriptor):
super(FnApiLogRecordHandler, self).__init__()
self._log_channel = grpc.insecure_channel(log_service_descriptor.url)
- self._logging_stub = beam_fn_api_pb2.BeamFnLoggingStub(self._log_channel)
+ self._logging_stub = beam_fn_api_pb2_grpc.BeamFnLoggingStub(
+ self._log_channel)
self._log_entry_queue = queue.Queue()
log_control_messages = self._logging_stub.Logging(self._write_log_entries())
diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py
index 565bedb..647b8b7 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler_test.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py
@@ -18,15 +18,17 @@
import logging
import unittest
-
-import grpc
from concurrent import futures
-from apache_beam.runners.api import beam_fn_api_pb2
+import grpc
+
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+from apache_beam.portability.api import endpoints_pb2
from apache_beam.runners.worker import log_handler
-class BeamFnLoggingServicer(beam_fn_api_pb2.BeamFnLoggingServicer):
+class BeamFnLoggingServicer(beam_fn_api_pb2_grpc.BeamFnLoggingServicer):
def __init__(self):
self.log_records_received = []
@@ -44,12 +46,12 @@
def setUp(self):
self.test_logging_service = BeamFnLoggingServicer()
self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
- beam_fn_api_pb2.add_BeamFnLoggingServicer_to_server(
+ beam_fn_api_pb2_grpc.add_BeamFnLoggingServicer_to_server(
self.test_logging_service, self.server)
self.test_port = self.server.add_insecure_port('[::]:0')
self.server.start()
- self.logging_service_descriptor = beam_fn_api_pb2.ApiServiceDescriptor()
+ self.logging_service_descriptor = endpoints_pb2.ApiServiceDescriptor()
self.logging_service_descriptor.url = 'localhost:%s' % self.test_port
self.fn_log_handler = log_handler.FnApiLogRecordHandler(
self.logging_service_descriptor)
@@ -73,7 +75,8 @@
num_received_log_entries = 0
for outer in self.test_logging_service.log_records_received:
for log_entry in outer.log_entries:
- self.assertEqual(beam_fn_api_pb2.LogEntry.INFO, log_entry.severity)
+ self.assertEqual(beam_fn_api_pb2.LogEntry.Severity.INFO,
+ log_entry.severity)
self.assertEqual('%s: %s' % (msg, num_received_log_entries),
log_entry.message)
self.assertEqual(u'log_handler_test._verify_fn_log_handler',
@@ -98,8 +101,9 @@
lambda self: self._verify_fn_log_handler(num_logs))
-if __name__ == '__main__':
- for test_name, num_logs_entries in data.iteritems():
- _create_test(test_name, num_logs_entries)
+for test_name, num_logs_entries in data.iteritems():
+ _create_test(test_name, num_logs_entries)
+
+if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/runners/worker/opcounters.py b/sdks/python/apache_beam/runners/worker/opcounters.py
index 2bb15fa..f4ba6b9 100644
--- a/sdks/python/apache_beam/runners/worker/opcounters.py
+++ b/sdks/python/apache_beam/runners/worker/opcounters.py
@@ -20,6 +20,7 @@
"""Counters collect the progress of the Worker for reporting to the service."""
from __future__ import absolute_import
+
import math
import random
@@ -47,10 +48,10 @@
def __init__(self, counter_factory, step_name, coder, output_index):
self._counter_factory = counter_factory
self.element_counter = counter_factory.get_counter(
- '%s-out%d-ElementCount' % (step_name, output_index), Counter.SUM)
+ '%s-out%s-ElementCount' % (step_name, output_index), Counter.SUM)
self.mean_byte_counter = counter_factory.get_counter(
- '%s-out%d-MeanByteCount' % (step_name, output_index), Counter.MEAN)
- self.coder_impl = coder.get_impl()
+ '%s-out%s-MeanByteCount' % (step_name, output_index), Counter.MEAN)
+ self.coder_impl = coder.get_impl() if coder else None
self.active_accumulator = None
self._sample_counter = 0
self._next_sample = 0
@@ -137,7 +138,8 @@
Returns:
True if it is time to compute another element's size.
"""
-
+ if self.coder_impl is None:
+ return False
self._sample_counter += 1
if self._next_sample == 0:
if random.randint(1, self._sample_counter) <= 10:
diff --git a/sdks/python/apache_beam/runners/worker/opcounters_test.py b/sdks/python/apache_beam/runners/worker/opcounters_test.py
index 74561b8..008720f 100644
--- a/sdks/python/apache_beam/runners/worker/opcounters_test.py
+++ b/sdks/python/apache_beam/runners/worker/opcounters_test.py
@@ -25,7 +25,6 @@
from apache_beam.transforms.window import GlobalWindows
from apache_beam.utils.counters import CounterFactory
-
# Classes to test that we can handle a variety of objects.
# These have to be at top level so the pickler can find them.
diff --git a/sdks/python/apache_beam/runners/worker/operation_specs.py b/sdks/python/apache_beam/runners/worker/operation_specs.py
index c03d9a2..bdafbea 100644
--- a/sdks/python/apache_beam/runners/worker/operation_specs.py
+++ b/sdks/python/apache_beam/runners/worker/operation_specs.py
@@ -305,7 +305,8 @@
assert coder_spec is not None
# Ignore the wrappers in these encodings.
- # TODO(silviuc): Make sure with all the renamings that names below are ok.
+ ignored_wrappers = (
+ 'com.google.cloud.dataflow.sdk.util.TimerOrElement$TimerOrElementCoder')
if coder_spec['@type'] in ignored_wrappers:
assert len(coder_spec['component_encodings']) == 1
coder_spec = coder_spec['component_encodings'][0]
@@ -325,23 +326,28 @@
assert len(coder_spec['component_encodings']) == 2
value_coder, window_coder = [
get_coder_from_spec(c) for c in coder_spec['component_encodings']]
- return coders.WindowedValueCoder(value_coder, window_coder=window_coder)
+ return coders.coders.WindowedValueCoder(
+ value_coder, window_coder=window_coder)
elif coder_spec['@type'] == 'kind:interval_window':
assert ('component_encodings' not in coder_spec
- or len(coder_spec['component_encodings'] == 0))
- return coders.IntervalWindowCoder()
+ or not coder_spec['component_encodings'])
+ return coders.coders.IntervalWindowCoder()
elif coder_spec['@type'] == 'kind:global_window':
assert ('component_encodings' not in coder_spec
or not coder_spec['component_encodings'])
- return coders.GlobalWindowCoder()
+ return coders.coders.GlobalWindowCoder()
elif coder_spec['@type'] == 'kind:length_prefix':
assert len(coder_spec['component_encodings']) == 1
- return coders.LengthPrefixCoder(
+ return coders.coders.LengthPrefixCoder(
get_coder_from_spec(coder_spec['component_encodings'][0]))
+ elif coder_spec['@type'] == 'kind:bytes':
+ assert ('component_encodings' not in coder_spec
+ or len(coder_spec['component_encodings'] == 0))
+ return coders.BytesCoder()
# We pass coders in the form "<coder_name>$<pickled_data>" to make the job
# description JSON more readable.
- return coders.deserialize_coder(coder_spec['@type'])
+ return coders.coders.deserialize_coder(coder_spec['@type'])
class MapTask(object):
diff --git a/sdks/python/apache_beam/runners/worker/operations.pxd b/sdks/python/apache_beam/runners/worker/operations.pxd
index 2b4e526..cb05c90 100644
--- a/sdks/python/apache_beam/runners/worker/operations.pxd
+++ b/sdks/python/apache_beam/runners/worker/operations.pxd
@@ -28,7 +28,7 @@
cdef class ConsumerSet(Receiver):
cdef list consumers
- cdef opcounters.OperationCounters opcounter
+ cdef readonly opcounters.OperationCounters opcounter
cdef public step_name
cdef public output_index
cdef public coder
@@ -71,6 +71,8 @@
cdef class DoOperation(Operation):
cdef object dofn_runner
cdef Receiver dofn_receiver
+ cdef object tagged_receivers
+ cdef object side_input_maps
cdef class CombineOperation(Operation):
cdef object phased_combine_fn
diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py
index 5dbe57e..6b5f024 100644
--- a/sdks/python/apache_beam/runners/worker/operations.py
+++ b/sdks/python/apache_beam/runners/worker/operations.py
@@ -28,6 +28,7 @@
from apache_beam.io import iobase
from apache_beam.metrics.execution import MetricsContainer
from apache_beam.metrics.execution import ScopedMetricsContainer
+from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.runners import common
from apache_beam.runners.common import Receiver
from apache_beam.runners.dataflow.internal.names import PropertyNames
@@ -35,11 +36,11 @@
from apache_beam.runners.worker import opcounters
from apache_beam.runners.worker import operation_specs
from apache_beam.runners.worker import sideinputs
+from apache_beam.transforms import sideinputs as apache_sideinputs
from apache_beam.transforms import combiners
from apache_beam.transforms import core
-from apache_beam.transforms import sideinputs as apache_sideinputs
-from apache_beam.transforms.combiners import curry_combine_fn
from apache_beam.transforms.combiners import PhasedCombineFnExecutor
+from apache_beam.transforms.combiners import curry_combine_fn
from apache_beam.transforms.window import GlobalWindows
from apache_beam.utils.windowed_value import WindowedValue
@@ -122,13 +123,15 @@
self.state_sampler = state_sampler
self.scoped_start_state = self.state_sampler.scoped_state(
- self.operation_name + '-start')
+ self.operation_name, 'start')
self.scoped_process_state = self.state_sampler.scoped_state(
- self.operation_name + '-process')
+ self.operation_name, 'process')
self.scoped_finish_state = self.state_sampler.scoped_state(
- self.operation_name + '-finish')
+ self.operation_name, 'finish')
# TODO(ccy): the '-abort' state can be added when the abort is supported in
# Operations.
+ self.scoped_metrics_container = None
+ self.receivers = []
def start(self):
"""Start operation."""
@@ -156,6 +159,24 @@
"""Adds a receiver operation for the specified output."""
self.consumers[output_index].append(operation)
+ def progress_metrics(self):
+ return beam_fn_api_pb2.Metrics.PTransform(
+ processed_elements=beam_fn_api_pb2.Metrics.PTransform.ProcessedElements(
+ measured=beam_fn_api_pb2.Metrics.PTransform.Measured(
+ total_time_spent=(
+ self.scoped_start_state.sampled_seconds()
+ + self.scoped_process_state.sampled_seconds()
+ + self.scoped_finish_state.sampled_seconds()),
+ # Multi-output operations should override this.
+ output_element_counts=(
+ # If there is exactly one output, we can unambiguously
+ # fix its name later, which we do.
+ # TODO(robertwb): Plumb the actual name here.
+ {'ONLY_OUTPUT': self.receivers[0].opcounter
+ .element_counter.value()}
+ if len(self.receivers) == 1
+ else None))))
+
def __str__(self):
"""Generates a useful string for this object.
@@ -225,24 +246,24 @@
class _TaggedReceivers(dict):
- class NullReceiver(Receiver):
+ def __init__(self, counter_factory, step_name):
+ self._counter_factory = counter_factory
+ self._step_name = step_name
- def receive(self, element):
- pass
-
- # For old SDKs.
- def output(self, element):
- pass
-
- def __missing__(self, unused_key):
- if not getattr(self, '_null_receiver', None):
- self._null_receiver = _TaggedReceivers.NullReceiver()
- return self._null_receiver
+ def __missing__(self, tag):
+ self[tag] = receiver = ConsumerSet(
+ self._counter_factory, self._step_name, tag, [], None)
+ return receiver
class DoOperation(Operation):
"""A Do operation that will execute a custom DoFn for each input element."""
+ def __init__(
+ self, name, spec, counter_factory, sampler, side_input_maps=None):
+ super(DoOperation, self).__init__(name, spec, counter_factory, sampler)
+ self.side_input_maps = side_input_maps
+
def _read_side_inputs(self, tags_and_types):
"""Generator reading side inputs in the order prescribed by tags_and_types.
@@ -257,6 +278,10 @@
either in singleton or collection mode according to the tags_and_types
argument.
"""
+ # Only call this on the old path where side_input_maps was not
+ # provided directly.
+ assert self.side_input_maps is None
+
# We will read the side inputs in the order prescribed by the
# tags_and_types argument because this is exactly the order needed to
# replace the ArgumentPlaceholder objects in the args/kwargs of the DoFn
@@ -281,7 +306,7 @@
# Backwards compatibility for pre BEAM-733 SDKs.
if isinstance(view_options, tuple):
- if view_class == pvalue.SingletonPCollectionView:
+ if view_class == pvalue.AsSingleton:
has_default, default = view_options
view_options = {'default': default} if has_default else {}
else:
@@ -307,7 +332,8 @@
# Tag to output index map used to dispatch the side output values emitted
# by the DoFn function to the appropriate receivers. The main output is
# tagged with None and is associated with its corresponding index.
- tagged_receivers = _TaggedReceivers()
+ self.tagged_receivers = _TaggedReceivers(
+ self.counter_factory, self.step_name)
output_tag_prefix = PropertyNames.OUT + '_'
for index, tag in enumerate(self.spec.output_tags):
@@ -317,11 +343,17 @@
original_tag = tag[len(output_tag_prefix):]
else:
raise ValueError('Unexpected output name for operation: %s' % tag)
- tagged_receivers[original_tag] = self.receivers[index]
+ self.tagged_receivers[original_tag] = self.receivers[index]
+
+ if self.side_input_maps is None:
+ if tags_and_types:
+ self.side_input_maps = list(self._read_side_inputs(tags_and_types))
+ else:
+ self.side_input_maps = []
self.dofn_runner = common.DoFnRunner(
- fn, args, kwargs, self._read_side_inputs(tags_and_types),
- window_fn, context, tagged_receivers,
+ fn, args, kwargs, self.side_input_maps,
+ window_fn, context, self.tagged_receivers,
logger, self.step_name,
scoped_metrics_container=self.scoped_metrics_container)
self.dofn_receiver = (self.dofn_runner
@@ -338,6 +370,15 @@
with self.scoped_process_state:
self.dofn_receiver.receive(o)
+ def progress_metrics(self):
+ metrics = super(DoOperation, self).progress_metrics()
+ if self.tagged_receivers:
+ metrics.processed_elements.measured.output_element_counts.clear()
+ for tag, receiver in self.tagged_receivers.items():
+ metrics.processed_elements.measured.output_element_counts[
+ str(tag)] = receiver.opcounter.element_counter.value()
+ return metrics
+
class DoFnRunnerReceiver(Receiver):
@@ -433,7 +474,7 @@
fn, args, kwargs = pickler.loads(self.spec.combine_fn)[:3]
self.combine_fn = curry_combine_fn(fn, args, kwargs)
if (getattr(fn.add_input, 'im_func', None)
- is core.CombineFn.add_input.im_func):
+ is core.CombineFn.add_input.__func__):
# Old versions of the SDK have CombineFns that don't implement add_input.
self.combine_fn_add_input = (
lambda a, e: self.combine_fn.add_inputs(a, [e]))
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py
index 596bb90..55ecbcc 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py
@@ -21,202 +21,37 @@
from __future__ import division
from __future__ import print_function
-import collections
-import json
+import functools
import logging
import Queue as queue
+import sys
import threading
import traceback
-import zlib
+from concurrent import futures
-import dill
-from google.protobuf import wrappers_pb2
+import grpc
-from apache_beam.coders import coder_impl
-from apache_beam.coders import WindowedValueCoder
-from apache_beam.internal import pickler
-from apache_beam.runners.dataflow.native_io import iobase
-from apache_beam.utils import counters
-from apache_beam.runners.api import beam_fn_api_pb2
-from apache_beam.runners.worker import operation_specs
-from apache_beam.runners.worker import operations
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-try:
- from apache_beam.runners.worker import statesampler
-except ImportError:
- from apache_beam.runners.worker import statesampler_fake as statesampler
-from apache_beam.runners.worker.data_plane import GrpcClientDataChannelFactory
-
-
-DATA_INPUT_URN = 'urn:org.apache.beam:source:runner:0.1'
-DATA_OUTPUT_URN = 'urn:org.apache.beam:sink:runner:0.1'
-IDENTITY_DOFN_URN = 'urn:org.apache.beam:dofn:identity:0.1'
-PYTHON_ITERABLE_VIEWFN_URN = 'urn:org.apache.beam:viewfn:iterable:python:0.1'
-PYTHON_CODER_URN = 'urn:org.apache.beam:coder:python:0.1'
-# TODO(vikasrk): Fix this once runner sends appropriate python urns.
-PYTHON_DOFN_URN = 'urn:org.apache.beam:dofn:java:0.1'
-PYTHON_SOURCE_URN = 'urn:org.apache.beam:source:java:0.1'
-
-
-class RunnerIOOperation(operations.Operation):
- """Common baseclass for runner harness IO operations."""
-
- def __init__(self, operation_name, step_name, consumers, counter_factory,
- state_sampler, windowed_coder, target, data_channel):
- super(RunnerIOOperation, self).__init__(
- operation_name, None, counter_factory, state_sampler)
- self.windowed_coder = windowed_coder
- self.step_name = step_name
- # target represents the consumer for the bytes in the data plane for a
- # DataInputOperation or a producer of these bytes for a DataOutputOperation.
- self.target = target
- self.data_channel = data_channel
- for _, consumer_ops in consumers.items():
- for consumer in consumer_ops:
- self.add_receiver(consumer, 0)
-
-
-class DataOutputOperation(RunnerIOOperation):
- """A sink-like operation that gathers outputs to be sent back to the runner.
- """
-
- def set_output_stream(self, output_stream):
- self.output_stream = output_stream
-
- def process(self, windowed_value):
- self.windowed_coder.get_impl().encode_to_stream(
- windowed_value, self.output_stream, True)
-
- def finish(self):
- self.output_stream.close()
- super(DataOutputOperation, self).finish()
-
-
-class DataInputOperation(RunnerIOOperation):
- """A source-like operation that gathers input from the runner.
- """
-
- def __init__(self, operation_name, step_name, consumers, counter_factory,
- state_sampler, windowed_coder, input_target, data_channel):
- super(DataInputOperation, self).__init__(
- operation_name, step_name, consumers, counter_factory, state_sampler,
- windowed_coder, target=input_target, data_channel=data_channel)
- # We must do this manually as we don't have a spec or spec.output_coders.
- self.receivers = [
- operations.ConsumerSet(self.counter_factory, self.step_name, 0,
- consumers.itervalues().next(),
- self.windowed_coder)]
-
- def process(self, windowed_value):
- self.output(windowed_value)
-
- def process_encoded(self, encoded_windowed_values):
- input_stream = coder_impl.create_InputStream(encoded_windowed_values)
- while input_stream.size() > 0:
- decoded_value = self.windowed_coder.get_impl().decode_from_stream(
- input_stream, True)
- self.output(decoded_value)
-
-
-# TODO(robertwb): Revise side input API to not be in terms of native sources.
-# This will enable lookups, but there's an open question as to how to handle
-# custom sources without forcing intermediate materialization. This seems very
-# related to the desire to inject key and window preserving [Splittable]DoFns
-# into the view computation.
-class SideInputSource(iobase.NativeSource, iobase.NativeSourceReader):
- """A 'source' for reading side inputs via state API calls.
- """
-
- def __init__(self, state_handler, state_key, coder):
- self._state_handler = state_handler
- self._state_key = state_key
- self._coder = coder
-
- def reader(self):
- return self
-
- @property
- def returns_windowed_values(self):
- return True
-
- def __enter__(self):
- return self
-
- def __exit__(self, *exn_info):
- pass
-
- def __iter__(self):
- # TODO(robertwb): Support pagination.
- input_stream = coder_impl.create_InputStream(
- self._state_handler.Get(self._state_key).data)
- while input_stream.size() > 0:
- yield self._coder.get_impl().decode_from_stream(input_stream, True)
-
-
-def unpack_and_deserialize_py_fn(function_spec):
- """Returns unpacked and deserialized object from function spec proto."""
- return pickler.loads(unpack_function_spec_data(function_spec))
-
-
-def unpack_function_spec_data(function_spec):
- """Returns unpacked data from function spec proto."""
- data = wrappers_pb2.BytesValue()
- function_spec.data.Unpack(data)
- return data.value
-
-
-# pylint: disable=redefined-builtin
-def serialize_and_pack_py_fn(fn, urn, id=None):
- """Returns serialized and packed function in a function spec proto."""
- return pack_function_spec_data(pickler.dumps(fn), urn, id)
-# pylint: enable=redefined-builtin
-
-
-# pylint: disable=redefined-builtin
-def pack_function_spec_data(value, urn, id=None):
- """Returns packed data in a function spec proto."""
- data = wrappers_pb2.BytesValue(value=value)
- fn_proto = beam_fn_api_pb2.FunctionSpec(urn=urn)
- fn_proto.data.Pack(data)
- if id:
- fn_proto.id = id
- return fn_proto
-# pylint: enable=redefined-builtin
-
-
-# TODO(vikasrk): move this method to ``coders.py`` in the SDK.
-def load_compressed(compressed_data):
- """Returns a decompressed and deserialized python object."""
- # Note: SDK uses ``pickler.dumps`` to serialize certain python objects
- # (like sources), which involves serialization, compression and base64
- # encoding. We cannot directly use ``pickler.loads`` for
- # deserialization, as the runner would have already base64 decoded the
- # data. So we only need to decompress and deserialize.
-
- data = zlib.decompress(compressed_data)
- try:
- return dill.loads(data)
- except Exception: # pylint: disable=broad-except
- dill.dill._trace(True) # pylint: disable=protected-access
- return dill.loads(data)
- finally:
- dill.dill._trace(False) # pylint: disable=protected-access
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+from apache_beam.runners.worker import bundle_processor
+from apache_beam.runners.worker import data_plane
class SdkHarness(object):
- def __init__(self, control_channel):
- self._control_channel = control_channel
- self._data_channel_factory = GrpcClientDataChannelFactory()
+ def __init__(self, control_address):
+ self._control_channel = grpc.insecure_channel(control_address)
+ self._data_channel_factory = data_plane.GrpcClientDataChannelFactory()
+ # TODO: Ensure thread safety to run with more than 1 thread.
+ self._default_work_thread_pool = futures.ThreadPoolExecutor(max_workers=1)
+ self._progress_thread_pool = futures.ThreadPoolExecutor(max_workers=1)
def run(self):
- contol_stub = beam_fn_api_pb2.BeamFnControlStub(self._control_channel)
- # TODO(robertwb): Wire up to new state api.
- state_stub = None
- self.worker = SdkWorker(state_stub, self._data_channel_factory)
+ control_stub = beam_fn_api_pb2_grpc.BeamFnControlStub(self._control_channel)
+ state_stub = beam_fn_api_pb2_grpc.BeamFnStateStub(self._control_channel)
+ state_handler = GrpcStateHandler(state_stub)
+ state_handler.start()
+ self.worker = SdkWorker(state_handler, self._data_channel_factory)
responses = queue.Queue()
no_more_work = object()
@@ -228,23 +63,49 @@
return
yield response
- def process_requests():
- for work_request in contol_stub.Control(get_responses()):
- logging.info('Got work %s', work_request.instruction_id)
+ for work_request in control_stub.Control(get_responses()):
+ logging.info('Got work %s', work_request.instruction_id)
+ request_type = work_request.WhichOneof('request')
+ if request_type == ['process_bundle_progress']:
+ thread_pool = self._progress_thread_pool
+ else:
+ thread_pool = self._default_work_thread_pool
+
+ # Need this wrapper to capture the original stack trace.
+ def do_instruction(request):
try:
- response = self.worker.do_instruction(work_request)
- except Exception: # pylint: disable=broad-except
+ return self.worker.do_instruction(request)
+ except Exception as e: # pylint: disable=broad-except
+ traceback_str = traceback.format_exc(e)
+ raise Exception("Error processing request. Original traceback "
+ "is\n%s\n" % traceback_str)
+
+ def handle_response(request, response_future):
+ try:
+ response = response_future.result()
+ except Exception as e: # pylint: disable=broad-except
+ logging.error(
+ 'Error processing instruction %s',
+ request.instruction_id,
+ exc_info=True)
response = beam_fn_api_pb2.InstructionResponse(
- instruction_id=work_request.instruction_id,
- error=traceback.format_exc())
+ instruction_id=request.instruction_id,
+ error=str(e))
responses.put(response)
- t = threading.Thread(target=process_requests)
- t.start()
- t.join()
+
+ thread_pool.submit(do_instruction, work_request).add_done_callback(
+ functools.partial(handle_response, work_request))
+
+ logging.info("No more requests from control plane")
+ logging.info("SDK Harness waiting for in-flight requests to complete")
+ # Wait until existing requests are processed.
+ self._progress_thread_pool.shutdown()
+ self._default_work_thread_pool.shutdown()
# get_responses may be blocked on responses.get(), but we need to return
# control to its caller.
responses.put(no_more_work)
self._data_channel_factory.close()
+ state_handler.done()
logging.info('Done consuming work.')
@@ -254,202 +115,141 @@
self.fns = {}
self.state_handler = state_handler
self.data_channel_factory = data_channel_factory
+ self.bundle_processors = {}
def do_instruction(self, request):
request_type = request.WhichOneof('request')
if request_type:
- # E.g. if register is set, this will construct
- # InstructionResponse(register=self.register(request.register))
- return beam_fn_api_pb2.InstructionResponse(**{
- 'instruction_id': request.instruction_id,
- request_type: getattr(self, request_type)
- (getattr(request, request_type), request.instruction_id)
- })
+ # E.g. if register is set, this will call self.register(request.register))
+ return getattr(self, request_type)(
+ getattr(request, request_type), request.instruction_id)
else:
raise NotImplementedError
- def register(self, request, unused_instruction_id=None):
+ def register(self, request, instruction_id):
for process_bundle_descriptor in request.process_bundle_descriptor:
self.fns[process_bundle_descriptor.id] = process_bundle_descriptor
- for p_transform in list(process_bundle_descriptor.primitive_transform):
- self.fns[p_transform.function_spec.id] = p_transform.function_spec
- return beam_fn_api_pb2.RegisterResponse()
-
- def initial_source_split(self, request, unused_instruction_id=None):
- source_spec = self.fns[request.source_reference]
- assert source_spec.urn == PYTHON_SOURCE_URN
- source_bundle = unpack_and_deserialize_py_fn(
- self.fns[request.source_reference])
- splits = source_bundle.source.split(request.desired_bundle_size_bytes,
- source_bundle.start_position,
- source_bundle.stop_position)
- response = beam_fn_api_pb2.InitialSourceSplitResponse()
- response.splits.extend([
- beam_fn_api_pb2.SourceSplit(
- source=serialize_and_pack_py_fn(split, PYTHON_SOURCE_URN),
- relative_size=split.weight,
- )
- for split in splits
- ])
- return response
-
- def create_execution_tree(self, descriptor):
- # TODO(vikasrk): Add an id field to Coder proto and use that instead.
- coders = {coder.function_spec.id: operation_specs.get_coder_from_spec(
- json.loads(unpack_function_spec_data(coder.function_spec)))
- for coder in descriptor.coders}
-
- counter_factory = counters.CounterFactory()
- # TODO(robertwb): Figure out the correct prefix to use for output counters
- # from StateSampler.
- state_sampler = statesampler.StateSampler(
- 'fnapi-step%s-' % descriptor.id, counter_factory)
- consumers = collections.defaultdict(lambda: collections.defaultdict(list))
- ops_by_id = {}
- reversed_ops = []
-
- for transform in reversed(descriptor.primitive_transform):
- # TODO(robertwb): Figure out how to plumb through the operation name (e.g.
- # "s3") from the service through the FnAPI so that msec counters can be
- # reported and correctly plumbed through the service and the UI.
- operation_name = 'fnapis%s' % transform.id
-
- def only_element(iterable):
- element, = iterable
- return element
-
- if transform.function_spec.urn == DATA_OUTPUT_URN:
- target = beam_fn_api_pb2.Target(
- primitive_transform_reference=transform.id,
- name=only_element(transform.outputs.keys()))
-
- op = DataOutputOperation(
- operation_name,
- transform.step_name,
- consumers[transform.id],
- counter_factory,
- state_sampler,
- coders[only_element(transform.outputs.values()).coder_reference],
- target,
- self.data_channel_factory.create_data_channel(
- transform.function_spec))
-
- elif transform.function_spec.urn == DATA_INPUT_URN:
- target = beam_fn_api_pb2.Target(
- primitive_transform_reference=transform.id,
- name=only_element(transform.inputs.keys()))
- op = DataInputOperation(
- operation_name,
- transform.step_name,
- consumers[transform.id],
- counter_factory,
- state_sampler,
- coders[only_element(transform.outputs.values()).coder_reference],
- target,
- self.data_channel_factory.create_data_channel(
- transform.function_spec))
-
- elif transform.function_spec.urn == PYTHON_DOFN_URN:
- def create_side_input(tag, si):
- # TODO(robertwb): Extract windows (and keys) out of element data.
- return operation_specs.WorkerSideInputSource(
- tag=tag,
- source=SideInputSource(
- self.state_handler,
- beam_fn_api_pb2.StateKey(
- function_spec_reference=si.view_fn.id),
- coder=unpack_and_deserialize_py_fn(si.view_fn)))
- output_tags = list(transform.outputs.keys())
- spec = operation_specs.WorkerDoFn(
- serialized_fn=unpack_function_spec_data(transform.function_spec),
- output_tags=output_tags,
- input=None,
- side_inputs=[create_side_input(tag, si)
- for tag, si in transform.side_inputs.items()],
- output_coders=[coders[transform.outputs[out].coder_reference]
- for out in output_tags])
-
- op = operations.DoOperation(operation_name, spec, counter_factory,
- state_sampler)
- # TODO(robertwb): Move these to the constructor.
- op.step_name = transform.step_name
- for tag, op_consumers in consumers[transform.id].items():
- for consumer in op_consumers:
- op.add_receiver(
- consumer, output_tags.index(tag))
-
- elif transform.function_spec.urn == IDENTITY_DOFN_URN:
- op = operations.FlattenOperation(operation_name, None, counter_factory,
- state_sampler)
- # TODO(robertwb): Move these to the constructor.
- op.step_name = transform.step_name
- for tag, op_consumers in consumers[transform.id].items():
- for consumer in op_consumers:
- op.add_receiver(consumer, 0)
-
- elif transform.function_spec.urn == PYTHON_SOURCE_URN:
- source = load_compressed(unpack_function_spec_data(
- transform.function_spec))
- # TODO(vikasrk): Remove this once custom source is implemented with
- # splittable dofn via the data plane.
- spec = operation_specs.WorkerRead(
- iobase.SourceBundle(1.0, source, None, None),
- [WindowedValueCoder(source.default_output_coder())])
- op = operations.ReadOperation(operation_name, spec, counter_factory,
- state_sampler)
- op.step_name = transform.step_name
- output_tags = list(transform.outputs.keys())
- for tag, op_consumers in consumers[transform.id].items():
- for consumer in op_consumers:
- op.add_receiver(
- consumer, output_tags.index(tag))
-
- else:
- raise NotImplementedError
-
- # Record consumers.
- for _, inputs in transform.inputs.items():
- for target in inputs.target:
- consumers[target.primitive_transform_reference][target.name].append(
- op)
-
- reversed_ops.append(op)
- ops_by_id[transform.id] = op
-
- return list(reversed(reversed_ops)), ops_by_id
+ return beam_fn_api_pb2.InstructionResponse(
+ instruction_id=instruction_id,
+ register=beam_fn_api_pb2.RegisterResponse())
def process_bundle(self, request, instruction_id):
- ops, ops_by_id = self.create_execution_tree(
- self.fns[request.process_bundle_descriptor_reference])
+ self.bundle_processors[
+ instruction_id] = processor = bundle_processor.BundleProcessor(
+ self.fns[request.process_bundle_descriptor_reference],
+ self.state_handler,
+ self.data_channel_factory)
+ try:
+ processor.process_bundle(instruction_id)
+ finally:
+ del self.bundle_processors[instruction_id]
- expected_inputs = []
- for _, op in ops_by_id.items():
- if isinstance(op, DataOutputOperation):
- # TODO(robertwb): Is there a better way to pass the instruction id to
- # the operation?
- op.set_output_stream(op.data_channel.output_stream(
- instruction_id, op.target))
- elif isinstance(op, DataInputOperation):
- # We must wait until we receive "end of stream" for each of these ops.
- expected_inputs.append(op)
+ return beam_fn_api_pb2.InstructionResponse(
+ instruction_id=instruction_id,
+ process_bundle=beam_fn_api_pb2.ProcessBundleResponse(
+ metrics=processor.metrics()))
- # Start all operations.
- for op in reversed(ops):
- logging.info('start %s', op)
- op.start()
+ def process_bundle_progress(self, request, instruction_id):
+ # It is an error to get progress for a not-in-flight bundle.
+ return self.bundle_processors.get(instruction_id).metrics()
- # Inject inputs from data plane.
- for input_op in expected_inputs:
- for data in input_op.data_channel.input_elements(
- instruction_id, [input_op.target]):
- # ignores input name
- target_op = ops_by_id[data.target.primitive_transform_reference]
- # lacks coder for non-input ops
- target_op.process_encoded(data.data)
- # Finish all operations.
- for op in ops:
- logging.info('finish %s', op)
- op.finish()
+class GrpcStateHandler(object):
- return beam_fn_api_pb2.ProcessBundleResponse()
+ _DONE = object()
+
+ def __init__(self, state_stub):
+ self._lock = threading.Lock()
+ self._state_stub = state_stub
+ self._requests = queue.Queue()
+ self._responses_by_id = {}
+ self._last_id = 0
+ self._exc_info = None
+
+ def start(self):
+ self._done = False
+
+ def request_iter():
+ while True:
+ request = self._requests.get()
+ if request is self._DONE or self._done:
+ break
+ yield request
+ responses = self._state_stub.State(request_iter())
+
+ def pull_responses():
+ try:
+ for response in responses:
+ self._responses_by_id[response.id].set(response)
+ if self._done:
+ break
+ except: # pylint: disable=bare-except
+ self._exc_info = sys.exc_info()
+ raise
+ reader = threading.Thread(target=pull_responses, name='read_state')
+ reader.daemon = True
+ reader.start()
+
+ def done(self):
+ self._done = True
+ self._requests.put(self._DONE)
+
+ def blocking_get(self, state_key, instruction_reference):
+ response = self._blocking_request(
+ beam_fn_api_pb2.StateRequest(
+ instruction_reference=instruction_reference,
+ state_key=state_key,
+ get=beam_fn_api_pb2.StateGetRequest()))
+ if response.get.continuation_token:
+ raise NotImplementedErrror
+ return response.get.data
+
+ def blocking_append(self, state_key, data, instruction_reference):
+ self._blocking_request(
+ beam_fn_api_pb2.StateRequest(
+ instruction_reference=instruction_reference,
+ state_key=state_key,
+ append=beam_fn_api_pb2.StateAppendRequest(data=data)))
+
+ def blocking_clear(self, state_key, instruction_reference):
+ self._blocking_request(
+ beam_fn_api_pb2.StateRequest(
+ instruction_reference=instruction_reference,
+ state_key=state_key,
+ clear=beam_fn_api_pb2.StateClearRequest()))
+
+ def _blocking_request(self, request):
+ request.id = self._next_id()
+ self._responses_by_id[request.id] = future = _Future()
+ self._requests.put(request)
+ while not future.wait(timeout=1):
+ if self._exc_info:
+ raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
+ elif self._done:
+ raise RuntimeError()
+ del self._responses_by_id[request.id]
+ return future.get()
+
+ def _next_id(self):
+ self._last_id += 1
+ return str(self._last_id)
+
+
+class _Future(object):
+ """A simple future object to implement blocking requests.
+ """
+ def __init__(self):
+ self._event = threading.Event()
+
+ def wait(self, timeout=None):
+ return self._event.wait(timeout)
+
+ def get(self, timeout=None):
+ if self.wait(timeout):
+ return self._value
+ else:
+ raise LookupError()
+
+ def set(self, value):
+ self._value = value
+ self._event.set()
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
index b891779..70e4c96 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
@@ -21,10 +21,9 @@
import os
import sys
-import grpc
from google.protobuf import text_format
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.api import endpoints_pb2
from apache_beam.runners.worker.log_handler import FnApiLogRecordHandler
from apache_beam.runners.worker.sdk_worker import SdkHarness
@@ -33,31 +32,34 @@
def main(unused_argv):
"""Main entry point for SDK Fn Harness."""
- logging_service_descriptor = beam_fn_api_pb2.ApiServiceDescriptor()
- text_format.Merge(os.environ['LOGGING_API_SERVICE_DESCRIPTOR'],
- logging_service_descriptor)
+ if 'LOGGING_API_SERVICE_DESCRIPTOR' in os.environ:
+ logging_service_descriptor = endpoints_pb2.ApiServiceDescriptor()
+ text_format.Merge(os.environ['LOGGING_API_SERVICE_DESCRIPTOR'],
+ logging_service_descriptor)
- # Send all logs to the runner.
- fn_log_handler = FnApiLogRecordHandler(logging_service_descriptor)
- # TODO(vikasrk): This should be picked up from pipeline options.
- logging.getLogger().setLevel(logging.INFO)
- logging.getLogger().addHandler(fn_log_handler)
+ # Send all logs to the runner.
+ fn_log_handler = FnApiLogRecordHandler(logging_service_descriptor)
+ # TODO(vikasrk): This should be picked up from pipeline options.
+ logging.getLogger().setLevel(logging.INFO)
+ logging.getLogger().addHandler(fn_log_handler)
+ else:
+ fn_log_handler = None
try:
logging.info('Python sdk harness started.')
- service_descriptor = beam_fn_api_pb2.ApiServiceDescriptor()
+ service_descriptor = endpoints_pb2.ApiServiceDescriptor()
text_format.Merge(os.environ['CONTROL_API_SERVICE_DESCRIPTOR'],
service_descriptor)
# TODO(robertwb): Support credentials.
assert not service_descriptor.oauth2_client_credentials_grant.url
- channel = grpc.insecure_channel(service_descriptor.url)
- SdkHarness(channel).run()
+ SdkHarness(service_descriptor.url).run()
logging.info('Python sdk harness exiting.')
except: # pylint: disable=broad-except
logging.exception('Python sdk harness failed: ')
raise
finally:
- fn_log_handler.close()
+ if fn_log_handler:
+ fn_log_handler.close()
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
index 996f44c..2532341 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
@@ -23,18 +23,17 @@
import logging
import unittest
-
-import grpc
from concurrent import futures
-from apache_beam.io.concat_source_test import RangeSource
-from apache_beam.io.iobase import SourceBundle
-from apache_beam.runners.api import beam_fn_api_pb2
-from apache_beam.runners.worker import data_plane
+import grpc
+
+from apache_beam.portability.api import beam_fn_api_pb2
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.runners.worker import sdk_worker
-class BeamFnControlServicer(beam_fn_api_pb2.BeamFnControlServicer):
+class BeamFnControlServicer(beam_fn_api_pb2_grpc.BeamFnControlServicer):
def __init__(self, requests, raise_errors=True):
self.requests = requests
@@ -64,103 +63,28 @@
class SdkWorkerTest(unittest.TestCase):
def test_fn_registration(self):
- fns = [beam_fn_api_pb2.FunctionSpec(id=str(ix)) for ix in range(4)]
-
- process_bundle_descriptors = [beam_fn_api_pb2.ProcessBundleDescriptor(
- id=str(100+ix),
- primitive_transform=[
- beam_fn_api_pb2.PrimitiveTransform(function_spec=fn)])
- for ix, fn in enumerate(fns)]
+ process_bundle_descriptors = [
+ beam_fn_api_pb2.ProcessBundleDescriptor(
+ id=str(100+ix),
+ transforms={
+ str(ix): beam_runner_api_pb2.PTransform(unique_name=str(ix))})
+ for ix in range(4)]
test_controller = BeamFnControlServicer([beam_fn_api_pb2.InstructionRequest(
register=beam_fn_api_pb2.RegisterRequest(
process_bundle_descriptor=process_bundle_descriptors))])
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
- beam_fn_api_pb2.add_BeamFnControlServicer_to_server(test_controller, server)
+ beam_fn_api_pb2_grpc.add_BeamFnControlServicer_to_server(
+ test_controller, server)
test_port = server.add_insecure_port("[::]:0")
server.start()
- channel = grpc.insecure_channel("localhost:%s" % test_port)
- harness = sdk_worker.SdkHarness(channel)
+ harness = sdk_worker.SdkHarness("localhost:%s" % test_port)
harness.run()
self.assertEqual(
harness.worker.fns,
- {item.id: item for item in fns + process_bundle_descriptors})
-
- @unittest.skip("initial splitting not in proto")
- def test_source_split(self):
- source = RangeSource(0, 100)
- expected_splits = list(source.split(30))
-
- worker = sdk_harness.SdkWorker(
- None, data_plane.GrpcClientDataChannelFactory())
- worker.register(
- beam_fn_api_pb2.RegisterRequest(
- process_bundle_descriptor=[beam_fn_api_pb2.ProcessBundleDescriptor(
- primitive_transform=[beam_fn_api_pb2.PrimitiveTransform(
- function_spec=sdk_harness.serialize_and_pack_py_fn(
- SourceBundle(1.0, source, None, None),
- sdk_harness.PYTHON_SOURCE_URN,
- id="src"))])]))
- split_response = worker.initial_source_split(
- beam_fn_api_pb2.InitialSourceSplitRequest(
- desired_bundle_size_bytes=30,
- source_reference="src"))
-
- self.assertEqual(
- expected_splits,
- [sdk_harness.unpack_and_deserialize_py_fn(s.source)
- for s in split_response.splits])
-
- self.assertEqual(
- [s.weight for s in expected_splits],
- [s.relative_size for s in split_response.splits])
-
- @unittest.skip("initial splitting not in proto")
- def test_source_split_via_instruction(self):
-
- source = RangeSource(0, 100)
- expected_splits = list(source.split(30))
-
- test_controller = BeamFnControlServicer([
- beam_fn_api_pb2.InstructionRequest(
- instruction_id="register_request",
- register=beam_fn_api_pb2.RegisterRequest(
- process_bundle_descriptor=[
- beam_fn_api_pb2.ProcessBundleDescriptor(
- primitive_transform=[beam_fn_api_pb2.PrimitiveTransform(
- function_spec=sdk_harness.serialize_and_pack_py_fn(
- SourceBundle(1.0, source, None, None),
- sdk_harness.PYTHON_SOURCE_URN,
- id="src"))])])),
- beam_fn_api_pb2.InstructionRequest(
- instruction_id="split_request",
- initial_source_split=beam_fn_api_pb2.InitialSourceSplitRequest(
- desired_bundle_size_bytes=30,
- source_reference="src"))
- ])
-
- server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
- beam_fn_api_pb2.add_BeamFnControlServicer_to_server(test_controller, server)
- test_port = server.add_insecure_port("[::]:0")
- server.start()
-
- channel = grpc.insecure_channel("localhost:%s" % test_port)
- harness = sdk_harness.SdkHarness(channel)
- harness.run()
-
- split_response = test_controller.responses[
- "split_request"].initial_source_split
-
- self.assertEqual(
- expected_splits,
- [sdk_harness.unpack_and_deserialize_py_fn(s.source)
- for s in split_response.splits])
-
- self.assertEqual(
- [s.weight for s in expected_splits],
- [s.relative_size for s in split_response.splits])
+ {item.id: item for item in process_bundle_descriptors})
if __name__ == "__main__":
diff --git a/sdks/python/apache_beam/runners/worker/statesampler.pyx b/sdks/python/apache_beam/runners/worker/statesampler.pyx
index 3ff6c20..1e37196 100644
--- a/sdks/python/apache_beam/runners/worker/statesampler.pyx
+++ b/sdks/python/apache_beam/runners/worker/statesampler.pyx
@@ -40,12 +40,13 @@
from apache_beam.utils.counters import Counter
-
+from apache_beam.utils.counters import CounterName
cimport cython
from cpython cimport pythread
from libc.stdint cimport int32_t, int64_t
+
cdef extern from "Python.h":
# This typically requires the GIL, but we synchronize the list modifications
# we use this on via our own lock.
@@ -73,12 +74,16 @@
class StateSamplerInfo(object):
"""Info for current state and transition statistics of StateSampler."""
- def __init__(self, state_name, transition_count):
+ def __init__(self, state_name, transition_count, time_since_transition):
self.state_name = state_name
self.transition_count = transition_count
+ self.time_since_transition = time_since_transition
def __repr__(self):
- return '<StateSamplerInfo %s %d>' % (self.state_name, self.transition_count)
+ return ('<StateSamplerInfo state: %s time: %dns transitions: %d>'
+ % (self.state_name,
+ self.time_since_transition,
+ self.transition_count))
# Default period for sampling current state of pipeline execution.
@@ -104,13 +109,17 @@
cdef pythread.PyThread_type_lock lock
cdef public int64_t state_transition_count
+ cdef int64_t time_since_transition
cdef int32_t current_state_index
def __init__(self, prefix, counter_factory,
sampling_period_ms=DEFAULT_SAMPLING_PERIOD_MS):
- self.prefix = prefix
+ # TODO(pabloem): Remove this once all dashed prefixes are removed from
+ # the worker.
+ # We stop using prefixes with included dash.
+ self.prefix = prefix[:-1] if prefix[-1] == '-' else prefix
self.counter_factory = counter_factory
self.sampling_period_ms = sampling_period_ms
@@ -118,6 +127,8 @@
self.scoped_states_by_name = {}
self.current_state_index = 0
+ self.time_since_transition = 0
+ self.state_transition_count = 0
unknown_state = ScopedState(self, 'unknown', self.current_state_index)
pythread.PyThread_acquire_lock(self.lock, pythread.WAIT_LOCK)
self.scoped_states_by_index = [unknown_state]
@@ -138,6 +149,7 @@
def run(self):
cdef int64_t last_nsecs = get_nsec_time()
cdef int64_t elapsed_nsecs
+ cdef int64_t latest_transition_count = self.state_transition_count
with nogil:
while True:
usleep(self.sampling_period_ms * 1000)
@@ -151,6 +163,10 @@
nsecs_ptr = &(<ScopedState>PyList_GET_ITEM(
self.scoped_states_by_index, self.current_state_index)).nsecs
nsecs_ptr[0] += elapsed_nsecs
+ if latest_transition_count != self.state_transition_count:
+ self.time_since_transition = 0
+ latest_transition_count = self.state_transition_count
+ self.time_since_transition += elapsed_nsecs
last_nsecs += elapsed_nsecs
finally:
pythread.PyThread_release_lock(self.lock)
@@ -178,23 +194,48 @@
"""Returns StateSamplerInfo with transition statistics."""
return StateSamplerInfo(
self.scoped_states_by_index[self.current_state_index].name,
- self.state_transition_count)
+ self.state_transition_count,
+ self.time_since_transition)
- def scoped_state(self, name):
- """Returns a context manager managing transitions for a given state."""
- cdef ScopedState scoped_state = self.scoped_states_by_name.get(name, None)
+ # TODO(pabloem): Make state_name required once all callers migrate,
+ # and the legacy path is removed.
+ def scoped_state(self, step_name, state_name=None, io_target=None):
+ """Returns a context manager managing transitions for a given state.
+ Args:
+ step_name: A string with the name of the running step.
+ state_name: A string with the name of the state (e.g. 'process', 'start')
+ io_target: An IOTargetName object describing the io_target (e.g. writing
+ or reading to side inputs, shuffle or state). Will often be None.
+
+ Returns:
+ A ScopedState for the set of step-state-io_target.
+ """
+ cdef ScopedState scoped_state
+ if state_name is None:
+ # If state_name is None, the worker is still using old style
+ # msec counters.
+ counter_name = '%s-%s-msecs' % (self.prefix, step_name)
+ scoped_state = self.scoped_states_by_name.get(counter_name, None)
+ else:
+ counter_name = CounterName(state_name + '-msecs',
+ stage_name=self.prefix,
+ step_name=step_name,
+ io_target=io_target)
+ scoped_state = self.scoped_states_by_name.get(counter_name, None)
+
if scoped_state is None:
- output_counter = self.counter_factory.get_counter(
- '%s%s-msecs' % (self.prefix, name), Counter.SUM)
+ output_counter = self.counter_factory.get_counter(counter_name,
+ Counter.SUM)
new_state_index = len(self.scoped_states_by_index)
- scoped_state = ScopedState(self, name, new_state_index, output_counter)
+ scoped_state = ScopedState(self, counter_name,
+ new_state_index, output_counter)
# Both scoped_states_by_index and scoped_state.nsecs are accessed
# by the sampling thread; initialize them under the lock.
pythread.PyThread_acquire_lock(self.lock, pythread.WAIT_LOCK)
self.scoped_states_by_index.append(scoped_state)
scoped_state.nsecs = 0
pythread.PyThread_release_lock(self.lock)
- self.scoped_states_by_name[name] = scoped_state
+ self.scoped_states_by_name[counter_name] = scoped_state
return scoped_state
def commit_counters(self):
@@ -235,3 +276,6 @@
def __repr__(self):
return "ScopedState[%s, %s, %s]" % (self.name, self.state_index, self.nsecs)
+
+ def sampled_seconds(self):
+ return 1e-9 * self.nsecs
diff --git a/sdks/python/apache_beam/runners/worker/statesampler_fake.py b/sdks/python/apache_beam/runners/worker/statesampler_fake.py
index 88ace8c..bc56021 100644
--- a/sdks/python/apache_beam/runners/worker/statesampler_fake.py
+++ b/sdks/python/apache_beam/runners/worker/statesampler_fake.py
@@ -23,9 +23,21 @@
def __init__(self, *args, **kwargs):
pass
- def scoped_state(self, name):
+ def scoped_state(self, step_name, state_name=None, io_target=None):
return _FakeScopedState()
+ def start(self):
+ pass
+
+ def stop(self):
+ pass
+
+ def stop_if_still_running(self):
+ self.stop()
+
+ def commit_counters(self):
+ pass
+
class _FakeScopedState(object):
@@ -34,3 +46,6 @@
def __exit__(self, *unused_args):
pass
+
+ def sampled_seconds(self):
+ return 0
diff --git a/sdks/python/apache_beam/runners/worker/statesampler_test.py b/sdks/python/apache_beam/runners/worker/statesampler_test.py
index 663cdec..44b2f72 100644
--- a/sdks/python/apache_beam/runners/worker/statesampler_test.py
+++ b/sdks/python/apache_beam/runners/worker/statesampler_test.py
@@ -16,6 +16,7 @@
#
"""Tests for state sampler."""
+from __future__ import absolute_import
import logging
import time
@@ -32,7 +33,7 @@
try:
# pylint: disable=global-variable-not-assigned
global statesampler
- import statesampler
+ from . import statesampler
except ImportError:
raise SkipTest('State sampler not compiled.')
super(StateSamplerTest, self).setUp()
@@ -40,7 +41,7 @@
def test_basic_sampler(self):
# Set up state sampler.
counter_factory = CounterFactory()
- sampler = statesampler.StateSampler('basic-', counter_factory,
+ sampler = statesampler.StateSampler('basic', counter_factory,
sampling_period_ms=1)
# Run basic workload transitioning between 3 states.
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf
new file mode 100644
index 0000000..e9b064b
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf
@@ -0,0 +1,23 @@
+##fileformat=VCFv4.0
+##fileDate=20090805
+##source=myImputationProgramV3.1
+##reference=1000GenomesPilot-NCBI36
+##phasing=partial
+##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
+##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
+##INFO=<ID=AF,Number=.,Type=Float,Description="Allele Frequency">
+##INFO=<ID=AA,Number=1,Type=String,Description="Ancestral Allele">
+##INFO=<ID=DB,Number=0,Type=Flag,Description="dbSNP membership, build 129">
+##INFO=<ID=H2,Number=0,Type=Flag,Description="HapMap2 membership">
+##FILTER=<ID=q10,Description="Quality below 10">
+##FILTER=<ID=s50,Description="Less than 50% of samples have data">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Read Depth">
+##FORMAT=<ID=HQ,Number=2,Type=Integer,Description="Haplotype Quality">
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA00001 NA00002 NA00003
+20 14370 rs6054257 G A 29 PASS NS=3;DP=14;AF=0.5;DB;H2 GT:GQ:DP:HQ 0|0:48:1:51,51 1|0:48:8:51,51 1/1:43:5:.,.
+20 17330 . T A 3 q10 NS=3;DP=11;AF=0.017 GT:GQ:DP:HQ 0|0:49:3:58,50 0|1:3:5:65,3 0/0:41:3
+20 1110696 rs6040355 A G,T 67 PASS NS=2;DP=10;AF=0.333,0.667;AA=T;DB GT:GQ:DP:HQ 1|2:21:6:23,27 2|1:2:0:18,2 2/2:35:4
+20 1230237 . T . 47 PASS NS=3;DP=13;AA=T GT:GQ:DP:HQ 0|0:54:7:56,60 0|0:48:4:51,51 0/0:61:2
+19 1234567 microsat1 GTCT G,GTACT 50 PASS NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/2:17:2 1/1:40:3
\ No newline at end of file
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.bz2 b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.bz2
new file mode 100644
index 0000000..dff64e6
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.bz2
Binary files differ
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.gz b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.gz
new file mode 100644
index 0000000..ac58128
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.gz
Binary files differ
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf b/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf
new file mode 100644
index 0000000..c470685
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf
@@ -0,0 +1,10000 @@
+##fileformat=VCFv4.1
+##fileDate=20121204
+##center=Complete Genomics
+##source=CGAPipeline_2.2.0.26
+##source_GENOME_REFERENCE=NCBI build 37
+##source_MAX_PLOIDY=10
+##source_NUMBER_LEVELS=GS01868-DNA_H02:7
+##source_NONDIPLOID_WINDOW_WIDTH=100000
+##source_MEAN_GC_CORRECTED_CVG=GS01868-DNA_H02:41.51
+##source_GENE_ANNOTATIONS=NCBI build 37.2
+##source_DBSNP_BUILD=dbSNP build 135
+##source_MEI_1000G_ANNOTATIONS=INITIAL-DATA-RELEASE
+##source_COSMIC=COSMIC v59
+##source_DGV_VERSION=9
+##source_MIRBASE_VERSION=mirBase version 18
+##source_PFAM_DATE=April 21, 2011
+##source_REPMASK_GENERATED_AT=2011-Feb-15 10:08
+##source_SEGDUP_GENERATED_AT=2010-Dec-01 13:40
+##phasing=partial
+##reference=ftp://ftp.completegenomics.com/ReferenceFiles/build37.fa.bz2
+##contig=<ID=1,length=249250621,assembly=B37,md5=1b22b98cdeb4a9304cb5d48026a85128,species="Homo sapiens">
+##contig=<ID=2,length=243199373,assembly=B37,md5=a0d9851da00400dec1098a9255ac712e,species="Homo sapiens">
+##contig=<ID=3,length=198022430,assembly=B37,md5=641e4338fa8d52a5b781bd2a2c08d3c3,species="Homo sapiens">
+##contig=<ID=4,length=191154276,assembly=B37,md5=23dccd106897542ad87d2765d28a19a1,species="Homo sapiens">
+##contig=<ID=5,length=180915260,assembly=B37,md5=0740173db9ffd264d728f32784845cd7,species="Homo sapiens">
+##contig=<ID=6,length=171115067,assembly=B37,md5=1d3a93a248d92a729ee764823acbbc6b,species="Homo sapiens">
+##contig=<ID=7,length=159138663,assembly=B37,md5=618366e953d6aaad97dbe4777c29375e,species="Homo sapiens">
+##contig=<ID=8,length=146364022,assembly=B37,md5=96f514a9929e410c6651697bded59aec,species="Homo sapiens">
+##contig=<ID=9,length=141213431,assembly=B37,md5=3e273117f15e0a400f01055d9f393768,species="Homo sapiens">
+##contig=<ID=10,length=135534747,assembly=B37,md5=988c28e000e84c26d552359af1ea2e1d,species="Homo sapiens">
+##contig=<ID=11,length=135006516,assembly=B37,md5=98c59049a2df285c76ffb1c6db8f8b96,species="Homo sapiens">
+##contig=<ID=12,length=133851895,assembly=B37,md5=51851ac0e1a115847ad36449b0015864,species="Homo sapiens">
+##contig=<ID=13,length=115169878,assembly=B37,md5=283f8d7892baa81b510a015719ca7b0b,species="Homo sapiens">
+##contig=<ID=14,length=107349540,assembly=B37,md5=98f3cae32b2a2e9524bc19813927542e,species="Homo sapiens">
+##contig=<ID=15,length=102531392,assembly=B37,md5=e5645a794a8238215b2cd77acb95a078,species="Homo sapiens">
+##contig=<ID=16,length=90354753,assembly=B37,md5=fc9b1a7b42b97a864f56b348b06095e6,species="Homo sapiens">
+##contig=<ID=17,length=81195210,assembly=B37,md5=351f64d4f4f9ddd45b35336ad97aa6de,species="Homo sapiens">
+##contig=<ID=18,length=78077248,assembly=B37,md5=b15d4b2d29dde9d3e4f93d1d0f2cbc9c,species="Homo sapiens">
+##contig=<ID=19,length=59128983,assembly=B37,md5=1aacd71f30db8e561810913e0b72636d,species="Homo sapiens">
+##contig=<ID=20,length=63025520,assembly=B37,md5=0dec9660ec1efaaf33281c0d5ea2560f,species="Homo sapiens">
+##contig=<ID=21,length=48129895,assembly=B37,md5=2979a6085bfe28e3ad6f552f361ed74d,species="Homo sapiens">
+##contig=<ID=22,length=51304566,assembly=B37,md5=a718acaa6135fdca8357d5bfe94211dd,species="Homo sapiens">
+##contig=<ID=X,length=155270560,assembly=B37,md5=7e0e2e580297b7764e31dbc80c2540dd,species="Homo sapiens">
+##contig=<ID=Y,length=59373566,assembly=B37,md5=1e86411d73e6f00a10590f976be01623,species="Homo sapiens">
+##contig=<ID=M,length=16569,assembly=B37,md5=c68f52674c9fb33aef52dcf399755519,species="Homo sapiens">
+##ALT=<ID=CGA_CNVWIN,Description="Copy number analysis window">
+##ALT=<ID=INS:ME:ALU,Description="Insertion of ALU element">
+##ALT=<ID=INS:ME:L1,Description="Insertion of L1 element">
+##ALT=<ID=INS:ME:SVA,Description="Insertion of SVA element">
+##ALT=<ID=INS:ME:MER,Description="Insertion of MER element">
+##ALT=<ID=INS:ME:LTR,Description="Insertion of LTR element">
+##ALT=<ID=INS:ME:PolyA,Description="Insertion of PolyA element">
+##ALT=<ID=INS:ME:HERV,Description="Insertion of HERV element">
+##ALT=<ID=CGA_NOCALL,Description="No-called record">
+##FILTER=<ID=URR,Description="Too close to an underrepresented repeat">
+##FILTER=<ID=MPCBT,Description="Mate pair count below 10">
+##FILTER=<ID=SHORT,Description="Junction side length below 70">
+##FILTER=<ID=TSNR,Description="Transition sequence not resolved">
+##FILTER=<ID=INTERBL,Description="Interchromosomal junction in baseline">
+##FILTER=<ID=sns75,Description="Sensitivity to known MEI calls in range (.75,.95] i.e. medium FDR">
+##FILTER=<ID=sns95,Description="Sensitivity to known MEI calls in range (.95,1.00] i.e. high to very high FDR">
+##FILTER=<ID=VQLOW,Description="Quality not VQHIGH">
+##FILTER=<ID=SQLOW,Description="Somatic quality not SQHIGH">
+##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
+##INFO=<ID=CGA_WINEND,Number=1,Type=Integer,Description="End of coverage window">
+##INFO=<ID=CGA_BF,Number=1,Type=Float,Description="Frequency in baseline">
+##INFO=<ID=CGA_MEDEL,Number=4,Type=String,Description="Consistent with deletion of mobile element; type,chromosome,start,end">
+##INFO=<ID=CGA_XR,Number=A,Type=String,Description="Per-ALT external database reference (dbSNP, COSMIC, etc)">
+##INFO=<ID=MATEID,Number=1,Type=String,Description="ID of mate breakend">
+##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant">
+##INFO=<ID=CGA_BNDG,Number=A,Type=String,Description="Transcript name and strand of genes containing breakend">
+##INFO=<ID=CGA_BNDGO,Number=A,Type=String,Description="Transcript name and strand of genes containing mate breakend">
+##INFO=<ID=CIPOS,Number=2,Type=Integer,Description="Confidence interval around POS for imprecise variants">
+##INFO=<ID=END,Number=1,Type=Integer,Description="End position of the variant described in this record">
+##INFO=<ID=IMPRECISE,Number=0,Type=Flag,Description="Imprecise structural variation">
+##INFO=<ID=MEINFO,Number=4,Type=String,Description="Mobile element info of the form NAME,START,END,POLARITY">
+##INFO=<ID=SVLEN,Number=.,Type=Integer,Description="Difference in length between REF and ALT alleles">
+##INFO=<ID=AN,Number=1,Type=Integer,Description="Total number of alleles in called genotypes">
+##INFO=<ID=AC,Number=A,Type=Integer,Description="Allele count in genotypes, for each ALT allele">
+##INFO=<ID=CGA_FI,Number=A,Type=String,Description="Functional impact annotation">
+##INFO=<ID=CGA_PFAM,Number=.,Type=String,Description="PFAM Domain">
+##INFO=<ID=CGA_MIRB,Number=.,Type=String,Description="miRBaseId">
+##INFO=<ID=CGA_RPT,Number=.,Type=String,Description="repeatMasker overlap information">
+##INFO=<ID=CGA_SDO,Number=1,Type=Integer,Description="Number of distinct segmental duplications that overlap this locus">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=CGA_GP,Number=1,Type=Float,Description="Depth of coverage for 2k window GC normalized to mean">
+##FORMAT=<ID=CGA_NP,Number=1,Type=Float,Description="Coverage for 2k window, GC-corrected and normalized relative to copy-number-corrected multi-sample baseline">
+##FORMAT=<ID=CGA_CL,Number=1,Type=Float,Description="Nondiploid-model called level">
+##FORMAT=<ID=CGA_LS,Number=1,Type=Integer,Description="Nondiploid-model called level score">
+##FORMAT=<ID=CGA_CP,Number=1,Type=Integer,Description="Diploid-model called ploidy">
+##FORMAT=<ID=CGA_PS,Number=1,Type=Integer,Description="Diploid-model called ploidy score">
+##FORMAT=<ID=CGA_CT,Number=1,Type=String,Description="Diploid-model CNV type">
+##FORMAT=<ID=CGA_TS,Number=1,Type=Integer,Description="Diploid-model CNV type score">
+##FORMAT=<ID=FT,Number=1,Type=String,Description="Genotype filters">
+##FORMAT=<ID=CGA_BNDMPC,Number=1,Type=Integer,Description="Mate pair count supporting breakend">
+##FORMAT=<ID=CGA_BNDPOS,Number=1,Type=Integer,Description="Breakend position">
+##FORMAT=<ID=CGA_BNDDEF,Number=1,Type=String,Description="Breakend definition">
+##FORMAT=<ID=CGA_BNDP,Number=1,Type=String,Description="Precision of breakend">
+##FORMAT=<ID=CGA_IS,Number=1,Type=Float,Description="MEI InsertionScore: confidence in occurrence of an insertion">
+##FORMAT=<ID=CGA_IDC,Number=1,Type=Float,Description="MEI InsertionDnbCount: count of paired ends supporting insertion">
+##FORMAT=<ID=CGA_IDCL,Number=1,Type=Float,Description="MEI InsertionLeftDnbCount: count of paired ends supporting insertion on 5' end of insertion point">
+##FORMAT=<ID=CGA_IDCR,Number=1,Type=Float,Description="MEI InsertionRightDnbCount: count of paired ends supporting insertion on 3' end of insertion point">
+##FORMAT=<ID=CGA_RDC,Number=1,Type=Integer,Description="MEI ReferenceDnbCount: count of paired ends supporting reference allele">
+##FORMAT=<ID=CGA_NBET,Number=1,Type=String,Description="MEI NextBestElementType: (sub)type of second-most-likely inserted mobile element">
+##FORMAT=<ID=CGA_ETS,Number=1,Type=Float,Description="MEI ElementTypeScore: confidence that insertion is of type indicated by CGA_ET/ElementType">
+##FORMAT=<ID=CGA_KES,Number=1,Type=Float,Description="MEI KnownEventSensitivityForInsertionScore: fraction of known MEI insertion polymorphisms called for this sample with CGA_IS at least as high as for the current call">
+##FORMAT=<ID=PS,Number=1,Type=Integer,Description="Phase Set">
+##FORMAT=<ID=SS,Number=1,Type=String,Description="Somatic Status: Germline, Somatic, LOH, or . (Unknown)">
+##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
+##FORMAT=<ID=HQ,Number=2,Type=Integer,Description="Haplotype Quality">
+##FORMAT=<ID=EHQ,Number=2,Type=Integer,Description="Haplotype Quality, Equal Allele Fraction Assumption">
+##FORMAT=<ID=CGA_CEHQ,Number=2,Type=Integer,Description="Calibrated Haplotype Quality, Equal Allele Fraction Assumption">
+##FORMAT=<ID=GL,Number=.,Type=Integer,Description="Genotype Likelihood">
+##FORMAT=<ID=CGA_CEGL,Number=.,Type=Integer,Description="Calibrated Genotype Likelihood, Equal Allele Fraction Asssumption">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total Read Depth">
+##FORMAT=<ID=AD,Number=2,Type=Integer,Description="Allelic depths (number of reads in each observed allele)">
+##FORMAT=<ID=CGA_RDP,Number=1,Type=Integer,Description="Number of reads observed supporting the reference allele">
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT GS000016676-ASM
+1 1 . N <CGA_NOCALL> . . END=10000;NS=1;AN=0 GT:PS ./.:.
+1 10001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=12000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:1.44:.:0:.:0:0.999:152
+1 10001 . T <CGA_NOCALL> . . END=11038;NS=1;AN=0 GT:PS ./.:.
+1 11048 . CGCACGGCGCCGGGCTGGGGCGGGGGGAGGGTGGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 11270 . AGAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 11302 . GGGCACTGCAGGGCCCTCTTGCTTACTGTATAGTGGTGGCACGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 11388 . AGGTGTAGTGGCAGCACGCCCACCTGCTGGCAGCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 11475 . ACACCCGGAGCATATGCTGTTTGGTCTCAGTAGACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 11528 . TGGGTTTAAAAGTAAAAAATAAATATGTTTAATTTGTGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 11650 . TGGATTTTTGCCAGTCTAACAGGTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 11707 . TGGGGCCTGGCCATGTGTATTTTTTTAAATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 11769 . TGAGAATGACTGCGCAAATTTGCCGGATTTCCTTTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 11841 . CCGGGTATCATTCACCATTTTTCTTTTCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 11891 . CTTTGACCTCTTCTTTCTGTTCATGTGTATTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 11958 . ACCGGGCCTTTGAGAGGTCACAGGGTCTTGATGCTGTGGTCTTCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 12001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=14000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.26:1.68:.:0:.:0:0.999:152
+1 12027 . ACTGCTGGCCTGTGCCAGGGTGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 12099 . AGTGGGATGGGCCATTGTTCATCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 12135 . TGTCTGCATGTAACTTAATACCACAACCAGGCATAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 12187 . AAGATGAGTGAGAGCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 12238 . CTTGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 12264 . ACGTGGCCGGCCCTCGCTCCAGCAGCTGGACCCCTACCTGCCGTCTGCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 12329 . GCCGGGCTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 12380 . TCTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 12551 . GGTAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 12656 . CCAGAGCTGCAGAAGACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 12734 . TAGACAGTGAGTGGGAGTGGCGTCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 12780 . GGCGTCTCCTGTCTCCTGGAGAGGCTTCGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 12829 . GATCTTCCCTGTGATGTCATCTGGAGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 12936 . CAGCAAACAGTCTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 12976 . TCAGAGCCCAGGCCAGGGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 13113 . AAGTGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 13175 . GGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 13270 . CCAGTGATACACCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 13299 . ACACGCTGTTGGCCTGGATCTGAGCCCTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 13358 . ATTGCTGCTGTGTGGAAGTTCACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 13410 . ACCACCCCGAGATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 13610 . GTGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 13653 . AAACAGGGGAATCCCGAAGAAATGGTGGGTCCTGGCCATCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 13747 . CTGCGTGGCCGAGGGCCAGGCTTCTCACTGGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 13810 . ACCTTCTTAGAAGCGAGACGGAGCAGACCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 13865 . ACTAAAGTTAGCTGCCCTGGACTATTCACCCCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 13954 . ACCTCCCCCACCTTCTTCCTGAGTCATTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 14001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=16000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.19:1.78:.:0:.:0:0.999:152
+1 14108 . ATCTTCTACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 14162 . ACTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 14207 . GGAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 14245 . GACTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 14351 . AGACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 14433 . GCCGTTTTCTCTGGAAGCCTCTTAAGAACACAGTGGCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 14495 . ATGGAGCACAGGCAGACAGAAGTCCCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 14539 . TCAAGCCAGCCTTCCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 14650 . CAACGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 14668 . TCTGGGGGGGAAGGTGTCATGGAGCCCCCTACGATTCCCAGTCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 14733 . GGCTGCTGCGGTGGCGGCAGAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 14809 . CAGGTCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 14904 . GGAAGAAAAAGGCAGGACAGAATTACAAGGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 14973 . TGCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 15012 . AGTGCCCACCTTGGCTCGTGGCTCTCACTGCAACGGGAAAGCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 15115 . GACACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 15186 . CACCGGGCACTGATGAGACAGCGGCTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 15237 . CTCGGGGCCAGGGCCAGGGTGTGCAGCACCACTGTACAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 15387 . GGCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 15422 . ACAGCAGGCATCATCAGTAGCCTCCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 15510 . GACCGCTCTTGGCAGTCGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 15588 . TCCCAAACCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 15814 . GCTGCTGCTTCTCCAGCTTTCGCTCCTTCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 15862 . TGCCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 15893 . TAGCAGAGTGGCCAGCCACCGGAGGGGTCAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 15953 . GCCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 15988 . TGCTCAGGCAGGGCTGGGGAAGCTTACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 16001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=18000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.06:1.76:.:0:.:0:0.999:152
+1 16055 . AAACGAGGAGCCCTGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 16100 . GTGTGGGGGCCTGGGCACTGACTTCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 16203 . CCCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 16254 . AGGGGTTTTGTGCCACTTCTGGATGCTAGGGTTACACTGGGAGACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 16374 . GGAATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 16484 . ATATTTGAAATGGAAACTATTCAAAAAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 16531 . TAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 16567 . GCACGCCAGAAATCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 16678 . GGGAGTGGGGGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 16715 . GGGGTGGTGGTGGGGGCGGTGGGGGTGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 16798 . AAGGTGTGTGACCAGGGAGGTCCCCGGCCCAGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 16853 . CCTACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 16954 . CATGAGGTCGTTGGCAATGCCGGGCAGGTCAGGCAGGTAGGATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 17219 . CCCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 17362 . CTTCTACCTACAGAGGCGACATGGGGGTCAGGCAAGCTGACACCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 17481 . GCCGAGCCACCCGTCACCCCCTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 17594 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 17713 . CACGCACACAGGAAAGTCCTTCAGCTTCTCCTGAGAGGGCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 17805 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 17882 . GTGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 17942 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 17995 . CGCCCGTGAAGATGGAGCCATATTCCTGCAGGCGCCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 18001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=20000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.79:1.98:.:0:.:0:0.999:152
+1 18083 . TGAGGGGGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 18140 . TGGAAGCCTGGGCGAGAAGAAAGCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 18183 . CAGGGCAGAGACTGGGCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 18253 . GGGTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 18350 . CTCCGGCTCTGCTCTACCTGCTGGGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 18496 . CCTACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 18562 . GTCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 18640 . GAAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 18691 . GGCTACTGATGGGGCAAGCACTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 18742 . ACAATGTGGCCTCTGCAGAGGGGGAACGGAGACCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 18831 . GAACTGCCCCTGCACATACTGAACGGCTCACTGAGCAAACCCCGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 18930 . GGTGCGGGGTGGGCCCAGTGATATCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 19001 . TGCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 19048 . CTGATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 19169 . CACAACATCCTCCTCCCAGTCGCCCCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 19265 . GCACTCACCGGGCACGAGCGAGCCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 19303 . GGATGAGAAGGCAGAGGCGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 19387 . AGGCGCGACTGGGGTTCATGAGGAAAGGGAGGGGGAGGATGTGGGATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 19589 . CCGTGCCCTAAAGGGTCTGCCCTGATTACTCCTGGCTCCTTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 19680 . AAGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 19772 . CGGGACCACCACCCAGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 19855 . TGTCTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 19939 . TTCGAGGTCCACAGGGGCAGTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 20001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=22000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.01:1.69:.:0:.:0:0.999:152
+1 20096 . AACAGAGAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 20133 . GAGTCCCAGGGGCCAGCACTGCTCGAAATGTACAGCATTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 20188 . TTATTAGCCTGCTGTGCCCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 20242 . CAGGATTTTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 20301 . GGAGAGAACATATAGGAAAAATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 20385 . CACGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 20463 . TAAGCTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 20508 . GCCAGAGGGTAGACTGCAATCACCAAGATGAAATTTACAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 20592 . TATAAATACAGAAGGTGGAGGGAACTTGCTTTAGACACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 20708 . AAAGGCAATGAGATCTTAGGGCACACAGCTCCCCGCCCCTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 20810 . AAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 20859 . CAGAGGGTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 20911 . AGCAGGAGGAGAGAGCACAGCCTGCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 20957 . CACCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 20982 . ACGCCAGTGAGGCCAGAGGCCGGGCTGTGCTGGGGCCTGAGCCGGGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 21073 . GAGGAGCATGTTTAAGGGGACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 21118 . ACCGAAAAAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 21169 . AGGAGGGGCAAGTGGAGGAGGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 21221 . GTCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 21316 . CTTGCAAGTCCCCTGTCTGTAGCCTCACCCCTGTCGTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 21408 . CTTGTCCCTTCCGTGACGGATGCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 21500 . CACGCCTGAATCAACTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 21580 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.83|rs526642;CGA_FI=653635|NR_024540.1|WASH5P|INTRON|UNKNOWN-INC;CGA_SDO=5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:126:126,168:126,168:48,48:-168,-126,0:-48,-48,0:1:1,1:0
+1 21623 . ACACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 21707 . CCCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 21766 . CCCCTCCCACCCCTGTGCAGGCCGGCCTTCGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 21837 . CCTCCCTCCAAGCCTGCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 21870 . CCCTGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 21988 . GCAATGGCCCCATTTCCCTTGGGGAATCCATCTCTCTCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 22001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=24000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.65:1.60:.:0:.:0:0.999:152
+1 22067 . GCTCCTCAGTCTAAGCCAAGTGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 22109 . CCCATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 22152 . GGGATGAGTGAGTGTGGCTTCTGGAGGAAGTGGGGACACAGGACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 22246 . CGAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 22371 . TTAATTTTTGCTTAGCTTGGTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 22408 . GGCGTGCCACCAATTCTTACCGATTTCTCTCCACTCTAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 22491 . TCTCGCCCTATGTGTTCCCATTCCAGCCTCTAGGACACAGTGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 22651 . TGAGAGGCATCTGGCCCTCCCTGCGCTGTGCCAGCAGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 22755 . CATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 22822 . AGACGCCAAAAATCCAGCGCTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 22861 . CCACGCAGTCCCCATCTTGGCAAGGAAACACAATTTCCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 22945 . CCATAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 22977 . TGCATCCTCTTCCCTAGGTGTCCCTCGGGCACATTTAGCACAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 23044 . GCACTTTGTTACTATTGGTGGCAGGTTTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 23094 . GTACGGGTCAAGATTATCAACAGGGAAGAGATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 23171 . TTTGCATGTTTTGATTAATTTAATATTTAAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 23252 . CACCGAGGCTTAGAGGGGTTGGGTTGCCCAAGGTTACAGAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 23356 . TCACTGTGTGTCCCCTGGTTACTGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 23391 . ACAAACTCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 23503 . CTGGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 23528 . GGCAGGGATGGCTTGGACCACGAGAGGCACCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 23581 . CCCACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 23643 . TCAGTTTGCTTATGGCCAAAGACAGGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 23697 . TTTACCAAAAAAAGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 23786 . AGCACTGCCAATACAAGAAGCTGCAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 23827 . CCCTCAATGGCCACTCCGTGCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 23878 . CCACCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 23975 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2748067&dbsnp.131|rs76046194;CGA_FI=653635|NR_024540.1|WASH5P|INTRON|UNKNOWN-INC;CGA_RPT=L2b|L2|53.1;CGA_SDO=5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:102,.:102,.:40,.:-102,0,0:-40,0,0:0:0,.:0
+1 24001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=26000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.46:1.70:.:0:.:0:0.999:152
+1 24031 . CCCCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 24095 . GAATCCTGGCTCTGTCACTAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 24127 . CAGCCCTTCTGTGCCTCAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 24194 . TGAGTTAATGCACTCAAATCAATGGTTGTGCACGGTTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 24266 . AGACCTTGTCACAACTGTTATTGAAGAACTAATCATCTATTGCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 24344 . TCCAGGTGGAGAGGTATGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 24416 . CACTGCTGGGTAAATATTTGTTGGCTGCAGGAAAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 24480 . AAAAGCATGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 24508 . CCACAGGAAACCAGGAGGCTAAGTGGGGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 24635 . GACCGGGATTCCCCAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 24701 . GCCCTCTCATCAGGTGGGGGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 24762 . TTCTGCAGGTACTGCAGGGCATCCGCCATCTGCTGGACGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 24829 . TGAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 24871 . TCCTCACAGGAGTCATGGTGCCTGTGGGTCGGAGCCGGAGCGTCAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 24936 . CACGCCCCCACCACAGGGCAGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 25041 . TTGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 25062 . ATAGATGGGACTCTGCTGATGCCTGCTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 25108 . CAGGGCCCGGGACTGGGGAATCTGTAGGGTCAATGGAGGAGTTCAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 25215 . TACCTTGTCTCAGTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 25305 . GGGGTAGCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 25344 . TACACAGTTCTGGAAAAGCACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 25437 . GGGCAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 25496 . CTTAGGGGGACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 25530 . GGTGGAGGACAGGAAGGAAAAACACTCCTGGAATTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 25609 . CTCTCCCTGGTGCCACTAAAGCAGCAATCACACTGCAGACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 25730 . CTCACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 25771 . ACTGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 25871 . TTTCACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 25931 . TTCGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 25970 . GAGACGTGGTTATTTCCAATAATAATTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 26001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=28000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.37:1.49:.:0:.:0:0.999:152
+1 26010 . TAACGCACCACACCAACATCTTCACCCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 26057 . CTCCCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 26263 . ACCCAACCCTCTGAGACCAGCACACCCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 26339 . GTTTGCTGGCTGTCCTAACTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 26443 . ATTTCTTGTTAGTGTGTGTGTGTTTGCTCACACATATGCGTGAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 26501 . ACAGATCTCCTCAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 26561 . TGAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 26595 . GATCATCTGTTAGGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 26648 . ACTAGCCAGGGAGAGTCTCAAAAACAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 26697 . CTACTCCAGTCATGGGTACAAAGCTAAGGAGTGACAAATCCCTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 26792 . GCCGGGCGCAGCGGCTCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 26835 . GGCGAAGGCAGGCAGATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 26887 . ACATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 26926 . GCCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 26954 . CCCCGCTACTCGGGAGGCTGAGGAAGGAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 26992 . AACCAGGAAGGTGGAGGTTGCAGTGTGCCAAGATCGCGCCATGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 27044 . CCTAGGCAACGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 27103 . AAAGAAACAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 27136 . AACCGCAAGCGGTCTTGAGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 27172 . TCCTTGGGGAAGTACTAGAAGAAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 27227 . CACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 27274 . CCTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 27313 . CATGCAGCCACTGAGCACTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 27361 . GCCATAAGTGTAAAATATGCACCAAATTTCAAAGGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 27428 . TTTATATTGATTACGTGCTAAAATAACCATATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 27486 . TATCACTAATTTCATCTGTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 27537 . TTAAATATTTCTTTTCTTTTTCTTTCCTCTCACTCAGCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 27604 . GCTGTTTTTGGGCAGCAGATATCCTAGAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 27664 . TCATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 27720 . TGACCATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 27777 . ACAGTATGACTGCTAATAATACCTACACATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 27840 . TTAACTCTTATTATCAGTGAATTTATCATCATCCCCTATTTTACATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 27904 . AGACCAAATAACATTTTTTCAACATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 27965 . CTGTCGTCTGAATTCCAAGCTTTTTGTTATTTATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 28001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=30000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.40:1.68:.:0:.:0:0.999:152
+1 28039 . GCCCAAACATTTTGTTAGTAGTACCAACTGTAAGTCACCTTATCTTCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 28110 . AATTAGATCTGTTTTTGATACTGAGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 28174 . TGTGGTCAACACTTTCGTTACTTTAGTATACATCACCCCAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 28244 . TAGGTAGTAGTATCTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 28324 . TAGTTGCTCATCTGAAGAAGTGACGGACCACCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 28364 . AGTGGACAGACAGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 28391 . GACAGGGGATTTTGTTGGCGGAAAAAAAAATTTATCAAAAGTCGTCTTCTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 28474 . AGTTCCACAGTGGGTAACTGTAATTCATTCTAGGTCTGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 28555 . CCACAAATACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 28582 . ATGGTGGTTTTTTTTTTTTTTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 28660 . CGCTCAATATTTCTAGTCGACAGCACTGCTTTCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 28748 . ACCGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 28811 . TCCAGGGTCTCTCCCGGAGTTACAAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 28860 . CAACGCGGTGTCAGAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 28906 . TCCGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 28950 . GAACCCGGCAGGGGCGGGAAGACGCAGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 29011 . CGGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 29061 . GCCGGGTGCAGGGCGCGGCTCCAGGGAGGAAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 29121 . GGCGGTCGGGGCCCAGCGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 29153 . GGAGCCGGGCACCGGGCAGCGGCCGCGGAACACCAGCTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 29253 . CGGGTCCCCTACTTCGCCCCGCCAGGCCCCCACGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 29359 . CGCTCTGCCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 29390 . GCCGCCCCCAGTCCGCCCGCGCCTCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 29430 . CCGCTCGCCCTCCACTGCGCCCTCCCCGAGCGCGGCTCCAGGACCCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 29495 . CCTGTCGGGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 29568 . CATGCGTTGTCTTCCGAGCGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 29630 . TCCTAGACCTCCGTCCTTTGTCCCATCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 29693 . CCAACCTCGGCTCCTCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 29721 . GCCCGGGGTGCGCCCCGGGGCAGGACCCCCAGCCCACGCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 29787 . TACGCCTTGACCCGCTTTCCTGCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 29846 . GGGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 29874 . CCCACCCCCCTTTAAGAATTCAATAGAGAAGCCAGACGCAAAACTACAGATATCGTATGAGTCCAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 29958 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 30001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=32000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.52:1.49:.:0:.:0:0.999:152
+1 30024 . AGCTCGTGTTCAATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 30076 . AAATGAGTGGTAGTGATGGCGGCACAACAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 30237 . TTTTAAAAAGTTAAATATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 30285 . GCAGTTGTCCCTCCTGGAATCCGTTGGCTTGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 30359 . AAAGACAGGATGCCCAGCTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 30408 . TTCGTAGCATAAATATGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 30485 . TTCAGAATTAAGCATTTTATATTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 30525 . CCACCCTACTCTCTTCCTAACACTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 30567 . TGTCCGCCTTCCCTGCCTCCTCTTCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 30633 . CTCGCTGGCTGCAGCGTGTGGTCCCCTTACCAGAGGTAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 30725 . AATGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 30776 . CCTTTGGTAGGTAATTACGGTTAGATGAGGTCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 30852 . TTGTCTCTGTGTCTCCCTCTCTCTCTCTCTCTCTCTCTCTCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 30907 . CATTTCTCTCTCTCTCGCTATCTCATTTTTCTCTCTCTCTCTTTCTCTCCTCTGTCTTTTCCCACCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 30982 . TGCGAAGAGAAGGTGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 31026 . ACCGGGAACCCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 31125 . TTTTGTTTTGACTAATACAACCTGAAAACATTTTCCCCTCACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 31248 . GCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 31282 . CACAGGCTCAGGGATCTGCTATTCATTCTTTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 31381 . GCCCTGCCTCCTTTTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 31431 . AATCTGGCTGGGCGTGGTGGCTCATGCCTGTAATCCTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 31484 . GACGCGAGAGGACTGCTTGAGCCCAAGAGTTTGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 31552 . TACAAAAATAAAATAAAATAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 31647 . GATCGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 31669 . GATTGTACCACTGCACTCCAGGCTGGGCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 31716 . TCAGAAAAAAAAAAAAAAGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 31790 . CACGATGCCTGTGAATATACACACACACCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 31856 . TGCACTGCTAGGCACCACCCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 31938 . GTTCCCTACCTAATCTACTGACAGGCTCATCCCCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 32000 . TGCAGTGGGAATCCTGGACCTCAGCCTGGACAAAGAACAGCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 32001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=34000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.55:1.39:.:0:.:0:0.999:152
+1 32064 . CACAGAAGCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 32093 . AGCTGGGCTGAGCGGGCCTGGGAATTAAGGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 32147 . TTGCTGAAGCTTGCCACATCCCCCAGCCTCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 32227 . GAGTGAAGAAAATGTGACAGGGTGTCCTAAGCCCCGATCTACAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 32322 . GCCTCTAGCTTTTGTGCTACAGTTCTGGGAACAGACTCCTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 32374 . CCACTTCCCTCCGCAGCATTAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 32455 . ATGGAAATGTCCTGCTCTCTAAACAGATAGACAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 32584 . TGCACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 32646 . CGGTGACTGTGTTCAGAGTGAGTTCACACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 32733 . CAGCCCAGGAACCTCCCCTTATCGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 32815 . TGCCATGTGGGTTGTTCTCTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 32891 . CCTCGGCTGGAGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 32971 . TATGTAATAACTGAATCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 33051 . CCATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 33108 . TCATAAAAAGGAAGGCAGAGCTGATCCATGGCACCATGTGACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 33176 . GGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 33258 . AGCCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 33329 . GAGCTGATGAAAATGTTTTGGAACTACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 33377 . CATGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 33398 . CACTGATTGTTCAATTTAAAATGGTCAAACTTATATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 33443 . CTCCATTAAAAAAAAAAAAAAAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 33492 . AATCCCAACACTTTGGAAAAAGGTGAAAGTTTTTTTTTCTTTTTTTTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 33552 . GTTCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 33653 . TCTTCTAATGCTATCCCTCCCCCAGCCCCCCACCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 33703 . TGTATGATGTTCTCTGCCCCATGTCCAAGCGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 33744 . TCAATTCCCACCTGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 33797 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 33868 . ATGGCTGCATAGTATCCCATGGTATATATGTGCCACATTCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 33925 . ATTGATGGACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 33958 . GCTATTGTGAATACTGCCACAATAAACATACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 34001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=36000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.23:1.45:.:0:.:0:0.999:152
+1 34026 . CTTTGGGTATATACCCTAAGACCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 34125 . TGGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 34291 . TCTCACATCTTCTTGGCCAGCACTGGACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 34358 . TATGAGAAAGAAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 34404 . CATCTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 34452 . AGAAGGCTTTCTGGACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 34513 . TGGTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 34581 . AGGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 34631 . AAGTTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 34704 . TGGTTCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 34740 . GCCGTGCTCCTTGGAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 34767 . AGGCGGAGGACACGTGGGAGGTTTTAGGGACAAGCCTGGAGGCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 34968 . GTCAGGCAGGGAGTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 35030 . GGTGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 35076 . GACCACGTGCTGGATGTCACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 35116 . GCCGGGTTAGGCACCTGGTGTTTTACGTACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 35160 . GTGAGGGCATCCGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 35229 . TTAGAGCTTAATCGTGTTCAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 35364 . GTCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 35407 . GGGAGGCTGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 35482 . CTATAAAAAATAAATAAATAAATAAAAACAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 35656 . GGCTTAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 35755 . TCACGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 36001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=38000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:1.37:.:0:.:0:0.999:152
+1 36070 . CCCCGTTGTGTGGGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 36125 . AGGGAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 36158 . TGCAGCTGGCTCATTCCCATATAGGGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 36229 . GGGGAGGCCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 36258 . GGGTGGCTCTGAGGGGGCTCTCAGGGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 36344 . AAGTTTGGAAAAAAAAAAAAACCCAGCCTGGCGGAAAGAATTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 36483 . GTCATCCTTCCCCAACACATCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 36521 . CAAGCCTCTCCCACCCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 36590 . AGACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 36722 . TGATTCTGTGGTATGTTAATGTTTATGCATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 36786 . GAGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 36851 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 36888 . GGCTAGTTGTTTGAATGTCTGCATGAAAAAGCGGACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 36960 . GACCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 36996 . CTGCACTATTAATTTGTTTTTTAGCTAAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 37052 . CCACCCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 37103 . TGAACCTACCTTTTCAATGTAAATTCAGTGAAATCTAAGTGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 37236 . AGGGATTTTTTTTTTGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 37296 . AATGATAATCTTTTGGATATATTGGGTTAAATAAATTTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 37362 . GTTTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 37506 . AGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 37560 . ACAGAGATAACTCCAACCCTTAAGAAGGTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 37624 . TACTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 37688 . GCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 37714 . GTGGTCTCACCTCCGGCAGTATCACCACCACTGGGCACAAGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 37769 . CAACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 37791 . GTACTCCCAGTGTTCACACCATGCTGCACTCACAGAAGACTCTTCGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 37882 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 37941 . GGGGAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 37962 . AGCCAGGAGTCTCATCCCCTGGGGAAGTTCCAGGGACCCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 38001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=40000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.37:1.47:.:0:.:0:0.999:152
+1 38049 . CAGAGCCTGCCTTCCACGTGGGTTTGACAGGAGCCTCCTAACTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 38140 . GCCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 38221 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 38232 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs77823476&dbsnp.86|rs806727;CGA_FI=645520|NR_026818.1|FAM138A|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:87:195,87:195,87:48,48:-195,-87,0:-48,-48,0:1:1,1:0
+1 38339 . GGACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 38378 . AAAGTGGTCTCCTGCAGTTACGTGGCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 38481 . TTCTTCTTACTGCTTATAATATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 38592 . GTCTCCCCACATGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 38736 . AGATTACAAGGGTGTACCATGCAGAACCTCTCCACCAAACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 38821 . TTGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 38907 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3874156&dbsnp.131|rs75829199&dbsnp.86|rs806726;CGA_FI=645520|NR_026818.1|FAM138A|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=MLT1E1A-int|ERVL-MaLR|38.5;CGA_SDO=6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:94:172,94:172,94:48,48:-172,-94,0:-48,-48,0:13:9,9:4
+1 38998 . AAAGAAGACTGTCAGAGACCCCAAACTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 39140 . AATCTTCCCACATCTTAAAACCTGTTTAGGGAACACCAGCATCTGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 39217 . TCCTTCCCCTGCTGCCTCTTTCTGAACAGCAATGTCTCAAGCTTTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 39289 . GGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 39403 . CAAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 39484 . GTGACCCCCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 39575 . AGGCGGTATATATGTGATTCATGTACTGATCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 39625 . GCTGGATGCAGTGGCTCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 39654 . CCAACACTTTGGGAGGCTGAGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 39702 . TCGAGACCAGGCTGGCCAACATGGCAAAACCCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 39827 . ACTCAGGAGGTGGAGGTGGCAGTGAGCCAAGATCGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 40001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=42000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.33:1.54:.:0:.:0:0.999:152
+1 40026 . GGAGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 40090 . CATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 40156 . GAACCCAGTGCTGGCTGACACCCTGATGGCACCTTACAGAGGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 40234 . CTGGGGAACACTGGGTCGTATTTGCAGCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 40295 . TGGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 40386 . CCGCAGCCACGCTGGCCCTCTGCTGTTCTTCGAAGCCACCAGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 40491 . GTATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 40604 . ACATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 40636 . TTCCTTTTTTTTTTTTTTTTTTTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 40703 . CTCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 40890 . ACTGGTCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 41025 . TCTTGGGAATATTAAGTGGAGAGGGGTACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 41215 . TTCTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 41253 . CCTCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 41376 . AAAGGTATAGCAATATTTCTATTTCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 41475 . GGAAGCCAAAATGACAGGGAGCTACTAAAACTTTATTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 41668 . AGGACCCAATATCTTACAATGTCCATTGGTTCAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 41757 . GGCATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 41839 . AGTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 41981 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.86|rs806721;CGA_RPT=ERVL-E-int|ERVL|47.4;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:109:109,238:109,238:48,48:-238,-109,0:-48,-48,0:6:2,2:4
+1 42001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=44000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.57:1.37:.:0:.:0:0.999:152
+1 42036 . GAACAAATTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 42198 . CATGCTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 42262 . ATTGTGCAAGCATAAGTGGCTGAGTCAGGTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 42343 . ACATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 42522 . TTATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 42574 . TTACGCTTTTCTTAAACACACAAAATACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 42662 . AATCAATTAGCAATCAGGAAGGAGTTGTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 42736 . AAATTATTCACAATAAAAAAAAAGATTAGAATAGTTTTTTTAAAAAAAAAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 42821 . AGGTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 43039 . TTAGGCAAGGAAGCAAAAGCAGAAACCATGAAAAAAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 43201 . ATATCAATAACCACAACATTCAAGCTGTCAGTTTGAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 43370 . GAAACCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 43583 . ACACTGGTAAAAAAAATGAAAGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 43662 . ATATCGTACTAAAAGTCCTAGCCAGGACAATTAGACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 43793 . CAGCAAAAAAAAAAAAAAAACTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 44001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=46000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.42:1.36:.:0:.:0:0.999:152
+1 44162 . TTTGACAGAAATAAAAAAAAAATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 44329 . AAACACACAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 44669 . AAATAGACAAATGAGACTATGCCAAATTAAAAAATTTCTAACAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 44752 . GAATGGGAGAAATATTTGCAAACTACTCATCCAACCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 44832 . AGTAAAATAAATAAATAAATAAATAAATAAATAAATTAAATAAATTATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 45028 . TGGCTATTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 45084 . GAACCCTTGCATCATGTACAAATTAAAAATAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 45244 . ACCTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 45517 . AAAGGAAAAAAATTCAATTAGTAGGATTACATTCAGGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 45623 . TATTGTAAATGTTAATATGAGGTAATATATGTGTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 45753 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 45758 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 45791 . GATTAAAAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 45853 . TTTTAAATATAATTTAAACCAAATTTAAAATAAGCATATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 45974 . TAAATTTTAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 46001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=48000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.56:1.63:.:0:.:0:0.999:152
+1 46051 . CACTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 46085 . TATGTCAGATCATGAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 46399 . TAACTTTTTTTTTTTTTTGAGCAGCAGCAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 46670 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2548905;CGA_RPT=MER45A|hAT-Tip100|29.0;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:24:24,24:24,24:5,24:-24,0,-24:-5,0,-24:22:0,22:22
+1 46868 . GCCAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 46922 . AAAGTAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 47108 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531241;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:75:75,75:75,75:32,36:-75,0,-75:-32,0,-36:16:1,15:15
+1 47489 . TCCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 47658 . AAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 47693 . GCATCAATGGGTCACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 47932 . TCTCCTCCACTTTTCTGTTTTCCTCCTATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 47995 . ACAATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 48001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=50000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.72:1.41:.:0:.:0:0.999:152
+1 48445 . GAACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 48934 . CAGTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 49240 . CAAGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 49269 . ACCGCATGTTCTCACTTATGAGCGTGAGATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 49431 . CTGTACAACGAACCCCCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 49479 . CACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 49506 . AGTCAAAAAGAAAAAGAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 49829 . AAAGCAAAACAAACAAACAAACAAAACAAAACACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 50001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=52000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.47:1.56:.:0:.:0:0.999:152
+1 50169 . GGTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 50482 . GTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 50888 . TGGTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 51456 . AGCGGAAGAGTAAGTCTTTGTATTTTATGCTACTGTACCTCTGGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 51617 . AGCACTTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 51665 . GACCATCCTGGCTAACACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 51742 . TGCGGTCCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 51803 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs62637812;CGA_RPT=AluY|Alu|7.7;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:82:112,82:111,81:38,38:-112,0,-82:-38,0,-38:20:9,11:11
+1 51861 . CCTCAAAAAAAAAAAAAGAAGATTGATCAGAGAGTACCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 52001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=54000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.20:1.19:.:0:.:0:0.999:152
+1 52058 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs62637813&dbsnp.131|rs76894830;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:24:24,26:24,26:3,25:-24,0,-26:-3,0,-25:9:4,5:5
+1 52134 . CTTGTCTAATTGTTATTAATAATTAATAAATAACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 52182 . TTATTAATAATAACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 52206 . ATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 52228 . AATAACTT AC . . NS=1;AN=2;AC=1;CGA_RPT=AT_rich|Low_complexity|3.1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:52228:PASS:43:121,43:118,40:24,20:-121,0,-43:-24,0,-20:27:11,16:16
+1 52238 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2691277&dbsnp.134|rs150021059;CGA_RPT=AT_rich|Low_complexity|3.1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:52228:PASS:75:275,75:272,72:52,48:-275,-75,0:-52,-48,0:26:26,26:0
+1 52727 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2691278;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:78:78,78:78,78:27,37:-78,0,-78:-27,0,-37:22:1,21:21
+1 52797 . CAGGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 53006 . AGACACTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 53184 . TTTTTATGCCATGTATATTTCTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 53385 . AGTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 53425 . ACATGGCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 53705 . AGTAAGCATATAGATGGAATAAATAAAATGTGAACTTAGGTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 53786 . TTAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 53955 . GTCAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 53998 . CCTACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 54001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=56000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.63:1.53:.:0:.:0:0.999:152
+1 54043 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531228;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:35:35,35:35,35:10,27:-35,0,-35:-10,0,-27:26:5,21:21
+1 54108 . AAGGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 54175 . AGCTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 54287 . ACACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 54351 . AACCGTACCTATGCTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 54586 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79600414;CGA_RPT=L2|L2|49.7;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:39:39,39:33,33:3,17:-39,0,-39:-3,0,-17:64:15,49:49
+1 54705 . CTTGTATTTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTTCTTCCTCCTTTTCTTTCCTTTTCTTTCTTTCATTCTTTCTTTCTTTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 54841 . GTTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 55082 . CAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 55164 . C A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3091274;CGA_RPT=L2|L2|49.7;CGA_SDO=2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:44,.:44,.:18,.:-44,0,0:-18,0,0:18:18,.:0
+1 55296 . ATGCGACCTTCCCACTTAAAATCCTACTATTTACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 55378 . GCTGAAGACACTTCACTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 55425 . CATGGTATAGTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 55484 . AACATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 55542 . ACTCCAAAATCTATCAACTCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 55813 . GTCGTGTTCACCTCTATCACATCATAAATATAGCAAACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 55926 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs3020698&dbsnp.121|rs13343114;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:31:65,31:65,31:22,30:-65,-31,0:-30,-22,0:23:23,23:0
+1 55973 . TTCTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 56001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=58000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.42:1.69:.:0:.:0:0.999:152
+1 56154 . GTTAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 56295 . ATACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 56378 . TTATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 56482 . TTTCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 56635 . ACACTTCTTATTCTGCTGCTGTTCTAGAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 56799 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2691309;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:31:31,31:31,31:8,26:-31,0,-31:-8,0,-26:23:2,21:21
+1 56984 . ATCCAAAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 57155 . CCTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 57246 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2691313;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:26:26,26:26,26:3,17:-26,0,-26:-3,0,-17:34:5,29:29
+1 57289 . TTCCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 57373 . TCCCTCCCCCTATTTCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 57792 . TTTCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 57853 . AACTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 57892 . TGTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 57952 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2691334&dbsnp.135|rs189727433;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:81:205,81:205,81:48,48:-205,-81,0:-48,-48,0:13:12,12:1
+1 57987 . TGTCTGATCTCAGCTATTTCCATCCTATTTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 58001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=60000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.85:1.47:.:0:.:0:0.999:152
+1 58176 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 58211 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 58595 . ACTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 58909 . TAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 58986 . TCTCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 59051 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2691352;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:113:147,113:147,113:44,42:-147,0,-113:-44,0,-42:26:11,15:15
+1 59131 . ATGCAAAAAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 59319 . ATCCCACCATACCTCATTATCACACCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 59498 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2854666&dbsnp.131|rs76479716;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:122:122,122:112,112:13,34:-122,0,-122:-13,0,-34:50:32,18:18
+1 59615 . CCTTCCCCTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 60001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=62000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.37:1.68:.:0:.:0:0.999:152
+1 60276 . TTGTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 60405 . TTACGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 60718 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 60726 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531295&dbsnp.131|rs77618875&dbsnp.135|rs192328835;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:60726:PASS:77:86,77:80,71:12,27:-86,0,-77:-12,0,-27:43:31,12:12
+1 60788 . AATACATGCATATTGTGGAGATAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 61018 . TTCGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 61216 . TGATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 61289 . A AG . . NS=1;AN=2;AC=1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:57:57,57:57,57:22,26:-57,0,-57:-22,0,-26:11:1,10:10
+1 61347 . TTGTAAAAAAAAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 61442 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2531261&dbsnp.129|rs62637818&dbsnp.131|rs74970982;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:31:31,62:0,9:14,0:-62,-31,0:-14,0,0:11:11,11:0
+1 61448 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:127,.:74,.:0,.:0:0
+1 61477 . TTTGTTTACCATTATTACTCTTGGTATTTTTAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 61576 . TCCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 61675 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 61848 . TAATAATTGTAAAACTTTTTTTTCTTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 61984 . CCCAAGTAGCTGGGACTACAGGCATGCACCACCATGCCCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 62001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=64000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.32:1.47:.:0:.:0:0.999:152
+1 62203 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28402963;CGA_RPT=L1M5|L1|41.7;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:90:90,90:90,90:37,39:-90,0,-90:-37,0,-39:18:5,13:13
+1 62236 . ACATACACACACACACACACATATCTGTATATACAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 62295 . ATTCTTCATTTCATTTGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 62536 . TTTTTAAAGATTCTGTATTTTTTAAACCATTTATTTGTATATGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 62678 . CCAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 62774 . TTAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 62877 . TTTTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 63071 . AAAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 63236 . TTGCATGATACAAAAGTTCTTTATCCATGTTATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 63356 . GAATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 63413 . TGCTATGTCTCAGTTTGTTTTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 63466 . ATGTGGGGAGCTTTTATTGTGATTTTCCTCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 63513 . TGCATGGACACTTATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 63693 . TTGTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 63733 . TCCCTACTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 63792 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2907079;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:57:57,57:57,57:20,33:-57,0,-57:-20,0,-33:23:8,15:15
+1 63908 . ACAGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 64001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=66000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.68:1.49:.:0:.:0:0.999:152
+1 64122 . CTACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 64168 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 64200 . AGTATTTTTATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 64277 . ATGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 64476 . GAAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 64510 . GGTGGGGGGAAGGGGGAGGGATAGCATTAGGAGATATAACTAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 64580 . CACACCCACATGGCACATGTATACATATGTAACTAACCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 64761 . TCTCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 64973 . ATTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 65250 . AAAAAGCACCTTTAGACTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 65526 . TGCCTCATTCTGTGAAAATTGCTGTAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 65588 . AGACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 65742 . GAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 65794 . TGCTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 66001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=68000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.27:1.67:.:0:.:0:0.999:152
+1 66094 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 66157 . C <CGA_NOCALL> . . END=66634;NS=1;AN=0 GT:PS ./.:.
+1 66734 . AAGGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 66790 . TCTAATTTTTTTTGAATAATTTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 67069 . TTAATTTTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 67223 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:67223:PASS:42,.:39,.:0,.:0:0
+1 67242 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531263&dbsnp.131|rs77818189;CGA_FI=79501|NM_001005484.1|OR4F5|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:67223:PASS:42:42,42:39,39:20,4:-42,0,-42:-4,0,-20:59:34,25:34
+1 67445 . AAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 67602 . ATGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 68001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=70000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.57:1.36:.:0:.:0:0.999:152
+1 68303 . CAGCTATTACCTATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 68384 . AGAGCTAAATTAAACAATCATTCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 68544 . TTTTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 68613 . ATATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 68893 . ATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 68905 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 68931 . GTAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 69060 . GAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 69267 . CTCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 69453 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2854682&dbsnp.134|rs142004627;CGA_FI=79501|NM_001005484.1|OR4F5|CDS|SYNONYMOUS;CGA_PFAM=PFAM|PF00001|7tm_1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:129:129,129:129,129:42,45:-129,0,-129:-42,0,-45:28:4,24:24
+1 69511 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2691305&dbsnp.131|rs75062661;CGA_FI=79501|NM_001005484.1|OR4F5|CDS|MISSENSE;CGA_PFAM=PFAM|PF00001|7tm_1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:135:135,317:135,317:48,48:-317,-135,0:-48,-48,0:5:4,4:1
+1 69552 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531266&dbsnp.129|rs55874132;CGA_FI=79501|NM_001005484.1|OR4F5|CDS|SYNONYMOUS;CGA_PFAM=PFAM|PF00001|7tm_1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:69552:PASS:130:130,178:130,178:42,49:-130,0,-178:-42,0,-49:23:4,19:19
+1 69569 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531267;CGA_FI=79501|NM_001005484.1|OR4F5|CDS|MISSENSE;CGA_PFAM=PFAM|PF00001|7tm_1;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:69552:PASS:177:177,178:177,178:48,49:-177,0,-178:-48,0,-49:29:8,21:21
+1 69732 . CCTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 69894 . TTCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 69965 . GACAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 70001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=72000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.24:1.30:.:0:.:0:0.999:152
+1 70242 . GAACAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 70297 . TGACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 70349 . TTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 70496 . ATACTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 70604 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2854679;CGA_RPT=LTR89|ERVL?|39.7;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:28:28,28:28,28:6,25:-28,0,-28:-6,0,-25:16:2,14:14
+1 70725 . AAGCACAGGCTTTAAAGTAAAAAACAAAGAGCTGGATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 71176 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 71378 . AATTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 71632 . AATATGGTAAAGATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 71666 . TTAATTTTTAATGCGTAATAAAACTATGAGAAAATTTAAAAGTGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 71779 . CCCAAAATATTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 71843 . AGTCTTTTTTTTTTTTTTTACAGTTGTAGGCAGAAAACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 71989 . AATTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 72001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=74000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:1.18:.:0:.:0:0.999:152
+1 72082 . TACTCTTTTATATATATACATATATGTGTGTATATGTGTATATATATATACACACATATATACATACATACATACATACATATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 72206 . GGGATACATGTGCAGAATGTACAGGTTTGTTACACAGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 72277 . CAACTCACCATCTACATTAGGTATTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 72336 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 72389 . TTGGTCAACTCCCATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 72452 . TGCGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 72520 . GCTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 72585 . TTAAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 72694 . GTGTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 72743 . CCATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 72787 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2854675&dbsnp.129|rs62641289;CGA_RPT=L1PA7|L1|7.3;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:54:169,54:169,54:48,45:-169,-54,0:-48,-45,0:7:7,7:0
+1 72859 . CCCGTCAATGTTAGACTAGATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 73230 . ATGTTTAGCTCCCCCTTGTTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 73343 . GACTTTCTTCTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 73486 . TGTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 73620 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 73633 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 73645 . TTCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 73695 . TATGAAAAAAATGTTCAAGTCTCTCAGATTAAGATGCATGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 73799 . TTTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 73838 . CACCCTTTTTTTTTTTTTTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 73874 . TAGGGTACATGTGCACCTTGTGCAGGTTAGTTACATATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 73932 . TGCGCTGAACCCACTAACTCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 73971 . ATCTCCCAATGCTATCCCTCCCCCCTCCCCCCACCCCACAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 74001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=76000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.20:0.87:.:0:.:0:0.999:152
+1 74021 . GAGTGTGATATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 74070 . CCACCTATGAGTGAGAATATGCGGTGTTTGGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 74112 . T <CGA_NOCALL> . . END=74372;NS=1;AN=0 GT:PS ./.:.
+1 74384 . TGGTATTTCCAGTTCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 74408 . GAGGAATCGCCACACTGACTTCCACAATGGTTGAACTAGTTTACAGTCCCACCAACAGTGTAAAAGTGTTCCTATTTCTCCACATCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 74503 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 74506 . TGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 74520 . TTTTTAATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 74535 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:74535:VQLOW:28,.:23,.:0,.:0:0
+1 74550 . AGATGATATCTCATTGTGGTTTTGATTTGCATTTCTCTGATGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 74610 . TTTTTCATGTGTTTTTTGGCTGCATAGATGTCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 74672 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 74676 . CACTTGTT . . . NS=1;AN=0 GT:PS .|.:74676
+1 74690 . GTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 74701 . TTTTCTTGTAAATTTGTTTGAGTTCATTGTAGATTCTGGATATTAGCCCTTTGTCAGATGAGTAGGTTGCAAAAATTTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 74822 . TTTTGCTGTGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 74927 . ATCAGAGAATACTACAAACACCTCTACGCAAATAAACTAGAAAATCTAGAAGAAATGGATAAATTCCTGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 75004 . CACTCTCCCAAGCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 75085 . TAGCTTACCAACCAAAAAGAGTCCAGGACCAGATGGATTCACAGCCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 75144 . GGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 75172 . TCTGAAACTATTCCAATCAATAGAAAAAGAGGGAGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 75222 . TTTATGAGGCCAGCATCATTCTGATACCAAAGCCAGGCAGAGACACAACAAAAAAAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 75301 . GATGAACATTGATGCAAAAATCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 75333 . TACTGGCAAAACGAATCCAGCAGCACATCAAAAAGCTTATCCACCAAGATCAAGTGGGCTTCATCCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 75409 . AGGCTGGTTCAATATACGCAAATCAATAAATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 75451 . TATAAACAGAGCCAAAGACAAAAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 75490 . AATAGATGCAGAAAAGGCCTTTGACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 75523 . A <CGA_NOCALL> . . END=75821;NS=1;AN=0 GT:PS ./.:.
+1 75828 . CCATTGTCTCAGCCCAAAATCTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 75876 . A <CGA_NOCALL> . . END=76275;NS=1;AN=0 GT:PS ./.:.
+1 76001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=78000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.60:0.90:.:0:.:0:0.999:152
+1 76285 . AAAGAACAAAGCTGGAGGCATCACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 76326 . TATACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 76416 . TAATGCCGCATATCTACAACTATCTGATCTTTGACAAACCTGAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 76536 . TGAAACTGGATCCCTTCCTTACACCTTATACAAAAATCAATTCAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 76597 . AAACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 76619 . AAAACCCTAGAAGAAAACCTAGGCTTTACCATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 76843 . CACTCAAGTCTATTCATTGAAGCATTGTTTTTCATAGTAAACGG . . . NS=1;AN=0 GT:PS ./.:.
+1 76918 . TGATCCCAGCATTTTGGGAGGCTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 76992 . TGGCGAAACCCCATCTCTACCAAAAATACAAAAATTAGCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 77081 . GGGAGGATCGCTTGAACCTGGGAGGCAGAAGTTTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 77129 . CGTGCCTCTGCACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 77175 . AAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 77276 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 77409 . ATAAGTATATATTTTATAAATGTTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 77459 . AACGTAATACATATATAATTTTCTTATGGCAGGAGGAGGAAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 77565 . ATCCTGTAGCTGTTTTATGTAATATAAAAATGTAATTAAATTAACAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 77688 . CATGGGACACTAACATACAGACAAATTCATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 77773 . CGAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 77916 . TTGTTAAATATTCTCTATTTTATGACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 77958 . GTCGAAGAGAGAAACATGCAAGAACACCGTAGGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 78001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=80000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:1.04:.:0:.:0:0.999:152
+1 78015 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 78035 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2691297;CGA_RPT=L1MC4a|L1|35.4;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:78035:VQLOW:23:23,23:23,23:2,24:-23,0,-23:-2,0,-24:8:1,7:7
+1 78093 . TATATTTTTAAAAACTAAAAAGATATATTAGCTGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 78193 . TATTAAAATAATTTAAAAATGACCAAGTATTTGATTATATCAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 78311 . TAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 78354 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 78388 . TGAAACCCTATCTCTACAAAAAACAAACAAAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 78437 . TTTAAAAAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 78507 . TGATCTGACTATGTGCTTCCCTGAACAAATGCACTTTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 78630 . CTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 78892 . CTCGCAGCCCTCACCCTGGAGAGTCCACAGGTACCAGGGGTTGGTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 78965 . CACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 79022 . CAAGCAGGGCCACCTGGCCTGGGACTCCGGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 79075 . GACGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 79150 . GCAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 79415 . CCCGTGTCACAGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 79663 . A G . . NS=1;AN=2;AC=1;CGA_RPT=L1PREC2|L1|19.7;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:79663:PASS:62:62,64:62,64:27,35:-62,0,-64:-27,0,-35:14:1,13:13
+1 79678 . T C . . NS=1;AN=2;AC=1;CGA_RPT=L1PREC2|L1|19.7;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:79663:PASS:54:54,64:54,64:24,35:-54,0,-64:-24,0,-35:17:1,16:16
+1 79769 . CCTCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 79876 . AACGACCATACTGCCAAAAGCAACCTACAAATTCAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 80001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=82000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.15:1.21:.:0:.:0:0.999:152
+1 80141 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4030287;CGA_RPT=L1PB|L1|9.0;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:106:106,106:106,106:26,33:-106,0,-106:-26,0,-33:32:7,25:25
+1 80383 . AACCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 80500 . AAATAAATAATCAGCAGAGTAAACAGACAACCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 81204 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2531303;CGA_RPT=Tigger5|TcMar-Tigger|32.1;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:40:40,40:40,40:16,28:-40,0,-40:-16,0,-28:17:2,15:15
+1 81257 . TGCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 81326 . CAGAGACTGACTGTGTCAAAGTATTAGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 81784 . AAAATGTAAAAAGTATCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 81856 . TAATAAAATAAGAAGCCAAAAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 82001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=84000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.25:.:0:.:0:0.999:152
+1 82030 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 82112 . ACAATTCACCACAACTGACTTCAAAAAAAAAAAAAAAAAAAAAGAAGTACCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 82210 . TATAACACACACACAAACACTAGGTTTAGATGTTTTCACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 82300 . AACTCTCAGCCATTTGAGGCAAAATATTACAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 82453 . GCCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 82537 . AAATGAAGGCTAAGGCAGAATTATATATGGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 82620 . AGAAGTTTTTCATATTTTTTTCTTTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 82673 . TTTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 82734 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4030331;CGA_RPT=L1M4c|L1|45.3;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:32:32,32:30,30:4,18:-32,0,-32:-4,0,-18:38:25,13:13
+1 82797 . CATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 82884 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 82961 . CCCTGAGTAAGCAGATATTGAAAATATTAGACAAAAACTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 83010 . GTCTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 83052 . ACATATAAATAAATAAGAAATATGAATTTTTTTAAAGGTACAAAAAAATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 83121 . TAAGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 83242 . AATGACAACAAAAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 83329 . CAGGAAGACTATTTGAAGAAATGTGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 83385 . AATATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 83412 . ACTTCATCAAGGAAATATACAAAGATATTCACACC . . . NS=1;AN=0 GT:PS ./.:.
+1 83511 . CAACGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 83556 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 83588 . TTAGGAAAAAGGCAACGCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 83642 . GAGAGCTCATTATAAACCATGGGTGCCAGAAGAGCTTAGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 83783 . G <CGA_NOCALL> . . END=84056;NS=1;AN=0 GT:PS ./.:.
+1 84001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=86000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.21:1.10:.:0:.:0:0.999:152
+1 84091 . CACTTTTAAAAAAAAGACTCCTTCAGATACAAACTAAAAAACACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 84195 . ATATAAAAGCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 84241 . TATATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 84283 . TATGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 84299 . GATGTATACAGATGTGGTTTGTGAAATTACCAACATAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 84379 . CTATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 84435 . AAATCCCCATGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 84585 . ACAGAAAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 84680 . ATTAACAGAATGGATTTTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 84750 . ACACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 85048 . AAAAGATAAAACATCTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 85118 . CTATAGATAACACTTCTCTCAAAAACTGCAGAGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 85197 . ATATGTTAGGCCATAAGATAAGCTCAATAAACTTAAAAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 85403 . AGACGATTGAAAACAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 85490 . AACATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 85801 . ATGCAAAAAAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 85994 . AAATGTACCAGAATCTGAAAACATCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 86001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=88000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.57:0.68:.:0:.:0:0.999:152
+1 86300 . TTAGTTCAAATTGACTTTTGAACATACTTGGACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 86417 . AAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 86429 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 86435 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 86448 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 86452 . GTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 86469 . AGTAAATATTAATATATTTGTATTGCTAGAACCCCAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 86515 . GTGAAAGGACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 86573 . TACATTAGAATCAGTATTATCAACATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 86636 . TCTTAAAAAAATATAATATGGACATATTATATATTATATGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 86687 . GTGTGTCTATACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 86758 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 86796 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 86809 . ATAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 86818 . CCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 86983 . ACATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 88001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=90000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.60:0.82:.:0:.:0:0.999:152
+1 88065 . CTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 88125 . TCTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 88239 . CCATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 88259 . GACCTTCTCCTGGGACCACAGGCCTGTGTCTCTATCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 88329 . TAAAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 88570 . TTTCACG . . . NS=1;AN=0 GT:PS ./.:.
+1 88773 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 88797 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 88915 . GCAGAGCCGGCCCCCATCTCCTCTGACCTCCCACCTCTCTCCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 89251 . AATATTGAGCACTATCAGTAAAATACATAAAACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 89346 . CAAATGGATTACACGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 89412 . TGGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 89575 . TAATGAATAATTTTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 89628 . GGTTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 89845 . AATACTCCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 89885 . AGCCTCCATCTTTCCACTCCTTAATCTGGGCTTGGCCAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 89948 . ATTAACAAGTCTGATGTGCACAGAGGCTGTAGAATGTGCACTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 90001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=92000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.44:0.93:.:0:.:0:0.999:152
+1 90025 . TGCCCCACGAAGGAAACAGAGCCAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 90062 . T <CGA_NOCALL> . . END=90300;NS=1;AN=0 GT:PS ./.:.
+1 90306 . GACAGTCCCTCTGTCCCTCTGTCTCTGCCAACCAGTTAACCTGCTGCTTCCTGGAGGAAGACAGTCCCTCTGTCCCTCTGTCTCTGCCAACCAGTTAACCTGCTGCTTCCTGGAGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 90452 . TTGACCGCAGACATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 90638 . TTCTCTGCTCATTTAAAATGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 90674 . TACATTTTTATAGGATCAGGGATCTGCTCTTGGATTTATGTCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 90777 . AGGCGCTGGGAGGCCTGTGCATCAGCTGCTGCTGTCTGTAGCTGAGTTCCTTCACCCCTCTGCTGTCCTCAGCTCCTTCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 90868 . CAGGAAATCAATGTCATGCTGACATCACTCTAGATCTAAAACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 90939 . ACATCTGTAATCCCAGCAATTTGGGAGGCCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 91004 . GATCCTGGCTAACACGGTGAAACCCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 91061 . GGTTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 91079 . TGTAGCCCCAGCTACTTGGGAGGCTGAAGCAGGAGAATGGCGTGAACCTGGGAGGTGGAGCTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 91160 . GCCACTGCACTCCAGACTGGGAGAGAGAGCGAGACTTTCTCAAAAAAAAAAAAATCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 91252 . CTAGAATCCTTGAAGCGCCCCCAAGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 91417 . TGTGTGGCACCAGGTGGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 91450 . GGCAAACCCGAGCCCAGGGATGCGGGGTGGGGGCAGGTACATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 91504 . TACAGCAGATTAACTCTGTTCTGTTTCATTGTGGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 91548 . TGCGTTTTTTTTTCTCCAACTTTGTGCTTCATCGGGAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 91613 . GAAGAAAAGGCCAAACTCTGGAAAAAATTTGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 91666 . GACCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 91727 . AAAAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 91801 . AGACAAAAAAGCTACATCCCTGCCTCTACCTCCATCGCATGCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 91886 . AACCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 91937 . TCCCCAATACCCGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 91983 . CAACCTTTGGGAAAAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 92001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=94000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.09:1.17:.:0:.:0:0.999:152
+1 92143 . GAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 92478 . GCTGAGGCTGCTATTCTTTTGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 92635 . CCAAACCTCAGTCCCTCAGTTGTAAAATTAAAAAAAAAAAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 92708 . GATTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 92833 . GAGGGACAGAAACAAGTGGGAGAAGGTAAAGAGATGGACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 92915 . TATGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 93120 . GAAATACAGAAGAGAGATTTCTCATGGTTAAAACGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 93279 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4265376;CGA_RPT=L5|RTE|30.4;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:36:36,36:36,36:13,27:-36,0,-36:-13,0,-27:16:1,15:15
+1 93491 . TCAATTTTATTGAAGTTCACTTCTGACCTCTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 93662 . TAGAGCTGAGACCATTTGCCACTCAGTTTCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 93725 . CCGGTTTTTTTGTTTTTGTTTTTGTTTTTAGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 93804 . GCTCACTGCAACCTCCGCTGCCTGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 94001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=96000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.22:.:0:.:0:0.999:152
+1 94011 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 94116 . CTCGTAAGTAGATTACTACAATCACCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 94221 . AAATGAAAATCTGACCACGTTACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 94257 . TCCGCCTATGGCCGCTGTTAGGATCAAGTCTAAACTCCCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 94461 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 94522 . GCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 94567 . GCCTCACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 94722 . AATCACATCACATTGCTTCCTTCATATTTTTTTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 94805 . GCTCCTTTTCTTTTCTTTTCTTTTTTTTTTTTTTTTTTTTTTTGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 94857 . TCTCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 94890 . TGCAATCTAGGCTCACTGCAAGCTCTGCCTCCTGGGTTCACGTCATTCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 94975 . ACCTACCACCACGCCTGGCTAATTTTTTTTTATTTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 95147 . GCATAAACTAAATGTTTTCCAAAGGGAATAGGGCAAAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 95290 . GGGCTCTCCACTTACAAGAAGAGAGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 95412 . CCTGTTAATTTAATCACACGGAACACTTCTATTTAAAATTCCCGAGAGTTAAGATGTAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 95532 . CATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 95575 . ATGGGGCAATTTCTTAAAAGCACCATGTATTTTATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 95695 . CCACTATAAAGAACCCAGCGTGGTTTTAACTAATGGATCAAAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 96001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=98000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.21:.:0:.:0:0.999:152
+1 96005 . TGGGAGGCACAGTGGAAGATCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 96150 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 96250 . TCACGGAGGAAAAAAATCTCTCAATGATCTTATCTTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 96363 . GAGGCAACCTCCAAAGGTGGGGCCCTCTGCTCACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 96476 . TTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 96589 . AAGTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 96636 . ATAAATTCGTTCAAGCAGCCATTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 96895 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 96941 . CGGTAGACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 97044 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 97127 . TTATAAAAAGGTGAGCTGTAATAAATACTAGTGCCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 97279 . AAGTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 97311 . ACCGGCAAATTCTGTTGTTTGTATAAACATCAGCCATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 97461 . ATAATTAATACATTATTAAATTGAATTGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 97563 . TCAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 97621 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 97624 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 97818 . AGACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 97866 . CCACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 97929 . TAACAATCTGAGAGACATTCATACATTTTCCATGTGCTGTAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 98001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=100000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.74:1.17:.:0:.:0:0.999:152
+1 98018 . CCCTGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 98103 . AATAAAGAATTCTATCAATGCTGAGGGAAGATGACTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 98307 . CCTGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 98378 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs3868675&dbsnp.108|rs4114931;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:37:37,37:37,37:14,27:-37,0,-37:-14,0,-27:17:1,16:16
+1 98447 . ACATGGGCACCCATATTTTTCTAGCCACTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 98507 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 98566 . GTGATTTTCTGTTGGTGTTCACTTCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 98615 . TTATTGACTGACTGACTAACTAATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 98652 . TTCATAAAGAAAGGCTCTCTACAAAAACGGAGGGATGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 98776 . AATGTGCCTTTCTAGTAACAGGTTTTTAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 98830 . TATTTGTGTGTGTGCATGTGGTAGTGGGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 98882 . AGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 98910 . ATACTGTATTCAGGGGGAAAAAATTTTCCCAAGGTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 98996 . TGCTTTTATTTATTTATTTATTTATTTATTTATTTATTTATTTATTTTTCCTTTTTTTTCTTTCTCTTTTTTTCTTCTTTTTTTTTTCTTTTCTTTCTTTTTTTTTTTTTTTTTTTTTTTTGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 99331 . CTCATGATCCACCCACGTTGGCCTCCCAAAGTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 99373 . CAGGCGTGAGCCACCGCCCCTGGCCAGGATTGCTTTTACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 99480 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 99488 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 99497 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 99584 . GTTAAAAGATATTATTTTGCTTTACACTTTTTCTCTCAGAAATAAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 99754 . AGGGGAGATTTTTCAGGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 99820 . GAAAGTGTATAATGATGTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 99871 . TGCCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 99938 . AAAAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 100001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=102000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:1.22:.:0:.:0:0.999:153
+1 100319 . TTTGGACAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 100481 . TCCGTGTTACTGAGCAGTTCTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 100538 . TTCTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 100583 . AGGTCAAATTCAAAGGAGAGAAAAAAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 100652 . ATGGCACAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 100750 . TACCCTTCTAATCTCTATCACAGCAAAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 100855 . ACATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 100942 . AATGTAGAAATGCTACAGATTATATTCTCTGATTATGACACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 101011 . TTTAAAAGCTTTCTCTTAAATAATTCTATGTCAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 101092 . CTTTGGGAGGCCAAGGTGGGCAGGTCACTTGAGGTCAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 101140 . CCAGCCTCGTCAACATGGTAACACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 101210 . TGCCTGTAATCCCAGCTACTTAGGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 101265 . AAGGTGGAGGTTGCAGTGAGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 101312 . TAGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 101376 . GTGGAAAATAGTGACAATAAAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 101425 . TTGAGATGCCAAGGTGGCAGGATCACTTGAGACCAGGAGTTCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 101509 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 101512 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 101547 . TTCCTGTAATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 101608 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 101618 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 101666 . TCCTATCTCAAAAAAAAAAAAAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 101792 . GGAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 101825 . TAGCCACGGTGACTCACATCTGTAATCCCAGCACTTTGGGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 101909 . GCCTGGCCAACATGGTGAAATCTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 102001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=104000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:1.32:.:0:.:0:0.999:153
+1 102133 . ATAAACTAGAAAACAGAAACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 102181 . ATGCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 102207 . GTGAATTAAGGAAGGGAAGAGATGGTTGGAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 102322 . AGAGATGCTTGACTGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 102412 . ATCTCCTCCCCTCCCCTACTCCTCACCCCACACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 102494 . GCATTCTTATTTCCCTGATTTCTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 102593 . CAAGGGCTTCACAGACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 102635 . TTCAGGTTTTATACCTACCTTATAGATAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 102693 . TGTTCCCAAAGCCTCGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 102766 . ACTCTACTGCCTCTCCATGGATAAAGACAGAGATCACATATTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 102856 . TGATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 102915 . AATTTGAATAACTCCCTGCGGGTGAAGTTCAAAGTACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 103007 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 103088 . TCTACTAAAAATAAAAAATTAGCCGGGCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 103165 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 103214 . CACTGCACTCCAGCCTGGGCGACAGAGCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 103257 . AAAGTAAAATAAAATAAAATAAAAAATAAAAGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 103304 . ATCAGGGAGGTCTGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 103336 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 103381 . TCAGGGTCCTAGCAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 103441 . AGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 103523 . TGTTGGAGGTGGGGCCTAATGGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 103737 . CAGAGTAGCTAGGATTACAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 103800 . AGACGGGGTTTCACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 103853 . GATACACCTGCCTCGGCCTCCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 103890 . CAGGTGTGAGCCACCATGCCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 103970 . AACCCCTCTCTCTCGCCACGTGATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 104001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=106000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:1.45:.:0:.:0:0.999:153
+1 104030 . GAGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 104104 . AGCCAAATAAACCTCTCTTCTTTAAAATTATTCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 104157 . AACAACACACACACACACACACACACACATACACACACACG . . . NS=1;AN=0 GT:PS ./.:.
+1 104222 . AATTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 104317 . TAATGGTTAAGTAATTATTTGCTCTTACTCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 104382 . TCAACTAGAATCTAGGAAGCAGAGAACCTGAGTGTTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 104519 . CAACAGAGCGACTCAGATGCTATAAAACTTGCTAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 104611 . CACAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 104672 . ACCTCACAGAGAAGGAAATTTACACGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 104756 . TTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 104819 . GACATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 104939 . ATACAAAGAGTAATACCATGTCACTTAAGAATAGAATCATGGACGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 105054 . TTCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 105120 . ACACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 105178 . GCTCAGATACCTTCTCCGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 105265 . GAGACTAATGAGTAGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 105295 . AAGCTGAGAATGCTTCTACCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 105330 . GGAATATTCATCAAAACACAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 105484 . TTCCAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 105631 . ACACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 105713 . TACAATAAACATGTGTTTTTAACAAGAAAAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 105785 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 105886 . TAGAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 105984 . ACATGGGTGTTAAAATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 106001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=108000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.19:1.25:.:0:.:0:0.999:153
+1 106040 . AGAGCAAGCTGGGAAAGCAGTGGCCTTTAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 106115 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 106124 . AGT . . . NS=1;AN=0 GT:PS ./.:.
+1 106130 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 106133 . TAT . . . NS=1;AN=0 GT:PS ./.:.
+1 106154 . AATAGTAAACTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 106462 . AGAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 106497 . GTAATTTTAATATATAACTGGGGTGAGAATCATTGACATAATTGTAACAGGATAATATTCAGGAAATATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 106608 . AAAAGTTTTATGTTTTCCCCTAACTCAGGGTCATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 106908 . CTTGAGCAAATGGTAAATTAACTCTCTCTTTTCTCTCTCTCTCTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 107122 . TTTACTGGAGTACACAATTGTGACTATTTTTAGCCATAGGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 107279 . CACCTTACACTTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 107329 . CACTTTTCAAAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 107514 . GGAGATTTGGACATAGAGAGAGGCACACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 107967 . TGGGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 108001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=110000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.13:1.11:.:0:.:0:0.999:153
+1 108016 . TCTTTCCTGGCTATGTTTCTGACATCCTCTTGTACCATGCTCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 108252 . CTATAACAACCTAATATATTCTCAATTGATTAACTGTTTTGCTGAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 108318 . GAAAGAAAACATGGCCAGGTGCAGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 108356 . AATCCCACCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 108401 . CTTCAAAAAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 108537 . GTCACTATCAAAAAAAAAAAAAAAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 108596 . TATCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 108672 . AATGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 108890 . TGACCCAGCATGGCTGAACACTCAGTGACTACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 108998 . TTATATTCAGAATTACTCAAGTCTTAGAAGCACCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 109061 . TCAAGTGATGGGCTGAAGTGAAGGGAGGGAGTCACTCACTTGAACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 109403 . ATATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 109568 . GTGTATGCGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGAAAGACAGAAGAAAGAGGGAGACCTTAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 109707 . AGGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 109773 . ATATATGCAATATATATACATATATACACACATATACATATGTATTTAAATATTTAAATTACATTTTCTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 109852 . AGATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 109951 . CAACCCTCCTGTATTAGTCTCCCCAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 110001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=112000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:1.31:.:0:.:0:0.999:153
+1 110004 . ATGTCCACCTTTATGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 110059 . AACTTAATAATAAAAACATTTCAAATGTAAAGAAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 110124 . AAATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 110148 . ACACTTTTCAAAAGAATACATGCATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 110217 . TTAGAGAAATGCAAATCAAAACCATAATGAGATACCATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 110416 . CCATTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 110438 . TACTGGGTATATACCCAGATGAATATAAACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 110504 . TTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 110606 . ATGCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 110700 . ATACAGCATACTCTCAGTTATAAGTGGGAGCTAAATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 110925 . AAATAAAAGTTAAAAAAAAAAGAAAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 110980 . TATGAAAAACACATATCTTTCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 111099 . ACACCTGTAATCCCAGCACTTTGGGAGGCCGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 111158 . TTCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 111201 . TAAAAATACAAAAATTAGCTGGGTGTGGTGGCAGGCACCTGTAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 111259 . GAGGCTGAGGCAGGAGAATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 111341 . GCAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 111363 . GGGGAAAAAAAAAAACAAAAAAAACCACCACCATCATTTTGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 111520 . ATTATTTTGTATGCGATGACAACAGAATATATTATCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 111570 . AATCTCATTCATAATATAAAGTATAAATTTGTGATTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 111708 . AAAATTTGAAACTAGTAACATGGAGGACTATTGTCATTGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 111779 . CAGTGTACATAAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 111916 . GGCTAATAGTAGGCACCTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 111987 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 112001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=114000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.16:1.52:.:0:.:0:0.999:153
+1 112014 . AATTACTGTTTAGAGAATAACATTTGATGGAATCATGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 112065 . TTACGACTCAATTGTTTGTACTGACATTAACATCCCAAATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 112134 . ATGTGGCACCTGCTGAAGCCTGCTGCCTCATTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 112200 . CTCTAACATTTTTTAGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 112248 . TGACCTTTGTACCTGTTCTTTATTCCTGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 112320 . AGGTTAAATGGCACTAACTCAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 112357 . CTGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 112402 . TAGTATCGAATCAAGTTTATAATTTTAAAATAATTGGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 112504 . CCTGTTTGTTCACTCCTGCCACAGTCAGAATAGTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 112656 . CACGTAGGTAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 112755 . TTCTTTAGAAAACTTGGACAATAGCATTTGCTGTCTTGTCCAAATTGTTACTAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 112824 . TCTGACATGAAATGACATTGGAAAACATTAAACACGATTGAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 112885 . TTATTATTAGAAACCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 112918 . AAAATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 113068 . GTGATTTTTCAGGTTCACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 113113 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 113179 . TATGAAAACAAGAGATAAATATACACAACTGAGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 113296 . TAACTTTTAATAGAACCAGTCACTACATTAAAAAAATGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 113584 . TACAATTTAGGAGAAGAAATTGTATGGAAGGAAGGTTCATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 113679 . GATAAGATATTGTGGCTGCTACCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 113965 . TTGACGATTACCTGTAGCCAACCCTAAGTGAAGAACTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 114001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=116000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:1.52:.:0:.:0:0.999:153
+1 114039 . TAGCTAAGAACCATGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 114079 . CCTCAGTTGAAATTTAAGATGACATATTGAGCAGACATACTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 114180 . ACTGATTTTGAGATTCTCACATAAGTATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 114237 . ATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 114284 . ATTTCCCTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 114376 . AACTCACACAATCTGTGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 114465 . TAGGAAAAAAAATCCTCTCTGGACAAATAAATCATCAAAGCAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 114654 . AAACACATATTTTAATGTGGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 114690 . ATCACAACTATGAGTAAAGACCAAGAAAATTGTGCTGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 114745 . GGCTCCCCTCCTATTTAAGTCTGGGTACTGTGTCACCCGAAGTCTTCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 114807 . GGTCTGGGTTTGCCTATGAAAGAAACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 114919 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 114985 . ATTCCTATAAGCTTGGGTTCTGTGCCCACACTCTAGACTGTCAGGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 115042 . ATATAAAACAGACCTCTTCTGATTTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 115115 . GCATAAGGCCCTAATTAATATTAAACTTTTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 115200 . TAAAATATAAAGAATTGTCCAGAAATATATAAAAAAAGAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 115268 . TATAACAATTGTATGGACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 115320 . TTTGAAGAAAAAAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 115436 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 115440 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 115471 . ACAATTTCCTAACAATTTTGGGGTTTATATTTTTGAAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 115662 . TGTCTGTCCACGATAAGCACTATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 115785 . AAAATTCCTCAAGACTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 115819 . CACCCTCACAAGAACACTTGCCTAGCAATGGCTGTTTCTGCCAGTAAGTTAACACCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 115886 . CAGACCCTGTGACCAATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 116001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=118000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.12:1.54:.:0:.:0:0.999:153
+1 116017 . AATCCTTGTTTATTTCCAAATAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 116131 . TCCAGAAGAATTTCTGTAACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 116164 . GTTCTTTGCATGTTTGCTAGAACTCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 116204 . TGAGCAACCAAAGCCTGGTTTTTGTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 116253 . GGAGGGGGGTTTATCGTACTGATTCAAGGTGTGAAGGTAACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 116486 . TACTGTAAAACATCCCATGGTTTCTCATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 116523 . AGTAAAAGTGAAATTTTTATGATGGCCTGAGAAACTTTTCCCATTAGATGCCCAAGTGCTGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 116624 . GGCAGTCACACTAGCCTCCTTGCTGCTCCACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 116878 . AAACTGTAAATATACATGTTCACTTTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 116926 . TGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 116985 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 116996 . TACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 117034 . CACATTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 117230 . ATATTATTTTCATGTATAAAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 117308 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 117325 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 117344 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 117365 . AGGGCCAAAAGAGTCAACTTCTGAAGAAGCGCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 117443 . TTGCAAAAATAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 117469 . ATTCATGAGTAGAAAAATAGACTAGTGGAATAACATAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 117619 . GGAGAATAATGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 117746 . TTACCCAGATGGGCCCAGTCTAATCACATGAGTTCTTAAAAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 117871 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 117913 . CTAGAAGATAGAAAAGGCCAGGATATGGATTCTACCCTAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 117983 . TTGATTTTAGTTCACTAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 118001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=120000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:1.53:.:0:.:0:0.999:153
+1 118056 . TTTAGGTCACTTAGTTTGTAGAAATTTGTTACAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 118098 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 118157 . AATTCAAGGTGAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 118183 . CTTAAAACATTTAGATTAAAAATAAATGAGAATTTTTGTTACTTTTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 118246 . AGAAAAACAAACATTAAGGAGGAAAAATGAACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 118313 . GGAAGATATCATAAGGTGACAAATCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 118351 . TTTACAACATATATATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 118385 . TTAGAATATATATGAACTCCCAAAAATCAACAGGAAAAATAAGACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 118442 . AAATGCATAAACAAAAGAAGGCAAAACAAAAATAATGACTCATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 118514 . GATGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 118526 . AATGCAAATTAAAACCACCCTGAGATGCTTTTTACATCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 118590 . AAAAGTAATAACAAAGATGGGAAGTAATAGAAAATCTTGTCCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 118721 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 118725 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 118794 . AACATTGTTTGTTATATCAAAAAATAAAAAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 118839 . CAGCAAAAAAAATAAGTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 118883 . ATGGAATAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 119029 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 119033 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 119041 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 119103 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 119126 . AGTTGAGGGAATTTCAATTGGAAAAAAATAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 119222 . AGCTACACTATATATTTTCAATGTATTTAATGTATTTTTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 119271 . AATATTATGCAATAAAAATGAGAAAACAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 119326 . AAAGAAATGGAGAAAAAATTATAATCTAGTTGAGTAATGGTATATTACATAGCTATTTTCTTAAGTAGATGTATGTACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 119434 . CTTAATTATATATAAATATATATGTACATATTTTTAATATAAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 119496 . AAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 119536 . TTTTGTATTTTAAGTTTTACATAGTAGGTGTATTTTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 119587 . CTATAAAGAACTGCCCAAGACTGGGTAATTTATAAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 119643 . CACCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 119694 . AGACAAAGAGGAAGCAAGCCAGCTTCTTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 119735 . GAAGAAGTGCCGAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 119766 . CTTATAAAACCATCAAATCTCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 119803 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 119827 . CCCCCATGATTCAATTACCTCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 119952 . TGTATTGAATTTTAAACTCAGAGAAAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 120001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=122000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.60:1.70:.:0:.:0:0.999:153
+1 120057 . GTTGTTTAGTTACAAGATAGAATGTGGCCTTGTAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 120367 . GAGAGCTGTTCCAAAGTTTAGGGAGTTTTTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 120410 . AAATAAAAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 120452 . ATACTGTCAGAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 120504 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 120508 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 120529 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 120606 . TGGTAAATCATTTTCTACCAAAAGAAAGAAATGTCTTGTCTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 120696 . TTAGAAAATTATATTTTATACGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 120775 . CTTCAAGTTTGCTCTTAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 120877 . TTCATTGAATCCTGGATGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 120908 . AATAAGAGGAATTCATATGGATCAGCTAGAAAAAAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 120964 . AAAGTTATATATTATATATCTATTATATATAATATATATCTATTACATATTATATATTGTATATCTATTACATATATATTATATATGTATTATATATATTATATATTATATATGTATTATATATATTATATATTATATATCTATTATATATATAATATTATATATTATATATCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 121145 . ATTCCCCAGCGTTCATATTTGTCAGTGCAAGTAAAGAGCCTTACTGCTGATGAGGTTTGAGGTATGACCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 121263 . G <CGA_NOCALL> . . END=121528;NS=1;AN=0 GT:PS ./.:.
+1 121542 . AGGTGTGAGCCACCACGCCCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 121594 . TTTGAAGGTCATAAAAAATATAATAAGAGATAAGGCTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 121648 . ATAAAATCCTTTAATAAAAATATAAAGGAATAATATAATAATTTTCTTTAATAAAATATAATAAGAGATAAGGCTAATTTCCTTTAATAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 121765 . TCCAAAAAAAGAAATGGAGAGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 121802 . ATTAATCTTGTCAAAAATATAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 121849 . ACTGTTTTCCTTGTCTGCGGCCATTGTGCTGCTGCTACACAACTACCGCAAGCAGCCCTTCACGCCCTCCTCCCAGTACAAAGCTAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 121948 . AAATGTTAAGCTTGGAAGAGTCAGCATCACTGCACTTATTTTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 122001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=124000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:1.55:.:0:.:0:0.999:153
+1 122021 . AGTGGGGGAAAGGTTAAAAACCCCCCTGGATAAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 122100 . TTTCCTTGTCCCTTGACATAAACTTGATAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 122177 . CAGGTACTTAAAGTTAGCTCCAAAAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 122226 . GCATTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 122432 . AGTGACATTGCCTTTTAGTTGTACTTTCACAAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 122546 . TCTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 122633 . CTTCTTTTCATATTTTTGAAAACTTTTGAAAAACTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 122769 . AAAACAATATGTTGTCTTTATCTTTACCTCTCTGTGGCATTTAATGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 122830 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 122854 . GAATTCAGTCAGACAACGTACTTACATTTTTCGTCTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 122909 . CACCTCAGCTTTCTCCATTCAGCTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 122962 . TCTGCCTCTCCTCTCACTCTATACTATCTCTGTTAGCTAATTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 123036 . TATACACATATGCATGTGTGTACATGTGCACACACACACTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 123085 . C <CGA_NOCALL> . . END=123432;NS=1;AN=0 GT:PS ./.:.
+1 123467 . ATTCTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 123508 . AGAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 123629 . TATCCTCATTTTTTTCAGATTCTTGCTTAGAAGTCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 123685 . GACATATTAAACATTGCAGTCCATTATAAGCTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 123727 . AGGGATTTTTGCCTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 123798 . TGTAATGAATATTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 123827 . AAATTCAATCAAATCACATCACCTGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 123881 . ACTTAGAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 123900 . TCTTTTTATACAATATACAATATATTTTATACAATATAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 123984 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 124001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=126000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:1.52:.:0:.:0:0.999:153
+1 124109 . GTCCCTCCACTTTTGCCAACTAATCCCTGCTCAACTTTTCATCTCAGCAGGAGGCCCATTCTCTTTGGCAATCCTCTGGCCTCCAGCCCATTTATTATATGCTCACATGTCAACATGTACTTCGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 124252 . GCACTTTTATATTTTAACAAATTATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 124310 . CAGGAATTTTGTTCTTGCTCATCATCAACTTTTTCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 124411 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 124426 . CCTTTAAATTAGGATGGCAAAGATCGTATATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 124494 . CCCAATTAAGGAGCACAGCTATGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 124552 . TTTTAAAAAGAAAACTGGCCAGGTACTGTGGCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 124674 . TGTCTCTACAAAAAATACAAAAATTAGCCAAGTTTGGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 124739 . GGGAGGCTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 124800 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 124842 . TCTCAAAAAAAAAAAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 124991 . TAGTATTAGAAAATTACATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 125183 . TATGTTTTTGAAATAAAATATATCTGAGTAGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 125265 . GACGTACATGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 125299 . CTTTTAGAAGTCAATCAGGAAGAGGGGAGCAGTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 125491 . TGCATGCTTATACATATAAACACAGCTGATAATTTATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 125563 . CCAGTTTTTTATTTAAATTGAAGATTAGTATACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 125611 . TCAAAATAAAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 125653 . TCAGAAAAAAAAAGTCAAAAGCTAGAGTATAGAGAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 125730 . ACAAGATTTAAATATTTTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 125799 . AAATAAACAGATTATATGGAGGATTTTTAGAAGATAAGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 125861 . AAACAAGGGAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 125906 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 125920 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 125954 . GATGCATAAATATATAAATAAACGATAAAAAATGTTGCATACATATATGACTTTTTCAGAATCAAAAAATTTAAATTTCTGTAATAAAATTTAAATGTTTATAAATTTAAAAAACTAGAAGAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 126001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=128000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:1.68:.:0:.:0:0.999:153
+1 126099 . CAAATAAATGACAACTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 126208 . CAATTATTTGTCTCAAAAACAAACAAAAAAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 126258 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 126322 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 126337 . AGGCCAGTAGGCAGTGGATTATATATTTAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 126394 . AATATATAGCTGGAAAACTTATCCTTCAAAAATGAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 126444 . ATTTCCGGATTTTTTTTTAAAACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 126622 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 126642 . TTTTGGTTTGTAACTCTGCTTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 126680 . TTAAAAGGCAAATGCATAAAATGTAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 126768 . AAAGAGTAGAGCTATATATATAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 126808 . GTGATTGAACTTAAGTTGAAATAAATTCAAATTAAAATGTTATAACTCTAGGATGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 126876 . TCATAGTAACCAAAAATGAAATATACATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 126935 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 126977 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 126987 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 127001 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 127048 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 127062 . TCACATAAAACTGGAGCTGAAAGAAACAAATATTTACCTATAAAGTTAAAAGTTATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 127136 . TTT . . . NS=1;AN=0 GT:PS ./.:.
+1 127152 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 127155 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 127161 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 127165 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 127266 . CCCTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 127289 . CCTAAAAAGTCTATTCTCAAATGCAGCAGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 127374 . TTCAATTTTATAACACTGGGTTAAGATGAAAGAATGAGAAGATAAAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 127436 . AACTCACAAACATGTTCAGAAGCAGTAAGAAGTTACATTAATTATCTTTTGAAAGTCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 127505 . CTTTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 127647 . GAGTTTCCATTAAAAGACAATTTAGTAAAACTTTTCTTCCCCCAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 127709 . ATGATTTAACAACATGTGTAAAAGTCATTGTGGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 127829 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 127864 . GCCTCAGCTTTCTGAGTAGCAAGGACTACAGGTGCACACCATCACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 127922 . TTGTACTATTAGTACAGACGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 127958 . GCCAGGCTGGTCTCGAACTCCTGACCTCAAATGATCCATCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 128001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=130000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:1.41:.:0:.:0:0.999:153
+1 128064 . ACTTTGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 128126 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 128195 . ACACAAAATCATGGGAGTTCTAATCAAAATCCAACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 128285 . GCCTATTAATTAGATTTGTCTTTGTAGCATTTAACTCTATAATAAATAATATTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 128495 . TGGCAAATATTGATTGTCATCTTCGTGTTTGTCTATGTCCTAAGTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 128588 . TCACCCCCTTTTTTTTTTTTTTTTGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 128798 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3926105&dbsnp.131|rs76554219;CGA_RPT=AluSq2|Alu|12.1;CGA_SDO=24 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:55:55,183:55,183:45,48:-183,-55,0:-48,-45,0:5:5,5:0
+1 128844 . CTCGGCCTCCCACAGTGCTGAGATTACAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 128883 . CCACGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 128979 . AAAGACAAACTCACAGGAAGATGGGATGTAGAATGATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 129083 . CAGCTGAGTCTGCAGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 129112 . CTTTCATTTTATAAAAATCTATGATTTCTCCTTCCAGTTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 129229 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 129271 . ACCTAAAAGAATACGCTTTTTTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 129312 . CAAATCATCACAGTAGACCACGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 129427 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 129455 . GCGATCTCAGCTCACTGCAACCTCCATCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 129571 . CTGGCTAATTTTTGTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 129602 . GGGTTTTGCCATGATGGCCAGGCTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 129674 . AGACTTTTTTTTTTTTTTTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 129739 . TCCTGAGCTCAAGTGATCCTCCCACCTCAGCTTCCCAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 129806 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 129947 . GAAATTATCCAGTCAGTGGACAGAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 130001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=132000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.51:.:0:.:0:0.999:153
+1 130118 . AAAATACAAAAATTAGCCGGGTGTGGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 130216 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 130278 . CTGTCTCAAAAAGAAAAAAAAAAGAGACAGAGAAAAGAAAGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 130333 . TTAAGCAAACCATTGTCAGGTTATGGGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 130430 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 130434 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 130437 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 130573 . CATTCAAAGTGCTGAAAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 130695 . CCACTAGGTCTACCTTAAAAAAATGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 130734 . TCAAGTAAAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 130758 . GAGCGGTGGCTCATGCCTGTAATCCCATTTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 130852 . AAAACCCCACCTCCAGTAAAAATACAAAAAATTAGCCAGGTATGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 130961 . ACAAAAAACAAAACAAACAAAAAAAACAAAACTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 131062 . GGGCGAGGAGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 131099 . CCCGGACCAAGTGCTCGGCCCCCAGGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 131143 . TCCCGTGGCGTCAGCATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 131200 . TTCTCCTGGTACTCCATCCCCTTCCTGACCCCTCCCTGCAGCCACACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 131281 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 131305 . AGTTGGCAGCTGTTGCTCATGAGCGTCCACCAGGTGGGACAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 131361 . GGGCGGCCCCCTGGAGCCACCTGCCCTGAAAGCCCAGGGCCCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 131413 . ACACTTTGGGGTTGGTGGAACCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 131455 . CCATGGAGGAGGAGCCCTGGGCCCCTCAGGGGAGTCCCTGCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 131549 . CCTGACACCCAGTTGCCTCTACCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 131597 . CTCAGTGCCCTGCGCAAGGAACAGGACTCATCTTCTGAGAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 131658 . AATCAGACAAGGACCACATCCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 131694 . GCGCTCATGATCTTCAGCAGGCGGCACCAGGCCCTGGCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 131746 . TCACCCCAACCAGGATAACCGGACCGTCAGCCAGATGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 131875 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 131889 . TCAGAGGCCAAGCCCACAAGCCAGGGGCTAGCAGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 131944 . GAGCGGAGCATATCAGAGACGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 132001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=134000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.16:1.77:.:0:.:0:0.999:153
+1 132027 . GAGCTCGGATACCAAGGAGCAGCTTCTGTGGGGCAGAACGGCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 132081 . GGGAACCTGGCTCAGCCTGGCCCAAGCCTTCTCCCACAGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 132135 . GGACGGCAGGGAAATAGACCGTCAGGCACTACGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 132264 . GGGAGGTGACCCGTGGGCAGCCCTGCTGCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 132327 . CAGCGAGGTCATAGCGAGTGACGAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 132366 . CCATGAGGAGGAGGGGGTGATGATGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 132401 . TGATGGCTTTAGCACCACCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 132441 . GAGTGGGTGACCGACTGAGAGTGGGGACAACTCTGGGGAGGAGCCAGAGGGCAACAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 132514 . GTATTTGCACCTGTCATTCCTTCCTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 132560 . GCTGGATCCTGAGCCCCCAGGGTCCCCCGATCCACCTGCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 132624 . CCTGTCCTCCTCCTACACATACTCGGATGCTTCCTCCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 132716 . AGTCTGGTCAACGCAGCAGAGCGGGCCCCCTACGGCCCCAACCCCTGGGGATGGGGGCCCAGGGACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 132835 . AGAGACCTGAAAGTGTGGGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 132936 . CCCGGGGGGCAGAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 133006 . TGGACCCCACACTGGAGGACCCCACCGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 133055 . ATGCTCCAGCTGCAGTCCAAAGCCCAACACCCCCAAGTGTGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 133119 . CCCTTTGCCTGTACAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 133187 . CTCTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 133236 . CTTCCAGGCCCACTGCTTCTTCCTGTCCACTAGGCCACAGCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 133305 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 133333 . CCCTGACTCCCAGCCCTGTGGGGGTCCTGACCGCACCTCACCTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 133456 . GGGCAGCAGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 133478 . GGGGCGGTCCAGTGGGAGGAGCCTCAGCCTCGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 133584 . ATGCTGGTGGTGGGTGCAGGGCCGCTGGGAGCTGCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 133668 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 133711 . GAAGATGTGTGCATAGCAGGTCCACTGCTGCTGCCCCTGCCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 133869 . ACACCCCAGCCCTGCCTCAACACCTGGGGGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 133971 . ACAGTATGTGGGGGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 134001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=136000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.36:.:0:.:0:0.999:153
+1 134047 . GGACGGAGTAAGGCCTTCCTTTTTTTTTTTTTTTTTTTTTTTTTTTTTGAGACCGAGTCTTGCTCTGTCGCCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 134231 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 134259 . TACAGACGGGGCTTCATCATCTTGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 134331 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 134367 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 134421 . GGGGGAAAGCTGGGCAGTTTCCCTCCTCCGAGCCCCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 134493 . TCACTTTTCGGAAAATAGCTCCTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 134527 . AAGATGGAGTGTGAAGAGGGCCTTGGGCCACAGGGAGGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 134597 . TTCTTTCCCCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 134624 . GTGAGTATGGGGGTGGGGGCTCCTGCACTTCGACACAGGCAGCAGGAGGGTTTTCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 134712 . TACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 134741 . TTTGCCCCCTTCCCCAGAACAGAACACGTTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 134873 . TGGCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 134929 . AAATATTCCAAAATTCAATATTTTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 134973 . AAACAAATTAGAGGCCAAGAGGCTGCCGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 135037 . GAATGAGCTGGGCCTAAAGAGGCCACTGGCAGGCAGGAGCTGGACCTGCCGAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 135176 . GCCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 135230 . GCCGACTGGAGATCAAGTTCTGCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 135462 . GGCCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 135536 . GGACGATTTGGGCCTGCGGAGGCCGCCGGGAGGCCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 135587 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 135660 . ACCGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 135801 . GCCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 135823 . GGAGGGGGCGCCGGGAGGCTGCAAGTGGGTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 135978 . CCCCAAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 136001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=138000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:1.27:.:0:.:0:0.999:153
+1 136014 . ATCCCTTCTCCCAGTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 136045 . CTCCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 136158 . GCAGCCCGAGTGCGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 136186 . TCACGGTGGCCTGTTGAGGCAGGGGGTCACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 136226 . GTCCGCGTGGGAGGGGCCGGTGTGAGGCAAGGGCTCACACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 136276 . CAGCGTGGGAGGGGCCGGTGTGAGGCAAGGGGCTCACGCTGACCTCTGTCCGCGTGGGAGGGGCCGGTGTGAGGCAAGGGCTCACACTGACCTCTCTCAGCGTGGGAGGGGCCGGTGTGAGGCAAGGGGCTCACGCTGACCTCTGTCCGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 136434 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 136437 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 136446 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 136467 . T <CGA_NOCALL> . . END=137271;NS=1;AN=0 GT:PS ./.:.
+1 137278 . GGGGCTCACGCCTCTGGGCAGGGTGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 137332 . CACCGTGAGGGAGGAGCTGGGCCGCACGCGGGCTGCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 137560 . CGGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 137606 . GCTGGGAGGCAGGGCCGGGAGAGCCCGACTTCAGGACAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 137656 . GGC . . . NS=1;AN=0 GT:PS ./.:.
+1 137685 . TGGAGGAGCCCACCGACCGGAGACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 137723 . AGATGCCATCGGAGGGCAGGAGCTCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 137776 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 137918 . GAGAACG . . . NS=1;AN=0 GT:PS ./.:.
+1 138001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=140000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.13:1.73:.:0:.:0:0.999:153
+1 138153 . TCCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 138260 . GGCCGCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 138304 . GGGAGGCAGGAGGAGCTGGGCCTGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 138353 . TCACCTGAGGATGCCACAGTGAGACACCATCTGGGTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 138444 . AGTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 138557 . GTGAGGGAGGAGCTGTGCCTGTTGAGGCTGCTGGCAGGCAGGCAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 138648 . AAAAGCCCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 138813 . AGGCTGTTGTGAGGCAGCAGTTGTGCCTGTAGACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 138887 . AGGCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 138921 . GCAGGAGCTGGGCCTGGACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 138970 . TAGGCCACCAGGAGGCAGCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 139054 . TTGGCCGTGGAGAGGCCACCGTGAGGCATAAGCTGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 139106 . GTGAGGCAAGACCTGGGCCTGTCTAGGCTGCTGGGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 139186 . TTGGGCCTGGAAAGGCCCTTGTGAAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 139223 . CCTAAAGAGGCCACTGGGTGGCAGGAGCTGGGTGTGTAGAAGCTGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 139479 . CCTGGAGAGAAGGCTGGGAGGCAGGAGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 139532 . ATGGAGCTGTGCCTGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 139556 . TTGTGAGGCAGTAGCCTCATCTGCGGAGGCTGCCGTGACGTAGGGTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 139757 . TGTAATATATAATAAAATAATTATGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 139903 . AGCGCAACCTCGATCCCTCACATGCACGGTTCACAACAGGGTGCGTTCTCCTATGAGAATCTAATGCTGCTGCTCATCTGAGAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 139994 . CTCAGGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 140001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=142000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:1.45:.:0:.:0:0.999:153
+1 140012 . CAAAGGGGAGTGGCTGTAAATACAGACGAAGCTTCCCTCACTCCCTCACTCGACACCGCTCACCTCCTGCTGTGTGGCTCCTTGCGGCTCCATGGCTCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 140127 . GCTCAAGTGCATCCAAAACGA . . . NS=1;AN=0 GT:PS ./.:.
+1 140350 . AAATCCCATCTGTGTGGGTTTACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 140407 . ACATACTTGAGAGGCTGAGGTGAGACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 140466 . GGACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 140512 . TCACTTGAGCCCAGGAATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 140559 . CCCCATCTGGCCAACATGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 140814 . GGCACAGAGCTTCTAAAGCTCTTACAAAGACCTCAGTGATAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 140884 . ATTTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 140923 . AAGAACTTTGGGATCTCCAGCATGGTAAGAATGCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 141029 . ATGCAACCACATGGTAAGAGGCTTGGAACTTTCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 141225 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 141260 . GCCCCTCCGAACTTAACTTGCCCTGGGTATCTTTCTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 141330 . GCTGGAGTGCAGTGGCACAATCTCAGCTTACTGTAACCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 141377 . CCAGTCCCCAGCTCAAGGTATCCTCTCATCTCAGCTTCCCTAGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 141462 . ATTATTTTTTAATTTTTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 141501 . GTTGCCCAGGCTGGTCTCAAACTCCTGAGTTTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 141611 . TCATTGACTGTTTCTGAGATGTATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 141730 . ACTTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 141754 . AGCCTGGCCAACACAACAAGACCCCATCTATACAAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 141929 . CAACAAAATAAGACCCTCTCTCTCAGAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 142001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=144000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.55:.:0:.:0:0.999:153
+1 142027 . TTACGGGAACCCCCGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 142119 . GGACTGAGCCCCTAACTTGTGGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 142258 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 142356 . CCAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 142466 . CTACATGTGATTCTGTGAGAATTAACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 142632 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 142675 . ACAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 142716 . GGTTTCCCTTCCCGGACAGTTTGCGCTATCCCATCCCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 142778 . CCCCTTCCCTCCCCACTCTCATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 142952 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 143002 . GTAGATCCAGCTGGAAGTGACAAAAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 143152 . CATGAAGGGTTAATTTGTATTTTATTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 143205 . CACCTAGGCTGGAGTGCAGTGGTGCAATCAGGCTCACTGCAGCCTTGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 143265 . AAGTAATCTCACTTAATTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 143347 . GGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 143411 . GACCCTGCCTCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 143634 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 143804 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 143807 . AGG . . . NS=1;AN=0 GT:PS ./.:.
+1 143818 . ATACAGATGAAGTTTCCCTTCACTCGCCTGCTGCTCACCTCCAGCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 143882 . AGACCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 143906 . AAGGAACCAACCCACGCCATTCTTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 143944 . CTGCTGCAGTGGTCAACTTGTAGCACCCCTAAGCTCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 144001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=146000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.79:1.57:.:0:.:0:0.999:153
+1 144047 . CACATCCTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 144135 . TCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 144183 . TTGTAGGAGTCCAATCAGGAGACACAAACCACTCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 144263 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 144290 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 144295 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 144299 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 144331 . TGGCGAAACCTCGTCTCTACAAAAAACACAAAAATCAGCTGGGTGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 144425 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 144427 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 144431 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 144435 . GCC . . . NS=1;AN=0 GT:PS ./.:.
+1 144459 . AGCAGAGGTTGTGCCACTGTACTCCAGCCTGGGTGACAGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 144524 . AACGTATATATATATATATATATATATATATATATATATATATATATATATATGTAAATTTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 144645 . TGGGAGGCCAAGGCAGACAGATCACCTGAGGTCAGGAGTTCGAGACCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 144702 . GCACAGAGAAACCCCATCTCTACTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 144761 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 144778 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 144809 . CCCAGAAGGTGGAGGTTGCGCTGAGCCGAGATAGCGCCATTGCACTCCAGCCTGGGCAACAAGAGTGAAACTCCATCTCAAAAAAAAAAAAGGGTATTAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 144993 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 145048 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 145103 . TACCAAAAAAAAGAGACATTAGCCAGGTGTGGTGGTGGTGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 145174 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 145182 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 145214 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 145248 . TGGGCGACAGAGTGAGACCCTGTCTTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 145364 . CCCACTTTCCTGTATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 145507 . TCCCGGTGCCTTCTCTACAGCAGCCTGAGCCATGTCTCTAATCTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 145642 . GCCTTGCAAGGCAGCCTCACTGCTTGCCCCTCTCCATTTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 145720 . GAACGCACACTCTTTCTCCTCTGGGAGTCTCTGAAGTGGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 145858 . TCATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 145884 . GATTCTCAGGAGCATGGCAGGTGAAGTGCTCCTCCCATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 145935 . TTAGGGAGTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 146001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=148000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.16:1.38:.:0:.:0:0.999:153
+1 146006 . TGGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 146055 . GTTACACAGCACAGTTACAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 146098 . GATAAAATTAATGTTGCTCATCAGCTGACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 146182 . CTTTAAAACTGGAAGAGGGAGGCAGAAGGTTAAGAACCAGAGACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 146477 . CTCCATCCTGAGTGACAGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 146537 . GGGCCCTCTCCATCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 146575 . CTCTGCAAACGAGTAAACATCACCCTCCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 146700 . AAGCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 146747 . TTCCTTTGGTTCTCAGTAGGCAGGGTAGGGGCCAGGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 146864 . CCTGGACAACATAGCAAGACCTGGGTGGCACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 146958 . TTTCAGGCTGCAGTGAGCCATGATCACACCACTGCACTTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 147024 . TCACAAAAAGTTAGAAAAAAAAAAGAGAGAGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 147066 . ATACACAGGCACCACCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 147129 . GTTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 147184 . TTGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 147278 . TATCAAAAATACAAAAATCAGCTGGGCAGTAGTGGCGTGTGCCTGTAGTCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 147430 . CTGTCAACAACAACAACAACAACAAAAACAAAAACAACAACAACAAAAAAAACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 147521 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 147587 . GGAAGAAAAAAAAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 147682 . ATCACTTGAGGCCAGGAGTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 147718 . AACATGGTAAAATCCCACCACTACAGAAAAATCTAAAAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 147841 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 148001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=150000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.34:.:0:.:0:0.999:153
+1 148022 . TGGATTTTTAAAAAATCAAGACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 148084 . GGCTCAAGCCATCCTCCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 148113 . CTGAGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 148151 . AACTGGTATAGCCACGTTAGAAAACATTCTGGCAGTTTCTCAAAAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 148250 . CCAGAAAAATAAAAATATATGTCCACACAAAAACTTGTACAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 148336 . ACATGGAAACAACCCAAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 148466 . ACACTGTGCTAAGAGGGAAAAAAAGCCACAAAAGATCACATATTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 148548 . GACAAAAAAATTAATCAATGGTTGCCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 148613 . AGTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 148655 . TTCTAAAAGTGACTGTGGTGATCGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 148690 . TGTGAATATTCTAAAACCTACTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 148741 . TGGTATGTGAATATTTTAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 148770 . ATTTAAAATAATAATAATAGGGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 148826 . CCAGCACTTTGGGAGGCTGAGGCAGGAGGATCACTTGAGGTCAGGAGTTTTGAGCCCAGTCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 148905 . CCCGTCTCTATGATAAAAAATTAGCTGGACATGGTGGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 149046 . CCTAAGCAACAGAGCAAGACGCTGTCTCTGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 149092 . AATGCAAGTTTTTATCACTTTGTGAGTGTAGCCAAGTTGGAGGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 149147 . TAATAAAAGAGCACTGAATAATGACAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 149319 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 149667 . AGACTCTTATCTTAAAAAAAAAAAGAAAAAAAAGAAATGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 149787 . CCAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 149933 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 150001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=152000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.23:.:0:.:0:0.999:153
+1 150018 . TTGAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 150072 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 150096 . GCTGTATGGTTACAAGGCCTACATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 150366 . GCCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 150448 . GCTATATTTATTTTATTTTATTAAATTTATTTTTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 150558 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 150670 . AGTGTTTTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 150764 . TCCTACTGACTGACTTCAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 150824 . CCCCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 150977 . CCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 151247 . ATTCACTTATGAGGCCAACCCTGACCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 151293 . CTGTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 151324 . CTTTCTTTTTGAAACAAGATCTTGCTTTATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 151481 . TTTATTTTTTTTTTTTTGAGACGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 151591 . CCCAAGTAGCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 151689 . GAACTTCTGACCTCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 151744 . GGCGTGAGCCACTGCGCCTGGCCTTTAAAAAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 151844 . TCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 151912 . TTCGTGTTCATTTTTTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 151999 . GGGGCCAGAAGCCCAACTAGGTCTATTAAGGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 152001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=154000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:1.36:.:0:.:0:0.999:153
+1 152053 . CTGCATTCCTTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 152120 . ACCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 152240 . CTACCTCATTCTCTTATAAAGATCCTTGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 152449 . CTAAGGTGGGAGGATTGCTTTAGCCTAGGAGGTCAAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 152584 . CCCGTGGTTCCTGGCATAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 152759 . AACGGGGTGGCTGCAAGCTCCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 152833 . TTTAAAACAGAGTTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 152873 . CAATGGCACAATCTCAGCTTGCTACAACCTCCACCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 152949 . GCTGGAATTATAGGGGTGTGCCACAATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 153013 . TTTCACCATGTTGGTCAGGCTGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 153089 . GGATTACAGGAGTGAGCCACCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 153240 . AAACACCCAAACAGCAGGGTTTGGAGAGCTTCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 153340 . CCCTCCCCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 153387 . TGAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 153413 . GTAATAGAAAATAAGGTGGCCAGATGCACTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 153520 . CTGGGCAACATAAGAAGACCCCATCTATACAAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 153622 . GGAGAATCACTTGAGCCCTGGACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 153670 . A G . . NS=1;AN=2;AC=2;CGA_RPT=AluJo|Alu|18.9;CGA_SDO=17 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:97,46:97,46:39,41:-97,-46,0:-41,-39,0:0:0,0:0
+1 153699 . AGCGAGGCCCTGTCTCTTAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 153850 . TTGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 153953 . GCGGTTGGCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 154001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=156000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:1.58:.:0:.:0:0.999:153
+1 154069 . GACTGGCTGAAGCCACAGCAGAAGAATATAAATTGTGAAGATTTCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 154222 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 154272 . CCGGTCATCTTCGTAAGCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 154549 . CTAAGGGGAGGCCTCTGAAATGGCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 154586 . GCTGTCTTTTACAGTCATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 154630 . TTCGCGTGGCGCTCCCAGGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 154772 . TTAATTTCGCCCCAGTCCTGTGGTCCTGTGATCTTGCCCTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 155000 . TCTCTTTTGTACTCTTTCCCTTTATTTCTCAGACTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 155046 . AGGGAAAATAGAAAAGAACCTACATGAAATATCAGGGGTGAATTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 155169 . AACAAGGCAAGCATTAAAGTCAGACCAGACTAACATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 155240 . AATCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 155424 . GGGCAAAATTCCATACAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 155487 . CATACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 155535 . GCTGGTTTCCCTGCCTGGGCAGCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 155589 . CCCTCCACCTCCCCCTTCCCTCCCCACTCTCATACAACTCTTCCTTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 155677 . TCTCTCCCTCTCCAGAAGAGCTTCCGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 155757 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 155797 . TTCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 155835 . GAAGTGACAAAAAGACATTTAAAAAAAAAAAAAAAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 155889 . CATCAGCACTTAAAAGTTTTAAACGATATGTGAAAAACAAAATTTAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 155960 . GGAAGGTGTTACTGGGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 155992 . TAATTTTTATTTTATTTTATTTTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 156001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=158000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:1.36:.:0:.:0:0.999:153
+1 156049 . ACTGCAGTGGTGCAATCACAGTTAACTGCAGCCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 156133 . GAAATGCAGTCTTGCTCTTAGCAAAGCTAAAGTGCAATGGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 156231 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 156276 . CTCATTTTTTTTTTTTAATTTTTAGTAGAGACAAAGTGTCACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 156350 . CTCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 156380 . AAATGCTGGGATTACAGGTGTGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 156433 . TTAATTATATAAAGAGCTCAAAGCAAATATTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 156483 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 156503 . GTAAATTGTGATACATCCATATAATAAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 156603 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 156671 . TTTCCTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 156725 . AAAAGTATTTATCATTTTTATAATTTAATAAAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 156837 . GTGGCTCATGCTTATAATACCAGTACTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 156875 . TGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 156927 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 156941 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 156945 . TAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 157026 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 157028 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 157045 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 157104 . TCTCAAAAAAAAAAAAAAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 157226 . GCCAAGGCGGGTAGATCTTGAGATCAGGAGTTCGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 157310 . AATTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 157400 . TACCTGTGTGGGCGAAGGTGCAGTGAAATGGCCATTTTCTTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 157492 . GAAATTTTTTTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 157635 . CATCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 157729 . GTCATATGCACAAACACAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 157761 . GTAATTTTTTTCTCTTTTTTTAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 157913 . GTGAAATAAGGAAGAATTATGGAGAATTTAAAAATCTATGCTATTTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 158001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=160000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:1.50:.:0:.:0:0.999:153
+1 158003 . ATTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 158049 . TTGCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 158085 . TAATTGTAGAAACAGATACAATTTGTCCCTTGGTATATGGGGGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 158361 . AGAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 158472 . ATGCTGGAATTGGGAACAGCAGAAGTGTCATCTCAGAGCTACTCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 158535 . GGGGCTCAGGTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 158582 . GTGGGATTTACTTGTCCATCCATTTTCTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 158696 . ATTGAAAAATCGTCGCAGGTCAGGTGAGGTGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 158743 . CCAGCCCACTGGGAGACTAAGGCAGGAGGATTCCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 158828 . CTACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 158986 . CCCTGTCTCAAGACACACACAAACACACACACACACACACACACACCCCCAATCTCACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 159068 . AGGGCCTTCTGGTTACAGAAGAGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 159237 . ATGCCTCCTTTGTCAATTAATAAATGGAACATCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 159360 . GCGCGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 159435 . AGATCAGCCTGGCCAACATGGTGAAACCCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 159525 . ACTTGGGAGGCTGAGGCAGGAAAATCGCTTGAACCCGGAAGGCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 159598 . CATTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 159948 . CTCCAGAAACAATAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 160001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=162000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:1.50:.:0:.:0:0.999:153
+1 160010 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 160116 . GGGCCCAGCTCCTCACTACTCACCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 160166 . AGAGGATGGGGAAACAAGGCTCCTGACTTTTTTTCCCTAATATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 160383 . TGGGATCATTCCAAATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 160435 . AAGGACCAACCATTCAAATGGGCCCTGCTGCCAAGCCTTTTTTTTTTTTTTTTAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 160541 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 160562 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 160815 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 160819 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 160823 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 160833 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 160925 . GAGCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 161075 . GTGACTGAAGCAAAAGCTTCATAACCAGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 161224 . TATATGCAGGGATGCAGGCTGTAGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 161283 . CAAGACCTCAAACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 161400 . AAGCCATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 161466 . AACCTAGATGAGACAATCTAAGCATCCAAAACAATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 161512 . TGGCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 161588 . TAGAGGGAAGGTTACTAGGTCACTAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 161654 . GGAAAATTAGCTATTTATTCAGTCTTTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 161716 . AGATGAAACAAATCTGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 161808 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 161815 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 161831 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 161870 . TCTGCCTACAAGAGACACTAGGATATGAGGGGTAGTTTTAGCCCTAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 162001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=164000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.14:1.55:.:0:.:0:0.999:153
+1 162016 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 162049 . CTATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 162140 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 162290 . GGGAAGGAGGAAGGGAGGGAAGGAGGGAGGGAGGGAGAGAGAGAGGGAGGGAGGGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 162378 . GAAGAAAAGGAAAGGAAAGGAATAAATTTTATTTCTTAACAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 162428 . GTTAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 162554 . TTTATAATAAGCCCACTCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 162588 . TTACCACAATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 162802 . AAGGAAAAAAGTCACAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 162853 . ACCTAAAAAAGATCTCATTAACTCCCCCAGCTCACCTCCACGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 162980 . GACACCCGACCTCAATAGCTCCAGAACAGCCCTAAAACATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 163184 . CTCTCGGGGTCCACCAAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 163268 . CACAAGGCATGTCGTCCTCAAAGATAAATGAGCAGGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 163405 . TCTGAGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 163512 . TCACGCTGACCCCAGCTCCCTGGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 163702 . AAAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 163738 . CTCTCACCTATTTCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 163799 . TTTTAAGCTGATAATGAAAAAAAAAGAAAAAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 163859 . GTGGCTCATGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 163964 . TACTAAAAATACAAAAAATTAGCCAGGCATGGTGGCGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 164001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=166000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.50:.:0:.:0:0.999:153
+1 164020 . ACTAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 164059 . AGGCAGAGAATGTGGTGACCTGAGATCACGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 164135 . AAAACAAAACAAAACAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 164305 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 164342 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 164374 . CACGTCTGTAATCTCAGCACTCTGGGAGGCCGAGGCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 164439 . GCGGGAAGATCACTTGACGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 164474 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 164670 . TCTCAAAAAAAAAAAAAAAAAAAAAATTCCTTTGGGAAGGCCTTCTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 164746 . AAAGGGTATGGGATCATCACCGGACCTTTGGCTTTTACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 164811 . AAGGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 164938 . AAATAAAAAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 165064 . TCTCCTGAAGTCAGGAGTTCAAGGCCAGCCTGGCCAACATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 165259 . GTGCGTGACAGAACAAAACTTCAACCTCCAAAAAAAAAAAAAAAAAAAAAAACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 165408 . TTTCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 165565 . GAATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 165614 . TGCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 165642 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 165700 . TAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 165706 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 165709 . GAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 165846 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 165886 . GACAAAGTTGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 165949 . GGTGAGAATAAATACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 166001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=168000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:1.51:.:0:.:0:0.999:153
+1 166044 . AGAGAAAGGACAGGCTGGGCACAGTGGCTCACACCTGTAATCCCAGCAGTTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 166162 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 166220 . CAGCTACTCGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 166358 . AAGAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 166484 . GGAGAAGAGACGTGGCCAGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 166567 . GAGATTTTTGCTTTAAAATGAACCAAAAAAAAACCAAAGGTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 166647 . AGAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 166787 . AATACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 166885 . CAAGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 166972 . GAGAGGATAATAACAAATCGCTAATTTCTTTCATCACTATATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 167104 . CAACAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 167131 . ACATTTTATTTATTTATTTATTTTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 167228 . TCCTGGGTTCAAGCGATTCTCCTGCCTTGGCCTCCCGAATAGCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 167285 . TGCGCCACCACACCCGTCTAATTTTGTATTTTTAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 167409 . TACAGGTGTGAGCCACCACGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 167514 . CACCTATAATCCCAGCACTTTGGGAGGCTGAGGTGAGTGGATCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 167596 . GAAACCCCGTCTCTACTAAAAATACAAAAATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 167664 . GCTCGGGAGGCTGAGGCAGAGAACTGCTTGAACCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 167770 . TCTCAAAAAAAATAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 167960 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 168001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=170000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.89:1.48:.:0:.:0:0.999:153
+1 168063 . AACCTAAGATTACAAGACTTTTCCAGTTTAGACATACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 168145 . CGATACG . . . NS=1;AN=0 GT:PS ./.:.
+1 168223 . AACATATAATGACTGATTTCATATATTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 168352 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 168373 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4083155&dbsnp.131|rs79285249&dbsnp.135|rs183198872;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:15:15,44:15,44:17,18:-44,-15,0:-18,-17,0:0:0,0:0
+1 168442 . TCCTAAAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 168463 . TGTGAAATAAGACTTTACAGCAGCCGGGTGCAGTGGTGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 168566 . AGCCAAAACCCCCCTCCCTAGCCCCACCCCCACCCCGTCCCTACCAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 168636 . TGGCGGGCGCCTGTAGTCCCAGCTACTCAGGAGGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 168750 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 168762 . ACCTCAAAAAAAACAAAAACAAAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 168857 . ACCCTTTTTCTCCCAATCATTGAAACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 168930 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 168964 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs80158785&dbsnp.135|rs187450123;CGA_RPT=MLT1D|ERVL-MaLR|40.2;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:50,113:50,113:43,42:-113,-50,0:-43,-42,0:2:1,1:1
+1 169221 . ACCTTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 169267 . GTAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 169489 . ATACAAAAAATACAAAAACTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 169578 . CCCAGGGGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 169631 . TGGGTGACAGAGCGACGCTCCATCTCGAAAACAAAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 169717 . CACCCAACCCCCAGAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 169755 . CCACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 169856 . ATTCTTTTTTTTTTTTTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 169919 . TCTCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 170001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=172000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.09:1.41:.:0:.:0:0.999:153
+1 170038 . ATGCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 170102 . CAGCCCCGCAAAGTGCTGCTATTATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 170333 . AGGTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 170372 . AAAGGATGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 170423 . CCAGGTTGGCAGGGCTGGGGAAGGGAAAGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 170466 . CAAGAAAAAAAAGAGGCAGCAGAGGGAGCAGGAGAGCGCTCACATGGAACTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 170525 . TGCCTGAGGGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 170556 . GACGTCAGGGGGCAGAGAGGCGCAGTTCCAGGGCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 170634 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4096701;CGA_SDO=15 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:48,123:48,123:42,43:-123,-48,0:-43,-42,0:1:1,1:0
+1 170702 . GAGTGGGGCAGAGCAGGGAGGAGTCCTGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 170778 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 170821 . TACCCCTGGGCTGATCACTTGGGGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 170966 . TTTCGCTCCTGTCGCCCAGGCTGGAGTGCAGTGGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 171035 . GTTCAAGTGATTCTCCTGCCTCAGCCTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 171146 . GGGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 171159 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 171236 . CACGATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 171411 . TCGAGCCCATCATCCCCTAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 171454 . GCAGAGCTGAGGGGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 171485 . TGTGAAATCGCCCTGAGATGACCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 171519 . GCTAGGAAGTAAGCGCTGCATCTCCTGCAGCGTCCTCCATCCCTAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 171612 . TTATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 171782 . GGACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 171828 . AGTGGGAGGGGGAACAGCATGAGCCAGGCCTCGAGGCAGAAGGACAACCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 171897 . TGCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 172001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=174000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.43:.:0:.:0:0.999:153
+1 172093 . AGGTTTTTTGTTTTTGTTTTGGAGACGGAGTTTCGCTCTTGTCACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 172273 . GGCAGGAAGCTAAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 172531 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 172536 . CAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 172561 . GCCTGGCAGCTCTCTCCAACTTTGGAAGCCCAGGGGCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 172627 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 172705 . GTCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 172874 . CGTGACTCCAAAGTCCCTGCCCTAGCCCCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 172915 . GCAGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 172934 . AGGCCAGCAGCACAGCCGGCCAAGACCAGGGAAACTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 172983 . AGCACCCCCAGGTATTCCAACCTAACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 173022 . CTCTCACCACCCTTCTTCCTGCTTTAACCTCAACCCCTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 173095 . AGACGCCTCAATAAATCAGTCTAATCTCGAAAATAAAAAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 173167 . GCTAAAAACCATAAACATATAACAACTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 173209 . TTCAATATATATCCAATCATTGTAACTATGACACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 173264 . TTTCAAAATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 173297 . ATTCAAACTATTTATTCAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 173386 . ATAAATTAGCAGCCAGCAGGCAGTGACACACCGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 173582 . CAATGTCTGTAACGTGACTTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 173649 . CATGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 173682 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 173709 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4117998&dbsnp.131|rs80085981&dbsnp.135|rs192722547;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:68:68,134:68,134:48,45:-134,-68,0:-48,-45,0:2:2,2:0
+1 173737 . TTCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 173769 . CATTTTCCGTTCCCCAGCATTGGCAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 173820 . CTCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 173870 . AAGAAAAAATGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 174001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=176000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.65:1.33:.:0:.:0:0.999:153
+1 174055 . TGAGTACTGCTTTTTCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 174090 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 174214 . GCCGCCTAAAGTTATACAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 174278 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 174353 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 174482 . AGTGAAACCCTCTCTCTACTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 174525 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 174531 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 174536 . ACG . . . NS=1;AN=0 GT:PS ./.:.
+1 174544 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 174556 . TTGAGAGGCCGAGGCGGGTAGATCACCTGAGGTCAGGAGTTTGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 174678 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 174721 . CCCAAAAGGCAAAGATTGTGGTGAGCCGAGATTGTGCCATTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 174777 . CAAAAACAGCGAAACTCCGTCTCAAAAAAAAAAAAAAGAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 174841 . TGCAGTGAGCTGAGACTGCACCATTGCACTCCAGCCTGGGTAGCAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 174899 . TCTCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAGAGAGAGAGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 175131 . TGCAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 175180 . GCTGACGCCTGTAATCCTAACACTTTGGGAAGCCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 175239 . GGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 175245 . CGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 175276 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 175312 . GGTGGCGCGCGCCTGTAATCCCAGCTACTCGGGAGCCTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 175360 . AATCGCTTGAACCCGGGAGGTGGAGGTTGCAGTGAGCCGAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 175427 . GGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 175443 . TGTCAAAAAAAAAAAACAGAAAAAGAAAAAGAAAAAAGAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 175578 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 175595 . AATCTTTTTTTTATTTTGAGACAGAGTTTTGCTCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 175707 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 175717 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 175726 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 175729 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 175763 . TGTATTTTCAGTTGAGACAGGGTTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 175808 . TCTCGAACTCCTGACCTCAGGTGATCCACTGACCTTGGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 175918 . AGGCGCGGTGGCTCATGCCTATAATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 176001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=177417 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.62:1.26:.:0:.:0:0.999:153
+1 176034 . AAAAATATATTTTAAAAATTAGCTGGGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 176082 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 176110 . AGAACCACTTGAACCTGGGAGGTGGAGGTTGCAGTGAGCGGAGATCACGCCACTGCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 176182 . CAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 176199 . CTCAAAAACAAAACAAAACAAAACAAAACAAAAAACCACTAAAAAAAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 176299 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 176309 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3897087;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:21:56,21:56,21:25,22:-56,-21,0:-25,-22,0:0:0,0:0
+1 176357 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:VQLOW:34,.:0,.:0,.:0:0
+1 176366 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:VQLOW:25,.:0,.:0,.:0:0
+1 176372 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3897088;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:30:30,55:8,20:15,0:-55,-30,0:-15,0,0:4:2,2:2
+1 176409 . AAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 176523 . AAGGTAAGAAATGTAAATTTGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 176620 . TTAAGGGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 176673 . TTGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 176762 . GTCACCAACATCGATGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 176929 . GCCGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 176977 . ACAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 177073 . CAAGAAAAGAGAGAAGAATGGAGACGGCAGCACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 177132 . TTGAGATCAGTTATATTTCTTCTGACAAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 177289 . TACTTTGATTCCAAATAAAACAAATATTTAAAAAATTTAATGAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 177376 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 177400 . TGGTGGTCTAGAGAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 177418 . N <CGA_NOCALL> . . END=227417;NS=1;AN=0 GT:PS ./.:.
+1 227418 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=230000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.55:.:0:.:0:0.999:167
+1 227418 . GATTCATGGCTGAAATCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 227455 . GTCTCTCAATCCGATCAAGTAGATGTCTAAAATTAACCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 227509 . CTGATTCATGGCTGAAATTGTGTTTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 227543 . TGTGTGTCTCTTAATCCACTCAAGTAGATGTCTAAAATTAACCATCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 227599 . TGCCTGATTCATGGCTGAAATCACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 227632 . GCTATGTGTGTCTCTTAATCCAGTCAAGTAGATGTCTAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 227695 . CTGATTCATGGCTGAAATCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 227737 . TCTCAATCCGATCAAGTAGATGTCTGAAATTAACCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 227782 . TTATGCCTGATTCATGGCTGAAATTTCAGGATGAAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 227839 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 227851 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 227880 . TCTTAACTCCAGAGAGCATTGCAAAATTCATTTATGAAAACCTCTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 227949 . TTGGAAAAAAATAAGCATTTATAAATAAATATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 228018 . TTTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 228070 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 228093 . CTATATCTTCAAAATTATCATTATTGAATATAAAACAAGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 228154 . GTTCTAGTCAAATAAGCTAATATTATACTTACTAGAAACGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 228211 . TAGATTTGATTCTAATTAAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 228250 . CATTATTTTTTTTATGCTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 228337 . ATCGAAAGCATCATAATCAGGAGCAAGTCGAACATATGCCTTCTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 228425 . TCATAGAGCTTCTTCACAGCCTGTCTGATCTGGTGCTTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 228491 . AGCGTGTTGTTTTCTTCTATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 228528 . CAGTGGTCAGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 228590 . TTCCGAGGATATCTGGGCTGCCTCCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 228644 . GTGAGTGACATGCGGATCTTCTTTTTTGCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 228687 . ACCTTTCAACACTGCCTTCTTGGCCTTTAAGGCCTTCGCTTTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 228780 . GTGAAAAGCGAAAAACATTATTTCAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 228874 . AAAGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 228947 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 229001 . AGAGAGACTAAAGATATTTTGGCCCGTTAATAAACATGTTTTTTTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 229136 . ATGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 229181 . CTGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 229202 . AGTATTAAAATTATAATCAATATATGTAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 229270 . CTATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 229327 . TTTTATAAGGAAAACCATACAGAAGATACAAATAAAAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 229401 . GTAAAATGTTATGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 229561 . TTTAGGAGGCTGAGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 229673 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6678242;CGA_RPT=AluY|Alu|10.4;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:11:110,11:110,11:37,15:-110,-11,0:-37,-15,0:25:25,25:0
+1 229801 . TCTCTAAATAAATAAATAAATAAATGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 229865 . CCCCGTCTCTACCAAAAATACAAAAATTAGATGGGCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 230001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=232000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.46:1.77:.:0:.:0:0.999:167
+1 230018 . CGATAGAGCCAGACCCTGTCTCAAAAAAAATTTTTTTAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 230430 . ACATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 230541 . TGGGATATCTGCCACAATGCATTTGTCGAAATATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 230630 . ACTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 230685 . CTACAAATTACGATGGTTTGGATGTGGTTTGTCCCCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 230768 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 230794 . GTCGTGGGGACGGATCCCTCATGAAAGGATTAATGTCCTCCATGGGGGTGAGTGAGTTCTGTTCTCACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 230921 . CCTCTTGCTTTCACTTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 230955 . TGCACCCCTTGCTCCCCTTCCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 230989 . AGGTGAAAAAGACTGAGGCCCCGCCAGATGCAACTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 231057 . TGAACCAAATGAAACTTTTTTACTTATAAATTACGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 231116 . AGCACAAAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 231139 . AATCTAGGTAAAAACTTTGAAAATGAATAGAATCTGTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 231202 . CATTATTGGATTCCATTTTATAAAGTTCTTTCCAACAGAAGCAATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 231477 . TTAGGTGTGTGTAGGTAGGTTAGACACGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 231523 . TTGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 231681 . ACTAAAAAAATTAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 231857 . GCATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 231984 . ATGCAGTACGGACAAGGAGGAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 232001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=234000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.92:1.67:.:0:.:0:0.999:167
+1 232063 . GAATATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 232104 . CAATGGGGGGCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 232212 . ATGACTACATGCCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 232367 . TGAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 232410 . TTGCGGCTACATGGGAAATCTCTGCTTTTTTTTTTTGACGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 232471 . AGACGTAAAATAAAACTTTATTTAAAACACAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 232566 . CTAGTCCTGTTTTTTAAAATAAGAGCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 232626 . ATATATAAATCAAAACAAATGTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 232672 . AGTAACAATATGTGTAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 232771 . AAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 232832 . TGTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 232874 . GATCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 232927 . TAGCCATAATACAACAGAATCAAATATTGGCCACTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 233001 . CTTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 233030 . CACAATGCTTTCTAAAACAAAAGAGTCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 233083 . TCAGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 233140 . TACGGGTGATTTTAAATGTTGCTATGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 233218 . AACCCAAACTCTGGAATGTTTGCAAATTTAGTTGAGCTTCTGTGTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 233551 . CATTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 233629 . CTTACACGTATTGATTGATCTCTCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 233687 . CCCGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 233769 . TTCTAAAAAATCTGAGAGCTGTCTCAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 233811 . ACATGTAATGTAGGATGTCAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 234001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=236000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.24:1.81:.:0:.:0:0.999:167
+1 234156 . GATATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 234426 . GGACACACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 234463 . GACCCACCCTCGGGTGGGTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 234539 . GACACTGAACCTAAATCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 234606 . TGATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 234710 . ATGTCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 234756 . CTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 235080 . TGACTTTTTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 235435 . ATATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 235500 . TTTAAAAGACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 235604 . CTGACTCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 235840 . AAAACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 235926 . TTCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 235973 . TGGCTCTTTTTCATGTCCTTTATCAAGTTTGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 236001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=238000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.65:1.95:.:0:.:0:0.999:167
+1 236197 . ACATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 236475 . TGGGCTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 236615 . CTGCTGCTTCCTGGAGGAAGACAGTCCCTCTGTCCCTCTGTCTCTGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 236682 . TCCTGGAGGGAGACAGTCCCTCAGTCCCTCTGTCTCTGCCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 236733 . CTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 236790 . ACATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 237006 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 237058 . AGGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 237158 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 237269 . ACCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 237328 . AGACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 237420 . ACTTGGGAGGCTGAGGCAGGAGAATGGCTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 237467 . CTTGCAGTGAGCCAAGATCACGCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 237525 . TCTCAAAAAAAAAAAAAAAACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 237663 . TATACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 237749 . TGCAGCACCAGGTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 237801 . GCGGAGTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 237869 . TTAGTTTGCGTTGTGTTTCTCCAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 238001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=240000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.41:1.51:.:0:.:0:0.999:167
+1 238055 . AAAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 238213 . AACAATTTGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 238263 . TTCCCCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 238307 . AGAAAATCTTTGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 238470 . GAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 238817 . ATTCTTTTGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 238987 . AATTAAAAAAAAAAAAAAAGAAGAAGAAGAGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 239036 . GATTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 239161 . GAGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 239243 . TATGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 239479 . AACGAAGCTCTCTTTATTTGCTTCTGCTAATTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 239693 . TTCTGTAGGTTGTACAATAACTTTTGGTGAGAAAAAATAAAAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 239819 . TCAATTTTATTGAAGTTCACTTCTGACCTCTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 240001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=242000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.16:1.55:.:0:.:0:0.999:167
+1 240017 . TTCCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 240082 . AGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 240144 . CTCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 240549 . AAATGAAAATCTGACCACGTTACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 240608 . TCAAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 240850 . GCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 240895 . GCCTCACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 241079 . TTTTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 241133 . GCTCCTTTTCTTTTCTTTTCTTTTCTTTTTTCTTTTTTTTTTTTTTTTGAGTCAGAATCTCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 241220 . TGGTGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 241271 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 241317 . CACACCCGGCTAATTTTTTTTTTTTTTTTGTATTTTTTAGTAGAGACTGTGTCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 241482 . GCATAAACTAAATGTTTTCCAAAGGGAATAGGGCAAAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 241625 . GGGCTCTCCACTTACAAGAAGAGAGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 241747 . CCTGTTAATTTAATCACACGGAACACTTCTATTTAAAATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 241833 . ACACTGCTTGGAGTGTCAGGCCTAGATCTCTATCCATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 241910 . ATGGGGCAATTTCTTAAAAGCACCATGTATTTTATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 241989 . AATAAATGTCTTCCACAATCCCATAGCCCAGAGCTAACTAACCACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 242001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=244000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.40:.:0:.:0:0.999:167
+1 242340 . TGGGAGGCACAGTGGAAGATCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 242461 . AGACTGTGGGTCCCCTCAGTCTTGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 242575 . AAACTGCCAATCATGGAGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 242695 . AGGGAGGCAACCTCCAAAGGTGGGGCCCTCTGCTCACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 242811 . TTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 242949 . AAACGGCCTTTTGAGTTGAGCAATAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 243186 . TAACATTACAGTAACTGTTACAGGTTCCAGCAGGCTAACTGGGTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 243397 . TGTGTGTTGGGATAGAGTGGTAAGAAAATGGGAAATAATAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 243496 . CACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 243572 . CTTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 243761 . ATTTGACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 243850 . CAAGT C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs140116466;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:322:322,322:269,269:32,45:-322,0,-322:-32,0,-45:67:17,50:50
+1 243898 . TCAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 244001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=246000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.46:.:0:.:0:0.999:167
+1 244285 . TACATTTTCCATGTGCTGTAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 244353 . CCCTGGTGAGCCACAGAGATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 244642 . CCTGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 244782 . ACATGGGCACCCATATTTTTCTAGCCACTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 244831 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 244920 . CACTTCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 244954 . TGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 245021 . ATGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 245088 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 245105 . TGTATTAATGTGCCTTTCTAGTAACAGGTTTTTAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 245165 . TATTTGTGTGTGTGCATGTGGTAGTGGGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 245217 . AGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 245258 . GGGGAAAAAATTTTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 245325 . TAGAGTTGCTTTTATTTATTTATTTATTTATTTATTTATTTTTCCTTTTTTTTCTTTCTCTTTTTTTCTTCTTTTTTTTTTCTTTTCTTTCTTTTTTTTTTTTTTTGGACAGAGTCTCACACTGTCACCTCGGCTGGAGTGCATTGGTGCAATCTCGACTCACTGCAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 245602 . TGAGGTTTCACTATGTTGGCCAGGCTGGTCTCAAACTCCTGACCTCATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 245658 . CACGTTGGCCTCCCAAAGTGCTGGGATTACAGGCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 246001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=248000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.35:.:0:.:0:0.999:167
+1 246056 . ATGCGAACTGGGAGGGGAGATTTTTCAGGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 246134 . GAAAGTGTATAATGATGTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 246186 . GCCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 246252 . AAAAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 246408 . TCGCCATGCCTAGTACAGACTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 246633 . TTTGGACAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 246795 . TCCGTGTTACTGAGCAGTTCTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 246852 . TTCTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 246900 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 246974 . ACAATTTTTATACATAAAGATTTCATAAAACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 247067 . TACCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 247172 . ACATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 247295 . GACACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 247406 . GTACTTTGGGAGGCCAAGGTGGGCAGGTCACTTGAGGTCAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 247527 . TGCCTGTAATCCCAGCTACTTAGGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 247582 . AAGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 247632 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 247652 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 247712 . AAAATATTACACATGTGTAATCCCAGCATTTTGAGATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 247935 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 247983 . TCCTATCTCAAAAAAAAAAAAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 248001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=250000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:1.52:.:0:.:0:0.999:167
+1 248108 . GGAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 248225 . GCCTGGCCAACATGGTGAAATCTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 248467 . ACATAGAACTAATTTATAAATCAAAGCACTATGCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 248735 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 248828 . TTTCTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 248951 . TTCAGGTTTTATACCTACCTTATAGATAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 249269 . ATTCGGGTTTTTTTTTTAAAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 249401 . CATTTCTACTAAAAATAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 249555 . GAGCGAGATTCCGTCTCAAAAAGTAAAATAAAATAAAATAAAAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 249698 . TCAGGGTCCTAGCAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 249840 . TGTTGGAGGTGGGGCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 249938 . TTTTCTTTTTGCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 250001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=252000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.19:1.52:.:0:.:0:0.999:167
+1 250054 . CAGAGTAGCTAGGATTACAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 250120 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 250287 . AACCCCTCTCTCTCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 250340 . CTGTCATGAGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 250421 . AGCCAAATAAACCTCTCTTCTTTAAAATTATTCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 250474 . AACAACACACACACACACACACACACATACACACACACG . . . NS=1;AN=0 GT:PS ./.:.
+1 250537 . AATTAGAAATGGTGATGCACCGAGGGATTGGCACCGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 250632 . TAATGGTTAAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 250697 . TCAACTAGAATCTAGGAAGCAGAGAACCTGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 250834 . CAACAGAGCGACTCAGATGCTATAAAACTTGCTAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 250926 . CACAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 251008 . ACACGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 251071 . TTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 251134 . GACATTAGACATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 251254 . ATACAAAGAGTAATACCATGTCACTTAAGAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 251369 . TTCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 251435 . ACACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 251493 . GCTCAGATACCTTCTCCGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 251591 . GTAGTGAGCAAATATCCTGAAGTTGAGAATGCTTCTACCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 251645 . GGAATATTCATCAAAACACAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 251799 . TTCCAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 251920 . GTCAATATATATATAGATATATACACACACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 252001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=254000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.32:1.32:.:0:.:0:0.999:167
+1 252028 . TACAATAAACATGTGTTTTTAACAAGAAAAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 252100 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 252201 . TAGAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 252299 . ACATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 252355 . AGAGCAAGCTGGGAAAGCAGTGGCCTTTAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 252469 . AATAGTAAACTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 252953 . TCATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 253126 . GCAGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 253223 . CTTGAGCAAATGGTAAATTAACTCTCTCTTTTCTCTCTCTCTCTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 253437 . TTTACTGGAGTACACAATTGTGACTATTTTTAGCCATAGGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 253598 . TTACACTTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 253829 . GGAGATTTGGACATAGAGAGAGGCACACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 254001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=256000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.21:1.27:.:0:.:0:0.999:167
+1 254109 . GATCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 254257 . TTAAGGCCTTGCTTTAAAGCTTCAATGGGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 254355 . TCCTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 254489 . TTCAAATGTCACTTCCCTGTAAAAGCTTCCTGGCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 254567 . CTATAACAACCTAATATATTCTCAATTGATTAACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 254639 . AAACGTGGCCAGGTGCAGTGGCTCACACCTGTAATCCCACCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 254716 . CTTCAAAAAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 254857 . TATCAAAAAAAAAAAAAAAAAAAAAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 254917 . TATCTATCACGTTCACCTCCCAAGAGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 254993 . AATGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 255123 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 255141 . TTTCCAGTTATATATCTGGTAGAGATTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 255211 . TGACCCAGCATGGCTGAACACTCAGTGACTACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 255319 . TTATATTCAGAATTACTCAAGTCTTAGAAGCACCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 255382 . TCAAGTGATGGGCTGAAGTGAAGGGAGGGAGTCACTCACTTGAATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 255500 . TGATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 255630 . GAAATTTAGAATAAATTAATAAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 255724 . ATATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 255816 . TGCTTGAGGGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 255889 . GTGTATGCGTGTGTGTGTGTATGTGTGTGTGTGTGTGTGTGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 256001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=258000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.27:.:0:.:0:0.999:167
+1 256019 . CATAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 256088 . ATATATGCAATATATATACATATATACACACATATACATATGTATTTAAATATTTAAATTACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 256266 . CAACCCTCCTGTATTAGTCTCCCCAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 256319 . ATGTCCACCTTTATGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 256374 . AACTTAATAATAAAAACATTTCAAATGTAAAGAAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 256439 . AAATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 256458 . AACGGACACTTTTCAAAAGAATACATGCATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 256532 . TTAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 256720 . AAGCAGAGCTACCATTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 256755 . CTGGGTATATACCCAGATGAATATAAACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 256819 . TTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 257015 . ATACAGCATACTCTCAGTTATAAGTGGGAGCTAAATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 257240 . AAATAAAAGTTAAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 257288 . CTGGTAATATGAAAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 257414 . ACACCTGTAATCCCAGCACTTTGGGAGGCCGATGCTGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 257473 . TTCGGGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 257527 . AAATTAGCTGGGTGTGGTGGCTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 257591 . ATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 257656 . GCAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 257678 . GGGGAAAAAAAAAAACAAAAAAAACCACCACCATCATTTTGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 257803 . ATTTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 257834 . GTTATTATTTTGTATGCGATGACAACAGAATATATTATCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 257887 . AATCTCATTCATAATATAAAGTATAAATTTGTGATTTTGCTTTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 258001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=260000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.00:1.59:.:0:.:0:0.999:167
+1 258025 . AAAATTTGAAACTAGTAACATGGAGGACTATTGTCATTGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 258086 . TTCTGCAAAGCAGTGTACATAAAAATAATTTCAAGAAATTTATAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 258141 . TTATGGTGTATAAACAACTTTAGATTCTTTGTTTAAGAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 258233 . GGCTAATAGTAGGCACCTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 258349 . AACATTTGATGGAATCATGCTTTTACTTTCTGCTTACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 258444 . CTGAGCAATGTGGCACCTGCTGAAGCCTGCTGCCTCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 258517 . CTCTAACATTTTTTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 258599 . TTTTCCCCTGACAAATTACTTATCATCTATCATAATTCAGGTTAAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 258674 . CTGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 258719 . TAGTATCGAATCAAGTTTATAATTTTAAAATAATTGGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 258845 . TCAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 258878 . GAACAAGCACTAAATAAATGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 258922 . CTCATTTTCAGAACAGAGTACTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 258973 . CACGTAGGTAATTTACAAGGGCTACAATTTCAGCTCAGATTTACCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 259060 . CACTTCAGATTCTTCTTTAGAAAACTTGGACAATAGCATTTGCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 259122 . CTAAGAATCAAGAGAGATATCTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 259163 . AAACATTAAACACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 259236 . AAATAGTAATACTTATTGCAGACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 259345 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 259385 . GTGATTTTTCAGGTTCACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 259430 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 259496 . TATGAAAACAAGAGATAAATATACACAACTGAGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 259613 . TAACTTTTAATAGAACCAGTCACTAAATTAAAAAAATGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 259664 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 259742 . GCTTACTGAATAAGCTGCTAAGGTTTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 259782 . GTGCGGTGAAATGATGTCTACATCACAGTCCAACATTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 259871 . AGACGTTTGTATATGATAAGAGAGCCAGAGTACAATTTAGGAGAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 259996 . GATAAGATATTGTGGCTGCTACCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 260001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=262000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.54:.:0:.:0:0.999:167
+1 260153 . GATTAGATTATAAGATACTGTGAATTTCTTCTTGTGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 260228 . ATGAAGCCAACTGGCATGCTGTCAGTGGCCCAGTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 260356 . TAGCTAAGAACCATGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 260396 . CCTCAGTTGAAATTTAAGATGACATATTGAGCAGACATACTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 260521 . GTATTACACCTTCAGTGAGCACGTGTACTAGAAATTTAAAAAATAAATAAAATAAACCTTCAAAGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 260679 . CAGGTTTCAGCCTGAACTCACACAATCTGTGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 260785 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 260820 . AGCAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 260850 . CATGAGCTTCTAACACACACACAAAAATCACACACACAAAATGGGGGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 260969 . AAACACATATTTTAATGTGGTTAATTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 261026 . CAAGAAAATTGTGCTGGATGGCCACTTCCACCATGGCTCCCCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 261077 . AGTCTGGGTACTGTGTCACCCGAAGTCTTCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 261122 . GGTCTGGGTTTGCCTATGAAAGAAACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 261160 . GAAATGAGGAGTGAAGAGGAGGTCTTCAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 261231 . GTGATGGCTTGCAGAATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 261293 . TTCAGAAATTCCTATAAGCTTGGGTTCTGTGCCCACACTCTAGACTGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 261357 . ATATAAAACAGACCTCTTCTGATTTTGTCTAGCTGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 261449 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 261455 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 261463 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 261515 . TAAAATATAAAGAATTGTCCAGAAATATATAAAAAAAGAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 261583 . TATAACAATTGTATGGACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 261635 . TTTGAAGAAAAAAGCAATAAGAAGCCTCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 261746 . TTGAGATGTTTATTATAATGAATTATCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 261786 . ACAATTTCCTAACAATTTTGGGGTTTATATTTTTGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 261849 . TTGTACTATTGTTAGGTAACTTTGATGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 262001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=264000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:1.55:.:0:.:0:0.999:167
+1 262069 . CTGTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 262089 . GGACCTATCATAAAAAATTCCTCAAGACTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 262136 . CACCCTCACAAGAACACTTGCCTAGCAATGGCTGTTTCTGCCAGTAAGTTAACACCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 262204 . AGACCCTGTGACCAATGATGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 262329 . ATTATAATCCTTGTTTATTTCCAAATAAATTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 262400 . ACATTGTTATTATGAAATTGGTTGGGTGATGTGTCTTATTTTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 262489 . CATGTTTGCTAGAACTCACCTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 262539 . TTTTTGTGTTTAGTTTTTCTTTTGTGATTGGGGAGGGGGGTTTATCGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 262608 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 262715 . CTGCTTCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 262826 . CTCGTAGTATTTATAGTAAAAGTGAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 262888 . TGCCCAAGTGCTGGTCTGGTCTGATCTTCTCATCTTCCCTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 262943 . CAGTCACACTAGCCTCCTTGCTGCTCCACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 263033 . CTTTTCTCCCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 263088 . TAAATGTCCCATTCTCTGTGAAGCTTTCCTGCCCACCCTATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 263195 . AAACTGTAAATATACATGTTCACTTTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 263243 . TGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 263302 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 263313 . TACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 263351 . CACATTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 263417 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 263547 . ATATTATTTTCATGTATAAAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 263625 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 263642 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 263661 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 263682 . AGGGCCAAAAGAGTCAACTTCTGAAGAAGCGCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 263760 . TTGCAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 263786 . ATTCATGAGTAGAAAAATAGACTAGTGGAATAACATAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 263936 . GGAGAATAATGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 264001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=266000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.43:.:0:.:0:0.999:167
+1 264063 . TTACCCAGATGGGCCCAGTCTAATCACATGAGTTCTTAAAAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 264188 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 264230 . CTAGAAGATAGAAAAGGCCAGGATATGGATTCTACCCTAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 264300 . TTGATTTTAGTTCACTAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 264373 . TTTAGGTCACTTAGTTTGTAGAAATTTGTTACAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 264415 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 264474 . AATTCAAGGTGAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 264500 . CTTAAAACATTTAGATTAAAAATAAATGAGAATTTTTGTTACTTTTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 264563 . AGAAAAACAAACATTAAGGAGGAAAAATGAACATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 264630 . GGAAGATATCATAAGGTGACAAATCATAAACTGTAATATTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 264680 . ATATAAGTGAATAAATATACATTTAGAATATATATGAACTCCCAAAAATCAACAGGAAAAATAAGACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 264759 . AAATGCATAAACAAAAGAAGGCAAAACAAAAATAATGACTCATAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 264831 . GATGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 264843 . AATGCAAATTAAAACCACCCTGAGATGCTTTTTACATCCATGAGCCTGATAAAAGTTAGAGTCTAAAAGTAATAATTAACAAAGATGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 265042 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 265046 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 265115 . AACATTGTTTGTTATATCAAAAAATAAAAAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 265160 . CAGCAAAAAAAATAAGTAAAAATAAATCCTGTTGTATTCTAACAATGGAATAATATATAGCCATTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 265258 . TAAGTATCAGCAAAACATATTGTTTAGTGAAAAACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 265330 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 265354 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 265421 . TAAGAGGGATAGCAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 265447 . AGTTGAGGGAATTTCAATTGGAAAAAAATAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 265490 . TAAGTCAGGTAGTGGGTATTAGCATTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 265538 . CTTATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 265554 . ATATTTTCAATGTATTTAATGTATTTTTTGCATAATTAAATATTATGCAATAAAAATGAGAAAACAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 265634 . GATAAATTACAATAAAGAAATGGAGAAAAAATTATAATCTAGTTGAGTAATGGTATATTACATAGCTATTTTCTTAAGTAGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 265755 . CTTAATTATATATAAATATATATGTACATATTTTTAATATAAAATACTAAACAAAGTACACCAAAATATTAGCTCCTATGTTAGTGAGATAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 265857 . TTTTGTATTTTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 265885 . TGTATTTTTCTGTTTTCATACTGCTATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 265989 . CTCAGGAAATCTACAATCATGGCGGAAGACAAAGAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 266001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=267719 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:1.33:.:0:.:0:0.999:167
+1 266038 . TTCTTCGCAAGGCAGCATGAAGAAGTGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 266105 . CTCGTGAGAACTCACTATCACAAGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 266148 . CCCCCATGATTCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 266229 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 266240 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 266273 . TGTATTGAATTTTAAACTCAGAGAAAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 266397 . GAATGTGGCCTTGTAAGAAAGCAAATTAACTTCTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 266494 . ACTTGAACTGCAGTAAAATATCCTCAGCAACATAGATGTGTGTGTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 266555 . ATACAAATTTAATGAAACTCCATTGGTGGTGTTTTTAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 266607 . AAGATGTCCTGGCTTATTCACAGATGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 266709 . GGAGTTTTTGTAAGGAATTAATTAATAAAAATGTTCTTGAAAGAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 266822 . ATCGCTATTTTTTTTTTGACACACACTTTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 266924 . TGATGGTAAATCATTTTCTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 266954 . GAAATGTCTTGTCTATTCAGGTTCTGCTCTACTTAAAAGTTTTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 267096 . CTTCAAGTTTGCTCTTAGCAAGTAATTGTTTCAGTATCTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 267198 . TTCATTGAATCCTGGATGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 267229 . AATAAGAGGAATTCATATGGATCAGCTAGAAAAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 267286 . A <CGA_NOCALL> . . END=267719;NS=1;AN=0 GT:PS ./.:.
+1 267720 . N <CGA_NOCALL> . . END=317719;NS=1;AN=0 GT:PS ./.:.
+1 317720 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=320000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.27:.:0:.:0:0.999:211
+1 317720 . GATCTACCATGAAAGACTTGTGAATCCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 317771 . ATGTTATTCAGGTACAAAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 317885 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 317926 . TGTATGAGAGTGGGGAGGGAAGGGGGAGGTGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 317988 . CGCAAACTGCCCGGGAAGGGAAACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 318093 . AGACACAGTAATTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 318135 . TCATTGCAGTTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 318233 . TTTCGTTAATTCTCACAGAATCACATATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 318366 . GGTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 318470 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 318493 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 318583 . AGACCCCACAAGTTAGGGGCTCAGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 318684 . AATCGGGGGTTCCCGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 318764 . TTCTTTTTTTCTGAGAGAGAGGGTCTTATTTTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 318932 . TTTATTTTTTGTATAGATGGGGTCTTGTTGTGTTGGCCAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 318992 . CTCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 319092 . GGATACATCTCAGAAACAGTCAATGAAAGAGACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 319193 . CTTAAACTCAGGAGTTTGAGACCAGCCTGGGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 319253 . AATTAAAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 319306 . CTACTAGGGAAGCTGAGATGAGAGGATACCTTGAGCTGGGGACTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 319422 . TCTCAAAAAAAAGAAAGATACCCAGGGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 319462 . GAGGGGCACAGAGCTCCCATGCCCTCTGTTGAACATGCGACCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 319656 . TGCATGAGGCTGAAAGTTCCAAGCCTCTTACCATGTGGTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 319755 . CAACAACATCCCCAAATGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 319799 . AGTTCTTAGAGGCTCTTGTGTTAGAAACCTGGGACCAAGATCAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 319871 . ATCTATCACTGAGGTCTTTGTAAGAGCTTTAGAAGCTCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 319949 . TTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 320001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=322000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:1.56:.:0:.:0:0.999:211
+1 320148 . TGGCCATGTTGGCCAGATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 320196 . CAAATTCCTGGGCTCAAGTGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 320252 . TTACGTCGTCCAGGCTGATCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 320294 . TTGTCTCACCTCAGCCTCTCAAGTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 320365 . CACAGATGGGATTTGGGCATAGGTTTGGTTTCCCAGGGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 320581 . TCGTTTTGGATGCACTTGAGCAGGGGTCCCCAACCCCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 320631 . CGCAAGGAGCCACACAGCAGGAGGTGAGCGGTGTCGAGTGAGGGAGTGAGGGAAGCTTCGTCTGTATTTACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 320743 . TCTCAGATGAGCAGCAGCGTTAGATTCTCATAGGAGAACGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 320795 . AACCGTGCATGTGAGGGATCTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 320896 . TGCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 321061 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 321103 . TCACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 321122 . CCCATACCCTACGTCACGGCAGCCTCCGCAGATGAGCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 321212 . TTTAGACCCAGCTCCTGCCTCCCAGCCTTCTCTCCAGGCTCTGAACTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 321288 . TAGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 321449 . TTTTCTAAAAAAAAAATGTCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 321539 . CACACATATTCTTTTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 321575 . TATGTACATTCCTTACAAACAAACAAAAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 321699 . TGTCGCTACCAGTATTTTGCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 321849 . TCTCTTTTTACTCTCTGTCTTTGTGTTCTGCATTTTCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 321913 . AAGTCAATCGTACTAATTTATCACGATTTGCTTTATTAATTTATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 321980 . CCAGCAGACCTCATTACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 322001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=324000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.44:.:0:.:0:0.999:211
+1 322006 . CCTGTTTTATTTTGTTTTTTTTTCTGAGACAGGGTCTCCCTCTGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 322062 . GGAGTGTAGTAGTGCTATCGCAGCTGACTGCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 322113 . AAGCGATCCTCCCACCTCAACCTCCCACGTGGCTGAGACTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 322192 . TTCGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 322206 . TTCCAGAGGGGTGACAGCGAAACGTGAGTAAGCATGGATTTTGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 322287 . ACTGAGGGACGACGACTATATGTTTTTACAATTATGCTGTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 322338 . TGTTGCATAGCCTTGAAAATAATAACTTTTAATTGAGTGGAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 322422 . GGCTCACACTGGTAATCGCAACACTTTGGGAGGCTGAGGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 322471 . GCTTGAGGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 322528 . ACAGAAAAATACATGAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 322569 . CTGTAGTCCCAGCTACTTGGGAGGCTGAGGTGGGAGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 322682 . GAGACCCTGTCTCAAAACAACAAAAAAGTAGCAGCTAACATCAACTGACCTTTTACCAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 322831 . GTGTGTACAGATGACCTTTTGTTTAGATTGAATTGTCTCCCCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 322888 . GTGAGTCATGGTGAATGGACATTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 323003 . GGCCAAGAGTTGGAGGAGGCAGTATGGCAGTATGGTGAGACCCTGTCTCCATTATTTTAAAAAATTGACAGGCTTTACCCGGGAAGGCTTATACACAATTTAAACACCCCTCATAGTATAAGAAGGTGCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 323170 . TTTAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 323199 . TTACATAGACAAAAGAACTCATATTACTTTACTTGTCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 323347 . GTAGTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 323371 . GCAGCCTTGACCTCCCAGGCTCAAACTTCAGCATTCCGAGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 323421 . TACAAGTGTGCACCACCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 323466 . GATAGAGACAGGGTCTCACTCTGTTGTCCAGACCGGTCTCTAGCTCCTGGCCTTAAGCAATCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 323542 . TCTGAAATTACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 323574 . CCATGCCTGGCCTGGGCTAGTCCCATATTCTCTAGAGTTCTCTTTACTCTGTGCTAGCCAATCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 323658 . TTATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 323751 . GGGTCCCAGGAGGTAAACCCACACAGATGGGATTTGGGCATAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 323805 . CAGGGGGCAGTGCTGAGCTCTTTGCCAGTGGGAAATGGGGTGCTGGTGATTTCCAGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 324001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=326000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:1.33:.:0:.:0:0.999:211
+1 324006 . AGCAGGGGTCCCCAACCCCTGAGCCATGGAGCCGCAAGGAGCCACACAGCAGGAGGTGAGCGGTGTCGAGTGAGGGAGTGAGGGAAGCTTCGTCTGTATTTAGAGCCACTCCCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 324132 . CCCGCCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 324155 . GATGAGCAGCAGCATTAGATGCTCATAGGAGAACGCACCCTGTTGTGAACCGTGCATGTGAGGGATCGAGGTTGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 324245 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 324263 . TGTCACTTTCTCCCATCACGCTCAGGTGGGACCATCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 324310 . AAACAAGCTTAACACGCCCACTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 324354 . TATAATTATTTTATTATATATTACAGTGTAATAATGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 324509 . CTCACAATGGCCTATTTAGGCCCATACCCTACGTCACGGCAGCCTCCGCAGATGAGGCTACTGCCTCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 324600 . CCATCGTTACAATGGCCTCTTTAGACCCAGCTCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 324650 . CTCCAGGCCCTGAACTTTCTCAAGTCGACCTCACCAGGCCCAGCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 324769 . GTCGGCCTCTGCAGTCCCAACGTCTGCCTCACAGCAGATTCTTCACGCCCAGCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 324899 . GTGGCCTCTTTAGGCCAAGCTCATGCTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 324968 . CTTCCCTGGCCAGATTCCTGCCTGTCTCCCAGCAGCCTAGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 325023 . GCCTCACACTGGCCTCTCTACATCCAGCTTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 325068 . CTCTCCAGGCCCAACTCCTGTCCCAGGACGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 325117 . TTACTCAAGTTAGACTCTCTAGTCCCAACTGCTGCCTCCTGGTGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 325199 . CAGGCCCAGCTCCTGCCTCCTGTCAGCGTCTACAGGCCCAACCTCTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 325290 . TCTACAGGCACAACTGCTGCCTCACAACAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 325345 . GCTCATGGCGGCCAATGTAGGCCCAAAACTTCCTCAAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 325475 . CCCAGGGGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 325525 . GCCAAATTTCTGCCTGCCTGCCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 325564 . CAGCTCCTCCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 325598 . ACTCATGACTGTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 325685 . CCCAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 325742 . TCCAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 325776 . CAGGCGAAGCTCCTGCCTTTCGGCAGCCTCTCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 325866 . CCCGGCGGCCTTCCCAAGCCCCGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 325975 . ACAGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 326001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=328000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:1.01:.:0:.:0:0.999:211
+1 326140 . CCTTCCGGCGGCCTCTCCGGGCCCAGAACCTCCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 326211 . CGTTCTCTCCGGGCCCAGCTCTTCTTCCTGGTTGGGTCTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 326320 . CGGCCCACAGCTTCCTCAAGCCAAGCTCCCCAGGCCCAGGTCAGGCCTCACGGTGGCCTCTCCAGGATGAGCTCCTGCCCTCCGATGGCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 326434 . GTCGGTGGGCTCCTCCACGCCAAGGTTGGGCCTCCCGGCGACCGCCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 326489 . AGTTGTCCTGAAGTCGGGCTCTCCCGGCCCTGCCTCCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 326566 . CTCCCAACCGCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 326676 . GCAGGCCCCTCCCTTGCCTCCCAGGGGCCTCTCCAGGCCCAGCTCTTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 326737 . CTCCCGGGGCCAAGTCCCTGCCTGCCTCCCAGCAGCCCGCGTGCGGCCCAGCTCCTCCCTCACGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 326824 . TGCCTCTGGCACCCTGCCCAGAGGCGTGAGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 326863 . CACACTGGCTCCTCCCACGCTGAGAGAGGTCAGTGTGAGCCCTTGCCTCACACCGGCCCCTCCCACGCGGACAGAGGTCAGCGTGAGCCCCTTGCCTCACACCGGCCCCTCCCATGCTGAGAGAGGTCAGTGTGAGCCCTTGCCTCACCCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 327022 . C <CGA_NOCALL> . . END=327233;NS=1;AN=0 GT:PS ./.:.
+1 327241 . GAGAGCTGGGCCTGGAGACTCCCCTGGGAGGCAACAGCGGGGTCTGCAGACGCCCTTCTCCAGCCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 327322 . AGTCACTGGGAGAAGGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 327367 . AACTTTGGGGTCTACAAACGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 327453 . TGGGCAATGCGGGAGGCAGAGGCCAGGCCTCCTTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 327498 . TCAGACCCACTTGCAGCCTCCCGGCGCCCCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 327547 . TCCCGGCTGCATCTCCAGGCCGGACTCTGGCCCGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 327688 . TCGCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 327764 . GCTCCTCTAGGCCCAGCTTGGGCCTCCCGGCGGCCTCCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 327812 . ATCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 327886 . GTCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 328001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=330000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:1.33:.:0:.:0:0.999:211
+1 328098 . AGGCGCAGAACTTGATCTCCAGTCGGCCTTTGCAGGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 328144 . TGCCTCTCGAAGGCCTGCACGGGCCCGGCCTCGGCCTCGGCCTCACAGCAGACTCTCCACGCCCAGCTAGCTCTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 328262 . CACTTCGGCAGGTCCAGCTCCTGCCTGCCAGTGGCCTCTTTAGGCCCAGCTCATTCCTCACGTCGGCCATTCCAGGCCCCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 328439 . AATATTTTGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 328475 . TGCGCCAAGCCCGAATTTTTTATTTTATTTTCCTTATTATTTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 328568 . AATATCGCCCACGATCAACGTGTTCTGTTCTGGGGAAGGGGGCAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 328630 . TTCTTAAAAAGTATAGCTCAAGTTGGGAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 328687 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 328702 . AGTGCAGGAGCCCCCACCCCCATACTCACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 328744 . CTCTGGGGAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 328773 . CCCCTAGTCCACAGGCGCCTCCCTGTGGCCCAAGGCCCTCTTCACACTCCATCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 328834 . CCAGCAGGAGCTATTTTCCGAAAAGTGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 328894 . TACAGGGGCTCGGAGGAGGGAAACTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 328948 . GGTAGGGGGTATAGATAAGAGGAGCAGGCCTTGGCCAGGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 329026 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 329036 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 329089 . GTCTGTACTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 329120 . CGTGGTAGCGTCCACCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 329206 . GCCAAGATCGCACCACTGCACTCCAGCCTGGGCGACAGAGCAAGACTCGGTCTCAAAAAAAAAAAAAAAAAAAAAAAAAGGAAGGCCTTACTCCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 329362 . GCTACCCCCACATACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 329436 . GTGGTAGTTATGGAGACCCCCAGGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 329471 . GGCTGGGGTGTCCCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 329501 . CAAGGCCCCAACTCTGGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 329596 . CAGGGCAGGGGCAGCAGCAGTGGACCTGCTATGCACACATCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 329682 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 329716 . CTCTGGGAACCATGCAGCAGCTCCCAGCGGCCCTGCACCCACCACCAGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 329801 . CCAGAAGATCATGCAGTCATCAGTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 329836 . GCCTGCGAGGCTGAGGCTCCTCCCACTGGACCGCCCCCCAACTGGCACCACTGCTGCCCCTGCCCCTACTCTCAGCCTCACGTGACTCTCGGGCAGAAGCAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 329971 . AGCCAGGTGAGGTGCGGTCAGGACCCCCACAGGGCTGGGAGTCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 330001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=332000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.25:.:0:.:0:0.999:211
+1 330059 . TGGGCCTGGAGGGCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 330136 . ACTGGTCCAGGGTACGTGCAGTGAAGAGGACAGCGCCTTCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 330201 . CCTCGGCTTCTCCACCTGTACAGGCAAAGGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 330240 . CCCCATCACACATGGCACACTTGGGGGTGTTGGGCTTTGGACTGCAGCTGGAGCATCTTCTCGTCTTGCATTTGGGCGCGGTGGGGTCCTCCAGTGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 330358 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 330368 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 330398 . AGGCTCTGCCCCCCGGGTGGCTCAGCCCAGCTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 330451 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 330493 . GTCGCCCACACTTTCAGGTCTCTTGCACCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 330542 . GGGAGGAAACAGGCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 330565 . AGGCGTCCCTGGGCCCCCATCCCCAGGGGTTGGGGCCGTAGGGGGCCCGCTCTGCTGCGTTGACCAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 330649 . GCTCCTGGGCCCAGTAAGAAGGAGGTGGGTGCCAAGGTTGAGGAGGAAGCATCCGAGTACGTGTAGGAGGAGGACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 330748 . AGCTGCAGGTGGATCGGGGGACCCTGGGGGCTCAGGATCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 330808 . AAGGAGGAAGGAATGACAGGTGCAAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 330852 . CTTGTTGCCCTCTGGCTCCTCCCCAGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 330887 . CACTCTCAGTCGGTCACCCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 330927 . TGTCGGTGGTGCTAAAGCCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 330956 . ATGACATCATCACCCCCTCCTCCTCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 330996 . CTCTTCGTCACTCGCTATGACCTCGCTGGCCATGTGCTGGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 331047 . TCACGTGGGCGGCAGCAGGGCTGCCCACGGGTCACCTCCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 331180 . TCCCGTAGTGCCTGACGGTCTATTTCCCTGCCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 331227 . CCCGCTGTGGGAGAAGGCTTGGGCCAGGCTGAGCCAGGTTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 331278 . TGCAGCCGTTCTGCCCCACAGAAGCTGCTCCTTGGTATCCGAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 331354 . TCAGAGGACACCCCAGGGGCAGTGGCCGTGCCCGTCTCTGATATGCTCCGCTCCCACGAGCCCTTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 331431 . CTAGCCCCTGGCTTGTGGGCTTGGCCTCTGAGCTGGACTTCTTTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 331496 . CACCTTCACCTGGAAGGCCAGGTTGTATTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 331558 . GCTCGCTCAGCATCTGGCTGACGGTCCGGTTATCCTGGTTGGGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 331615 . CCCCGCCAGGGCCTGGTGCCGCCTGCTGAAGATCATGAGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 331669 . ACCGGATGTGGTCCTTGTCTGATTTGTTGGGGCTGCGTCCATCCTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 331725 . GAGTCCTGTTCCTTGCGCAGGGCACTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 331773 . CTGAGTGGTAGAGGCAACTGGGTGTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 331851 . TCCAGCAGGGACTCCCCTGAGGGGCCCAGGGCTCCTCCTCCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 331913 . CCAGGTTCCACCACCCCCAAAGTGTGTGGGGTTGCGGGCCCTGGGCTTTCAGGGCAGGTGGCTCCAGGGGGCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 332001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=334000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:1.20:.:0:.:0:0.999:211
+1 332001 . TCCCTGTCCCACCTGGTGGACGCTCATGAGCAACGGCTGCCAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 332104 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 332115 . GGGAGGGGTCAGGAAGGGGATGGAGTACCAGGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 332188 . AACATGCTGACGCCACGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 332222 . CAGGCCTGGGGGCCGAGCACTTGGTCCGGGCAGGGGGTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 332275 . ACACCTCCTCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 332354 . TCAAGTTTTGTTTTTTTTGTTTGTTTTGTTTTTTGTTTTTGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 332451 . CCTTCATACCTGGCTAATTTTTTGTATTTTTACTGGAGGTGGGGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 332550 . CCTCAGCCTCCCAAAATGGGATTACAGGCATGAGCTACCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 332610 . TACTTGAAAAACTCCGTTAAGCATTTTTTTAAGGTAGACCTAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 332668 . TCAGCTTTGTTTGTCGAGGAAACACGTTATTTCTTTTTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 332756 . TTTTCTTTCAGCACTTTGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 332886 . AAGCTTCATGGATTTAGATGTCTAAATCTTTCCCATGATTTAGGCAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 333033 . TTTCTTTTCTCTGTCTCTTTTTTTTTTCTTTTTGAGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 333131 . CCAACCTCCGCCTCCTGGGTTCAAGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 333164 . CTGCCTCAGCCTCCTAAGTAGCTGGGACTACAGGTGTGTGCCACCACACCCGGCTAATTTTTGTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 333284 . CCTCTTAATCTGCCTGCCTCGGCCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 333526 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 333576 . GGGAAGCTGAGGTGGGAGGATCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 333630 . CAACGTACTGAGAACTTGTCTCTATATTAAAAAAAAAAAAAAAAAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 333721 . GAGACCAGCCTGGCCATCATGGCAAAACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 333763 . AAATACAAAAATTAGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 333799 . TGTAGTGGTGGTGCATGCCTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 333850 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 333868 . GAGAGGGAGGTTGCAGTGAGCTGAGATCGCACCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 333912 . GCCTGGGCAACAGAGTGAGACTCCATCTTATAAAAGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 333988 . ACTCCTGAGTTTTTGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 334001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=336000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.68:1.14:.:0:.:0:0.999:211
+1 334016 . GATCGTGCTCTACTGTGATGATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 334055 . TCAGAAAAAAAGCGTATTCTTTTAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 334121 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 334157 . TGATTCAACAGGAGGAGATAAGGAAGCTCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 334220 . TAGATTTTTATAAAATGAAAGCTGCCTCTGAAGCACTGCAGACTCAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 334315 . CTCTTACTATTCTGAGAGCCTTATCATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 334365 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 334406 . TAAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 334462 . GGGCGTGGTGGCTCATGCCTGTAATCTCAGCACTGTGGGAGGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 334586 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 334606 . GGCTGGGACCTGTAATCCCAGCTACTTGGGAGGCTGAGGCAGGAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 334659 . GAACCCTGGAGGTGGAGGTTGCAGTGAGCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 334698 . CCATTACACTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 334736 . ATCTCAAAAAAAAAAAAAAAAAGGGGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 334841 . GACAATCAATATTTGCCAAAATGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 334922 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 334945 . ATCCAGTAGTGACTTTTACAGTTTGTATCTAAATAGAAGCTGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 335010 . GCATAAAATATTATTTATTATAGAGTTAAATGCTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 335054 . ATCTAATTAATAGGCCTATTTTCCTTTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 335256 . ACAGATTTGGGGTATATGCTAAAGTTACCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 335345 . AGGCCGAGGTGGGCGGATCATTTGAGGTCAGGAGTTCGAGACCAGCCTGGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 335405 . GAAACTCCGTCTGTACTAATAGTACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 335441 . AGGCGTGATGGTGTGCACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 335476 . CAGAAAGCTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 335521 . TTGCAGTGAGCAGAGATTGTGCCACTGCACTCCAGCCTGGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 335626 . ACACATGTTGTTAAATCATCTTACAGATTTTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 335668 . GAAGAAAAGTTTTACTAAATGGTCTTTTAATGGAAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 335743 . ATCCATTCCTAGGCCTAGAAAAATGTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 335808 . CAGCGATAGTACATTAGCTATGCTATATGCATACATTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 335859 . TCGACTTTCAAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 335889 . CTTACTGCTTCTGAACATGTTTGTGAGTTATATTGCTGAGGGACCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 335958 . AACCCAGTGTTATAAAATTGAAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 336001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=338000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:1.30:.:0:.:0:0.999:211
+1 336003 . AAAATTAATATCTACCTTGTAAAAAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 336039 . CTGCATTTGAGAATAGACTTTTTAGGTAATAATGATGCAATCCATAGGGTTTTTGGGGGCACAGAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 336179 . CATATTTTTACTCTTTTTATAATTTTTTCTAAAAAAAATTAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 336232 . TATATAACTTTTAACTTTATAGGTAAATATTTGTTTCTTTCAGCTCCAGTTTTATGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 336304 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 336365 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 336369 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 336408 . CACATTTTGTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 336435 . TTTGTTTATATTCTATATATATTTCATTTTTGGTTACTATGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 336536 . TTCAATCACATACCAAAATTCTACTGCTATATATATAGCTCTACTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 336600 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 336610 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 336644 . CAATTACATTTTATGCATTTGCCTTTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 336682 . AAAATAAAAAGCAGAGTTACAAACCAAAATTACAATAGGACTGTTTTTATGTTTATGTATTTACCTTTACCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 336793 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 336803 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 336831 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 336846 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 336880 . CAGTTTTAAAAAAAAATCCGGAAATGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 336915 . CTCCTTCATTTTTGAAGGATAAGTTTTCCAGCTATATATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 336976 . ATTATTTTAAATATATAATCCACTGCCTACTGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 337022 . TGCCGAGAAATCAGCTGCTAATGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 337083 . TGAGTTTTCAACATTCTCCCATTATCTTTTTTTTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 337134 . ATAATTGTACATATTCATGGGATACAGAGTGATATTTTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 337213 . AGCATATCCATCACCTCAAATATTTGTCATTTATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 337268 . TTCTTTCTTCTAGTTTTTTAAATTTATAAACATTTAAATTTTATTACAGAAATTTAAATTTTTTGATTCTGAAAAAGTCATATATGTATGCAACATTTTTTATCATTTATTTATATATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 337421 . TCTATTTTATCATTATTTCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 337479 . CCCTTGTTTTTTCCTAGTATATTAATTTATTTACTTATCTTCTAAAAATCCTCCATATAATCTGTTTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 337573 . AATAATTAGTTCTGTTCTATTTTCCATTAAAATATTTAAATCTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 337655 . TTAATTTCTCTATACTCTAGCTTTTGACTTTTTTTTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 337731 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 337772 . AAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 337796 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 337811 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 337815 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 337833 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 337844 . TGTATAAGCATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 337933 . CGTTGGGGAAAAAATAAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 338001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=340000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:1.19:.:0:.:0:0.999:211
+1 338130 . TGCTGCTACTCAGATACATTTTATTTCAAAAACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 338190 . TTCCAAAAACATATTCACACTGAACTTTCAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 338253 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 338305 . AGTATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 338336 . GTAATGTAATTTTCTAATGCTAAATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 338421 . CCCCTAATTTTTCTTACCTTTGACATGATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 338474 . GTGATTTTTCTTTTTTTTTTTTTTTTTTGAGACAGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 338533 . GAGTGCAGTGGTGCAATCACAGCTCATTGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 338597 . CCTCAGCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 338633 . GCCACCAAACTTGGCTAATTTTTGTATTTTTTGTAGAGACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 338685 . AATTCTCAGGCTGGTCTGGAATTTCTTGGCTCAAGTAATCCTGCCTTGGCCTCCCAACATGCTGATATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 338761 . TAAGCCACAGTACCTGGCCAGTTTTCTTTTTAAAAAAATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 338833 . TAGCTGTGCTCCTTAATTGGGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 338874 . AACTTAGCCAATTTTCTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 338969 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 338983 . TAAATGGTGCACTGGATGTTGAAAAAGTTGATGATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 339036 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 339038 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 339066 . GAAATATAATTTGTTAAAATATAAAAGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 339148 . ATGGGCTGGAGGCCAGAGGATTGCCAAAGAGAATGGGCCTCCTGCTGAGATGAAAAGTTGAGCAGGGATTAGTTGGCAAAAGTGGAGGGACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 339289 . TGCGACAAAGTCTATATAAAAAACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 339326 . AATGTGGCTTAAATACAGAAGCTAGTAGGAGAGGAGTCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 339416 . GTATAAAAAGAATTTCTCTTTATTCTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 339468 . CTTTAAACAGGTGATGTGATTTGATTGAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 339522 . CATTACATGCCCACTGTTTGTCAGATATTGCTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 339582 . AAACAGGCAAAAATCCCTGTCCTCTTGCAGCTTATAATGGACTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 339635 . ATATGTCAGAGGAGGTCCACGGAGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 339681 . TGAAAAAAATGAGGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 339812 . CTTCTCTCCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 339842 . TGTCTATATATATAGAATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 339880 . T <CGA_NOCALL> . . END=340145;NS=1;AN=0 GT:PS ./.:.
+1 340001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=342000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.60:1.27:.:0:.:0:0.999:211
+1 340154 . TTTCCACTAATTTAAAATGCCACCTTTATGTTATTGTAATTTATATATATACTATATATACACACACACACATATATATATACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 340246 . TACAGTGTGTGTGTGCACATGTACACACATGCATATGTGTATAGAATGCCCAGTATAAGCAATGTGCACAAATAAAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 340342 . TAGAGTGAGAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 340389 . TATAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 340409 . TGAGGTGGTTTCTAAGATGGAGAATAAGACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 340449 . AGTACGTTGTTTGACTGAATTCAAGAAAGAAGGGTAAAAGAGAAGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 340506 . TTATCATTAAATGCCACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 340548 . ATTGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 340568 . TTAAGGGTGACCAAATTCCGTTTTGGAGGAGGAACAGATTCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 340654 . GGTAGTTTTTCAAAAGTTTTCAAAAATATGAAAAGAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 340723 . AATGGGAGAGACTATGGTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 340772 . ATTGAGATAGAGATATTGACTATATAAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 340854 . TGAATTTTTGTGAAAGTACAACTAAAAGGCAATGTCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 341091 . AACAATGCATGACAATTTACAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 341123 . TTTTGGAGCTAACTTTAAGTACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 341191 . TTATTTATCAAGTTTATGTCAAGGGACAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 341267 . ACACTTATCCAGGGGGGTTTTTAACCTTTCCCCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 341330 . AAT . . . NS=1;AN=0 GT:PS ./.:.
+1 341337 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 341347 . GCG . . . NS=1;AN=0 GT:PS ./.:.
+1 341369 . TAACATTTCTCACAAGTCAATTAGCTTTGTACTGGGAGGAGGGCGTGAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 341425 . TTGCGGTAGTTGTGTAGCAGCAGCACAATGGCCGCAGACAAGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 341495 . ATAATTTTATATTTTTGACAAGATTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 341535 . CTTCCTCTCCATTTCTTTTTTTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 341584 . ATTTTATTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 341605 . CTTATCTCTTATTATATTTTATTAAAGAAAATTATTATATTATTCCTTTATATTTTTATTAAAGGATTTTATTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 341687 . GGAAATTAGCCTTATCTCTTATTATATTTTTTATGACCTTCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 341739 . TCTGCTTAAAAGTGTACCCTGGCCGGGCGTGGTGGCTCACACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 341793 . C <CGA_NOCALL> . . END=342057;NS=1;AN=0 GT:PS ./.:.
+1 342001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=344000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.62:1.17:.:0:.:0:0.999:211
+1 342069 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 342105 . A <CGA_NOCALL> . . END=342360;NS=1;AN=0 GT:PS ./.:.
+1 342377 . TTAATTTTTTTCTAGCTGATCCATATGAATTCCTCTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 342426 . AAAGCATCCAGGATTCAATGAAGAACTGACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 342487 . GCAGGCTTAAGCCATTTTTGATATAGATACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 342529 . TTGCTAAGAGCAAACTTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 342573 . CTTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 342625 . CTAACCACTTGCTCGCCAACAAGGAAAACTTTTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 342674 . GAATAGACAAGACATTTCTTTCTTTTGGTAGAAAATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 342745 . GTAATTTTAACTTTGTGATTTATTGCCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 342782 . TCTTCTGTACTGTAAAGTGTGTGTCAAAAAAAAAAAATAGCGATTTTGGAGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 342889 . TTTCTCTCTTTCAAGAACATTTTTATTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 342926 . TTACAAAAACTCCCTAAACTTTGGAACAGCTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 343011 . CTTGCATCTGTGAATAAGCCAGGACATCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 343052 . TGATTAAAAACACCACCAATGGAGTTTCATTAAATTTGTATTGCTCTGACTAGTGAAACATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 343124 . TTGCTGAGGATATTTTACTGCAGTTCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 343222 . TTAATTTGCTTTCTTACAAGGCCACATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 343333 . TAAAAACACTAGTATTTTTCTCTGAGTTTAAAATTCAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 343383 . GATATGGTTAGGCTTTGTATCCCCACCTGAATCTCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 343483 . TAATTGAATCATGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 343531 . AGTTCTCACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 343578 . CTCGGCACTTCTTCATGCTGCCTTGCGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 343629 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 343711 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 343714 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 343730 . TTCTTTATAGCAGTATGAAAATAGAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 343781 . AAATACAAAAAAACAAAACATTATCTCACTAACATAGGAGCTAATATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 343845 . AGTATTTTATATTAAAAATATGTACATATATATTTATATATAATTAAGAACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 343929 . ACATCTACTTAAGAAAATAGCTATGTAATATACCATTACTCAACTAGATTATAATTTTTTCTCCATTTCTTTATTGTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 344001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=346000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:1.21:.:0:.:0:0.999:211
+1 344022 . CTTTTTTGTTTTCTCATTTTTATTGCATAATATTTAATTATGCAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 344104 . TAT . . . NS=1;AN=0 GT:PS ./.:.
+1 344128 . AACAAATGCTAATACCCACTACCTGACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 344166 . GATATTATTTTTTTCCAATTGAAATTCCCTCAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 344222 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 344293 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 344315 . AAATGTAACCATATTGTATATATTCTTCTTCAGCTTCTTAGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 344375 . TTTTGCTGATACTTACATTCATATGTACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 344423 . AATGGCTATATATTATTCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 344473 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 344475 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 344498 . TGTCTTTTTTATTTTTTGATATAACAAACAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 344601 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 344603 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 344627 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 344691 . TAATGGACAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 344711 . TACTTCCCATCTTTGTTAATTATTACTTTTAGACTCTAACTTTTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 344765 . TGGATGTAAAAAGCATCTCAGGGTGGTTTTAATTTGCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 344810 . GCTCATCTATGAAGATGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 344849 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 344877 . GTTTATGCATTTTGCTTGTTCTATGTCTTATTTTTCCTGTTGATTTTTGGGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 344939 . ATTCTAAATGTATATTTATTCACTTATATATATGTTGTAAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 344995 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 345009 . ATATCTTCCAAATAGAGAAGCTTTATATTTTGATGTAGTCATATGTTCATTTTTCCTCCTTAATGTTTGTTTTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 345098 . TACCAAAAGTAACAAAAATTCTCATTTATTTTTAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 345158 . GAATTCACCTTGAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 345218 . GATAACCACTTGTTCTATTACTGCTGTAACAAATTTCTACAAACTAAGTGACCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 345313 . GAAATCAGGCATGAATTTTAGTGAACTAAAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 345375 . TGGCTAGGGTAGAATCCATATCCTGGCCTTTTCTATCTTCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 345458 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 345539 . TCCATTTTTAAGAACTCATGTGATTAGACTGGGCCCATCTGGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 345811 . ATTGTTTTTATTTTTATGTTATTCCACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 345855 . CATGAATTATGGTACATGAGTTTATTTTTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 345930 . TTTTGCGCTTCTTCAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 345956 . TTTTGGCCCTTTGGTCTTCTATACACATTTTAGAAATGCTTTGTTGAGGACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 346001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=348000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.01:1.17:.:0:.:0:0.999:211
+1 346077 . GTGCTTTATACATGAAAATAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 346287 . GCCCAATGTGATATTTATTCAACAAATATTCATTGAGTATACCTAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 346398 . CATTCCAGTATTTGGAGACAGATGATAAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 346505 . AGTCTGTAATTTAAATAGGGTGGGCAGGAAAGCTTCACAGAGAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 346601 . ATATGGGAGAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 346686 . AAGGAGGCTAGTGTGACTGCCACAGAATCACCCAAGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 346743 . AGACCAGCACTTGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 346790 . AAAAATTTCACTTTTACTATAAATACTATGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 346838 . TACAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 346867 . CATGTTTTAAACAAACTCTATAGCTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 346922 . GGCAGAAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 347059 . GTACGATAAACCCCCCTCCCCAATCACAAAAGAAAAACTAAACACAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 347120 . TTGCTCAGACAATTTTACAGGTGAGTTCTAGCAAACATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 347212 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 347297 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 347421 . AAACATCATTGGTCACAGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 347453 . GCTGGTGTTAACTTACTGGCAGAAACAGCCATTGCTAGGCAAGTGTTCTTGTGAGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 347525 . GCTGCAGTCTTGAGGAATTTTTTATGATAGGTCCTATTATAAAACACCTACAGGATGAGCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 347623 . GTGAGTTTATAGAAAGTCCTTGTGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 347666 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 347750 . TTCACTGTAGTATAATCTGCACATCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 347791 . AGTACAAAGAAAGAAAATTAAAGGTATATCTCTTTCAAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 347883 . TTATAATAAACATCTCAAGCTTCACAGAATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 347927 . CACTCTCATCCACAATCTTTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 347978 . GTTGCTGAGGCTTCTTATTGCTTTTTTCTTCAAATAACAGTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 348001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=350000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.19:.:0:.:0:0.999:211
+1 348042 . CCTAGTCCATACAATTGTTATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 348088 . GTGATTCTTTTTTTATATATTTCTGGACAATTCTTTATATTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 348186 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 348197 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 348202 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 348250 . AAAAGCAGCTAGACAAAATCAGAAGAGGTCTGTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 348304 . TGACAGTCTAGAGTGTGGGCACAGAACCCAAGCTTATAGGAATTTCTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 348387 . AGCTCATAAAGAATTCTGCGAGCCATCACATCTGTCAAACCTGCTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 348455 . TATTTGAAGACCTCCTCTTCACTCCTCATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 348496 . TGAGTTTCTTTCATAGGCAAACCCAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 348537 . CCTGAAGACTTCGGGTGACACAGTACCCAGACTTAAATAGGAGGGGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 348601 . CATCCAGCACAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 348631 . ATAGTTTTGATTCCTTAAAAAAATTAACCACATTAAAATATGTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 348746 . GCTACCCCCATTTTGTGTGTGTGATTTTTGTGTGTGTGTTAGAAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 348817 . TGAGCTTGCTTTGATGATTTATTTGTCCAGAGAGGATTTTTTTTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 348915 . CAGCTTGCAAATTTGGAAGCCACACAGATTGTGTGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 349057 . CTCACTTTGAAGGTTTATTTTATTTATTTTTTAAATTTCTAGTACACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 349119 . GTAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 349208 . TCAGTATGTCTGCTCAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 349244 . ACTGAGGGTGGATCTTCTTCCCAGCTCACTCACATGGTTCTTAGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 349386 . CTGGGCCACTGACAGCATGCCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 349420 . CAAATGAGAGGGCAAGAGAAAGAGAGAGAGGGAGAGGGCACAAGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 349544 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 349562 . CATG . . . NS=1;AN=0 GT:PS .|.:349562
+1 349624 . TTCTGGTAGCAGCCACAATATCTTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 349767 . CAAATGTCTATGTTAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 349809 . TAGGCTTTTAAACTCTGTGAATGTTGGACTGTGATGTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 349890 . GCTTATTCAGTAAGCACATACTTGGCTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 349982 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 349994 . GTCATTTTTTTAATTTAGTGACTGGTTCTATTAAAAGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 350001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=352000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.23:.:0:.:0:0.999:211
+1 350107 . AAACCACAGCCCTCAGTTGTGTATATTTATCTCTTGTTTTCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 350215 . ATAAATATTTAATTCACAAGTTTAGAAAAGTGAACCTGAAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 350301 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 350364 . TTGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 350404 . ACTATTTTTAAGTTGAAAATGTAATTGGTTTCTAATAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 350468 . ATCGTGTTTAATGTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 350499 . TGTCAGATATCTCTCTTGATTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 350539 . AGACAGCAAATGCTATTGTCCAAGTTTTCTAAAGAAGAATCTGAAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 350644 . AAATTGTAGCCCTTGTAAATTACCTACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 350699 . ATTCAGTACTCTGTTCTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 350751 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 350754 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 350762 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 350795 . ATTCTGACTGTGGCAGGAGTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 350887 . TCTCCCAATTATTTTAAAATTATAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 350966 . GAGGCAGTTAGGGAAGCCTTCCCTGAGTTAGTGCCATTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 351016 . TGATAGACGATAAGTAATTTGTCAGGGGAAAAATACTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 351075 . AAGGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 351098 . TGTCTTGGTCCAGGAGCTAAAAAATGTTAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 351152 . AAAGAGTTATTAAATGAGGCAGCAGGCTTCAGCAGGTGCCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 351258 . GTCGTAAGCAGAAAGTAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 351291 . AAATGTTATTCTCTAAACAGTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 351333 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 351392 . TATTAGGTGCCTACTATTAGCCAGGTACAGCCCTTAGCTACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 351461 . CAGAATTTCTTAAACAAAGAATCTAAAGTTGTTTATACACCATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 351514 . TTTTATAAATTTCTTGAAATTATTTTTATGTACACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 351611 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 351657 . TTTACTGTCAATAGTTTGTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 351695 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 351701 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 351712 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 351721 . CAAAATCACAAATTTATACTTTATATTATGAATGAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 351769 . AGCATGATAATATATTCTGTTGTCATCACATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 351810 . AACATATAGAGTATGAATCAATAATTTTTCAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 351903 . TAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 351924 . CTTGCAAAATGATGGTGGTGGTTTTTTTTTTTTTTTTTTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 351982 . TTGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 352001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=354000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:1.13:.:0:.:0:0.999:211
+1 352005 . TGGCGTGATTTTGGCTCACTGTAAACTCCACCTCCTGGGTTCAAGCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 352082 . GTATTACAGGTGCCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 352105 . ACCCAGCTAATTTTTGTATTTTTAGTAGAGATGGGGGTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 352162 . TGGTCCCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 352185 . GTGATCCACCAGCATCGGCCTCCCAAAGTGCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 352227 . GTGTGAGCCACTGCGTCCAGCCAGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 352301 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 352361 . ATGATAAGAAGATTTTAATTTTCTTTTTTTTTTAACTTTTATTTTAAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 352442 . TCACAGGGGTTTATTGTACAGATTATTTCATCATCCAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 352515 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 352558 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 352623 . GCTGTATTTGGTTTTCTGTTCCTGCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 352679 . CCACATTCCAGCAAAAGACATGATATCATTTTTTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 352778 . TCACCAGAGCAGTCTGACAGAACCTCTCTGAAAGACTTCTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 352853 . CCCTAAATAAATCTAACTTTAATTTCTTAAAAGCTTAATTTTTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 352943 . TTCGGTATTTTTCCTATTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 353005 . ATATGACTTTGGTGCCTTTTCCTCAGCCTCCACTGATGATTTTTTCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 353147 . ATGCTTTTACAGGTTTTGCTATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 353316 . ACCTCAAAACTTCGCAGCTTAAAACAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 353391 . CTACATGATTCTGCTCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 353424 . AGCCATCTGTGGTATTCAGCTGGCAGCTGGGTAGTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 353574 . GATCTATTCAGCAGCAATATGGTTTGGCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 353643 . AATTTCCGTGTCTTGTGGGAGGGATCCAACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 353714 . CTCGTGGTCTTGAATAAGTCTCACAAGAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 353768 . CTTTCACTCAGCTCTCATTCTGTCTTGTCTGCCATCATGTAGAGATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 353884 . TTTTTCTTCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 353924 . CAGCAGCATGAAAACAGACTAATACAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 354001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=356000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:1.14:.:0:.:0:0.999:211
+1 354025 . AACAGGCAGGGGTTGAAACAGTTTGGAGGGCTCAGAAAACAACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 354129 . TCAAATAATGATATAGACAATGAAATCCAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 354189 . CTCAATGGGAACTGGAGTAAAGGTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 354327 . CAAATCATTCAAGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 354381 . AACAGAGCATAAACGTTCAGAAAATTTGTAGCATGACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 354563 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 354756 . GTTGAGCCTGTGGATTCACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 354801 . CCTCCATCTAGATTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 354910 . GAAATGTGGGGTTGAAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 354998 . CCCGGAATGGTAGATCCACCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 355073 . AACCGGGAGGAAGGCTCTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 355240 . AGCCTTTAGTCCCTTTGTTTTGGGTAAATTTTACCATTTGGAATGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 355323 . GAAAATAACTAAATTGCTTTTGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 355401 . TGCCAACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 355444 . GGACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 355477 . GTGAGGACATGAGATTTGGGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 355531 . CC GT . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs75706905&dbsnp.131|rs78394685;CGA_RPT=THE1B|ERVL-MaLR|25.6;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:42,82:42,82:33,19:-82,-42,0:-33,-19,0:3:3,3:0
+1 355823 . CCCGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 355867 . ATGCAAGCAGAATAAGCAGAATTCTTCGATACTGACTCAGCACCCACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 355936 . AAATGGAAGCTTCAAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 356001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=358000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.89:1.07:.:0:.:0:0.999:211
+1 356055 . CCCCACGTTTTCAACGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 356237 . AAAATGAGTTCATTACATGGTAAGGATGAGGGAGAAAGAAAAGATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 356350 . CCACATATAGCTGTGTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 356443 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs75715938&dbsnp.132|rs112222596;CGA_RPT=L2|L2|50.1;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:66,131:66,131:48,44:-131,-66,0:-48,-44,0:3:1,1:2
+1 356537 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs75249042&dbsnp.135|rs183318141;CGA_RPT=L2|L2|50.1;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:56:114,56:114,56:42,45:-114,-56,0:-45,-42,0:3:3,3:0
+1 356561 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 356598 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 356934 . TCCTCACAATGTGGTTTCAGGCACAACTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 357016 . TTTGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 357082 . GGAAACTGACTCACATCAAATCCCTCTCTTATAGTAATTTTCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 357307 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 357311 . TATTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 357328 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 357373 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 357453 . GGGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 357525 . TCATCCAACTTTATTTGATTTTGTTATTTTTAATTTTTCTAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 357583 . TTAAAATAATATTTAATCACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 357653 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 357842 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 358001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=360000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:1.16:.:0:.:0:0.999:211
+1 358013 . TCTATTTTTTTTTTTTTTTGTGGGTTGTTTATTTTGTGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 358105 . CACATTTTTTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 358191 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 358283 . TTTCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 358346 . ATTATTCTCTCCACTTCCCACTTTTTTTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 358422 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 358431 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 358440 . TTTATTTTTT . . . NS=1;AN=0 GT:PS .|.:358440
+1 358485 . TCTAAATATCATTTCAGAGAGAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 358517 . GAGAAGGCATATTTTTAAAAAACATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 358553 . TCCTGTTTTTTACTGCTTCATGGTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 358607 . TTTCCCTTTTTCTTCTAGTATTTTTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 358645 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 358659 . AATACAATTATTATAAATAATTATATAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 358721 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 358769 . CAGACGAGTTTTGGAATCTTGTCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 358894 . ATCCTTTTCTTTTCATAATAGAATGAAAATTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 358944 . CCCCATCTTTTTGCCTTCTTTCATTACACCACACACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 359245 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 359372 . CTGAATTTATTTACACGATTGAGATTGATCTAGAGTTTTATTTTTTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 359455 . TTTTGTGAATATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 359518 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 359523 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 359530 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 359678 . AGCATTAGTTTTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 359722 . TCTATTTTTTCCATGGAACATTATATGATTTCATGTCCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 359822 . ATGTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 359877 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 359990 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 360001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=362000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:1.06:.:0:.:0:0.999:211
+1 360197 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 360321 . CCTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 360399 . GGCACCG . . . NS=1;AN=0 GT:PS ./.:.
+1 360695 . CATTCCCCTTCTCAGCCCTTTTCATTCTCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 360747 . TTTCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 360853 . ACAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 360880 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 360950 . CACTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 361108 . TTGGCTCCTCTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 361165 . TTGATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 361206 . TCTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 361271 . GCACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 361439 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 361487 . TATTTATTCTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 361544 . CTCCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 361590 . TATTTCTATTTCTGTTTCTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 361731 . ATCGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 361762 . TTACTGGCTTCATCCTCAATTCCCGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 361804 . GGCTCCCTAAATTTATTTAATAATTCTCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 361871 . AATATTGACTACCTTTCTTATTTTCCTTTCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 361961 . CCATTTTTGAAATATTTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 362001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=364000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:1.00:.:0:.:0:0.999:211
+1 362017 . ATGTATTGTAATATTTATCCTCTTTTATGGCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 362119 . TTGTAGAGGATTTGTTACACTTCTTAAACTGCTGATCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 362174 . TTGACCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 362215 . CACATGGCCTGATTTGTTCCACTTCTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 362439 . TTTTCTGATATGCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 362532 . GGGAGAAAAGTCCATTTTCCTGAGGATAAGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 362672 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 362742 . AAGACATTTTTTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 362819 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 362891 . AGCAATTTTTTTTTTGGGGGGGGTGCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 363059 . TAGGCCTCAGATTATAATAAATAACGTTTATTGTGCTATACTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 363167 . AGGGAGCAAGCAAGCAAGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 363196 . TATCCCTATTTTATGTATTTATGTATTTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 363241 . TTTTGGTTTTTTTGTTGTTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 363285 . ATGTTTTGGGGTACATGTGATATTTTTGATACATGTATACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 363369 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 363420 . GCTATTTTAGAATATATAATAAATTATTGTTCCCTATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 363469 . GTACAATTGAATACTAAAATTTATTATGTCTATCTACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 363519 . AATAACTATAATTTCTCTGCTGTGCAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 363567 . TTCGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 363633 . TACGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 363685 . GTCGACCTCAAAGTCCAGATTATTGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 363770 . CAACCTCAAAGTCCAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 363875 . AAATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 363963 . TGACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 364001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=366000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.95:.:0:.:0:0.999:211
+1 364215 . GGCAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 364247 . CCCCGTTAAATGCAACCCAATGTCACAGTCCGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 364287 . CCAGATGAGTGTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 364336 . CATCCTTGAATGAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 364364 . TAAATTTATTTTATTTATGTTCTTTTTAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 364422 . ATTTCATCCTACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 364475 . TTTATTTTTAAATCATTGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 364505 . AATTTCCCCTTTCTCCCATTCTTTCCATATTTTTAATCTTTTTTTAACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 364564 . TCTGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 364676 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 364851 . ATCAAGAACTATCCACTTTCTATTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 364932 . AGGTCTCCTCCTGTTATATCAATTTAGGTATTTTTCACACATTGTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 365069 . AATTGTATCTTCTGCCCTCCAATATCTATAACACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 365203 . CAACTTTTTTTGGTTCTTTCCATTTACAGGAATTAGAGAAAGAACTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 365314 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 365325 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 365400 . TGACCAACAAAATATGAAGAATGGGTAAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 365544 . CTATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 365620 . ACTCTTTTGAGTTTTGCTGTTTTTACCAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 365835 . TTTATATAATTCCATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 365886 . ACAATTATAGAGTTTTCCGGTCCCTCTTCCCAGTGCCTCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 365965 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 365979 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 366001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=368000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:0.92:.:0:.:0:0.999:211
+1 366040 . TGTAAAAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 366157 . AGATATGATGTGTGCCTCTGTGAGTTTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 366240 . GGAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 366314 . TCTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 366357 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 366433 . ACAATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 366604 . TCCGAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 366686 . AGGGTGGGGAATGCAATGAACATAAAAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 366839 . GTGTCCATTTTATGCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 366952 . TCATTGCAGGATGGGAAGCCATTTAGGGCTTCATAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 367027 . AGTAAGCATGAGAAAATGTGGTAGATAATGGTGGTGATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 367091 . TCTAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 367130 . CCCCAGTCTCTATATTCTCTCAATCCACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 367196 . TGCAGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 367240 . ACTTTCTGTCAGAGAATTGTGGGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 367342 . ATCACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 367412 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 367470 . TCACATGTTGTGATAATATTATGTTGATTTTTTGTTTTTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 367564 . TTGATTTTAGTACATATTATTATCTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 367651 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 367828 . CCCCCATGTACTTTCTACTGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 367941 . GGAGGCTGCATCGCTCAAATCTTCTTCATCCACGTCGTTGGTGGTGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 368001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=370000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.03:.:0:.:0:0.999:211
+1 368021 . CAGATATGTGGCCCTATGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 368110 . TTGGTGTCAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 368154 . TTAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 368203 . CTCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 368292 . TTCATACTTCTAATCTCCTACGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 368392 . CAGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 368433 . ACACGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 368584 . AGAGGATCTCATAAATGATATAATAAGCCCTTCTCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 368653 . GATATTTTAGATTCAGGAACTATGAGACATTATGTATTGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 368724 . TTATCTGATGAATATATGATGAATATATTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 368800 . TATGCCCATTTAATTTCTTTCAGCAATGTTTTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 368927 . CAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 369008 . TATAAGTTCCATCAGTTTTTTATAGGTTATGTAGGATTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 369088 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 369151 . GAGTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 369194 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 369265 . TCT . . . NS=1;AN=0 GT:PS ./.:.
+1 369334 . TTTTTCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 369472 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 369520 . ATACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 369562 . ATTATTTTAACACTGGAGATAGAATCTGGTGGAATGACGTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 369644 . TAAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 369810 . TCTATCTTATTGACATACATATCTTTTTTGTGGTGAAAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 369915 . TTCTGTGCAATAGTTCACTGAAACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 369961 . CCCTTTTATCAACATCTACCTTTTCCATGTCTACCCCCAACTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 370001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=372000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:1.09:.:0:.:0:0.999:211
+1 370054 . TTTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 370096 . TTTCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 370197 . ATTGTATATATACACTACATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 370387 . TTTTAGTTTTTTGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 370503 . TATCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 370564 . TGCATTTCCCTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 370613 . TGTATTTTTTTTGAGAAATGTCTATTTAGGACCTTGCCCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 370735 . CTCTGCAAATATTTTCTCACAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 370820 . AATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 370859 . GGTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 370902 . TTTTCCCCTATGTTTTCATCTAGTAGTTTTACAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 370985 . GTGAGATAAGGATACACACCATACACATTCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 371107 . AAAATTTATTGGTCATAAATGCATGAGTTTATTTCTGGGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 371180 . TTTTGTGCAAGTGTCATATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 371223 . ATATGTACTTGTTTTGGGGGGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 371301 . TGCCTCACCAAGATTGCCCAGAAAACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 371476 . ATGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 371604 . TAGTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 371681 . CACTATTCCTTCAAATTCCCTATTTCTATCTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 371767 . AGCTTAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 371799 . TTGTAAAATATTGTAGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 371976 . GAGATTTTTTTTTAAATTATATTTTAAGTTCTGGGGTACATATGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 372001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=374000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.54:1.08:.:0:.:0:0.999:211
+1 372089 . TCTACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 372123 . CCCCAGCCCCTACCCCCAGACAGGCCCCGGTGTGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 372189 . ATTGTTCAACTCTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 372262 . TGATGGTTTCCAGCTTAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 372335 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 372344 . TATATGCCACATTTTCTTTATGCAGTCTATCACTGAATGGGCATTTTGGTTGGTTCCAAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 372411 . TATTGTGAACAGTGCCACAATAAACATATGTGTGCATGTGTCTTTATAGTAGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 372472 . TAATCCTTTGGATATATACCCAGTAATGCAATTACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 372565 . AATGGTTGAACTAATTTACACTCCCACCAACAGTGTAAAAGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 372697 . GTGGTATTGATATGCATTTCTCTGATGACCAGTGATGATGAGCTTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 372807 . CCACTGATGGGTTTGTTTGTTATTTTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 372860 . ATTCTGGATATTAGCCCTTTGTCAGATGGATAGATTGCAAAAATTTTCACCCATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 372921 . GTTGCCTGTTCACTCTGATGATAGTTTCTTTTGCTGTGCAGGAGCTCTTTAGTTTAATTAGATCCCATTTGTCAATTTTGGCTTTTGTTGCCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 373038 . AGTTTTTGTCCATGCCTATGTACTGAATGGTATTGCCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 373126 . TAATACATCGTGAGTTAATTTTTGTGTAAAGTGTAAGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 373198 . GCCGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 373223 . AAAAAGGGAATCGTTTCCCCATTGCTTGTTTTTGTCAGGTTTGTCAAAGATCAGATAGTTGTAGATGTGTGGTGTTATTTCTGAGGCCTCTGTTCTGTTCCATTGGTCTACATATCTGTTTTGGTACCAGTACCATGCTGTTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 373375 . AAGACTTGTAGTATAGTTCGAAGTCAGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 373459 . CTCATTTTTGGTTCCATATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 373490 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 373516 . TCAGTGGTAGCTTGATGGGGACAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 373558 . TTTGGGCAGTATGTCCATTTTCATGATATTGATTCTTCCTATCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 373618 . TTTCCATTTGTTTGTGTCCTCTCTTATTTCCTTGAGCAGTGGTTTGTAGTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 373702 . TTGGATTCCTAGGTATTTTATTCTCTTAGTAGCAATTGTGAATGGGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 373817 . TTGATTTTGTATCCTGAGACTTTTCTGAAGTTGCTTATTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 373868 . TTTTGGGCTGAGACCATGGGGTTTTCTAAATACACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 373959 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 373967 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 373985 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 374001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=376000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:1.18:.:0:.:0:0.999:211
+1 374019 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 374023 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 374066 . TGGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 374123 . TATTGAGAGTTTTTAGCATGAAGGGCTGTTGAATTTTTTCGAAGGCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 374265 . TTGTATCCCAGGGATGAAGCCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 374300 . GGACAAGCTTTTGATGTGCTGCTGGATTTGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 374340 . ATTTTATTGAGGATTCTTGCATCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 374392 . TTTCTTTTTTTTTGTTGTGTCTCTGCCAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 374444 . GCCTCATAAAATGAGTTAGGGAGGATTCCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 374513 . ACCAGCTCCTCTTTGTACCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 374569 . ACTGTTTTTTGGTTGGTAGGCTATTAATTCTGCCACAATTTCAGACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 374635 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 374650 . CTGGTTTAGTCTTGGGAGGGTGTATGTGTCCAGGAATTTGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 374711 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 374742 . GATAGTAGTTTGTATTTCTGTGGGATCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 374788 . TCATTTTTTATTGCATCTGTTTGATTCTTCTCTGTTTTCTTCTTTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 374896 . TTGATTTTTTTTGAAGGTTTTTTTGTGTCTCTATCTCCTTCAGTTCTGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 375005 . CTTCTAATTGAGATGTTAGGGTGTCAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 375187 . ACCCAGTAGTTGTTCAGGAGCAGGTTGTTCAGTTTACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 375235 . TGGGTTTGAGTCAGTTTCTTAATCCTGAGTTCTAATTTAATTGCACTGCGATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 375306 . CCCATTTTTTTTTGCATTTGCTGAGGAGTGTTTTACTTCCAAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 375371 . GTGCAATGTGGTGCTGAGAAGAATGTATATTCTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 375471 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 375494 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 375516 . TTGACAGTGGGATGTTAAAGTCTCCCACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 375590 . TACTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 375613 . TGTATTGGGTGCATATATATTTAGGATAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 375663 . ATCCTTTTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 375712 . TGGTTTAAAGTCTGTTTTATCAGAGACTAGGATTGCGACCCCTGCTTTTTTTTGCCTTCCATTTGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 375789 . TCCTCCATCCCTTTATTTTGAGCCTATTTGTGTCTTTGCACGTGAGATGGGTCTCCTGAATACAGAACACTGATGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 375876 . TTAGCCAATTTGCCAGTCTGTGTTTTTTAATTGGAGCATTTAGCCCATTTACATTTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 376001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=378000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.58:1.23:.:0:.:0:0.999:211
+1 376021 . GTCAATGGTCTTTACAATTTGGTATGTTTTTGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 376134 . ATCTCTCAGCATTTGCTTGTCTGTAAAGATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 376174 . CTTCACTTATGAAGCTTAGTTTGGCTGGATATGAAATCCTGGGTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 376253 . CCCCCATTCTCTTCTGGCTTGTAGAATTTCTGCTGATAGATCTGCTGTTAGTCTGATGGGCTTCCCTTTGTGGGTAACCTGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 376356 . AACATTTTTTCCTTCATTTCAACCTTGCTCAATCTGATGACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 376435 . GTGATGTTCTCCGTATTTCTGAATTTGAATATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 376502 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 376511 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 376519 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 376535 . CCATTCTCCCCATCACTTTCAGGTACAGCAATCAAACGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 376606 . TTGGAGGCTTTGTTCATTTCTTTTCATTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 376702 . TCCGCTTGATCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 376924 . TCTCTCAATTCATCAAACTCATTCTCCGTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 376984 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 377094 . TGGTAACCTTCTGATGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 377153 . CCTTTTTGTTTGTTAGTTTTCCTTCTAACAGTCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 377209 . TGGTGTTTGCCCTAGGTCTACTCTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 377248 . GGGCATCACCAGCGGAGGCTGGAGAACAGCAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 377292 . GTTTCTTCCTCTGGAAGTTTTGTCCAAGAGGGGCACCCACCAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 377362 . GTGCCTGTTGGCACCTACTGGGAGGTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 377411 . GGTTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 377458 . AATACTGTGCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 377489 . CAGAGCCATCAGGCTTTTCAAAGATGCTTTAAGTCTGCTGAAGCTGTGCCCACAGGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 377601 . CTGACTGGGGCTGCTGCCCTTTTTTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 377658 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 377735 . TACACTGTGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 377784 . CCCTCCCGCCACCAAGCTCGAGTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 377826 . CTGCTGTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 377856 . CTTAGCTTGCTGGGCTCTGTATGGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 377971 . CCATTGTGGCATGAAAAAAAAAAAAAAACTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 378001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=380000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:1.04:.:0:.:0:0.999:211
+1 378072 . GAGGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 378112 . ACCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 378218 . TCCCGGGTGAGGCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 378261 . GTGCGCTGCACCCACTGTCCAACCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 378318 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 378409 . GAGATTTTTTTTAAAAGTGCAAAGAAAGACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 378493 . GCCCCTGTTGGATTCAGCAGAGGGAGATAGGCCTTGCCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 378572 . GGGAGTGCTGATACCTGGGCCACAGTTAGTCCAAGTTTATCACTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 378673 . TGTGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 378765 . CACCGGCCTCCGAGAGTCCATGGAGCAATGGGAAAATTGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 378864 . GGGGACTGCAGTCGACAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 378920 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 379003 . GGCCGCAATTCAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 379174 . CAGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 379201 . GCTGGACTTGAGGGCTGGCTTGGCTGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 379235 . GACACAGCAGCAGCTCAGGATGATGGTGATGGTCCACGCGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 379292 . TTCATAGTAGTAGTTACAACACTGAGACTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 379473 . TCCTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 379575 . ATACGTTTTGAAGTCAGATTGTGAGGCCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 379628 . AGTGCTTTAGTTATTCAGGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 379721 . AATGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 379795 . ACATAAAATATCTTTCCATGTATTTGTGTCATCTACAATTTTTCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 379935 . TACAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 380000 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 380001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=382000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.61:1.14:.:0:.:0:0.999:211
+1 380025 . GTATATTGATTTTGAATTCTGCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 380089 . GGAGTTTTTAGGGTTTCCTATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 380159 . TGAATATCTTTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 380255 . GTTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 380358 . CTACGCCCAATTTGTTGAGAGGTTTTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 380421 . CTGCATATATAGAGATAGCTATTTTTTTATCCTTCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 380544 . AATATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 380559 . TCTGTTTGCTAGTACTTATTTTGAGGACTTTTGTATCTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 380609 . GATATTGGTTGGCCCATACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 380655 . T <CGA_NOCALL> . . END=380854;NS=1;AN=0 GT:PS ./.:.
+1 380867 . C <CGA_NOCALL> . . END=381153;NS=1;AN=0 GT:PS ./.:.
+1 381166 . TGTGGAAGTCAGTGTGGCGATTTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 381207 . GAAATACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 381224 . AGCCATCCCATTACTGGGTATATACCCAAAGGATTATAAATCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 381297 . TTTATTGCAGCACTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 381369 . AAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 381376 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 381379 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 381425 . GTTCATGTCCTTTGTAGGGACATGGATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 381473 . CAGCAAACTATCACAAGGACAAAAAACCAAACAGTGCATGTTCTCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 381543 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 381563 . GGGGAACATCACTTTAAAAAAAAAACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 381619 . AAATACTCTTTCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 381670 . ATTATTTTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 381904 . TCTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 381973 . CTCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 382001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=384000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:1.05:.:0:.:0:0.999:211
+1 382017 . TTTTCAAAGAACAAATTCTTAGTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 382069 . GTCTTTTTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 382119 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 382247 . ATTTTTTTGCTGTGTTTTCTTAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 382294 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 382348 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 382375 . TCAGGGTAATGCTGGTTTTGAAAAATGAATTTGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 382489 . CACTGAGCCATTCAATCCTGGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 382557 . TCACGATTTGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 382617 . TTCTTCTAGGTTATCTAATTTGCTGGTGAATAATTAATTATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 382692 . TCAGTTTTAATGTCTCTTCTTTCATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 382729 . TTTCTTTTTTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 382771 . TATCTTTTCAAAAAACAATTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 382941 . TTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 382973 . AGT . . . NS=1;AN=0 GT:PS ./.:.
+1 382978 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 382981 . TCT . . . NS=1;AN=0 GT:PS ./.:.
+1 383078 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 383101 . ATATAGGTATATATAAAAACAATACACACATTGTATATAAACTATGGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 383167 . GAACATAAAGTGTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 383361 . AAGATAAACAGAATAACTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 383425 . CAAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 383455 . TTTCTTTTTTTTTTAATTTGTGTAGCTTTTTATTTTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 383526 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 383534 . TGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 383566 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 383606 . ACCTAAAAATCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 383625 . GGGAGCGTTTCACTGAAGACAGTTGTTAGAGAAACGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 383687 . GTCCCTTCTTCTCTGCTTTCTTCTTTTCTCCTCCTCCTCCTCCTCCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 383744 . TTCTTCTCTCTCTGTTTTTCTAATCATGAAAACAAACGAAAAAAACTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 383817 . AGCGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 383849 . ACAGAAAATAAAATAACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 383888 . AAACAAAAAATTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 383971 . CATAGAGAATAACATAATTGAAATTTAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 384001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=386000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.10:0.96:.:0:.:0:0.999:211
+1 384043 . GTTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 384086 . CTATCCAAAATGAACACAGGAATAAGGAAATGGAAAATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 384153 . ACAGAGGGGGAAATGCTAAAAAGTTTTAAAGAGTGTTTCAGAAGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 384231 . TTTTAAATTTTATTTTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 384320 . CCATCAACCCATCACCTAGGTATGAGGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 384390 . T <CGA_NOCALL> . . END=386531;NS=1;AN=0 GT:PS ./.:.
+1 386001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=388000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.25:0.99:.:0:.:0:0.999:211
+1 386537 . CCTACCCCCTGTCCCCCTGAGAGGCCCTGGTGTGTGTTGTTCCCCTCCATGTACCCACGTGTTTGTCTTGATGGTCTCCTACCCCCTGTCCCCCTGAGAGGCCCTGGTGTGTGTTGTTCCCCTCCATGTATCCACGTGTTTGTCTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 386691 . CCTACCCCTGTCCCGCTGAGAGGCCCTGGTATGTGTTGTTCCCCTCCATGTATCCATGTGTTTGCTCTCATTGTTCAACTCCCTCTTACGACTGAGAACATGTGGTGTTTGGTTTTCTGTTCCTGTGTTAGTTTGCTGAGGGTGATGGCTTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 386855 . T <CGA_NOCALL> . . END=387191;NS=1;AN=0 GT:PS ./.:.
+1 387210 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 387270 . CAGCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 387322 . CACTATACCTGGCTAATTTTTTTATTTTTAGTAGAGATGGAGTTTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 387398 . TGTCCTCAGGTGATCCACCCGCCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 387492 . AGAATGATTTATATTCCTTTGGGCATATATCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 387576 . ATTTTCACACTGCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 387628 . GTGTAAAAACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 387728 . GTGGTTTTGAGTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 387803 . CTTCTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 387845 . TGATGTTTTTTTTTTTTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 387922 . TGTGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 388001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=390000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:1.15:.:0:.:0:0.999:211
+1 388004 . GGTACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 388202 . CCCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 388294 . TTTTCTGTTGCCCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 388352 . GGTGTTTTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 388437 . TGCAGGCATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 388461 . AAAGAAAAAAAAAAGCTGACTTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 388671 . TACAGAGGGAGGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 388809 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 388873 . GGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 388938 . TTTTGCAAATTGTTGTTTGTGGGGGTCTGTCCTGCAGACCCCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 389051 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 389129 . CAGGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 389183 . GAGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 389297 . ACTCAAGGAAGGATTAGGCCGCCTTCATTCAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 389349 . AACACCCCCAGCCTTCCATGAAGGTTTGTGTCGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 389417 . GCCTGACTGAACTCCCACAGTTGTTGCTACTTTTTGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 389577 . TGGAAAACTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 389690 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 389729 . ATCTTATATATAGAAAACACCAATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 389783 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 389900 . CAACAAAAAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 389960 . CTACAAAACATTGATGAAAACAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 390001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=392000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.99:.:0:.:0:0.999:211
+1 390009 . GATATATCATGTTCATGGATTGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 390052 . TGTCCATACTATCCAAAGTGATGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 390110 . GACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 390176 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 390379 . ACATGGGGAAAGAACAGTCTCTTCAAAAAATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 390465 . ATATAAAAATATAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 390526 . CCTGGAAGAAAACAGGGGAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 390654 . TGCACAGCAAAAGAAAGTCAACAGAGTGAAGTGATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 390758 . AGGAACTCAAACAACACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 390799 . AATGAAAAATTGACAAAGGATCTAAATAGACATTTCTCAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 391030 . AAACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 391060 . TAAATGATTCTGTAAAAATAGAATTACCATATGATTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 391130 . TATCAAATCAGTGTGTCAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 391246 . AAATGTGGTACACATACACAATAGAATAGTATACAACCTTAAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 391466 . GATGTTGGTCAAAGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 391606 . TTTATGGGGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 391657 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 391717 . CCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 391811 . TTTCTTTTTTTGTACCCATTAACCATCCCCCACCTCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 391961 . TGTCTTTTTGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 392001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=394000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.07:.:0:.:0:0.999:211
+1 392041 . TTCCTTTTTATGGCTGAATAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 392091 . CTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 392206 . TTTGGGGGGTATATATGCAGCAGTGGGATTGCTGGACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 392254 . TCTATTTTTACTTTTTTGAGGAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 392293 . CATAGGGGATGTACTTGTATTCTTCAAAATTGCTAAGAGTAGATTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 392350 . CAT . . . NS=1;AN=0 GT:PS ./.:.
+1 392395 . CATGGCTATTTTACAATGAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 392427 . A <CGA_NOCALL> . . END=392699;NS=1;AN=0 GT:PS ./.:.
+1 392734 . ACATAACACTTTCCAACTTGTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 392831 . ACATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 392862 . ATATAATCCAACAATATACTCCAAAGAAAAATACATCATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 392948 . CAAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 393008 . CTGTGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 393042 . TTTGTAATAATAACAAACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 393099 . ACACAAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 393141 . TAAACAGTTTCATTCTAACATGAGGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 393203 . CATTAAAATAATACTGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 393286 . TCAACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 393332 . CTAACAACACATCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 393380 . TCCAGGGGTGCTTGAATGGTTCAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 393562 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 393617 . GTTAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 393677 . ATTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 393797 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 393883 . TGTATTTCTATACACCAAATACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 394001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=396000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:1.08:.:0:.:0:0.999:211
+1 394177 . TTACCAACGTTATTTCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 394229 . AGCCAAAAAAAGATCCCTAATAGCCAAAGCACTTCTAAGCAAAAAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 394314 . ATATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 394389 . AA . . . NS=1;AN=0 GT:PS ./.:.
+1 394454 . ACCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 394582 . GTCCGACATATTCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 394618 . ATACGAATGGATGAATTACACCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 394794 . TATTAAAAAACACATACAATGCAATGGAAGGACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 394880 . AATGGGAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 394944 . TTTATTTTTCTGAGACATGGTGGCCCAGGCTGGAGTACGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 395131 . TCTCTTTTTTATTATTCCTTGAACACTAAGTATATGTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 395229 . CCCCATCCCCTTTTTACTTTTCGTCATGTCATATAGTTTCTGAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 395303 . TCAAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 395352 . TTCATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 395388 . CACCAGGTCATGTTAATTTTACTGTCACGCACATATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 395472 . GATAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 395560 . GCTAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 395597 . AACAAAAAACCAAACACCGCATGTTTTCACTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 395644 . ACAATGAGAACACTTGGACACAGGAAGGGGAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 395714 . GGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 395777 . ACACGGCACATGTATACATATGTAACCTGCACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 395830 . ACTTAAAGTATAATAAAAATAAATATTAAAAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 395895 . TCCTACT . . . NS=1;AN=0 GT:PS ./.:.
+1 395911 . GCTATAAATTTCTTATAATTTAATAAATATTTTTCCATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 395982 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 395996 . ATTATCAGGCAGCAGTACAAAGCTGCTTACACCTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 396001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=398000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.77:0.98:.:0:.:0:0.999:211
+1 396043 . TTTATTTTAGGCCTCCTTGCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 396089 . TGAAAAACTTACACATTTCTTCATTTGGTCAATCCTTGCACACTTTCAAGATTCATTTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 396163 . TGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 396169 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 396182 . CTTGTATTATTAATTTATCTTAGTTTTCACTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 396262 . GACAGTAAGCTTTTGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 396375 . TGGAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 396486 . AACCTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 396555 . CTGCCTCTCTCTCTTTCTATATATATAACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 396668 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 396709 . AAATAAAAATGAATCATTAATACACCACTTGGCACAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 396794 . GGGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 396862 . TATCTTTGCTCTGAATTTGTTTTTTTTCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 396932 . TGATATTTTATTTAAAAACTGTTACTATAGAAAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 397014 . CTGTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 397060 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 397093 . CCTTGATTTTTGTGGCTATCATTTTATTCATTCCATTGCTTTTTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 397179 . AACGATAGATTGTTTTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 397215 . GTGCCTGTGATGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 397275 . TTTAAAAACATAAATTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 397327 . TAAAGTATGATAAAAATATACATATATAACAAATTAAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 397398 . GCTATTTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 397490 . GTAAATGTGATTTACTGATGAAGATGCCACCCCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 397546 . CCTGCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 397948 . TATTTATTTTTAAGTTGAAAAACAAAACACACACACACAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 398001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=400000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:0.93:.:0:.:0:0.999:211
+1 398095 . CTCCTCAACTGCAGCCACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 398126 . AGTGACGATAAACATGACTGTTCCAGTAGAGGCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 398221 . TGCCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 398322 . GGGATTTTTGTCACTTGGTGTCCACACTGTGAATTTTGCTGGTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 398406 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 398436 . TTTGACTAGGACACGTGCTTGATTCTATTTTTGGTAAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 398575 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 398577 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 398583 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 398612 . GTACATGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 398627 . GTATAGACACACGTGTGTCTATGCATATAATATATAATAAGTACATATTATATATTTTTTAAGAGAAGGAAAGAGCATGTGAGAGAGCACATTGCTTATTTATGTTGATAATACTGATTCTAATGTAACTCAGTGGGCTGCTTCTTTCCTTGTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 398791 . TCTGTATCTCTGTCCTTTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 398826 . GTTCTAGCAATACAAATATATTAATATTTACTCATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 398871 . TTACAATGCACATACATTTGTTTCAGAATTGTTATATTTATACTAATATAAAAAGAAAAAATCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 398941 . GAAGATTTAATAATTTTTTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 399024 . TAAGAGAACAGTCAAAAATTAATCAGATTAATTATTATTTTCCCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 399177 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 399187 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 399280 . ATTTCCCTCTAATTCTTTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 399394 . ATCCACAGCAATGTTTTATTTCCATGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 399492 . GTGAGGTAGGGGTCCAGCTTCATTTTTTTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 399627 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 399668 . TTTTAAAAGAACCAACTTTTAGTTTTGTTGATCTCTGTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 399744 . AATGTTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 399831 . ACATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 399904 . TATCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 400001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=402000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:1.11:.:0:.:0:0.999:353
+1 400088 . AATCTTTTTAAGTTTATTGAGCTTATCTTATGGCCTAACATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 400203 . TCTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 400363 . TATTTTGGAGCCCTGTTAGTTGGTGTACATATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 400415 . CTTGTTTAATTTTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 400570 . ATTGTGTATTTTGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 400624 . TTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 400725 . TTGCTTTTGTTTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 400878 . TACCATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 400911 . TGGAACAATTTAGAATAAATTGAAACAACTTCAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 401021 . ATACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 401072 . TTTTAAAACATATAGAATATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 401110 . AATATGATAATTGCTTTTATATTTACCTATGTAGTTATCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 401190 . TAGTGTTTTTTAGTTTGTATCTGAAGGAGTCTTTTTTTTTAAAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 401271 . T <CGA_NOCALL> . . END=401471;NS=1;AN=0 GT:PS ./.:.
+1 401519 . TTACAGAATTTTTGGTTGACAGTGTTTTTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 401565 . AATGTCATTCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 401599 . TATAATGAGCTCTCTGCTGTCAAACTTACTGAGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 401676 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 401686 . ATT . . . NS=1;AN=0 GT:PS ./.:.
+1 401692 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 401695 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 401709 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 401737 . CGCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 401780 . TTTGACAATTTGAAACGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 401829 . TTCCTTGATGAAGTGCACTGAGCTTTTTGAATGTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 401896 . TCAAACACATTTCTTCAAATAGTCTTCCTGCCCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 401962 . TCTTGATGTTGTTACACAGGTCTTTAGGCTCTATTCTTTTTTGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 402001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=404000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.98:.:0:.:0:0.999:353
+1 402046 . TCACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 402110 . AAGTAAAAAGTAACTTTTTAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 402176 . AAAAATTCATATTTCTTATTTATTTATATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 402242 . TTAAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 402369 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 402406 . CTGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 402455 . TGGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 402581 . ATTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 402613 . CAAAAAAGAAAAAAATATGAAAAACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 402687 . TAGTGGAGCCATATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 402802 . CCTTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 402926 . TCAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 402975 . TGGATGATCTGATTTCTAAAGGTTTGGTGGAATTCTCTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 403033 . TGTTTGTGTGTGTGTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 403103 . CTTCTTTTTTTTTTTTTTTTTTGAAGTCAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 403284 . TCTCTTTTTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 403389 . CTTATTTTATTAATTTTCACTTTTTTCTACTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 403451 . TTTTAGATACTTTTTACATTTTTCAGCTGATAATTTAATTCCTACACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 403699 . TTTTGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 403763 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 403770 . ATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 403922 . AGTGTCTGTCTCTTCCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 404001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=406000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:1.01:.:0:.:0:0.999:353
+1 404027 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 404113 . TTTCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 404156 . TCTCGTGCGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 404243 . CTGCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 404285 . GATCAGACCCTGAGACTGCGTTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 404333 . TCTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 404406 . CAATATTCCCTAGGAAGCTGACCTCTGCTGAATGCAACACTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 404470 . TAGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 404529 . TACTTAACCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 404567 . TCTATCCATGGCCTCAGTTCCTGTTGGGGAGCCTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 404628 . TTTCTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 404658 . TTGCGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 404727 . AGGCTGAATAATTTTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 404757 . ATTTGGTTCACAGTTCTGAAGGTGTGCAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 404799 . ACCATTTGCTTCTGGTGAGGGCTTTAGTCTGTTTCCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 404872 . GAGATCACGTGGCAAGAGAGAGGGGTTTGTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 404932 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 404968 . GCTTACACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 404988 . CACTTTGGGAGGCCGAGGCAGGTGGATCACCTGAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 405052 . ATGGTGAAACCCCGTCTCTACTAAAAATACCAAAAATTAGCTGGGCATAGTGGTGGGTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 405328 . AGGCCCCACCTCCAACAATGGGGATCACATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 405391 . ACCCTAGCAGTTCCCTTAACCCTGGAAAGAGACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 405479 . GACCCTGACGGATAGAGGGACCATACAGATCACTAAAATGCTGAGGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 405576 . AAACTTTTATTTTTTATTTTATTTTATTTTATTTTTTGAGACGGAATCTCGCCCTGTCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 405669 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 405704 . CCTGCCTTAGCCTCCCGAGTAGCTGGGACTACAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 405748 . GCAGGCCCGGCTAATTTTTTATTTTTAGTAGAGATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 405800 . ACCGGGCTGGTCTTCAACTCCTGACTTCATGATCCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 405873 . TGAGCCGCTGCACCCAGCCAAACTTAAAAAAAAAAACCCAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 405947 . TCAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 406001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=408000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:1.03:.:0:.:0:0.999:353
+1 406006 . TTAATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 406089 . AGAGGCAGTATAGTATCATGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 406140 . TAGGTTTGAATTACTAGCCACGAGGCTTTGGGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 406199 . ATTTCTTTATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 406219 . TAGATACCTTACCTTTATCTATACCTTATAGATACCTTTATCTATAAGGTATAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 406288 . ACCTGAAAGTTTTGGCATGAGTTTAGTAAAACTGTCTGTGAAGCCCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 406367 . AAATGGTGGCTTTATATAGAGTAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 406405 . TCTCAAAAAGAAATCAGGGAAATAAGAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 406481 . TTGAGTGTGGGGTGAGGAGTAGGGGAGGGGAGGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 406547 . GGAGAAAAATGATGTCACTGGGAACTGCAGTCATTTGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 406681 . TCCCACCTCCTCTCCAACCATCTCTTCCCTTCCTTAATTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 406742 . AAGGCATAGTGCTTTGATTTATAAATTAGTTCTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 406908 . AGAGATTCTCCTGCCTCAGCCTCCCAAGTAGCTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 406971 . CTAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 406993 . AGACAAGATTTCACCATGTTGGCCAGGCTGGTCTGGAACTCCTGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 407123 . TAACACATTATTTCCACTTTCCTAAGGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 407175 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 407235 . GTGATTTTTTTTTTTTTTTTTGAGATAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 407358 . CTCTCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 407406 . AAAATTTTTTTTTGGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 407462 . TGCGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 407497 . GGCATCTCAAAATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 407529 . AATATTTTTATTGTCACTATTTTCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 407564 . GAAATTTTATTTGGATTTCTTTTTTTTTTTTTTTTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 407648 . AGCTCACTGCAACCTCCACCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 407687 . TCCTGCCTCAGCCTCCTAAGTAGCTGGGATTACAGGCATGCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 407750 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 407753 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 407775 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 407812 . CAAGTGACCTGCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 407883 . TTTCTTTTTGACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 407920 . TAAATTTTCATGCTGTAATTTCTAGTTTTGTTGTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 408001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=410000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:0.91:.:0:.:0:0.999:353
+1 408008 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 408073 . TCAATGTTTACTTTTTCAGGCTATAGGCTTTGCTACATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 408124 . TTTTTGGTCCTCATATAGATTTTTTAATTACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 408178 . AAGGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 408204 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 408207 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 408223 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 408271 . AAATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 408393 . GAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 408425 . TTGTGTTGCTGAGAACTGCTCAGTAACACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 408487 . TTCAGTGTCTTGAGTATTGTGAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 408600 . ATGCAGGTCTACTGTCCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 408713 . GAGGTATATTTTTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 408820 . GGAGAGTCTGTACTAGGCATGGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 408858 . GCCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 408993 . CAACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 409063 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 409097 . TCTGACATCATTATACACTTTCCAATGAAAGCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 409164 . CACTCCTGAAAAATCTCCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 409193 . ACAGTGACTTATTAACCAACACTCATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 409482 . GGGCTAAATGACATTGTTCCTACAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 409527 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 409573 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 409580 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 409584 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 409589 . AAC . . . NS=1;AN=0 GT:PS ./.:.
+1 409604 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 409655 . TACTAAAAATATAAAAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 409723 . TGAGACAGGAGAATCACTTGAACCTGGGAGGCAGAAGTTGCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 409798 . AGGCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 409812 . GACTCTGTCCAAAAAAAAAAAAGAAGAAAAAAAGAGAAAGGAAAAAAAAAGGAAAAATAAATAAATAAGTAAATAAATAAATACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 409906 . CCCTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 409971 . TTTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 410001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=412000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.77:1.04:.:0:.:0:0.999:353
+1 410012 . TCTTTCTCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 410045 . CACTACCACATGCACACACACAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 410093 . TTTCTAAAAACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 410149 . TAATCAGGCAACTCTGGTTTCTATCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 410208 . AGGGCATCCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 410259 . CACATTAGTTAGTCAGTCAGTCAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 410306 . AGCCGAAGTGAACACCAACAGAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 410393 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 410417 . CTAATGAAAGTGGCTAGAAAAATATGGGTGCCCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 410471 . TGTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 410586 . CTGGCAGGCTAAGGGTTCTGAGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 410872 . GCTCAACATGGAGCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 410928 . GGCTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 410965 . ATTCTTAAAGCTTCTGCTGGAAGTGACATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 411028 . CACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 411157 . GGTATAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 411208 . AAATCACAGTAAATTTCCTTTAAAAGATCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 411270 . GAGTCCTTAGTAATGGACTCCATCTCTTCCATCAGATAAAATGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 411330 . AATTTGAATAATGAAACCAAAGGAAAAAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 411390 . GAACAACTGTGGCATCAGCATAATTCAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 411426 . TGTATTAAATATTTTGCAGAAAAGTGAAAACAAATTGATAGCCAAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 411490 . ACCATGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 411527 . AAACACTAGTTTAGTTATATAAACATGGCTGATGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 411616 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 411731 . AAATGTGGCACTAGTATTTATTACAGCTCACCTTTTTATAATGAAGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 411792 . ATTCTTATTATTTCCCATTTTCTTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 411935 . TTTTATTAGGTATTAGTCTACCAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 412000 . TTCCACCCAGTTATCCTGCTGGAACCTGTAACAGTTACTGTAATGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 412001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=414000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:0.92:.:0:.:0:0.999:353
+1 412124 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 412179 . CAGTAGCTAGAGAAGTGCTGGAATGCCCCTGTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 412236 . AGAATGGCTGCTTGAACGAATTTACTGCTCAACTCGAAAGGCCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 412390 . CACCCATTCATTATTATCTAATGAGGAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 412497 . CAGGTGAGCAGAGGGCCCCACCTTCGGAGGTTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 412607 . TATAAAGATAAGATCATTGAGAGATTTTTTTCCTCCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 412797 . CTGCCCTACCCCTAGCCTGTGTGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 412841 . TGGCAGAGGAGTCCCAGGGGAAGGATGCATGATCTTCCACTGTGCCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 412920 . AGCAGGTCTCCCTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 412979 . GTTCCCTCCTTGTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 413127 . CTCAGGAAAGCTCAGAGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 413162 . TGATCCATTAGTTAAAACCACGCTGGGTTCTTTATAGTGGTTAGTTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 413285 . CGATAAAATACATGGTGCTTTTCAGAAATTGCCCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 413358 . TCTGATGGATAGAGATCTAGGCCTGACACTCCAAGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 413423 . TTCTTACATCTTAACCCTCAGGAATTTTAAATAGAAGTGTTCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 413543 . CACAGCGTGTTGGGAATGGTGATTATAAATGTAACCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 413592 . GTAAGTGGAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 413708 . TTTTGTTTTGCCCTATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 413867 . TGGTCGGGCACGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 413925 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 413945 . GGAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 413991 . ATACAAAAAAAAAAAAAAATTAGCCAGGCGTGGTGGTAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 414001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=416000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.82:1.07:.:0:.:0:0.999:353
+1 414068 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 414108 . TAGATCGCGCCACTGCACTCCAGCCTGGGCAACAGAGCAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 414155 . ACTCAAAAAAAAAAAAAAAAAAAGAAAAGAAAAGGAGCCTCTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 414239 . TGCCAAAAAAAATATGAAGGAAGCAATGTGATGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 414393 . ATCAAGGCTCCCATCAACAGCCAGTCCTGTGAGTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 414464 . TGCTGAACACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 414503 . AATCATGTGGTCAGTTTGCAGGGTGGTTATTACACAGCAGTAGATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 414611 . CGGTGAGCAGAATATGGATGTGGGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 414663 . AAGCACTGAACAGAGCACAAAGACCTGATGTTCCAGGGTCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 414722 . TAACAGCGGCCATAGGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 414752 . AGAGTAACGTGGTCAGATTTTCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 414790 . CTGACATCCATGTGGAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 414860 . GATCGTAGTAATCTACTTAAGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 414899 . GAATGGGGAGGTGTTAGAGAAGAGAAAATGGATTTGAAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 414988 . CCTGTAATCCCAGCACTTTGGGAGGCCAAGGTGGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 415167 . AACCCAGGCAGTGGAGGTTGCAGTGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 415233 . AAGACTCCGTCTAAAAAAAAAAAACAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 415286 . ATTGGGTGAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 415463 . TAAAGAGGTCAGAAGTGAACTTCAATAAAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 415551 . GGAACACACCTATGAGTCAGAAAGCCAGACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 415736 . TTCATTAAGCCAATAGACAACCATGGCATTTTAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 415795 . CTGATTTTTTAATTAGCAGAAGCAAATAAAGAGAGCTTCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 415864 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 415919 . CAACATAATGGTTAATTTTCTGAGAAGTAAGTTCATGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 416001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=418000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:0.99:.:0:.:0:0.999:353
+1 416026 . TTGCTTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 416066 . ACACATAATAAAGAATTGTGTTAGTGCCAGAGAGACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 416139 . CTTCTTTCTGTCCCTCCACCCCACCAGCTCTGATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 416208 . T G . . NS=1;AN=2;AC=1;CGA_RPT=MIR|MIR|38.2;CGA_SDO=14 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:26:26,26:26,26:6,25:-26,0,-26:-6,0,-25:25:1,24:24
+1 416276 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 416292 . TAGGTACTCTTCTTCTTTTTTTTTTTTTTTTTAAATTTTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 416560 . ATCGAATAAAGAGCTGATCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 416697 . GCCGCGCACCCAGCAGTCGGCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 416932 . TTTGCTCTTACTTTCAACATTTCTGCTGGGGCCTTGCATTGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 417055 . TTATTTATTATTTATTTATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 417366 . ATCTTGTAGGAAAGACAAAGGTAGAGAATCTGTCTGATGGCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 417461 . GCTATGAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 417516 . CTGTAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 417832 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 417914 . TTACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 417965 . AATTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 418001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=420000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:1.01:.:0:.:0:0.999:353
+1 418066 . CATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 418284 . GAAGTACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 418476 . GCTTAAAACAATAGTCAAGAAAATTAGAGCCACAAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 418610 . TAAAGTATAATTAAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 418788 . AACACTCACAACTGTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 418819 . CTTCCTGAGGGACCTGGAGGAAGTCACACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 418879 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 418932 . GGTTTTTGTTTGTTTGTTTGAGACAGAGTCTCACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 418974 . CCAGGCTGGAGTGCAGTGGCACGATCTCAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 419024 . CCACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 419086 . CCATGCCCAGATAATTTTTGTATTTTTAGTAGAGATGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 419135 . GTTTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 419170 . TTATCCACCCACCTTGGCCTCCCAAAGCGCTGAGATTACAGGTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 419336 . GTATTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 419383 . CCCTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 419638 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 419789 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 420001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=422000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:1.01:.:0:.:0:0.999:353
+1 420131 . GCCCTGTATCAGGGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 420165 . AAACAAAAAAGTCCCACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 420367 . ATACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 420436 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 420500 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 420555 . GGA . . . NS=1;AN=0 GT:PS ./.:.
+1 420634 . AAGGACACCTAGGGTTAGTCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 420759 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 420773 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 420974 . TTTGTGTCTACCTCAAAGGTACGTTGCAAGGATCGAGGGACAGAGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 421087 . TACCAGAGAATGTGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 421126 . TTAGAAAAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 421171 . CATAGATGGATGATCAGAATTTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 421208 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 421225 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 421325 . AGCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 421459 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 421580 . TCTGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 421737 . AGCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 421860 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 421958 . ATCCTGCCTTCAAGGAATCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 422001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=424000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:1.00:.:0:.:0:0.999:353
+1 422026 . AAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 422112 . AGCCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 422228 . TTGCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 422310 . AGCACTTTGGGAGGCCGAGGTGGGCAGACCACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 422362 . AGCCTGGCCAACATGGTGAAACCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 422392 . TACTACACACACACACACACACACACACACACACACACACACACCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 422447 . GCTACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 422557 . TCTCAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAAAGATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 422675 . AGTGAGACTCTGTCTCAAAAAAAAAAAAAAAAAATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 422757 . CATATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 422980 . AGGCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 423120 . AGCGCCGTGTACCTGATACGCTACCCTGGGCACAGGCGATCAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 423254 . TTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 423366 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 423370 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 423382 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 423440 . CTATTAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 423482 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 423500 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 423529 . ACCAGGGAGGTGGAGGTTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 423639 . AAAATAGGATGACTGCACGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 423811 . CAAATGATTTTTGTCCTGTAAAAAGATTTTATTGCTCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 424001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=426000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.74:1.08:.:0:.:0:0.999:353
+1 424221 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4951897&dbsnp.131|rs78102372;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:39,64:39,64:36,28:-64,-39,0:-36,-28,0:1:1,1:0
+1 424324 . TGGGAAAACAGGAGAAAGTGTTGTTGGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 424455 . GAAATGATAATATCAGTAGTAGCAATACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 424621 . CGTATTTTTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 424674 . CAAATCCACTGTCTATGTCCCTCCCTCCCTCCCTCCCTCCCTCCCTTCCTTCCTTCCTTCCTTCCTTGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 424775 . TTTCTGCCTCCCTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 425118 . GAAGTGGACATGGTGGCAGACAAAGCTGCTGGAAGCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 425186 . CACATTCCAAAGGCAGTGTGTGTGTGATGTTGCATCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 425380 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 425506 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 425525 . GCTCTACACACACACACACACACACACACACGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 425713 . TTGAGTGTGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 426001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=428000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.89:0.86:.:0:.:0:0.999:353
+1 426037 . CTTACAGTGGGTCTCCTGATGAAGCTGAGGTCATGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 426133 . AAACTTGGCTGCACATTGGAATTCCTAGGGAGAGTTTTAAACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 426351 . GAAGTCCATCCTCACAGCGGTCAGGCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 426403 . GGCATATGCCTAAGTTTCTGTGCAATGAATGCATGCCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 426773 . AACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 426894 . GGTGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 427077 . GATGGAGAGAAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 427200 . AGGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 427315 . GAGCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 427342 . CCACGACAGTGGGGAGTGAGGTGAGGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 427390 . ACTCACCCAGCACATTCCCACCTCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 427432 . AGGGACAAACTGACCAGAAGGCTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 427492 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 427612 . TTCCCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 427855 . AGCAGTTTTTTTTTTTTTTAGAGCTGGTGTCTTGCTCTGTTGCCCAAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 428001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=430000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:1.16:.:0:.:0:0.999:353
+1 428247 . TATAGGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 428502 . TCTTCAAAACCAGTAGGAGGATTTCTCTTGCACTGAATCTCTTTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 428830 . GGCCTTGCATTTTATTTCATTACCAGTGTCTATTGGCTCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 428944 . TCACGGTTGTTAGGAGGATTAAAAGTGTATAAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 429049 . ACAGAGCATTTTAAGTGAAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 429134 . TGCTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 429200 . TTTACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 429359 . AACTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 429439 . ATCAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 429509 . TCCTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 429543 . GCACCTGCTGGACCTGTGGCAAGATCATCAGGGATTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 429609 . ACCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 429702 . GACTCTTTCTTATTCTGGGCCTCTGCTGTTATCTGGGTCTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 429796 . TCTCGGCAGGCTGAATGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 429869 . ACACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 429941 . CTATTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 430001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=432000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:1.02:.:0:.:0:0.999:353
+1 430049 . GGGTGGGGTGACTTCCAACCACACTGGGAGCCATCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 430109 . TCCGCAGCACAGATGGGTGCAACCCAATGTCAGGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 430184 . GTAGACGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 430251 . CACGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 430306 . GGGGTACAGCTTGGTTTTATGCATTTTAGAAAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 430416 . GGGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 430523 . TTTTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 430567 . AATGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 430622 . CACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 430734 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 430943 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 431221 . AGATAAAAAAAAAAATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 431346 . AATGGCCTTGCCTTCCTGTGCAGTCAGATGGAGCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 431400 . CCTGCTGAGTAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 431777 . TGTATGTCACAGATCACCAGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 431986 . ATATTGGTTAAAAACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 432001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=434000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:0.88:.:0:.:0:0.999:353
+1 432024 . ACTGAAGGATGAGAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 432063 . CTACGCTGTTGTGAAGTTCCCCACAGGGATCTCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 432132 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 432193 . GGCTCTGGGGTTGGGCCCTTCTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 432325 . TTGTAAAAGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 432404 . CTCTGCCTCCTCACTTGGCTACATTTTTCTCTCACTTGTGCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 432496 . GTACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 432741 . GGGATGTGGGGGGAGGGCGTGTGTGTGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 432847 . GCAGGTCATTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 432885 . ACAGTCTAGCCTTTTAAGAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 432936 . ACGTGGGGATATGTGGGGGCGGCCATGTTGCCAGCCACCTGTTGGGGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 433036 . TGCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 433183 . ACTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 433257 . TGCTGAGTTACCCCAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 433320 . GAGATTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 433430 . ACTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 433523 . CCCAGAAGCAAAGCAGAAATTTCTTTTGAGATCCAGGGTGGGAAATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 433649 . AGTCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 433798 . CATTCCCCACCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 433938 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 434001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=436000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:0.99:.:0:.:0:0.999:353
+1 434150 . CTGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 434203 . TTTTGTTGTTGTTATTTCAGATGGAGTTTCACTCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 434312 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 434478 . AGGCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 434696 . ACTCCTCATTGACAGCAGATGGGATTTTTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 435059 . AGCGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 435283 . TTATAAAAAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 435563 . TTTGTTTTCTTGAACCCATTATATGAATAATTTTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 435803 . GTAATTTTTTTTTTCTTTTTTGAGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 435947 . CAGGTGCCTGCCACCATGCCTGGCTAATTTTTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 436001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=438000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.62:0.97:.:0:.:0:0.999:353
+1 436189 . GATATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 436231 . TCTCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 436272 . CTTTACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 436333 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 436350 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 436449 . TCCTCCCACCTCAGCCTCCGGAGTAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 436512 . TTTTAAAATTTTTTTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 436750 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 436768 . GCCCCGTCACACCCATGAACTCACACCGAGAACACAACATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 436821 . TCACGCACAGAGGACATGTGAACACTACACACACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 436867 . CACACCCCGTCACACCCATGAACTCACACCAAGAACACAACATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 436924 . CACGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 436970 . ACACGCCCCGTCACACTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 437005 . ACACAACATCACACACAGGGACTCACGCACAGAGGACATGTGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 437057 . ACACGTTCTCACACAGCACACACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 437095 . CCCCCACATGCCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 437231 . TGGGCCCTGCGCTGGCCCCGGCCGCAGACGCCCACCTGCTGCTGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 437297 . GGGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 437345 . TGAAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 437380 . GCAGAAGCATGAGCCCCAGAATGTGCACGAAGGAAGAGAGAGCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 437501 . CCATCCCCGCACCCACTGGTGTGGCCTGACCCTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 437560 . ACTAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 437642 . C <CGA_NOCALL> . . END=438074;NS=1;AN=0 GT:PS ./.:.
+1 438001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=440000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.77:0.89:.:0:.:0:0.999:353
+1 438085 . GTGCGCCGGCCGCCTGGCCTGGGCATCTCCTCTCCTGCAGCGCCGCCTGCTGGCCACAGAGAACCCGCGTGCGCCGGCCGCCAGGCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 438182 . CCCGGGCCCTAGTTCCCCCCCTCACCTAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 438246 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 438441 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 438506 . AGCTGGCCCAGGGGTTCAGGCTTTCCTTTCATAAAGTGGGGTCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 438747 . GGTGCATAGAAAACAAACACTCTGGGAAGCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 438818 . GTGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 438894 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 439001 . CTCACTGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 439129 . CCCTGGGGGAGGTTTTCCCCTTACTTGAAATGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 439224 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 439261 . TTGGCCCCCTCTGGCCAGCTGGTTCCCCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 439321 . CCCCATGGCCTGCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 439380 . AAAATCTGCTTTATAGATGAGGAAAGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 439460 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 439468 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 439473 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 439660 . TAGGTGGTATATGGGTGATAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 439922 . GAAAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 439968 . CTCACCCCTATTAGCCCCAGTGTTTGGCCTGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 440001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=442000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:0.98:.:0:.:0:0.999:353
+1 440158 . CTCCATGGGCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 440350 . GAAAGAATATGGTAAAGGAAAGCTTTGAGCCCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 440573 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 440694 . GGGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 440894 . ACCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 441007 . CCCTGGGGAACTCTCAGCACGGGGGTTGCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 441055 . TGCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 441221 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 441239 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 441244 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 441295 . CCAGTCCTCTGACTTGTCATTTTTCTACCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 441338 . TTAACCCCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 441450 . AAGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 441506 . TGGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 441681 . GCAGGACTGGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 441734 . ATGAGGTTTTTTTTTTTCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 441827 . TTTTGGTGTACAGTTCTATGAGTTTAACACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 441927 . CACGCTCCCCGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 441989 . GCAGCTGGCGGCACCTGGAGACCGGCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 442001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=444000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.77:.:0:.:0:0.999:353
+1 442027 . CGACCGCGCGTGCGCGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 442137 . GTTGTTGGATAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 442191 . AGCTTTTGTTTCTCTCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 442271 . AACACCACAGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 442301 . TCCCGCTAATCGGGAATGAGTGCCTGCTGCTCCGCGTTCTTGCGGGCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 442450 . TTTCCCATCCATATCCCTTGCTTGGTGACGC . . . NS=1;AN=0 GT:PS ./.:.
+1 442562 . TAGATACAATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 442668 . TTTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 442699 . ACTATATTTAAAATTTTAATTTAAAAACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 442939 . CACGCAGCGCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 442983 . GCCTCCATACCCGCTTGGCACCCACCAAAGGGTCCGGGGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 443114 . GGTCTTTTTTTTTTTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 443152 . GTCGCCCAGGCTGGAGTGCAGTGGCGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 443235 . CCTCGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 443261 . GCCTGCCACCATGCCTGGCTAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 443397 . GAGCCACCGCACCTGGCCAGGCTGGTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 443463 . GGAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 443529 . AGAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 443594 . CTAGATCCCCTGCCTGGAGGCCCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 443632 . TCTGGATAGTTGCAGCCACTGTGGGGAAGCTTCAGTTTGGGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 443691 . TATTGTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 443711 . TGCAGAGAGGAGAATGGAGCGGATGGAGGTGCGGTGGGGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 443823 . CTTCTTCTTGTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 443856 . GTCGCCCATCCTGGTTAGGAGGCACAAGGAAGTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 444001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=446000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.93:.:0:.:0:0.999:353
+1 444034 . CTTCGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 444065 . CCCCTGGAGGGAGAGGCTTCCTTCCTTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 444129 . CCTCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 444187 . CTCAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 444253 . GAAATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 444290 . ACCTAAAAATATTACTCAGTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 444353 . ATCCAACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 444414 . CACGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 444505 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 444536 . TTCATACGTTATTAAACCTAAACATAAAAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 444601 . AAGGTTCTTATATGTACACATCACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 444690 . TTTAGGGGCCAAGGGGCCCCACACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 444772 . GGAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 444865 . TAATGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 444920 . CACATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 444954 . GGTCTGCTTTGCTTACAAGGTCCAGAAACCCAGCAGAAAACCCTCGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 445039 . TCCATGTGTGACACAAGACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 445091 . AGTTGAAGCTGAATAAAGCCAGCAGGCAGCGAAAATCACCTGCACTCAGTTCAGTTGTGGCAGGAAAGGGAAAGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 445192 . AGAGGGAGAAAGGAGAGGGGGATGGAGAGGAGAAAGGAGAGGGGGATGGAGAGGGAGAAAGGAGAGGGGGATAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 445282 . GGGGCTAGAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 445305 . GGGAGATGGAGAGGGGGATGGAGAGGGGGATGGAGAGGAGAAAGGAGACGGGGATGGAGGGGGGATGGAGAGAAGAAAGGAGAGGAGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 445426 . GGCTGGCTGGGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 445458 . GCAGGGGAGGCCGGCTGGGCATGGACACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 445500 . AGCCACTCTGGGAACAGCAGCCAGTGGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 445543 . GCTGTTCCCTCCTGTCCAGGATGGGCCTGTCTCTGCAGACAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 445613 . CACTCCCCATGTGCATGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 445898 . AGGGGAGAGGAGTTTACTACCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 445982 . GTACGTGATGCTGTGCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 446001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=448000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.23:0.91:.:0:.:0:0.999:353
+1 446007 . CCCACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 446073 . GTCAGGGCCCGGGGGTCCTGACGGGTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 446142 . CACAGCAGGACACGCCTGCGCTGAAAGAGTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 446279 . ATTGTTTCCCACAGCAATACATGTTAGCAACTTTGAAACTACTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 446355 . TACAGTAGTAAGAGTCAGGGTTCTCACAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 446425 . G <CGA_NOCALL> . . END=447446;NS=1;AN=0 GT:PS ./.:.
+1 447472 . CAGTAGGAAATCATGTTTACATGATACATATATACAGATCAGAATGGACCCTGAGGTGGTCGGTTACAGTCAGATATGCCAGTAGGAAGTCGTGTTTACATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 447581 . TATACAGATCAGAATGGACCCTGAGGTGGTCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 447630 . CAGCAGGAACTCATGTTTACATAATACATATATACAGATAGGTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 447693 . TGCGTGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 447818 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 447881 . GATGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 447929 . AAATAAAATTTTTAAAAAGCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 448001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=450000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.35:0.84:.:0:.:0:0.999:353
+1 448006 . CTACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 448029 . CACTCACACAGACATACACACACACACACACACAAATCGGATTATGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 448164 . TGGCGAGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 448188 . ACTCGGGAGGCTGAGGCAGGAGAATGGTGTGAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 448252 . TCACGCCAATGCACTCCAGCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 448287 . AGACTCTGTCTCAAAAACAAAACAAAACAAAACAAAACAAAAAACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 448404 . GGCTGGGCACAGTGGCTCGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 448463 . ATCTCCTGAGGTCAGGAGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 448510 . ACCCTGTCTCTACAAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 448634 . TGTGCCACTGCACTCCAGCCTGGGCAATAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 448677 . TCTCAAAAAAAAGAAAGAAAGAAAGAAAAGAAAAGAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 448738 . TGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 448843 . ACCATGGTGCCCAAATTCTTTGATGCTCCTTCCTGTGGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 448938 . GAAAACAGTCACTCTCCCGTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 448974 . CCTCACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 449070 . CTCTGCCGTGTTCTTACGCAAACGCACAGTTCCAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 449142 . GGGCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 449187 . GGCAGTGAAACGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 449219 . CACAGGTGCAGGGGACTAAGGAGGCGTGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 449270 . T <CGA_NOCALL> . . END=450577;NS=1;AN=0 GT:PS ./.:.
+1 450001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=452000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.48:0.71:.:0:.:0:0.999:353
+1 450587 . CACGTGCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 450646 . GGGAAGTTGCAGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 450679 . GGACATCTTTTTTTTTTTTTTAAATAAAACATTTTTAACATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 450728 . GCAGAGCACGGTGGCTCGCACCTGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 450848 . AATACAAAAATTAGCTGGGCGTGGTGATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 450946 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 450989 . AGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 451113 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 451244 . GACGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 451283 . GTGAGTGTGTATGACTGTGTGTATGAGTGTGTATGATTTGTGTGTGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 451417 . TGAGTGTGAATGTGAACATGTGTGTGTGAGTGGGTATATGATTTGGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 451661 . CTGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 451676 . TGAGCATGAATGTGAACATGTGTGCATGAATGTGAACATGTGTGCATGAATATATGATTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 451768 . GTACAATCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 451922 . TGCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 451992 . TGGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 452001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=454000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.98:.:0:.:0:0.999:353
+1 452058 . GTGAGCATGAATGTGTATGCACAAGTGTGTGTATGTGTGTATGATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 452122 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 452159 . AGTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 452258 . AGCATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 452429 . GAAGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 452559 . CTCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 453065 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 453163 . CCATGCCGCAAAGATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 453264 . AAAGTATAAATCTGGAAAAAATAATGTCGATGTTATTTATTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 453334 . ACATAAAAACACAACAGAACATTTCATAGGCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 453466 . CCATAAAGAAATAAGTCAAAAGAGAAACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 453718 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 453752 . AAATGCTCACCTTTTGACAGGGTACTTTTAGTTCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 453987 . GCTGTTCCACCTGAGCCTAAGGTTCCTCTGCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 454001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=456000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.93:.:0:.:0:0.999:353
+1 454044 . AAGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 454087 . GGGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 454156 . AACGTTTCCATCTGTTAATAAAGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 454225 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 454359 . CTCCGCATGACTTGGATAACACGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 454528 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 454733 . GATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 454777 . CTTGGCTCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 454862 . GGATGTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 454948 . TGATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 455168 . GGCGTGGGCAGGGGGCTGACTCCATGTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 455257 . TTTCTCTCCTCCCTATAAAGCCTATTTTTGTATTAGGGTGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 455715 . TGTTCATGGTCCTCCCCGTGGGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 456001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=458000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.96:.:0:.:0:0.999:353
+1 456108 . TAGTGGGCGAGGATTGTTTAGCCGCCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 456145 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 456152 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 456201 . ATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 456218 . ATAAATATATATTTATATTATTTATATTTAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 456332 . CTCCTACT . . . NS=1;AN=0 GT:PS ./.:.
+1 456501 . CAAATCAGCCTACTTGGAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 456570 . TTCTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 456609 . TTGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 456684 . TCTCTTTCTCCCCCTCTCCCCGCTACATTTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 456742 . CTCTGTCTTCACGAGAGCTACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 456839 . TAGTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 456913 . CACGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 457090 . TTTAGATGGTTTTTTTTTTTTTTGTTTGTTTTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 457234 . TCCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 457284 . TTTGTATTTTAGTAGAGACGGGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 457331 . CTCGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 457583 . CTAATTTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 457711 . GCCCGGCCCCTTTTGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 457794 . GAGGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 458001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=460000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:0.98:.:0:.:0:0.999:353
+1 458009 . ATTCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 458055 . CTTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 458211 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 458693 . ACACCTGTAGTCCCAACTACGCAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 458753 . TTGAGGCTGCTCTGAGCTGTGATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 458820 . TCTTAAAAAAAAAAAAAAAAAACTATTGCAAGAGGAGAGAGAGAGACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 459164 . CTTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 459222 . GTGGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 459249 . TGCATGTCTCTTTGAATCCGTATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 459485 . ATGCAAGTCCTACTGTTTCTGTAACTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 459661 . CTCGTCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 459678 . CCCGCTCGAGCCTCTCCACATGCAGCAGGAAGGAAAGTGGAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 459732 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 459746 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 459921 . CCAGTTTTTCCTGCCTGTCCTGTTTGGGCAGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 460001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=462000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.04:.:0:.:0:0.999:353
+1 460084 . TTCTACACCATTCCGGGATGCTGGTGTCCACCACTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 460365 . CATTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 460403 . CTCGCCTTCAGGGTCGTCTGTGTCTGTTAAAGTCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 460663 . AAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 460720 . TGATTCCAGTGCAGCATCACATGACAGACAGAGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 460819 . CTGTCTCTTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 461079 . CAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 461230 . CACGTTCTCAGGACCTCCTGAAGCTGCGTCACAGGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 461421 . GGAGAAGTGTGGGTGTTGGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 461539 . ATGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 461645 . TGGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 461710 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 461811 . AAGTTGGAGAAACAAAACGCAAACTAAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 461895 . CCCCACCCCGCATCCCTGGGCTCGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 461941 . TGCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 462001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=464000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.22:1.35:.:0:.:0:0.999:353
+1 462037 . CTGTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 462100 . CTTGGGGGCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 462158 . TAAGTTTTTTTTTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 462213 . AGTGGCGTGATCTTGGCTCACTGCAAGCTCCACCTCCCAGGTTCAAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 462280 . CCCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 462321 . CTAATTTTTTTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 462372 . ATCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 462431 . TACAGGTGTGAGCCACCACACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 462535 . GACAGCAGAGGGGTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 462562 . TACAGACAGCAGCAGCTGATGCACAGGCCTCCCAGCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 462618 . GGGAAGTGCTCAGAAGCTTACAAAGCTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 462676 . GAGTAGATCCCTGATCCTATAAAAATGTACTAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 462736 . GGGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 462910 . TGCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 462927 . CTGGGTTGGCAGAGACAGAGTGACTGTCTTCCTCCAGGAAGCAGCAGGTTAACTGGTTGGCAGAGACAGAGGGACTGAGGGACTGTCTCCCTCCAGGAAGCAGCAGGTTAACTGGTTGGCAGAGACAGAGGGACAGAGGGACTGTCTTCCTCCAGGAAGCAGCAGGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 463224 . CAAGCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 463503 . GACGTGTCAGAAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 463557 . AAAATTATTCATTAAAAACATCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 463644 . ACAATACCCTTCAGACTTTGAGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 463699 . GATCCAAACTTGATAAAGGACATGAAAAAGAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 463859 . AGGGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 463921 . ATACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 463965 . CTGTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 464001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=466000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.35:1.25:.:0:.:0:0.999:353
+1 464012 . GCCATTGAGCACCCTGGTGTTGAGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 464066 . GGGTCACTGTGAGTGGGCTGCCCCCAACATGAGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 464194 . ACCGTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 464265 . AACATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 464574 . AAAAGTATGCGGAATAGAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 464614 . CCTGAAAAAGTCACATGTTATTTCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 464651 . TGGCAAAAAAAAAGTCACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 464937 . AAATAATTTAAAAGTGCTTTTGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 464976 . CTAGAGGGTAAGATTAGACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 465018 . GGTTAGGGATAGGATTAGGATCTGGGTCAGAGTCAGGGCCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 465094 . GAGATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 465135 . TCAGGATTTAGGTTCAGTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 465165 . GGACAGGGTTAGGGTTAGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 465194 . GAGCTTTGTTCTCCTCAGGACCCACCCGAGGACGGGTCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 465244 . GAGCACCTGGTAGTGTGGCATGTCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 465286 . TTCATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 465306 . CCTGGGGAGACGTGGCTGCAGGCCATTGAGGAAGGTGAGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 465365 . CCCGTGTGCTGAGGAGGGAGCTCTGCCGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 465531 . AGAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 465577 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 465592 . AAGTTGGGTATAGGCAGAGGCTGGAGGAAACATGTGCATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 465660 . ATTACTCATTTTTCTTACAGTGTTAAATTAGTAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 465858 . AACATTGACATCCTACATTACATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 465895 . AATCTGAGACAGCTCTCAGATTTTTTAGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 465977 . TCCTGACAACATGGGCCCAAGGTGGTCGGGGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 466001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=468000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.54:1.22:.:0:.:0:0.999:353
+1 466036 . GACACGAGAGATCAATCAATATGTGTAAGATGTACATTGGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 466102 . GACAGGGGGCTTCCAGGTCACAGGTAGGTAAGAGACAAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 466176 . CACGTGAGGCAATCAGGTATGCATTTATCTCGGTGATCAGATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 466253 . CTAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 466356 . GTGTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 466470 . TTGGGTTTCCAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 466677 . CATCGTGGTCACAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 466945 . TTTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 467152 . GTATAAATAATTAAACAAGGAAGTGTTAAAAAAAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 467234 . AATCGTCAAAAAAAAAAAGAGATTTCCCATGTAGCCGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 467450 . ATCCCACACGACATGTAGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 467501 . TGTTAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 467554 . GGATTTTTGTAGAATGTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 467693 . ACTGCATGTGTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 467800 . ATTCTTTTGCAAAGGAGATTTCTATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 468001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=470000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.15:1.21:.:0:.:0:0.999:353
+1 468102 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 468154 . TTAGCAATGCTAGGAAGCATATGTGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 468191 . ACCTACACACACCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 468434 . ACAATTGCTTCTGTTGGAAAGAACTTTATAAAATGGAATCCAATAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 468491 . TCACGTGCCTTCAGCCTACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 468523 . TTTCAAAGTTTTTACCTAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 468557 . CATTTTGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 468589 . CTGCGTAATTTATAAGTAAAACAGTTTCATTTGGTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 468634 . GGTAGCTGGAATGTCCGAGATTGGGCAGTTGCATCTGGCGGGGCCTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 468688 . TTCACCTCATGGTGGAAAGTGGAAGGGGAGCAAGGGGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 468738 . ACATAGCAGAAGTGAAAGCAAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 468777 . CAGACTCTTTTTAATTACCTACTCCTGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 468816 . ATTCCTGTGAGAACAGAACTCACTCACCCCCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 468860 . ATCTATTCATGAGGGATCCGTCCCCACGACCCAAACACCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 468910 . CCCACCGCCCCACACTGACACAGTGGGAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 468962 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 468984 . CATCGTAATTTATAGCATAAATTCTTTTTCACATGATGTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 469037 . TACTCCACATCCTGAGTAATTTGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 469112 . TTTCGACAAATGCATTGTGGCAGATATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 469604 . CTCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 469626 . TTTAAAAAAATTTTTTTTGAGACAGGGTCTGGCTCTGTCGCCCAGGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 469689 . AATCTCAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 469739 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 469772 . CATGCCGTCATGCCCATCTAATTTTTGTATTTTTGGTAGAGACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 469854 . CAACATTTATTTATTTATTTATTTAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 469940 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 470001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=471368 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.60:0.97:.:0:.:0:0.999:353
+1 470001 . CAGGTGCCTGCCATCACGCCCGGCTAATTTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 470065 . TTAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 470103 . ACCTGCCTCAGCCTCCTAAAGTGCTGGGATTATAGGCATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 470200 . TATAGTGGCACATAGCATGGATAAGGAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 470267 . AAACATAACATTTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 470302 . GGAAGGTTAGGTATCTCTTTTTATTTGTATCTTCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 470349 . TTATAAAAAATGCAACCTACTTTACTTGCGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 470401 . ATGCTTTGCATAGAGTTGTTTCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 470448 . TTTATTTACATATATTGATTATAATTTTAATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 470497 . TTC . . . NS=1;AN=0 GT:PS ./.:.
+1 470502 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 470516 . GTAGACAGTTATAAACTGTCATATATTAGCATTCTATAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 470603 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 470609 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 470630 . TGTGGCAGAAAAAAACATGTTTATTAACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 470674 . AGTCTCTCTGTAAAAACAGGAAGCCAAAAGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 470715 . GAATTATTTATGTTCAGTAATTAATGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 470756 . TTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 470863 . GTAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 470884 . TAATGTTTTTCGCTTTTCACAAGACGGCACCGAAAGCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 470951 . AGCCAAAGTGAAGGTTTTAAAGGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 470999 . CCGCAGCCACACGCAAAAAAGAAGATCCGCATGTCACCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 471059 . CTGCGACTCCGGAGGCAGCCCAGATATCCTCGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 471142 . TCCGCTGACCACTGAGTCGGCCGGAAGAAGATAGAAGAAAACAACACGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 471216 . CAACAAGCACCAGATCAGACAGGCTGTGAAGAAGCTCTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 471287 . TTTGTCCTGATAAAGAGAACAAGGCATATGTTCGACTTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 471347 . ATGTTGTAACAAAATTGGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 471369 . N <CGA_NOCALL> . . END=521368;NS=1;AN=0 GT:PS ./.:.
+1 521369 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=524000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.58:0.75:.:0:.:0:0.999:426
+1 521369 . GATCCTTGAAGCGCCCCCAAGGGCATCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 521421 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 521429 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 521542 . GGTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 521582 . ATGCGGGGTGGGGGCAGCTACGTCCTCTCTTGAGCTACAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 521648 . TTGCTTAGTTTGCGTTTTGTTTCTCCAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 521776 . GACCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 521838 . AAAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 521941 . TCCCTCGCATGTAAAATGTGTATTCAGTGAACACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 522046 . TTCCCCAATACCCGCACTTTTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 522079 . CTGAGGCCCCCAGACAATCTTTGGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 522173 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 522210 . GTTCTTTGATCAGCGCCTGTGACGCAGCTTCAGGAGGTCCTGAGAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 522363 . GGGCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 522393 . TTTTCTGATTGTCAATTGATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 522462 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 522482 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 522523 . GCCACCCTTAGAGAAAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 522564 . GACCCATAGAAGGTGCTAGACTCTCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 522659 . CACAAGAGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 522730 . TCACGGCCTCTGTCTGTCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 522820 . TCTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 522961 . GCGGGGTCCACGCAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 523011 . CGGAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 523074 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 523079 . AAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 523118 . CTTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 523181 . ATTGTGGCACACGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 523278 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 523349 . CTGTTCCCCGTGCCCAGGCAGCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 523388 . TCCCGGAATGGTGTAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 523414 . CCCGTCATAGCCAAAGCCTGGGGTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 523461 . CCCACTCCTCTCCCGACCCCTCCCTCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 523536 . GCCTGCCCAAACAGGACAGGCAGGAAAAACTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 523743 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 523751 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 523799 . AGGCTCGAGCGGGGCACAGTCCATGACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 523976 . GCCAGTTACAGAAACAGTAGGACTTGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 524001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=526000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.66:0.74:.:0:.:0:0.999:426
+1 524106 . AGAGGCCAGCAGGAGGGAAACACCGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 524145 . AGACGGGGATTGGGAGAGAAATTCAGAAAAGATTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 524216 . AGATACG . . . NS=1;AN=0 GT:PS ./.:.
+1 524261 . AAGCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 524316 . TTTTGGTAAGTTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 524412 . CCATTCAGGCTCCTTTGAGCCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 524460 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 524624 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 524641 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 524644 . AATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 524668 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 524692 . AGGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 524719 . GCTCACAGCAGCCTCAAACTCCTAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 524771 . CCTGCGTAGTTGGGACTACAGGTGTGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 524853 . GCCAAGGCTGGTTTTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 525132 . CCCATCTCTCTTACCATACACAGAAATCAAATAAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 525259 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 525272 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 525337 . CTGCAGATCAAAGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 525461 . ATCGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 525531 . CATGAAAAAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 525665 . GAAAGGGGGACCCTCACACACTGTTGTGGGAACATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 525745 . AACTAAAAGGGGCTGGGCCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 525877 . ATATAAAAAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 526001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=528000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.37:0.59:.:0:.:0:0.999:426
+1 526028 . TCCACCTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 526111 . GGGCGGATCACAAGGTCAGGAGATCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 526160 . AAATCCCGTCTCTACTAAAATACAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 526229 . CTCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 526343 . CTCCAAAAACAAACAAACAAACAAAAAACACCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 526387 . TGCTGTATGATCCAGTAATTTCACTAACTGGGCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 526549 . GAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 526595 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 526887 . GGCGGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 527036 . CCCGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 527110 . TCTCAAAAAAAAAAAAAAAAAAGAAGGAATAAGACCTAGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 527164 . AGTTGACAATTGCCTACTGTATATTTCAAAATAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 527231 . AAGCAAAAACAAATATTTAAGGCGATAGATATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 527319 . TACCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 527365 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 527377 . CTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 527398 . TCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 527406 . TGGGCTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 527418 . AAATATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 527429 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 527439 . CAAAGTAGGAGGATTCCTTGAGCCCGGGAGCTTGAGGCTGCAGTGAGATCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 527512 . CGACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 527530 . GCTCTAAATATAAATAATATAAATATATATTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 527577 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 527613 . ACAGTCCGCCCTTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 527636 . TGACACAGCCAAGGCAGCTGAACAATCCTCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 527673 . AGACAGTGGAGGTCGCCCTCCAGAGGACCTTATCAGATGTACGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 527739 . TTTCTATTCAGAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 527967 . CAGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 528001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=530000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.67:.:0:.:0:0.999:426
+1 528043 . ATCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 528094 . GTGCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 528148 . TAAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 528358 . AGATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 528432 . ATGGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 528476 . TTTAACAAACACCCTAATACAAAAATATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 528583 . GCCCACATGGAGTCAGCCCCCTGCCCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 528633 . CCCTCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 528793 . ACTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 528827 . TACATCATGGCTGAGGGCACATACGTGCACGCACATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 528911 . ATAACATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 528943 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 528958 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 528967 . GGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 528998 . AGCCAAGCCTGGGGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 529042 . CCCCATCCCTGTCCGGAATAGCACGGGTGCTTCTCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 529126 . TCCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 529232 . AGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 529397 . TACCGTGTTATCCAAGTCATGCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 529598 . TTGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 529675 . GCGTCCCCTCCCTGGCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 529731 . CTCGCTTGGCCCCCACCTGATTCCTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 529788 . GAACAGCCTCAACTGATTCAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 529992 . CCCAGAACTAAAAGTACCCTGTCAAAGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 530001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=532000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.65:.:0:.:0:0.999:426
+1 530062 . CTCCCCTGCTGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 530098 . GGCAGGTAGGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 530237 . CTATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 530269 . TAGTGATATACGTGTACACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 530309 . TTTATGGTCTGTCTTCTTATAACTGCTACACCCATGCCACCGTCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 530389 . ATCGTTGCCTATTTTATTGTGTAAAGTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 530474 . GTAAATAAATAACATCGACATTATTTTTTCCAGATTTATACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 530601 . CCCATCTTTGCGGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 530653 . CTGCCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 530763 . CAAGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 530889 . CCCAGCAGTGCAGACCCCTCTCTAGAGCCGAGATGCTCCCGGCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 530960 . GCTCCCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 531172 . TGAGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 531216 . CACTGAGCTCAGATCCCACGTCTGAGCCTCCGCCTTTCCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 531517 . CAATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 531616 . ACATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 531677 . AGATCATACACACATACACACACTTGTGCATACACATTCATGCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 531783 . ATACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 531891 . AGATCATACACACATACACACACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 532001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=534000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.40:0.64:.:0:.:0:0.999:426
+1 532003 . ACCGATTGTACACTCGTGCACACATTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 532098 . CACCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 532151 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 532239 . CACACTCATACACAGCCCAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 532304 . ACACCCAAATCATATACCCACTCACACACACATGTTCACATTCACACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 532399 . CCCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 532436 . CACTCACACACACAAATACACACTCATACACAGTCATACACACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 532515 . CACCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 532643 . CTGCTTTTTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 532770 . TTGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 532887 . CCCATCACCACGCCCAGCTAATTTTTGTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 532926 . AGATGGGGTTTCACCATGTTGGCTAGGCTGGTCTTGAACTCCTGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 532987 . CCTCGGCCTCCCAAAGTGCTGGGATTACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 533031 . GCTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 533058 . TTTTATTTAAAAAAAAAAAAAAGATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 533106 . TACTGCAACTTCCCACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 533160 . CACGCAATGAGGCACGTGTAGAAACTGCGACACTCACACGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 533222 . G <CGA_NOCALL> . . END=533973;NS=1;AN=0 GT:PS ./.:.
+1 533982 . ACTGCGACACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 534001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=536000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.58:0.89:.:0:.:0:0.999:426
+1 534005 . G A . . NS=1;AN=1;AC=1;CGA_SDO=6 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:534005:VQLOW:31,.:30,.:8,.:-31,0,0:-8,0,0:7:3,.:4
+1 534014 . GCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 534025 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 534028 . AACTGCGACACTCACGC . . . NS=1;AN=0 GT:PS ./.:.
+1 534050 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 534053 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 534057 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 534067 . GGTGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 534082 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 534085 . CTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 534096 . GCCGTCTCAGCAGCTCACGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 534138 . CCCTCACGCCTCCTTAGTCCCCTGCACCTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 534189 . TCGCTTCACCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 534311 . CGGCGGGGGGGGGGGCGGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 534411 . AGGTGAGGCCTGCCAGGTCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 534445 . GACTGTTTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 534542 . ACCGTGGTAATTACTGAACATTTAGGGGAGACACTTTGAGACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 534648 . GCCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 534666 . GATTTTCTTTTTTTTCTTTTCTTTTCTTTCTTTCTTTCTTTTTTTGAGACAGAGTTTTGCTCTCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 534806 . GCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 534859 . GTATTTTTTTGTAGAGACAGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 534908 . GAACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 534939 . CAGCCTCCCGAAGTGTTGAGATTACAGGCACGAGCCACTGTGCCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 535057 . TCAGTTTTTTGTTTTGTTTTGTTTTGTTTTGTTTTTGAGACAGAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 535133 . GGCGTGATCTTGGCTCACTGCAAGCTCCACCTCCCGGGCTCACACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 535196 . TCCCGAGTAGCTGGGACTACAGGCGCTCGCCACCTCGCCTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 535314 . GGGCATAATCCGATTTGTGTGTGTGTGTGTGTGTATGTCTGTGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 535379 . ATTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 535434 . ATGTCCACAGCTTTTTAAAAATTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 535504 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 535507 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 535523 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 535672 . GTGTGCTAAGCCATGTATGTACACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 535715 . TTTAACCTATCTGTATATATGTATTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 535755 . CCTGCTGGCATATCTGACTATAACTGACCACCTCAGGGTCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 535803 . TCTGTATATATGTATCATGTAAACACGACTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 535843 . ATATCTGACTGTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 535864 . CCTCAGGGTCCATTCCGATCTGTATATATGTATCATGTAAACATGATTTCCTACTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 535945 . T <CGA_NOCALL> . . END=536546;NS=1;AN=0 GT:PS ./.:.
+1 536001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=538000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.49:0.90:.:0:.:0:0.999:426
+1 536585 . CAGCTGTGAGAACCCTGACTCTTACTACTGTATTGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 536670 . CATGTATTGCTGTGGGAAACAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 536800 . ACTCTTTCAGCGCAGGCGTGTCCTGCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 536863 . CCCATCCTGTACCCGTCAGGACCCCCGGGCCCTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 536958 . CTGTGGGGTTTGACAAACACAGCATCACGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 537049 . CCTCGGTAGTAAACTCCTCTCCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 537188 . TGGCGGGGGTGGGCACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 537338 . CCACCATGCACATAGGCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 537415 . CAGGAGGGAACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 537441 . TCCGCCACTGGCTGCTGTTCCCAGAGTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 537484 . CAGCGTCCATGCCGGCCTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 537526 . CACGCCCAGCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 537571 . CACCCTCCTCTCCTTTCTTCTCTCCATCCCCCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 537629 . CCATCCCCCTCTCCATCCCCCTCTCCATCTCCCTCTCCTTTCTCCTCTCTAGCCCCCTCTCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 537699 . CTCCATCCCCCTCTCCTTTCTCCCTCTCCATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 537749 . GGCTCTTTCCCTTTCCTGCCACAACTGAACTGAGTGCAGGTGATTTTCGCTGCCTGCTGGCTTTATTCAGCTTCAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 537852 . AAATGTGTGTCTTGTGTCACACATGGAAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 537914 . GCCTCGAGGGTTTTCTGCTGGGTTTCTGGACCTTGTAAGCAAAGCAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 537991 . CGATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 538001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=540000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.68:0.76:.:0:.:0:0.999:426
+1 538046 . CTCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 538139 . CAGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 538194 . CCGCTGGAGAGTGTGGGGCCCCTTGGCCCCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 538290 . TATGTGATGTGTACATATAAGAACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 538348 . TCCATTTTTATGTTTAGGTTTAATAACGTATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 538472 . TTCATCTCTGAGCATTTTCTTCTCTGGACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 538556 . ATGTTGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 538600 . GACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 538658 . TGATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 538687 . GAGAGTCTGGTTTCTACAGCGCCTTCAGGGAGAATGAGACTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 538780 . CTGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 538823 . CAGAAGGAAGGAAGCCTCTCCCTCCAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 538875 . CCCTCGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 538957 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 538968 . TCT . . . NS=1;AN=0 GT:PS ./.:.
+1 539025 . CAACACTTCCTTGTGCCTCCTAACCAGGATGGGCGACACCAGCCCATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 539080 . TGGGACAAGAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 539164 . TCTCCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 539179 . CTCCATCCGCTCCATTCTCCTCTCTGCACATCAGCTTCCCAGACAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 539245 . CCCAAACTGAAGCTTCCCCACAGTGGCTGCAACTATCCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 539297 . ACTTGGGCCTCCAGGCAGGGGATCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 539382 . CTCTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 539447 . TTTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 539490 . AAAAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 539632 . AAATTAGCCAGGCATGGTGGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 539665 . TCCCAGCTACGCGCAAGGCTGAGGCAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 539719 . GCTTGCAGTGAGCCGAGATGGTGCCACTGCACTCCAGCCTGGGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 539778 . TCTCAAAAAAAAAAAAAAAAAAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 539892 . GGGTCCCCGGACCCTTTGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 539946 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 539972 . CTGCGTGGGGAGGGCGAGCTCAGAGAGCAGGGGAGCCTGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 540001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=542000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:0.91:.:0:.:0:0.999:426
+1 540165 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 540193 . TTTTAAATTAAAATTTTAAATATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 540243 . ACATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 540343 . GGAATTGTATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 540437 . GCGTCACCAAGCAAGGGATATGGATGGGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 540708 . CCCGAGAGAAACAAAAGCTTATGTTCACACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 540767 . CTCTATCCAACAACCCTGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 540863 . GAGCCGCGCACGCGCGGTCGGCTCGGCGAGGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 540903 . CCAAGTGCCGCCAGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 540969 . GGTGGGGGAGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 541049 . AAATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 541154 . AGAGAAAAAAAAAAACCTCATTTCCTCCCCACAAAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 541218 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 541238 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 541396 . TCAACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 541451 . ATGGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 541560 . TATGGGGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 541598 . CAAGTCAGAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 541665 . TCCCAGGCCATGTGGAAGACCTCACAGGGGGACCAACTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 541847 . GACAGCAGCTGTCTAAACAGGAGCATGCAACCCCCGTGCTGAGAGTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 542001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=544000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.91:.:0:.:0:0.999:426
+1 542008 . CCATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 542518 . ATATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 542732 . GGGTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 542902 . GCTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 542958 . ACCACTGTCCCCTTCTCACCTTTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 543221 . ACCTATCACCCATATACCACCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 543435 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 543565 . GTGAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 543635 . GGGGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 543768 . CCCGGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 543865 . AGTGTCTTAGGTAGACGGTTACACTTGTTTTCAGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 543993 . CTGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 544001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=546000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.39:0.85:.:0:.:0:0.999:426
+1 544001 . GAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 544079 . AGCGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 544123 . AGCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 544391 . GCCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 544520 . CTGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 544697 . GTGAGGGGGGGAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 544733 . G <CGA_NOCALL> . . END=545023;NS=1;AN=0 GT:PS ./.:.
+1 545035 . C <CGA_NOCALL> . . END=545972;NS=1;AN=0 GT:PS ./.:.
+1 545980 . GGCCAGCAGGCGGCGCTGCAGGAGAGGAGATGCCCAGGCCTGGCGGCCGGCGCACGCGGGTTCTCTGTGGCCAGCAGGCGGCGCTGCAGGAGGGGAGATGCCCAGGCCTGGCGGCCGGCGCACGTGGGCTCTCTGTGGCCAGCAGGCGGCGCTGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 546001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=548000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.18:1.04:.:0:.:0:0.999:426
+1 546189 . ACCGCGGCGGCCTCTCCTGAGGTTCCCTAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 546267 . GTGCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 546354 . CCGGCTCTCTCTTCCTTCGTGCACATTCTGGGGCTCATGCTTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 546428 . CCTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 546501 . AGCACAGCAGCAGGTGGGCGTCTGCGGCCGGGGCCAGCGCAGGGCCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 546644 . TGCGCGCAGAGTAAGGATGTGTGTGTCTACGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 546693 . TGACAGGGTGTGTTCTGTGTGAGAACATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 546729 . TGTCCACATGTCCTCTGTGCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 546768 . GTTGTGTTCTCGGTGTGAGTTCATGGGTGTGATGGGGTGTGTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 546820 . AACGTGTGTGTAGTGTCCACATGTCCTCTGTGCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 546872 . GTTGTGTTCTTGGTGTGAGTTCATGGGTGTGACGGGGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 546922 . AACGTGTGTGTAGTGTTCACATGTCCTCTGTGCGTGAGTCCCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 546974 . GTTGTGTTCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 546996 . ATGGGTGTGACGGGGTGTGTGCTGTGTGAGAACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547039 . TGTCCACATGTCCTCTGTGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 547065 . CCCCGTGTGTGATGTTGTGTTCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547100 . ATGGGTGTGACGGGGTGTGTGCTGTGTGAGAACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547143 . TGTCCACATGTCCTCTGTGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 547182 . GTTGTGTTCTCGGTGTGAGTTCATGGGTGTGACGGGGTGTGTGCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547234 . AACGTGTGTGTAGTGTCCACATGTCCTCTGTGCGTGAGTCCCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547286 . GTTGTGTTCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547308 . ATGGGTGTGACGGGGTGTGTGCTGTGTGAGAACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547351 . TGTCCACATGTCCTCTGTGCGTGAGTCCCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547390 . GTTGTGTTCTCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547412 . ATGGGTGTGACGGGGTGTGTGCTGTGTGAGAACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547455 . TGTCCACATGTCCTCTGTGCGTGAGTCCCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547494 . GTTGTGTTCTCGGTGTGAGTTCATGGGTGTGACGGGGTGTGTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 547545 . GAACGTGTGTGTAGTGTCCACATGTCCTCTGTGCATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 547598 . GTTGTGTTCTCGGTGTGAGTTCATGGGTGTGACGGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 547675 . CTGTGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 547700 . GTTGTGTTCTCGGTGTGAGTTCATGGGTGTGACGGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 547750 . AACGTGTGTGTAGTGTTCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 547777 . CTGTGCGTGAGTCCCTGTGTGTGATGTTGTGTTCTCGGTGTGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 547831 . TGACGGGGCGTGTGCTGTGTGAGAACATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 547875 . TGTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 548001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=550000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.99:.:0:.:0:0.999:426
+1 548081 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 548087 . ATT . . . NS=1;AN=0 GT:PS ./.:.
+1 548092 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 548130 . CAGCTACTCCGGAGGCTGAGGTGGGAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 548274 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 548281 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 548291 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 548328 . CAGTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 548457 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 548491 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2792860&dbsnp.131|rs75892356;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:28:28,63:28,63:27,28:-63,-28,0:-28,-27,0:0:0,0:0
+1 548511 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 548585 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 548622 . AAAATACAAAAATTAGCCAGGCATGGTGGCAGGCACCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 548701 . TGAACCCGGGAGGTGGAGCTTGCAGCGAGCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 548776 . TCCGTCTCAAAAAAGAAAAAAAAAATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 549007 . AAACAAAATTATTCATATAATGGGTTCAAGAAAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 549337 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2491327;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:54:54,54:54,54:24,32:-54,0,-54:-24,0,-32:17:3,14:14
+1 549538 . AAAGTGCCACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 549879 . AATTAAAAATCCCATCTGCTGTCAATGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 550001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=552000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.87:.:0:.:0:0.999:426
+1 550122 . TCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 550295 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 550368 . GGCGAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 550398 . AACAAAAAGCCAACCATGGGATCTGTGGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 550451 . CAGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 550669 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 550796 . CAAGGTGGGGAATGTTCTCTTAACCTGCAGCTTTCTCCTTCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 550882 . CTCAGGGCCTTCCAGCCAGACCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 550952 . TGTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 551036 . ACCATTTCCCACCCTGGATCTCAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 551333 . GACTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 551401 . CCACCCCCTTATTGTTATAGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 551540 . GAGTCACTGGCAAGCTTTGATATGCAAACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 551622 . TTGCCCCAACAGGTGGCTGGCAACATGGCCGCCCCCACATATCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 551718 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 551753 . GACCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 551814 . TCCTCATATAACTAGCTGATTACACCACACACACGCCCTCCCCCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 552001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=554000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.79:0.94:.:0:.:0:0.999:426
+1 552100 . CACCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 552159 . GATGCACAAGTGAGAGAAAAATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 552270 . GAACTTTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 552389 . ATGAGAAGGGCCCAACCCCAGAGCCCAGGCCAGTCAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 552471 . CCAGGGGCTAGGTCCATGGCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 552507 . GCAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 552537 . AGCGTAGATTCCCACTCATTCCCACAGCCAATTCTCATCCTTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 552604 . TGGTTTTTAACCAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 552823 . ACATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 553193 . CCCTACTCAGCAGGCCCAGATGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 553223 . GGGTGCTCCATCTGACTGCACAGGAAGGCAAGGCCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 553331 . GAACACAGGGGCACTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 553367 . CTGATTTTTTTTTTTATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 553673 . TCCTAAAGAGATTAACTGAAAGTCTAGCACTTTGTTTTTTTTTTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 553848 . GCCCACGACCACACACAGCTAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 554001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=556000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:1.04:.:0:.:0:0.999:426
+1 554034 . AAACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 554077 . CTTCAAAAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 554188 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 554350 . GGCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 554413 . TTTTCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 554462 . GCTCCTGACATTGGGTTGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 554492 . CTGTGGACTCTTCCCTCGGAATGAGAGAGGGAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 554659 . TGAAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 554732 . GTCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 554792 . AAGCACTCAGCCTGCCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 554862 . AGCTATACC . . . NS=1;AN=0 GT:PS ./.:.
+1 554899 . AAGAGTCTGTGAGTGGGCAGAATTCCCTCCAGGGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 554992 . AACTGGTCAGCCTCACTCCCTTGCTGAGACCAATAGCAACCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 555092 . ACCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 555162 . TGTTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 555242 . TCACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 555401 . CTGTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 555466 . CAGCAGCAACAGACTTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 555533 . CTTCTTTTTCACTTAAAATGCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 555660 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 555735 . CTCAGAGCCAATAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 555771 . CAAGGCCCCTTATCTTTGGAGCCCAGTGTTCCTTCCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 556001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=558000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:0.88:.:0:.:0:0.999:426
+1 556057 . ATGCAAAAGAGATTCAGTGCAAGAGAAATCCTCCTACTGGTTTTGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 556352 . TGACCTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 556703 . AACTTGGGCAACAGAGCAAGACACCAGCTCTAAAAAAAAAAAAAACTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 556827 . GTGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 556973 . AATCATTCCAGGAAGGTAGGGAAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 557112 . AGGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 557191 . GAGGAGGTGGGAATGTGCTGGGTGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 557236 . CCTGCCTCACCTCACTCCCCACTGTCGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 557286 . TGCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 557401 . TCCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 557517 . CCTTTCTCTCCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 557707 . GACCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 557793 . GAGCTATGGGAACCAATGGAATTGGATCTAAGGTTTTGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 558001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=560000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:1.00:.:0:.:0:0.999:426
+1 558166 . AAGGCATGCATTCATTGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 558226 . GTGAGGCCTGACCGCTGTGAGGATGGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 558433 . GTTTAAAACTCTCCCTAGGAATTCCAATGTGCAGCCAAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 558533 . ACCCATGACCTCAGCTTCATCAGGAGACCCACTGTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 558720 . CCCCAGCAGAAGGGAGAGGCGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 558753 . GCCAGGACTGTGTAAGGCCTTTGAAGGTTGACCATCCATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 558882 . CATTCACACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 559020 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 559048 . AATCGTGTGTGTGTGTGTGTGTGTGTGTGTGTAGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 559127 . GGCACCTCTTTTCACTGCTGTCATGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 559228 . GT . . . NS=1;AN=0 GT:PS ./.:.
+1 559385 . AGAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 559408 . CTGTCTTTGGAATGTGTGATTAAGCTTTTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 559452 . GACAGCTTCCAGCAGCTTTGTCTGCCACCGTGTCCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 559818 . AGGGAGGGAGGCAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 559865 . GAGCAAGGAAGGAAGGAAGGAAGGAAGGGAGGGAGGGAGGGAGGGAGGGACATAGACAGTGGATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 559975 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 560001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=562000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:0.97:.:0:.:0:0.999:426
+1 560042 . GGCTGGGGTAAAGAGTGATACATGTAAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 560129 . CTACTACTGATATTATCATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 560373 . CAGCAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 560788 . TCATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 560946 . GACGTGCAGTCATCCTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 561056 . TGCAACCTCCACCTCCCTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 561089 . CCTGCCTCAGCCTTCCCAGCAGCTGGGATTACAGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 561151 . TGTATTTTTTAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 561239 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 561441 . GTCTGATCGCCTGTGCCCACGGTAGCGTATCAGGTACACGGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 561619 . AATGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 561826 . ACCATTGTTTAGACAGTTATATGAAATGGGGTATTTTCTAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 561888 . ATATGGGGATTTTTTTTTTTTTTTTTTGAGACAGAGTCTCACTCTGTCGCCTAGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 562001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=564000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:0.95:.:0:.:0:0.999:426
+1 562013 . TTTCTTTTTTTTTTTTTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 562144 . GAGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 562158 . ACAGGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTGTTTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 562222 . CCATGTTGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 562255 . CTCGTGGTCTGCCCACCTCGGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 562365 . ATAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 562481 . AAGGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 562567 . TTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 562620 . CCAGATTCCTTGAAGGCAGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 562739 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 562856 . CCTCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 562907 . GA GG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.131|rs78231499;CGA_FI=100131754|XR_108278.1|LOC100131754|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L2c|L2|35.5;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:50,.:50,.:22,.:-50,0,0:-22,0,0:4:2,.:2
+1 562972 . A T . . NS=1;AN=2;AC=1;CGA_FI=100131754|XR_108278.1|LOC100131754|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:34:34,34:34,34:12,27:-34,0,-34:-12,0,-27:14:1,13:13
+1 563013 . CATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 563124 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 563268 . AGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 563374 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 563458 . TTTCTTTTCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 563491 . ACCTCACATTCTCTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 563573 . CACGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 563615 . GCACAAAAATTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 563819 . TAAAATCCAGTTTGTGCCTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 563938 . TTTTGACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 564001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=566000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:92.54:0.98:.:0:.:0:0.999:426
+1 564038 . TCC . . . NS=1;AN=0 GT:PS ./.:.
+1 564067 . CAACAATGATTTCAAATATTTCACTTTTTAAGTCAGTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 564151 . AAACAGTCGTCATTACCATAGCTGTGACAGGGAGACTGTTGAATTTATAATCTATTGGCCATTCACAGCATAGCGTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 564248 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 564270 . TCATCACATTCCCTTCACAACTTACTCACCAGATCAGACTTTGAGCTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 564326 . GCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 564338 . TCGTTTGAAATGGTCATCCATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 564365 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 564373 . ACC . . . NS=1;AN=0 GT:PS ./.:.
+1 564383 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 564385 . GTCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 564399 . AGATGATTTTCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 564415 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 564423 . TTTTGTTTAATATATTAGATTTGACCTTCAGCAAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 564463 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 564583 GS000016676-ASM_2060_L C [12:41757451[NC . . NS=1;SVTYPE=BND;MATEID=GS000016676-ASM_2060_R;CGA_BF=0.10;CGA_BNDGO=NM_001164595|+ GT:FT:CGA_BNDMPC:CGA_BNDPOS:CGA_BNDDEF:CGA_BNDP 1:TSNR;SHORT;INTERBL:12:564583:[41757451[NC:IMPRECISE
+1 564621 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs10458597;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:145,.:145,.:11,.:-145,0,0:-11,0,0:467:464,.:3
+1 564654 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 564714 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 564862 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.92|rs1988726;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:222:467,222:467,222:20,50:-467,-222,0:-50,-20,0:451:448,448:2
+1 564868 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 565006 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs142650224&dbsnp.92|rs1856864;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:270:270,460:270,460:53,20:-460,-270,0:-53,-20,0:313:311,311:2
+1 565148 . T A . . NS=1;AN=1;AC=1;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:166,.:166,.:11,.:-166,0,0:-11,0,0:495:494,.:1
+1 565286 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs140432825&dbsnp.88|rs1578391;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:234,.:234,.:12,.:-234,0,0:-12,0,0:322:308,.:14
+1 565406 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6594029&dbsnp.134|rs144191907;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:422:926,422:926,422:41,53:-926,-422,0:-53,-41,0:259:254,254:5
+1 565454 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 565464 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6594030;CGA_FI=100131754|XR_108278.1|LOC100131754|UTR|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:380,.:380,.:16,.:-380,0,0:-16,0,0:388:388,.:0
+1 565490 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs7349153&dbsnp.134|rs141292355;CGA_FI=100131754|XR_108278.1|LOC100131754|INTRON|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:319,.:319,.:15,.:-319,0,0:-15,0,0:353:351,.:1
+1 565508 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9283150&dbsnp.134|rs145079224;CGA_FI=100131754|XR_108278.1|LOC100131754|INTRON|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:288,.:288,.:14,.:-288,0,0:-14,0,0:296:294,.:1
+1 565541 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6594031&dbsnp.134|rs145787647;CGA_FI=100131754|XR_108278.1|LOC100131754|INTRON|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:374,.:374,.:16,.:-374,0,0:-16,0,0:272:269,.:3
+1 565591 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7416152&dbsnp.134|rs141914161;CGA_FI=100131754|XR_108278.1|LOC100131754|INTRON|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:249:249,367:249,367:53,16:-367,-249,0:-53,-16,0:262:255,255:7
+1 565697 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs3021087&dbsnp.134|rs143799743;CGA_FI=100131754|XR_108278.1|LOC100131754|INTRON|UNKNOWN-INC;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:218:218,469:218,469:50,20:-469,-218,0:-50,-20,0:202:196,196:6
+1 565870 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9326619&dbsnp.134|rs148172508;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:276,.:276,.:14,.:-276,0,0:-14,0,0:347:344,.:3
+1 565937 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 565976 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9283151&dbsnp.134|rs138694460;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:172,.:172,.:11,.:-172,0,0:-11,0,0:321:311,.:9
+1 566001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=568000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:98.61:1.03:.:0:.:0:0.999:426
+1 566010 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 566021 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6421778;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:224,.:224,.:12,.:-224,0,0:-12,0,0:396:395,.:1
+1 566024 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6421779;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:224,.:224,.:12,.:-224,0,0:-12,0,0:397:397,.:0
+1 566048 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6421780&dbsnp.134|rs145777875;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:105,.:105,.:9,.:-105,0,0:-9,0,0:209:208,.:1
+1 566130 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs147381188&dbsnp.92|rs1832730;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:517:517,1336:517,1336:53,44:-1336,-517,0:-53,-44,0:340:338,338:2
+1 566371 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs56133209&dbsnp.134|rs149516726&dbsnp.92|rs1832731;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:416:416,987:416,987:53,43:-987,-416,0:-53,-43,0:256:254,254:2
+1 566390 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7418044&dbsnp.134|rs144069128;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:407:897,407:897,407:39,53:-897,-407,0:-53,-39,0:298:297,297:1
+1 566573 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs143522810&dbsnp.92|rs1856866;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:265:265,730:265,730:53,33:-730,-265,0:-53,-33,0:133:130,130:3
+1 566771 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs146545403&dbsnp.96|rs2185536;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:222,.:222,.:12,.:-222,0,0:-12,0,0:453:452,.:1
+1 566778 . C T . . NS=1;AN=1;AC=1;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:236,.:236,.:12,.:-236,0,0:-12,0,0:502:502,.:0
+1 566792 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9283152&dbsnp.134|rs140024420;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:129,.:129,.:11,.:-129,0,0:-11,0,0:301:301,.:0
+1 566816 . C A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs151278223&dbsnp.96|rs2185537;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:41,.:41,.:5,.:-41,0,0:-5,0,0:120:118,.:2
+1 566849 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs140536240&dbsnp.96|rs2185538;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:305:305,713:305,713:53,32:-713,-305,0:-53,-32,0:155:152,152:3
+1 566916 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3949348&dbsnp.134|rs145370722;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:372:372,906:372,906:53,40:-906,-372,0:-53,-40,0:240:235,235:5
+1 566933 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.132|rs113120793&dbsnp.134|rs142636660;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:342:342,832:342,832:53,37:-832,-342,0:-53,-37,0:180:176,176:4
+1 566960 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs146905624&dbsnp.96|rs2185540;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:420:1045,420:1045,420:44,53:-1045,-420,0:-53,-44,0:175:174,174:1
+1 567002 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9285834&dbsnp.134|rs138612401;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:567002:PASS:61,.:193,.:11,.:-61,0,0:-11,0,0:486:486,.:0
+1 567005 . C T . . NS=1;AN=1;AC=1;CGA_SDO=10 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:567002:PASS:66,.:207,.:11,.:-66,0,0:-11,0,0:541:541,.:0
+1 567033 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 567037 . TTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 567061 . CC CTGCT,CT . . NS=1;AN=2;AC=1,1;CGA_XR=.,dbsnp.119|rs9326621&dbsnp.134|rs143484473;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/2:.:PASS:75:75,240:22,187:4,11:-240,-240,-240,-75,0,-75:-11,-11,-11,-4,0,-4:120:16,65:1
+1 567092 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9326622&dbsnp.134|rs148002581;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:53:53,587:159,534:43,24:-587,-53,0:-43,-24,0:116:116,116:0
+1 567119 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9283153&dbsnp.134|rs141701738;CGA_SDO=10 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:221:221,583:221,583:50,26:-583,-221,0:-50,-26,0:123:117,117:6
+1 567191 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs139593516&dbsnp.96|rs2185541;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:341:341,816:341,816:53,36:-816,-341,0:-53,-36,0:141:136,136:5
+1 567230 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 567239 . CG C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs60652689&dbsnp.131|rs78150957&dbsnp.134|rs147555239&dbsnp.134|rs149882676;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:28,.:28,.:4,.:-28,0,0:-4,0,0:358:357,.:1
+1 567486 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.92|rs1972377;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:204:204,498:204,498:48,22:-498,-204,0:-48,-22,0:134:133,133:1
+1 567489 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.92|rs1972378;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:174:393,174:393,174:17,44:-393,-174,0:-44,-17,0:136:134,134:2
+1 567575 . TAGCCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 567697 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs148529803&dbsnp.92|rs1972379;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:617:617,1464:617,1464:53,44:-1464,-617,0:-53,-44,0:365:365,365:0
+1 567783 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs142895724&dbsnp.92|rs2000095;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:337:635,337:635,337:28,53:-635,-337,0:-53,-28,0:240:240,240:0
+1 567807 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2853819&dbsnp.134|rs151077676;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:471:1005,471:1005,471:44,53:-1005,-471,0:-53,-44,0:362:360,360:2
+1 567867 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs142547273&dbsnp.92|rs2000096;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:119:256,119:256,119:26,38:-256,-119,0:-38,-26,0:65:64,64:1
+1 568001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=570000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:100.32:0.91:.:0:.:0:0.999:426
+1 568072 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2853820&dbsnp.134|rs147963388;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:293:713,293:713,293:32,53:-713,-293,0:-53,-32,0:193:192,192:1
+1 568201 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4098611&dbsnp.134|rs150293476;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:287:287,489:287,489:53,21:-489,-287,0:-53,-21,0:229:228,228:1
+1 568235 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 568256 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.108|rs4098612&dbsnp.134|rs137953150;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:131,.:131,.:11,.:-131,0,0:-11,0,0:336:336,.:0
+1 568361 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4098613&dbsnp.134|rs143080150;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:580:580,1458:580,1458:53,44:-1458,-580,0:-53,-44,0:354:354,354:0
+1 568404 . G A . . NS=1;AN=1;AC=1;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:349,.:349,.:15,.:-349,0,0:-15,0,0:467:467,.:0
+1 568419 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 568442 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs7419218&dbsnp.134|rs146264256;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:127,.:127,.:11,.:-127,0,0:-11,0,0:273:272,.:0
+1 568463 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 568572 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7413388&dbsnp.134|rs139256206;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:490:490,1198:490,1198:53,44:-1198,-490,0:-53,-44,0:219:216,216:3
+1 568616 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7411884&dbsnp.134|rs144294451;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:408:408,972:408,972:53,42:-972,-408,0:-53,-42,0:147:142,142:5
+1 568691 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.117|rs8179256&dbsnp.134|rs141850957;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:257:257,459:257,459:53,20:-459,-257,0:-53,-20,0:170:169,169:1
+1 568703 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7411906&dbsnp.134|rs143001019;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:163:163,245:163,245:43,13:-245,-163,0:-43,-13,0:166:166,166:0
+1 568718 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7417964&dbsnp.134|rs150918512;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:152:152,247:152,247:42,13:-247,-152,0:-42,-13,0:130:129,129:1
+1 568745 . C CCA . . NS=1;AN=1;AC=1;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:126,.:126,.:7,.:-126,0,0:-7,0,0:140:140,.:0
+1 568752 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9326624;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:146:180,146:180,146:14,41:-180,-146,0:-41,-14,0:102:102,102:0
+1 568941 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.117|rs8179289&dbsnp.134|rs150104574;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:208:208,356:208,356:49,15:-356,-208,0:-49,-15,0:162:158,158:4
+1 569004 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 569010 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9285836;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:264,.:264,.:14,.:-264,0,0:-14,0,0:291:291,.:0
+1 569052 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs138605248&dbsnp.96|rs2153588;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:166,.:166,.:11,.:-166,0,0:-11,0,0:237:235,.:2
+1 569094 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9283154&dbsnp.134|rs140303844;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:351:623,351:623,351:28,53:-623,-351,0:-53,-28,0:319:319,319:0
+1 569204 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.132|rs112660509&dbsnp.134|rs145433303;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:198,.:198,.:11,.:-198,0,0:-11,0,0:368:367,.:1
+1 569226 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 569267 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 569492 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6594033&dbsnp.134|rs147253560;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:255:522,255:522,255:23,53:-522,-255,0:-53,-23,0:218:218,218:0
+1 569609 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 569624 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6594035&dbsnp.134|rs144764545;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:472,.:472,.:21,.:-472,0,0:-21,0,0:381:379,.:2
+1 569717 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs147510063&dbsnp.96|rs2096044;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:40:272,40:272,40:14,29:-272,-40,0:-29,-14,0:301:301,301:0
+1 569803 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs150050719&dbsnp.96|rs2096045;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:312:312,823:312,823:53,36:-823,-312,0:-53,-36,0:285:282,282:3
+1 569874 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.96|rs2096046;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:391:391,834:391,834:53,37:-834,-391,0:-53,-37,0:326:325,325:1
+1 569878 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9283155;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:478:1099,478:1099,478:44,53:-1099,-478,0:-53,-44,0:352:351,351:1
+1 569983 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs145414098&dbsnp.96|rs2096047;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:437:437,1105:437,1105:53,44:-1105,-437,0:-53,-44,0:334:333,333:1
+1 570001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=572000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:9.98:0.94:.:0:.:0:0.999:426
+1 570076 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 570079 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9283156;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:122,.:122,.:10,.:-122,0,0:-10,0,0:367:366,.:1
+1 570094 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 570097 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2298012;CGA_SDO=9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:61,.:61,.:6,.:-61,0,0:-6,0,0:209:207,.:2
+1 570178 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9326626&dbsnp.134|rs146675873;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:412:412,981:412,981:53,43:-981,-412,0:-53,-43,0:208:206,206:2
+1 570301 . C <CGA_NOCALL> . . END=570596;NS=1;AN=0 GT:PS ./.:.
+1 570623 . CA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:570623:VQLOW:25,.:0,.:0,.:0:0
+1 570626 . G GA . . NS=1;AN=2;AC=1;CGA_SDO=9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:570623:VQLOW:25:25,25:0,0:5,0:-25,0,-25:0,0,-5:11:10,1:10
+1 570634 . TTAATGATAAACCCATTCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 570662 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 570668 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 570672 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 570679 . ATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 570814 . GCTCTTTTCATTACACATGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 570918 . AAAAGGTCACCAAACCAAATTTGGGTCCACCCACCCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 571203 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 571253 . TCACACCTGTAATCTCAGCGCTTTGGGAGGCCAAGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 571319 . GACCAGCCAGACCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 571344 . CACCATCTCTACTAAAAATACAAAAATTATCTGGGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 571439 . AACGTGGGAGGCAGAGGGTTCAGTGAGCTGAGATCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 571489 . AGCCTGGGCAGCAGAGTGAGACTCTGTCTCAAACAAACAAACAAAAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 571590 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 571620 . GGGTGTGACTTCCTCCAGGTCCCTCAGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 571662 . AAAGAACAGTTGTGAGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 571834 . TTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 571948 . TTTAAGATTAGTTTGTGGCTCTAATTTTCTTGACTATTGTTTTAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 572001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=574000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.74:0.92:.:0:.:0:0.999:426
+1 572315 . AGCGTGCCCCCAATTTTGCATGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 572398 . AAGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 572468 . ACACACCAGATTCATTCCCTGATTAGAGCTGCTGAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 572550 . CTCATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 572638 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 572947 . ACCTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 573002 . TTCATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 573098 . ACAAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 573212 . TGGCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 573393 . TCTGATAAATAAATAATAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 573493 . TTGCTCAATGCAAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 573532 . GAGCAAATAGATAGTTAACCACTCTTTAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 573752 . AGCCGACTGCTGGGTGCGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 573888 . TCCTGATCAGCTCTTTATTCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 574001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=576000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.01:.:0:.:0:0.999:426
+1 574140 . AAAATTTTTTTAAAAAAAAAGAAGAAGAGTACCTACTGTATAGCATTGATTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 574312 . GAGGGACAGAAAGAAGTAGGAGAAGGTAAAGAGATGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 574389 . TTTATTATGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 574433 . AGAAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 574556 . TGTGATTATTTGAACTTCAGCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 574599 . GAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 574661 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 574695 . AGTTAAAATGCCATGGTTGTCTATTGGCTTAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 574841 . CTCTTCCGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 574882 . AAATGTCTGGCTTTCTGACTCATAGGTGTGTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 574970 . TCAATTTTATTGAAGTTCACTTCTGACCTCTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 575023 . AACTGCCCAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 575127 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 575168 . TTCCTCACCCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 575206 . GTTGTTTTTTTTTTTTTAGACGGAGTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 575276 . CACTGCAACCTCCACTGCCTGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 575476 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 575658 . ATTCTCCACATGGATGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 575690 . AAATGAAAATCTGACCACGTTACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 575726 . TCCGCCTATGGCCGCTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 575881 . GTGCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 575926 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 575930 . GTG . . . NS=1;AN=0 GT:PS ./.:.
+1 575945 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 575958 . CATGATTTCATTTTGCAAGGGTTCCTTCCTTGGGCTGTGTTCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 576001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=578000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:0.94:.:0:.:0:0.999:426
+1 576036 . GCCTCACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 576190 . AAATCACATCACATTGCTTCCTTCATATTTTTTTGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 576291 . TTTCTTTTTTTTTTTTTTGAGTCAGAATCTTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 576346 . AGTGGCGCGATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 576436 . ACCTACCACCACGCCTGGCTAATTTTTTTTTTTTTTTGTATTTTTTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 576519 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 576552 . GGCCTCCCAAAGTGCTGGGATTACAGGCGTGAGCCACCGTGCCTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 576719 . ATAAACTAAATGTTTTCCAAAGGGAATAGGGCAAAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 576863 . GCTCTCCACTTACAAGAAGAGAGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 576913 . CAACACGCTGTGAGTGCAGGCAGCTACCAGGAGGAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 576983 . CCTGTTAATTTAATCACACGGAACACTTCTATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 577071 . ACTGCTTGGAGTGTCAGGCCTAGATCTCTATCCATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 577145 . GATGGGGCAATTTCTGAAAAGCACCATGTATTTTATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 577258 . CTAACTAACCACTATAAAGAACCCAGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 577472 . CTGGACAAGGAGGGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 577532 . TTAAGGGAGACCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 577576 . TGGGAGGCACAGTGGAAGATCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 577644 . CCTGCACACAGGCTAGGGGTAGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 577721 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 577821 . TCACGGAGGAAAAAAATCTCTCAATGATCTTATCTTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 577942 . CTCCGAAGGTGGGGCCCTCTGCTCACCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 578001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=580000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:0.99:.:0:.:0:0.999:426
+1 578047 . TTTCTCCTCATTAGATAATAATGAATGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 578098 . GTGAGGAAATCTACAAAATTAATTTCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 578185 . AAATGGCCTTTCGAGTTGAGCAGTAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 578253 . TTTAACAGGGGCATTCCAGCACTTCTCTAGCTACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 578333 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 578343 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 578365 . CTGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 578420 . TAACATTACAGTAACTGTTACAGGTTCCAGCAGGATAACTGGGTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 578506 . AAGTTGGTAGACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 578630 . ATGTGTGTTGGGATAGAGTGGTAAGAAAATGGGAAATAATAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 578696 . TTATAAAAAGGTGAGCTGTAATAAATACTAGTGCCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 578883 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 578903 . TAAACATCAGCCATGTTTATATAACTAAACTAGTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 578969 . CCACATGGTGGCTTAATGCTGCATTGATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 579008 . TTTGTTTTCACTTTTCTGCAAAATATTTAATACATTATTAAATTGAATTATGCTGATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 579085 . AA . . . NS=1;AN=0 GT:PS ./.:.
+1 579102 . TTTAATTTTTTTTTCCTTTGGTTTCATTATTCAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 579153 . CAACATTTTATCTGATGGAAGAGATGGAGTCCATTACTAAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 579239 . TAAAGGAAATTTACTGTGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 579384 . AGAGACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 579434 . CCACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 579472 . GACATGTCACTTCCAGCAGAAGCTTTAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 579534 . TGTAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 579586 . CCCTGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 579857 . TTCCCTCAGAACCCTTAGCCTGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 579991 . TCATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 580001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=582000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.66:0.90:.:0:.:0:0.999:426
+1 580015 . ACATGGGCACCCATATTTTTCTAGCCACTTTCATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 580075 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 580085 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 580110 . CCAGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 580134 . GTGATTTTCTGTTGGTGTTCACTTCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 580183 . TTATTGACTGACTGACTAACTAATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 580231 . AGGCTCTCTACAAAAACGGAGGGATGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 580276 . TACGTAAGAAATTGCCTCCGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 580362 . CAGGTTTTTAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 580398 . TATTTGTGTGTGTGCATGTGGTAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 580491 . GGGGAAAACATTTTCCCAAGGTTCTAACAGAAGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 580555 . TGTGAGGGTTGCTTTTATGTATTTATTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 580617 . CTTTCTCTTTTTTTCTTCTTTTTTTTTTTTTGGACAGAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 580664 . TGTCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 580792 . ATAATTTTTTTATATTTTTAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 580882 . GCCTCCCAAAGTGCTGGGATTACAGGCGTGAGCCACCGCCCCTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 580939 . TTATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 580981 . TTAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 581142 . TTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 581285 . AGGGGAGATTTTTCAGGAGTGCCACAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 581338 . CCTGCTTTCATTGGAAAGTGTATAATGATGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 581400 . ACTGCCGTCATAGGGATGCCTTAGTGAATCAATCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 581469 . AAAAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 581612 . TGACATGTAAGCATTGCCATGCCTAGTACAGACTCTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 581740 . TTATAAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 581829 . CACTGGGAATAACCTCTGTACTTTGGACAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 581938 . CATGTTTCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 582001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=584000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.74:1.01:.:0:.:0:0.999:426
+1 582013 . TCCGTGTTACTGAGCAGTTCTCAGCAACACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 582073 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 582192 . ACAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 582253 . AAAGAAAAATGATGGTAAATGAGACATTAATTTACCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 582377 . CCTAAAAAAGTAAACATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 582460 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 582515 . CACGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 582570 . TCTATGTCAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 582633 . GAGGCCAAGGTGGGCAGGTCACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 582691 . GGCGACACCCTGTCTCTACTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 582738 . TGGCGCATGCCTGTAATCCCAGCTACTTAGGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 582799 . GAAGGTGGAGGTTGCAGTGAGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 582850 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 582867 . TGTCAAAAAAAAAAAAAAAAAAGAAATCCAAATAAAATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 582916 . GTGGAAAATAGTGACAATAAAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 582965 . TTGAGATGCCAAGGTGGCAGGATCACTTGAGACCAGGAGTTCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 583046 . ACACGCCAAAAAAAAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 583149 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 583159 . CAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 583207 . TCCTATCTCAAAAAAAAAAAAAAAAAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 583335 . GGAAATAATGTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 583372 . CACGGTGACTCACATCTGTAATCCCAGCACTTTGGGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 583433 . GGTCAGGAGTTCCAGACCAGCCTGGCCAACATGGTGAAATCTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 583498 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 583541 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 583558 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 583562 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 583573 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 583693 . AACATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 583724 . ATGCCTTGAAAAGAGGGAGAAAAATTGTGAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 583777 . GGAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 583888 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 583955 . ATCTCCTCCCCTCCCCTACTCCTCACCCCACACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 584001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=586000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:0.98:.:0:.:0:0.999:426
+1 584058 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 584136 . CAAGGGCTTCACAGACAGTTTTACTAAACTCATGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 584201 . TATACCTTATAGATAAAGGTATCTATAAGGTATAGATAAAGGTAAGGTATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 584261 . TAGATAAAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 584297 . TGTTCCCAAAGCCTCGTGGCTAGTAATTCAAACCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 584361 . CCTCATGATACTATACTGCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 584460 . TGATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 584519 . AATTTGAATAACTCCCTGCGGGTGAAGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 584592 . CGG . . . NS=1;AN=0 GT:PS ./.:.
+1 584633 . GGGTGGATCATGAAGTCAGGAGTTGAAGACCAGCCCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 584687 . CCATCTCTACTAAAAATAAAAAATTAGCCGGGCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 584769 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 584834 . GGGTGACAGGGCGAGATTCCGTCTCAAAAAATAAAATAAAATAAAATAAAAAATAAAAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 584944 . ATTCCTCAGCATTTTAGTGATCTGTATGGTCCCTCTATCCGTCAGGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 585045 . AGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 585109 . TTGAAATGTGATCCCCATTGTTGGAGGTGGGGCCTAATGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 585340 . CAGAGTAGCTAGGATTACAGGTACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 585403 . AGACGGGGTTTCACCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 585433 . GTCTCAAACTCCTGACCTCAGGTGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 585466 . CCTCGGCCTCCCAAAGTGCTGGGATTACAGGCGTAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 585566 . TGGTACAAACCCCTCTCTCTTGCCACGTGATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 585632 . TGAGTGGAAACAGACTAAAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 585705 . TGAACCAAATAAACCTCTCTTCTTTAAAATTATTCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 585760 . AACAACACACACACACACACACACACACACACACACACACACACGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 585815 . CTAAAACAGGAACTAATTAGAAATGGTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 585863 . CGAGGCTCCCCAACAGGAACTGAGGCCATGGATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 585924 . TAATGGTTAAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 585965 . GCCAAGGCCTCCCATGGACCAAACTCAACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 586001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=588000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.97:.:0:.:0:0.999:426
+1 586015 . CCTGAGTGTTGCATTCAGCAGAAGTCAGCTTCCTAGGGAATATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 586126 . CAACAGAGCGACTCAGATGCTATAAAACTTGCTAACGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 586192 . CAAGCCAGGTTTTAGTCATCAGAAATCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 586301 . CACACGA . . . NS=1;AN=0 GT:PS ./.:.
+1 586346 . TTCGAAAACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 586437 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 586525 . ATCGGAAGAGACAGACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 586689 . ATTTAATAAATATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 586782 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 586786 . GTA . . . NS=1;AN=0 GT:PS ./.:.
+1 586956 . TTATTAATAAAAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 586993 . AAAATGTAAAAAGTATCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 587042 . CCATAGTAGAAAAAAGTGAAAATTAATAAAATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 587168 . AATAAAAAAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 587337 . CTTCAAAAAAAAAAAAAAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 587413 . TATAACACACACACAAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 587447 . TTCACAGAGAATTCCACCAAACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 587482 . ATCATCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 587532 . CAATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 587656 . GCCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 587762 . ATATATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 587824 . GATGTTTTTCATATTTTTTTCTTTTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 587876 . TTTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 588001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=590000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.16:.:0:.:0:0.999:426
+1 588003 . CATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 588052 . TTTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 588090 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 588167 . CCCTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 588216 . GTCTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 588258 . ACATATAAATAAATAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 588331 . TAATTAAAAAGTTACTTTTTACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 588382 . TGGAAAAAAGAATCAGTGAACTTGATAGATCAAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 588452 . AATGACAACAAAAAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 588541 . GGAAGACTATTTGAAGAAATGTGTTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 588595 . AATATATACATTCAAAAAGCTCAGTGCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 588653 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 588721 . CAACGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 588755 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 588769 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 588772 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 588774 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 588777 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 588788 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 588860 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 588888 . TAGAATGACATTTTAAAGTTCTGAAAGAAAAAAACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 588939 . TCTGTAACTTGGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 588993 . GATTAAAAAAAAAAGAGAGAGAGAAAGAGAAAGAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 589043 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 589058 . AAGAAAAAGAAAGAAAGAAAGAAAGAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAGAAAAAGAAAGAAAAGAAAGAAAGAAAAGCAAGCAAGCTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 589220 . CACTTTTAAAAAAAAAGACTCCTTCAGATACAAACTAAAAAACACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 589325 . ATATAAAAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 589360 . TTTAAATATTCTATATGTTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 589429 . GATGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 589509 . CTATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 589569 . CCCCATGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 589715 . ACAGAAAACAAAAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 589880 . ACACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 590001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=592000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.95:.:0:.:0:0.999:426
+1 590025 . TTTCAAAATTAAACAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 590081 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 590088 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 590187 . ACACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 590247 . CTATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 590326 . ATATGTTAGGCCATAAGATAAGCTCAATAAACTTAAAAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 590546 . AAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 590619 . AACATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 590703 . AATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 590748 . TATACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 590931 . ATGCAAAAAAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 591056 . TGTGGATTGGCAATGCATTCTTAGATAATACAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 591156 . TATAAAGAATTAGAGGGGAATTTGGTGAAAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 591269 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 591279 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 591391 . GAAAATAATAATTAATCTGATTAATTTTTGACTGTTCTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 591499 . AAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 591529 . TTCTTTTTATATTAGTATAAATATAACAATTCTGAAACAAATGTATGTGCATTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 591593 . CCAATGAGTAAATATTAATATATTTGTATTGCTAGAACCCCAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 591645 . GTGAAAGGACAGAGATACAGATATGGAATAAGACAAGGAAAGAAGCAGCCCACTGAGTTACATTAGAATCAGTATTATCAACATAAATATACAATGTGCTCTCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 591766 . TCTTAAAAAATATATAATATGTACATATTATATATTATATGCATAGACACACGTGTGTCTATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 591835 . CTACATGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 591873 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 591878 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 592001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=594000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.99:.:0:.:0:0.999:426
+1 592004 . CACGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 592088 . TCTAACCAGCAAAATTCACAGTGTGGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 592232 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 592265 . GCAGGAAGGACACAGCCGTGAAAATGCAAGGACGCCTCTACTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 592480 . TGTGTTTTGTTTTTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 592906 . ACGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 592933 . TTGGGGGTGGCATCTTCATCAGTAAATCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 592988 . TA . . . NS=1;AN=0 GT:PS ./.:.
+1 593048 . GAATAAAAATAGCAGGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 593088 . TATGTTTTTAATTTGTTATATATGTATATTTTTATCATACTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 593148 . TGCACAATGTGCAGGAATAAAATTTATGTTTTTAAAATTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 593227 . AGCCTCATCACAGGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 593260 . TGCCAAAACAATCTATCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 593317 . GGAGAAAAAAGCAATGGAATGAATAAAATGATAGCCACAAAAATCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 593438 . AATACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 593494 . TTTCTATAGTAACAGTTTTTAAATAAAATATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 593566 . AGTGAAAAAAAACAAATTCAGAGCAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 593658 . TAAACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 593711 . ATCTGTGCCAAGTGGTGTATTAATGATTCATTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 593775 . GGTGTAGCCTGCAACTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 593873 . ATGTTATATATATAGAAAGAGAGAGAGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 593965 . AACAGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 594001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=596000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:1.11:.:0:.:0:0.999:426
+1 594077 . TGCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 594179 . GTCCAAAAGCTTACTGTCTAGTGGGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 594243 . ATAAGTGAAAACTAAGATAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 594286 . AAGGTTTCCAAAGTCAATGAGGCCTCAAATGAATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 594377 . AGGAACAGCATGAGCAAATGCAAGGAGGCCTAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 594425 . AAAGAGGTGTAAGCAGCTTTGTACTGCTGCCTGATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 594499 . GAGTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 594526 . TAAATTATAAGAAATTTATAGCATAAGGAATAGTAGGACCGTTAAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 594591 . CTTCTTTTTTTAATATTTATTTTTATTATACTTTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 594647 . AACGTGCAGGTTACATATGTATACATGTGCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 594777 . GTGATGTTCCCCTTCCTGTGTCCAAGTGTTCTCATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 594828 . ATGAGTGAAAACATGCGGTGTTTGGTTTTTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 594892 . CTCTAGCTGCATTGTGGGAGGAAAAAAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 594980 . GATTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 595033 . AGATATGTGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 595087 . AAGTGAAAGGAAAGGATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 595149 . GCTTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 595183 . TATTTCAGAAACTATATGACATGACGAAAAGTAAAAAGGGGATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 595502 . TCAGAAAAATAAAAAAACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 595553 . CTTTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 595629 . ACTGTCCTTCCATTGCATTGTATGTGTTTTTTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 595778 . ATTCAATAATAAGTTTGCATATTACAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 595817 . TTGGTGTAATTCATCCATTCGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 595870 . GTCGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 595928 . AACTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 595998 . CTGGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 596001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=598000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:1.06:.:0:.:0:0.999:426
+1 596138 . ACCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 596180 . TTTGTTTTTTGCTTAGAAGTGCTTTGGCTATTAGGGATCTTTTTTTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 596262 . GTGAGAAATAACGTTGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 596355 . CGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 596369 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 596375 . CCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 596526 . TCTCAACAAACATTCAAACAGCTTGAATGTATTTGGTGTATAGAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 596658 . TTCTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 596775 . ATTGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 596833 . ACTTCTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 596891 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 597049 . TCTATTTTGAACCATTCAAGCACCCCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 597110 . TTTTGATGTGTTGTTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 597166 . GTGTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 597236 . TGCACAGTATTATTTTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 597289 . TGTTCCTCATGTTACAATGAAACTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 597350 . GTTTTTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 597397 . TAGTTTGTTATTATTACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 597444 . CCCACAGCTATTGAAATAACCATATTTTGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 597504 . TTGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 597621 . ATTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 597675 . TTACTATCATGGAGGTACCACCATATAAAACAAGTTGGAAAGTGTTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 597759 . GGCATTATTTCTTTATTAAATATTTGGTAATATTTCTTTATTAAATATTGCATCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 597821 . CCTGGAGTTCTTTCTACAGGAAAAAAAAATTTTCTAAATAAAATTTCTACAATGAAAAAAAAACTACT . . . NS=1;AN=0 GT:PS ./.:.
+1 597897 . CTAGTTTTTTTCTGATCATTTCATAAAAGTAGGTATTTTTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 597961 . GATTGTCAAATTTATTAATATAAAGTTTCATATTTTATATTTATTTTATCAGATAAATAAAATTATATGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 598001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=600000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.01:.:0:.:0:0.999:426
+1 598057 . AGCCATGTTAAGCTAACATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 598095 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 598098 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 598106 . ATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 598116 . TTA . . . NS=1;AN=0 GT:PS ./.:.
+1 598121 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 598149 . CAAGTACATCCCCTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 598178 . AAGGCTCCTCAAAAAAGTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 598213 . GTGGTCCAGCAATCCCACTGCTGCATATATACCCCCCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 598360 . GAATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 598393 . AGTATTATTCAGCCATAAAAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 598485 . GCACAAAAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 598607 . TGGGAGAGGTGGGGGATGGTTAATGGGTACAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 598735 . TAATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 598840 . TGTACCTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 598889 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 598976 . ACCCTTTGACCAACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 599169 . TTTTAAGGTTGTATACTATTCTATTGTGTATGTGTACCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 599306 . CTCATTGACACACTGATTTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 599360 . CTGAATCATATGGTAATTCTATTTTTACAGAATCATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 599422 . ATAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 599485 . ATCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 599540 . GTAGTTTTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 599624 . GAAATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 599682 . TTGTGTTGTTTGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 599767 . GTTATCACTTCACTCTGTTGACTTTCTTTTGCTGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 599912 . ATTCCCCTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 599987 . TTTATATGGTATGAAATAAGGGCCTAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 600001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=602000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:1.14:.:0:.:0:0.999:218
+1 600073 . CCCATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 600287 . AGAGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 600342 . AAATGTCATAGGAATTTTGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 600381 . GTACATCACTTTGGATAGTATGGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 600475 . ATCGCTTTCATCAATGTTTTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 600545 . AGTATTTTTTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 600703 . GTTATTGGTGTTTTCTATATATAAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 601003 . TTTCAAAAAGTAGCAACAACTGTGGGAGTTCAGTCAGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 601063 . TAGTTTTAAGAAATCGACACAAACCTTCATGGAAGGCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 601127 . GATCTGAATGAAGGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 601269 . CTCTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 601323 . GCCCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 601580 . CTCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 601773 . AGAGCCTCCCTCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 601979 . GTCAGCTTTTTTTTTTTCTTTTTGTGACCCAGCAGAATGCCTGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 602001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=604000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.51:1.23:.:0:.:0:0.999:218
+1 602110 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 602152 . CGGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 602252 . TGCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 602380 . TTTGTAAACATAGGTTGTGGTGCAGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 602450 . GAGTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 602523 . CCCTCTCATCTCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 602595 . CAAGAAAAAAAAAACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 602642 . AATCCTCAAAAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 602715 . ATGCAACTCAAAACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 602868 . AAGGCAGTGTGAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 602935 . TGGATATATGCCCAAAGGAATATAAATCACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 603017 . TCCCAGCACTTTGGGAGGCCAAGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 603183 . ATCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 603280 . ATATATATACATATACATACATATATATACATATATATACACATATATATATACATATATACATATATTATATAGGTAAATGTATATATATGTGTATATATATACACACATATATATACACATATATATACATATTATAACTACATATATATACACACACACATACATATACATGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 603480 . TTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 603490 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 603499 . GGAATCAACCCAAATGCCCATCAATGATATATTGGATAAAGAAAATGTGATATATATTCACCATGGAATACTATGCAGCCGTTAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 603631 . CCATCACCCTCAGCAAACTAACACAGGAACAGAAAACCAAACACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 603685 . CAGTCGTAAGAGGGAGTTGAACAATGAGAGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 603720 . ACG . . . NS=1;AN=0 GT:PS ./.:.
+1 603726 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 603739 . AACAACACACACCAGGGCCTCTCAGGGGGACAGGGGTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 603788 . GGACAAACACGTGGATACATGGAGGGGAACAACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 603828 . AGGGCCTCTCAGGGGGACAGGGGTAGGAGACCATCAGGACAAACACATGGATACATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 603892 . ACAACACACACCAGGGCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 603918 . GGACAGGGGTAGGAGACCATCAGGACAAACACGTGGATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 603967 . AACAACACACACCAGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 603995 . GACAGGGGGTAGGAGACCATCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=606000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.38:0.92:.:0:.:0:0.999:218
+1 604057 . AGGACCTCTCAGCGGGACAGGGGGTAGGAGACCATCAGGACAAACACGTGGATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604121 . AACAACACACACCAGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 604149 . GACAGGGGGTAGGAGACCATCAGGACAAACACGTGGGTACATGGAGGGGAACAACACACACCAGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 604223 . GGGGACAGGGGTAGGAGACCATCAGGACAAACACGTGGATACATGGAGGGGAACAACACACACCAGGACCTCTCAGCGGGACAGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 604317 . ACCATCAGGACAAACACGTGGGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604349 . GGAGCAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604412 . TGGATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604429 . ACAACACACACCAGGGCCTCTCAGGGGGACGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 604471 . ACCATCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604488 . GTGGGTACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAGGACAAACACGTGAGTACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 604583 . ACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 604642 . GTGGGTACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAGGACAAACACGTGGATACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 604734 . GGAACAACACACACCAGGGCCTCTCAGCGGGACAGGGGGTAGGAGACCATCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 604794 . ACGTGAGTACATGGAGGGGAACAACACATACCAGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 604840 . GGACGGGGGGTAGGAGACCATCAGGACAAACACGTGGGTACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 604951 . TGGATACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 605028 . TGGATACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 605085 . AGACCATCAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 605105 . TGGATACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAGGACAAACACGTGGGTACATGGAGGGGAACAACACACACCAGGGCCTCTCAGGGGGACGGGGGGTAGGAGACCATCAAGACAAACACGTGGGTACATGGAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 605281 . ACACACCAGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 605345 . AGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 605396 . ACACATTTACCTATGTATCAAACCTACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 605456 . AAATTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 605496 . TCTCCTTCTGAAACACTCTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 605579 . CATTTCCTTATTCCTGTGTTCATTTTGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 605647 . CTTCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 605700 . AAATTTCAATTATGTTATTCTCTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 605793 . TTTTAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 605820 . TACTAAATATAGTTATTTTATTTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 605873 . CTTCGCTAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 605903 . CATAGTTTTTTTCGTTTGTTTTCATGATTAGAAAAACAGAGAGAGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 605960 . AAAGGGAGGAGGAGGAGGAGGAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 605994 . GCAGAGAAGAAGGGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 606001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=608000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:1.14:.:0:.:0:0.999:218
+1 606033 . TAACGTTTCTCTAACAACTGTCTTCAGTGAAACGCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 606078 . TGGATTTTTAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 606130 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 606169 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 606201 . AAAATAAAATAAAAAGCTACACAAATTAAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 606265 . CCTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 606310 . TTCTTTTTAGTTATTCTGTTTATCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 606514 . CAGACACTTTATGTTCTCTTTTCTTTACAAGCATGCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 606559 . TATACAATGTGTGTATTGTTTTTATATATACCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 606618 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 606700 . AAAAGCAGTTATAAGAGGGACACTTATAGCAATAAATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 606902 . TAAGAATTGTTTTTTGAAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 606954 . TAAGAAAAAAGAAAACAAACTCAGAAATGAAAGAAGAGACATTAAAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 607036 . ACTATAATTAATTATTCACCAGCAAATTAGATAACCTAGAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 607093 . TACCAAAACTGAATCATGAAGAATTCAAAATTTAGAACAAATCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 607184 . GACCCAGGATTGAATGGCTCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 607299 . TTTCAAAACCAGCATTACCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 607421 . TTATTTTTAAGAAAACACAGCAAAAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 607619 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 607653 . TGAAACTAAGAATTTGTTCTTTGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 607717 . AAACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 607786 . ATATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 608001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=610000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.58:1.04:.:0:.:0:0.999:218
+1 608013 . TTTTAAAAAATAATAATACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 608060 . GAAGGAGAAAGAGTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 608105 . ATTGTTTTTTTTTTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 608176 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 608208 . TTGTGATAGTTTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 608265 . CATGAACTCATCATTTTTTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 608317 . CACATTTTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 608382 . GAATAGTGCTGCAATAAACATACGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 608433 . TTTATAATCCTTTGGGTATATACCCAGTAATGGGATGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 608480 . ATGGTATTTCTAGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 608502 . CCTAAGAAATCGCCACACTGACTTCCACAATGGTTGAACTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 608556 . C <CGA_NOCALL> . . END=608822;NS=1;AN=0 GT:PS ./.:.
+1 608831 . T <CGA_NOCALL> . . END=609042;NS=1;AN=0 GT:PS ./.:.
+1 609061 . TAAGAAAAGTATGGGCCAACCAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 609096 . ACATAGATACAAAAGTCCTCAAAATAAGTACTAGCAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 609146 . ACATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 609193 . ATGTTTCAGCAAACACAAATCAAATGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 609244 . GGATAAAAAAATAGCTATCTCTATATATGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 609313 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 609319 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 609335 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 609435 . CAAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 609523 . AATAAAAGATATTCAAATTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 609578 . ATATTATATATAGGAAACCCTAAAAACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 609665 . AATGTACAAAACTCAGTAGTTTCTTTACACTCACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 609754 . AAACTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 609855 . GATGAAAAATTGTAGATGACACAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 609969 . CAACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 610001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=612000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:1.12:.:0:.:0:0.999:218
+1 610046 . GGACCCTGAATAACTAAAGCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 610091 . GAAGGCCTCACAATCTGACTTCAAAACGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 610189 . CCCGCGCTGGGTCGGAGGAGCAGGAGTATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 610240 . ATGGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 610371 . GCAGTCTCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 610415 . GGCTCGCGTGGACCATCACCATCATCCTGAGCTGCTGCTGTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 610464 . CCACAGCCAAGCCAGCCCTCAAGTCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 610513 . CTGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 610676 . CTTTGAATTGCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 610731 . CCCTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 610771 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 610810 . CATTGTCGACTGCAGTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 610915 . CTCAGAGGCCGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 611014 . GTCCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 611075 . TTCAGTGATAAACTTGGACTAACTGTGGCCCAGGTATCAGCACTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 611160 . TATGGCAAGGCCTATCTCCCTCTGCTGAATCCAACAGGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 611272 . TA . . . NS=1;AN=0 GT:PS ./.:.
+1 611371 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 611406 . ACTGGTTGGACAGTGGGTGCAGCGCACGG . . . NS=1;AN=0 GT:PS ./.:.
+1 611459 . TGTCGCCTCACCCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 611575 . TCACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 611615 . GTGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 611675 . CACCGAGCTAGCTGCAGGAGTTTTTTTTTTTTTTTCATGCCACAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 611809 . CCCACCCATACAGAGCCCAGCAAGCTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 611861 . ACAGCAGTCTGAGGTTGACCTAGGACACTCGAGCTTGGTGGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 612001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=614000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.57:1.17:.:0:.:0:0.999:218
+1 612035 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 612064 . TCTGAAAAAAGGGCAGCAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 612130 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 612160 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 612164 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 612167 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 612173 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 612276 . CCTAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 612310 . CCCAGTAGGTGCCAACAGGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 612356 . CATCTGGTGGGTGCCCCTCTTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 612429 . CTCCGCTGGTGATGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 612478 . AACACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 612504 . GCCTGACTGTTAGAAGGAAAACTAACAAACAAAAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 612580 . ACCTCATCAGAAGGTTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 612907 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 612934 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 612979 . TCGATCAAGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 613060 . AATGAAAAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 613118 . CTACATTTGATTGCTGTACCTGAAAGTGATGGGGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 613165 . GTTAGAAAACACTCTTCAGGATATTATCCAGGAGAACTTCCCTAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 613243 . ATACGGAGAACATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 613294 . ATAGTCATCAGATTGAGCAAGGTTGAAATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 613342 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 613358 . GTCAGGTTACCCACAAAGGGAAGCCCATCAGACTAACAGCAGATCTATCAGCAGAAATTCTACAAGCCAGAAGAGAATGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 613459 . AAAGAAAAGAATTTTCCACCCAGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 613493 . CAGCCAAACTAAGCTTCATAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 613523 . AAATAAAATCTTTACAGACAAGCAAATGCTGAGAGATTTTGTCACCACCAGGCCTGCCTTAAAGGAGCTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 613621 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 613647 . CATACCAAATTGTAAAGACAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 613718 . ATCGTAATGACAGGATCAAATTCACACATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 613766 . GTAAATGGGCTAAATGCTCCAATTAAAAAACACAGACTGGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 613836 . TGTTCTGTATTCAGGAGACCCATCTCACGTGCAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 613881 . GGCTCAAAATAAAGGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 613923 . GAAGGCAAAAAAAAGCAGGGGTCGCAATCCTAGTCTCTGATAAAACAGACTTTAAACCAACAAAGATCAAAAGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 614001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=616000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.61:1.06:.:0:.:0:0.999:218
+1 614019 . TGGTAAAAGGATCAATGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 614049 . TAATTATCCTAAATATATATGCACCCAATACAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 614091 . GATGCATAAAGTAAGCTCTTAGAGACTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 614171 . CTGTCAATACTAGACAGATCAACGAAACAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 614222 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 614239 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 614271 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 614278 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 614290 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 614311 . ACCACATTGCACTTATTCTAAAATTGACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 614350 . AAGTAAAACACTCCTCAGCAAATGCAAAAAAAAATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 614406 . GATCGCAGTGCAATTAAATTAGAACTCAGGATTAAGAAACTGACTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 614500 . ACTGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 614547 . ACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 614557 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 614571 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 614575 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 614615 . ATTTATAGCACTAAATGCCCACAAGAGAAAGCAGGAAAGATCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 614685 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 614758 . AAGGAGATAGAGACACAAAAAAACCTTCAAAAAAAATCAATGAATCCAGGAGCTGGTTTTTTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 614836 . TAGATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 614948 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 615001 . GGACAAATTCCTGGACACATACACCCTCCCAAGACTAAACCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 615095 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 615121 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 615124 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 615145 . AGCCGAATTCTACCAGAGGTACAAAGAGGAGCTGGTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 615195 . AACTATTCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 615243 . ATGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 615269 . AAATTTGGCAGAGACACAACAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 615334 . GCAAAAATCCTCAATAAAATACTGGCAAACCAAATCCAGCAGCATATCAAAAGCTTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 615444 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 615522 . AAGGCCTTCGAAAAAATTCAACAGCCCTTCATGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 615564 . CTCAATAAACTAGGTACTGATGGAACATATCTCAAAATAATAATACCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 615666 . TTTGAAAACCAGCACAAGACAAGGATGCCCTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 615705 . ACTCCTATTCAACGTAGTATTGGAAGTTCTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 615790 . TGTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 615826 . TCTTCTTAAGCTAATAAGCAACTTCAGAAAAGTCTCAGGATACAAAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 615924 . AGCCAAATCATGAGTGAACTCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 615966 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 616001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=618000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.57:1.09:.:0:.:0:0.999:218
+1 616038 . TGCTCAAGGAAATAAGAGAGGACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 616138 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 616191 . ATTGGAAAAAACTACTTTAAATTTCATATGGAACCAAAAATGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 616352 . ACAGATATGTAGACCAATGGAACAGAACAGAGGCCTCAGAAATAACACCACACATCTACAACTATCTGATCTTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 616434 . TGACAAAAACAAGCAATGGGGAAACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 616492 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 616561 . TGTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 616593 . CATAAAAAACCCTAGAAGAAAACCTAGGCAATACCATTCAGTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 616678 . AATGGCAACAAAAGCCAAAATTGACAAATGGGATCTAATTAAACTAAAGAGCTCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 616741 . AAAAGAAACTATCATCAGAGTGAACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 616782 . GGGTGAAAATTTTTGCAATCTATCCATCTGACAAAGGGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 616857 . CAAGAAAATAACAAACAAACCCATCAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 616968 . GTCATCAGAGAAATGCATATCAATACC . . . NS=1;AN=0 GT:PS ./.:.
+1 617079 . ATAAGAATGCTTTTACACTGTTGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 617126 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 617185 . CAGTAATTGCATTACTGGGTATATATCCAAAGGATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 617232 . TACTATAAAGACACATGCACACATATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 617276 . CACAATAGCAAAGACTTGGAACCAACCAAAATGCCCATTCAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 617335 . AAAATGTGGCATATATACACCATGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 617400 . TTCAGGGACATGGATTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 617428 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 617489 . ATGAGAGTTGAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 617539 . CACCGGGGCCTGTCTGGGGGTAGGGGCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 617598 . ATGTAGATGATGGGTTGATGGGTGCAGCAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 617673 . GCATATGTACCCCAGAACTTAAAATATAATTTAAAAAAAAATCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 617901 . GAACAGGCAACCTGCAGACCTAAGCTTGATTCCCAAGTCACAGTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 617979 . AAGAGATAGAAATAGGGAATTTGAAGGAATAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 618001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=620000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.08:.:0:.:0:0.999:218
+1 618083 . GTTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 618211 . TCCGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 618310 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 618364 . TCAGTTTTCTGGGCAATCTTGGTGAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 618453 . CCCAAAACAAGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 618493 . CAATATGACACTTGCACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 618544 . GGAGCCCAGAAATAAACTCATGCATTTATGACCAATAAATTTTTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 618675 . ACACGAATGTGTATGGTGTGTATCCTTATCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 618756 . AACTGTAAAACTACTAGATGAAAACATAGGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 618828 . TATCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 618867 . TGGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 618934 . AAGTTGTGAGAAAATATTTGCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 619038 . AAATGGGCAAGGTCCTAAATAGACATTTCTCAAAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 619116 . ATCAGGGAAATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 619184 . AATGATAAAAGATAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 619474 . AAAATGTAGTGTATATATACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 619591 . CACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 619633 . TCTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 619687 . TAGTAGTTGGGGGTAGACATGGAAAAGGTAGATGTTGATAAAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 619752 . AAAGTTTCAGTGAACTATTGCACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 619843 . TGTTTTCACCACAAAAAAGATATGTATGTCAATAAGATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 620001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=622000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.02:.:0:.:0:0.999:218
+1 620043 . TCTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 620089 . TTGACGTCATTCCACCAGATTCTATCTCCAGTGTTAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 620167 . TGCGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 620217 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 620350 . AAACGAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 620426 . AGA . . . NS=1;AN=0 GT:PS ./.:.
+1 620446 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 620536 . TCTACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 620600 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 620642 . AAAGAAAATCCTACATAACCTATAAAAAACTGATGGAACTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 620763 . ATTGAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 620858 . CTACAAAACATTGCTGAAAGAAATTAAATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 620957 . ATTCATCAGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 620999 . AATCAATACATAATGTCTCATAGTTCCTGAATCTAAAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 621071 . TAATGAGAAGGGCTTATTATATCATTTATGAGATCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 621254 . GCCGTGTATACACAAACATGGGTGGACCAAAGAACAAAAGGACCACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 621377 . AGACGTAGGAGATTAGAAGTATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 621484 . AGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 621530 . AGAAGGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 621571 . TGACTGACACCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 621652 . TTACATAGGGCCACATATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 621703 . TCCACACCACCAACGACGTGGATGAAGAAGATTTGAGCGATGCAGCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 621841 . CTGGCCAGTAGAAAGTACATGGGGGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 622001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=624000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.83:0.92:.:0:.:0:0.999:218
+1 622042 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 622102 . TGAAGATAATAATATGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 622181 . TATTAAAAACAAAAAATCAACATAATATTATCACAACATGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 622276 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 622345 . CTGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 622428 . GCACCCACAATTCTCTGACAGAAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 622489 . ATCACTGCAGTAAGAGGTGCCTGGGACAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 622534 . CTGTGGATTGAGAGAATATAGAGACTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 622595 . AGACTAGAGTAAGTGGACTTTCACAAGAAATAGAATCACCACCATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 622660 . GCTTACTGCTATTTAAGTGCCTCAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 622707 . TACGAAGCCCTAAATGGCTTCCCATCCTGCAATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 622977 . CAGCTTTTATGTTCATTGCATTCCCCACCCTTTTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 623083 . ATTCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 623253 . CATATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 623373 . TAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 623447 . CAACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 623533 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 623643 . CACTTTTTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 623710 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 623728 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 623801 . TAATTGTTGAAAAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 624001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=626000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.18:1.12:.:0:.:0:0.999:218
+1 624049 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 624143 . TATATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 624285 . TTGGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 624456 . ATTCCTGTAAATAGAAAGAACCAAAAAAAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 624812 . AAACGAATAGAAAGTGGATAGTTCTTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 625120 . AAAGTAGAACAGAGATAAAGTTAAAAAAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 625196 . GAACAATGATTTAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 625255 . AAGGTAGGATGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 625297 . GGTTAAAAAGAACATAAATAAAATAAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 625339 . ACACTCATTCAAGGATGCTACTGAGTTTGACTTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 625389 . TAATACACTCATCTGGGGATGCTACGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 625469 . TATCGCCCTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 625721 . AAGATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 625794 . TTCAAAAGTCCAGAAACCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 625900 . ATAATTTGGACTTTGAGGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 626000 . GTCGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 626001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=628000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.06:.:0:.:0:0.999:218
+1 626051 . TAACGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 626117 . GTTCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 626165 . AGTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 626182 . TATGTAGATAGACATAATAAATTTTAGTATTCAATTATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 626231 . ATCGTAGGGAACAATAATTTATTATATATTCTAAAATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 626302 . AAAGAAAAGATAAATATTCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 626363 . ATTGTATACATGTATCAAAAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 626427 . AAAAAACAACAAAAAAACCAAAAGAATAGAAATCAAAAATAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 626576 . CATGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 626621 . TCTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 626929 . AAAGAAAAAAAATGTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 627016 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 627240 . CATCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 627410 . CATTAAAATATGCATCAGGCATGTGTGATGCATACAGTAGAAGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 627638 . ATAGCCATAAAAGAGGATAAATATTACAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 627706 . TTGTAAAATATTTCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 627791 . AGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 627920 . CCAGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 627953 . GTGATGTATCAAATGGAGGAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 628001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=630000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:1.12:.:0:.:0:0.999:218
+1 628077 . TATAGAAACAGAAATAGAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 628137 . AGCTGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 628189 . GCAAGAATAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 628411 . AGAGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 628476 . CTATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 628517 . GAATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 628735 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 628791 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 628829 . GAGTTGTCAAAAGCTGAATTAATTAAAAGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 628935 . CTTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 628962 . CATGAGAATGAAAAGGGCTGAGAAGGGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 629283 . CGGTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 629361 . ACTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 629487 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 629698 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 629811 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 629860 . TTTACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 629925 . ATGTGGACATGAAATCATATAATGTTCCATGGAAAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 629996 . ATCAAAACTAATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 630001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=632000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:1.19:.:0:.:0:0.999:218
+1 630158 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 630220 . ACATATTCACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 630267 . GCAGAAAAAATAAAACTCTAGATCAATCTCAATCGTGTAAATAAATTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 630443 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 630705 . ACACGTGTGTGGTGTAATGAAAGAAGGCAAAAAGATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 630762 . GCAATTTTCATTCTATTATGAAAAGAAAAGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 630893 . TGGAGACAAGATTCCAAAACTCGTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 630964 . CTTTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 631000 . TATTATATAATTATTTATAATAATTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 631040 . AACTTGATGACCTTTATAAAAATACTAGAAGAAAAAGGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 631109 . TATACCATGAAGCAGTAAAAAACAGGATCAAATTTTCTATGTTTTTTAAAAATATGCCTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 631178 . GTTTTCTCTCTGAAATGATATTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 631266 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 631311 . CAGGAAAAAAAGTGGGAAGTGGAGAGAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 631399 . AACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 631488 . GGTTAAAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 631570 . CCAGAAAAAATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 631634 . CAAACACAAAATAAACAACCCACAAAAAAAAAAAAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 631846 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 631956 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 632001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=634000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:1.12:.:0:.:0:0.999:218
+1 632083 . TGGGTGATTAAATATTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 632118 . ATTATTAGAAAAATTAAAAATAACAAAATCAAATAAAGTTGGATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 632229 . AGCACCCTTGGAAATAACACAAAAATTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 632339 . CATACAGAGAAATATTAATACCATGAAAATATTAAAATAAAAGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 632540 . CATAAAAATTACTCTGAAATACAATGAAAATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 632655 . CAGTTAACTGAGTGGAGAACAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 632706 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 632729 . GGGAGTTGTGCCTGAAACCACATTGTGAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 632946 . TTAGAAAGCAACATGCTTGCTAAGGCAAGAAATTTGATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 633092 . CTAATGCACCATCTAATGCTTCAAGTCTAATAATTATTCAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 633324 . TACCACACAGCTATATGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 633375 . TATACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 633582 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs11485756&dbsnp.131|rs75160317&dbsnp.135|rs192643069;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:48,121:48,121:42,43:-121,-48,0:-43,-42,0:0:0,0:0
+1 633623 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 633740 . GCCTTGAAGCTTCCATTTCTACATCTTGGAACCAACGTGTGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 633801 . AAGAATTCTGCTTATTCTGCTTGCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 633998 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 634001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=636000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:1.17:.:0:.:0:0.999:218
+1 634137 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 634196 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 634243 . ATAGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 634286 . GTTGGCAGTCCAAAGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 634335 . TAAACCTGTAAAATCAAAAGCAATTTAGTTATTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 634407 . GCCGTTCCAAATGGTAAAATTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 634450 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 634458 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs78339854&dbsnp.132|rs111275359;CGA_RPT=THE1B-int|ERVL-MaLR|13.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:114,48:114,48:42,42:-114,-48,0:-42,-42,0:1:1,1:0
+1 634583 . CCCCTGTGGTTTTGCAGGGGAGAGCCTTCCTCCCGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 634672 . TGTCGGTGGATCTACCATTCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 634708 . GCA . . . NS=1;AN=0 GT:PS ./.:.
+1 634715 . TCT . . . NS=1;AN=0 GT:PS ./.:.
+1 634725 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 634730 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 634733 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 634738 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 634764 . GGGCTTCAACCCCACATTTCCATTCCCCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 634915 . TTCCGTGAATCCACAGGCTCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 634995 . TGTACCTTGACCCCTTTTAGCTGTGGCTGGAGCAGCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 635362 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 635388 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 635478 . GACACCTTTACTCCAGTTCCCATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 635532 . AGCCTGGATTTCATTGTCTATATCATTATTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 635657 . CCCCTGCCTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 635747 . ATTAGTCTGTTTTCATGCTGCTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 635798 . TATGAAGAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 635877 . CACATCTCTACATGATGGCAGACAAGACAGAATGAGAGCTAAGTGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 635948 . CAGCTCTTGTGAGACTTATTCAAGACCACGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 636001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=638000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:1.03:.:0:.:0:0.999:218
+1 636018 . CCCGTTGGATCCCTCCCACAAGACACGGAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 636113 . ATAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 636231 . CCAGACTACCCAGCTGCCAGCTGAATACCACAGATGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 636295 . TCATGTAGCTAAGCCCTGCTTGCGCTAATACAAGTCCACAATTTTTTTTAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 636362 . TGCTAAGTTTTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 636465 . GAAGCCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 636523 . TGATAGCAAAACCTGTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 636646 . AAAATCATCAGTGGAGGCTGAGGAAAACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 636698 . AGAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 636725 . ATATAAAAAATAGGAAAAATACCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 636792 . AAAGAAAAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 636872 . TAGGAGAAGTCTTCCAGAGAGGTTCTGTCAGACTGCTCTGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 636976 . CCATTAAAAAATGATATCATGTCTTTTGCTGGAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 637043 . AATGCAGGAACAGAAAACCAAATACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 637289 . AAATAAAAGTTAAAAAAAAAGAAAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 637327 . ATCATCTACCTGGTAATATGAAAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 637391 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 637437 . CACCACTGGCTGGACGCAGTGGCTCACACCTGTAATCCCAGCACTTTGGGAGGCCGATGCTGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 637521 . TTCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 637562 . ACTAAAAATACAAAAATTAGCTGGGTGTGGTGGCAGGCACCTGTAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 637639 . ATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 637680 . TCACGCCATTGCACTCCAGCCTGGGCAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 637725 . CGGGGAAAAAAAAGAAAAAAAAAAACCACCGCCATCATTTTGCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 637851 . ATTTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 637874 . CTCTATATGTTATTATTTTGTATGTGATGACAACAGAATATATTATCATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 637935 . AATCTCATTCATAATATAAAGTATAAATTTGTGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 637993 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 638001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=640000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.27:.:0:.:0:0.999:218
+1 638073 . AAAATTTGAAACTAGTAACATGGAGGACTATTGTCATTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 638144 . CAGTGTACATAAAAATAATTTCAAGAAATTTATAAAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 638189 . TTATGGTGTATAAACAACTTTAGATTCTTTGTTTAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 638281 . GGCTAATAGTAGGCACCTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 638352 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 638379 . AATTACTGTTTAGAGAATAACATTTGATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 638430 . TTACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 638499 . ATGTGGCACCTGCTGAAGCCTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 638565 . CTCTAACATTTTTTAGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 638616 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 638685 . AGGTTAAATGGCACTAACTCAGGGAAGGCTTCCCTAACTGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 638767 . TAGTATCGAATCAAGTTTATAATTTTAAAATAATTGGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 638869 . CCTGTTTGTTCACTCCTGCCACAGTCAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 638926 . GAACAAGCACTAAATAAATGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 638989 . ACTGAATGGATCATGAACACTATCTGGTATGTCACGTAGGTAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 639107 . TCACTTCAGATTCTTCTTTAGAAAACTTGGACAATAGCATTTGCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 639170 . CTAAGAATCAAGAGAGATATCTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 639209 . GAAAACATTAAACACGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 639250 . TTATTATTAGAAACCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 639282 . AAAAATAGTAATACTTATTGCAGACTCAAATGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 639433 . GTGATTTTTCAGGTTCACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 639478 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 639544 . TATGAAAACAAGAGATAAATATACACAACTGAGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 639598 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 639607 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 639620 . C T . . NS=1;AN=2;AC=1;CGA_SDO=20 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:639620:VQLOW:34:34,34:34,34:12,27:-34,0,-34:-12,0,-27:15:1,14:14
+1 639661 . TAACTTTTAATAGAACCAGTCACTAAATTAAAAAAATGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 639910 . ACCTAACATAGACATTTGTATATGATAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 639964 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 640001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=642000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.24:.:0:.:0:0.999:218
+1 640022 . GGCCTGAATAAGAAATATTCTGGATAAGATATTGTGGCTGCTACCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 640204 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 640226 . TTCTTCTTGTGCCCTCTCCCTCTCTCTCTTTCTCTTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 640279 . AAGCCAACTGGCATGCTGTCAGTGGCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 640404 . TAGCTAAGAACCATGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 640444 . CCTCAGTTGAAATTTAAGATGACATATTGAGCAGACATACTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 640569 . GTATTACACCTTCAGTGAGCACGTGTACTAGAAATTTAAAAAATAAATAAAATAAACCTTCAAAGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 640727 . CAGGTTTCAGCCTGAACTCACACAATCTGTGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 640833 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 640868 . AGCAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 640898 . CATGAGCTTCTAACACACACACAAAAATCACACACACAAAATGGGGGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 641017 . AAACACATATTTTAATGTGGTTAATTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 641074 . CAAGAAAATTGTGCTGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 641108 . GGCTCCCCTCCTATTTAAGTCTGGGTACTGTGTCACCCGAAGTCTTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 641170 . GGTCTGGGTTTGCCTATGAAAGAAACTCATGAGAGCTGGAAATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 641233 . TCAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 641279 . GTGATGGCTTGCAGAATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 641341 . TTCAGAAATTCCTATAAGCTTGGGTTCTGTGCCCACACTCTAGACTGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 641405 . ATATAAAACAGACCTCTTCTGATTTTGTCTAGCTGCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 641497 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 641503 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 641511 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 641563 . TAAAATATAAAGAATTGTCCAGAAATATATAAAAAAAGAATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 641629 . AATATAACAATTGTATGGACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 641683 . TTTGAAGAAAAAAGCAATAAGAAGCCTCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 641785 . CTGTGAAGCTTGAGATGTTTATTATAATGAATTATCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 641834 . ACAATTTCCTAACAATTTTGGGGTTTATATTTTTGAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 641897 . TTGTACTATTGTTAGGTAACTTTGATGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 642001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=644000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:1.23:.:0:.:0:0.999:218
+1 642025 . TGTCTGTCCACGATAAGCACTATCACAAGGACTTTCTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 642117 . CTGTAGGTGTCTCTATAATAGGACCTATCATAAAAAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 642164 . CTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 642184 . CACCCTCACAAGAACACTTGCCTAGCAATGGCTGTTTCTGCCAGTAAGTTAACACCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 642252 . AGACCCTGTGACCAATGATGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 642375 . AGATTATAATCCTTGTTTATTTCCAAATAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 642458 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 642483 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 642537 . CATGTTTGCTAGAACTCACCTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 642587 . TTTTTGTGTTTAGTTTTTCTTTTGTGATTGGGGAGGGGGGTTTATCGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 642656 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 642763 . CTGCTTCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 642874 . CTCGTAGTATTTATAGTAAAAGTGAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 642936 . TGCCCAAGTGCTGGTCTGGTCTGATCTTCTCATCTTCCCTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 642991 . CAGTCACACTAGCCTCCTTGCTGCTCCACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 643081 . CTTTTCTCCCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 643136 . TAAATGTCCCATTCTCTGTGAAGCTTTCCTGCCCACCCTATTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 643243 . AAACTGTAAATATACATGTTCACTTTTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 643326 . TTTGGTTCACTGGTGTATTCTTAAAGCATGTTACATACTAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 643402 . ATTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 643453 . CAAATGTAATTCCTTACATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 643571 . GATACTAGGAAAAGAGGAAGGGATATATTATTTTCATGTATAAAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 643730 . AGGGCCAAAAGAGTCAACTTCTGAAGAAGCGCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 643808 . TTGCAAAAATAAACTCATGTGCCATAATTCATGAGTAGAAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 643997 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 644001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=646000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.21:.:0:.:0:0.999:218
+1 644111 . TTACCCAGATGGGCCCAGTCTAATCACATGAGTTCTTAAAAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 644232 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 644278 . CTAGAAGATAGAAAAGGCCAGGATATGGATTCTACCCTAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 644348 . TTGATTTTAGTTCACTAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 644458 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 644463 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 644522 . AATTCAAGGTGAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 644548 . CTTAAAACATTTAGATTAAAAATAAATGAGAATTTTTGTTACTTTTGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 644611 . AGAAAAACAAACATTAAGGAGGAAAAATGAACATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 644660 . TATAAAGCTTCTCTATTTGGAAGATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 644712 . AATATTTACAACATATATATAAGTGAATAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 644756 . TATATATGAACTCCCAAAAATCAACAGGAAAAATAAGACATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 644807 . AAATGCATAAACAAAAGAAGGCAAAACAAAAATAATGACTCATAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 644879 . GATGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 644891 . AATGCAAATTAAAACCACCCTGAGATGCTTTTTACATCCATGAGCCTGATAAAAGTTAGAGTCTAAAAGTAATAATTAACAAAGATGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 644993 . TCTTGTCCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 645090 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 645094 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 645163 . AACATTGTTTGTTATATCAAAAAATAAAAAAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 645208 . CAGCAAAAAAAATAAGTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 645252 . ATGGAATAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 645290 . ACTGTACATATGAATGTAAGTATCAGCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 645340 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 645360 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 645363 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 645472 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 645495 . AGTTGAGGGAATTTCAATTGGAAAAAAATAATATC . . . NS=1;AN=0 GT:PS ./.:.
+1 645538 . TAAGTCAGGTAGTGGGTATTAGCATTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 645588 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 645607 . TTCAATGTATTTAATATATTTTTTGCATAATTAAATATTATGCAATAAAAATGAGAAAACAAAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 645695 . AAAGAAATGGAGAAAAAATTATAATCTAGTTGAGTAATGGTATATTACATAGCTATTTTCTTAAGTAGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 645803 . CTTAATTATATATAAATATATATGTACATATTTTTAATATAAAATACTAAACAAAGTACACCAAAATATTAGCTCCTATGTTAGTGAGATAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 645905 . TTTTGTATTTTAAGTTTTACATAGTAGGTGTATTTTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 645956 . CTATAAAGAACTGCCCAAGACTGGGTAATTTATAAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 646001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=648000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.56:1.14:.:0:.:0:0.999:218
+1 646063 . AGACAAAGAGGAAGCAAGCCAGCTTCTTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 646104 . GAAGAAGTGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 646153 . CTCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 646167 . CTATCACAAGAACAGCACAGGGGAAACTGCCCCCATGATTCAATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 646277 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 646306 . CCATATCAGTAGGCATGTATTGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 646445 . GAATGTGGCCTTGTAAGAAAGCAAATTAACTTCTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 646542 . ACTTGAACTGCAGTAAAATATCCTCAGCAACATAGATGTGTATGTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 646622 . CCATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 646648 . ATTTCTGAAGATGTCCTGGCTTATTCACAGATGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 646736 . GAGAGCTGTTCCAAAGTTTAGGGAGTTTTTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 646776 . AATAAATAAAAATGTTCTTGAAAGAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 646873 . GCTATTTTTTTTTTTGACACACACTTTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 646914 . AATGTCTCCGGCAATAAATCACAAAGTTAAAATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 646976 . TGGTAAATCATTTTCTACCAAAAGAAAGAAATGTCTTGTCTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 647038 . AAAGTTTTCCTTGTTGGCGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 647115 . CAGGAAGATGACTCAGGGCCTTATCCATACCTTCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 647175 . TCAGTATCTATATCAAAAATGGCTTAAGCCTGCAACATGTTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 647247 . TTCATTGAATCCTGGATGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 647278 . AATAAGAGGAATTCATATGGATCAGCTAGAAAAAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 647334 . AAAGTTATATATTATATATCTATTATATATAATATTATATATCTATTATATATTATATATTGTATATCTATTACATATATATTATATATGTATTATATATATTATATATCTATTATATATATAATATTATATATTATATATCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 647484 . ATTCCCCAGCGTTCATATTTGTCAGTGCAAGTAAAGAGCCTTACTGCTGATGAGGTTTGAGGTATGACCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 647594 . T <CGA_NOCALL> . . END=647867;NS=1;AN=0 GT:PS ./.:.
+1 647881 . AGGTGTGAGCCACCACGCCCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 647933 . TTTGAAGGTCATAAAAAATATAATAAGAGATAAGGCTAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 647987 . ATAAAATCCTTTAATAAAAATATAAAGGAATAATATAATAATTTTATTTAATAAAATATAATAAGAGATAAGGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 648001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=650000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:1.30:.:0:.:0:0.999:218
+1 648068 . CTTTAATAAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 648104 . TCCAAAAAAAGAAATGGAGAGGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 648141 . ATTAATCTTGTCAAAAATATAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 648188 . ACTGTTTTCCTTGTCTGCGGCCATTGTGCTGCTGCTACACAACTACCGCAAGCAGCCCTTCACGCCCTCCTCCCAGTACAAAGCTAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 648287 . AAATGTTAAGCTTGGAAGAGTCAGCATCGCTGCACTTATTTTTTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 648360 . AGTGGGGGAAAGGTTAAAAACCCCCCTGGATAAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 648441 . TCCTTGTCCCTTGACATAAACTTGATAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 648516 . CAGGTACTTAAAGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 648565 . GCATTGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 648771 . AGTGACATTGCCTTTTAGTTGTACTTTCACAAAAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 648883 . TCTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 648917 . AGCCACCATAGTCTCTCCCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 648970 . CTTCTTTTCATATTTTTGAAAACTTTTGAAAAACTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 649079 . TTTGGTAACCCTTAAATTACTAAACCCAAAACAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 649149 . ATGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 649164 . CTTTCTTCTCTTTTACCCTTCTTTCTTGAATTCAGTCAAACAACGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 649219 . TTTCGTCTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 649246 . CACCTCAGCTTTCTCCATTCAGCTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 649307 . TCCTCTCACTCTATACTATCTCTGTTAGCTAATTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 649377 . CACATATGCATGTGTGTACATGTGCATACACACACTGTATGTGGACATGTATATATATATATGTGTGTGTGTATATATATAGTATATATATAAATTACAATAACATAAAGGTGGCATTTTAAATTAGTGGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 649521 . TGATCACTACACATTCTATACATGTAAAGAAAATATCACTCTGTATCCCAAGAATATGTACAATTATGGTTTGTCAAATGAAAAAGTTCATACATTGAAAAATTTTAGATAAATATCAAACTTTCTCTGAAACTGTAACTGTAAAATGTAAAAAACAGTAATTGCTATATTGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 649704 . GTAGAAGAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 649721 . ATTTCCCTAATCATTATGTGTAATTACAATTACATAGAAGAATATGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 649775 . CCCTAATCATTATGTGTAATTACAATTACATATATATATGTAATTGTAATTACACATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 649851 . CATATTCTATATATATAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 649890 . GAGGGAGAGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 650001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=652000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.24:.:0:.:0:0.999:218
+1 650016 . TATCCTCATTTTTTTCAGATTCTTGCTTAGAAGTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 650059 . GTGGACCTCCTCTGACATATTAAACATTGCAGTCCATTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 650114 . AGGGATTTTTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 650156 . CTACAGCAATATCTGACAAACAGTGGGCATGTAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 650214 . AAATTCAATCAAATCACATCACCTGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 650268 . ACTTAGAATAAAGAGAAATTCTTTTTATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 650347 . TTCGACTCCTCTCCTACTAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 650381 . CCACATTAGACCTTTCTTCAGTTTTTTATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 650472 . ATCGTCCCTCCACTTTCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 650510 . TTTTCATCTCAGCAGGAGGCCCATTCTCTTTGGCAATCCTCTGGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 650618 . GCACTTTTATATTTTAACAAATTATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 650694 . TCATCATCAACTTTTTCAACATC . . . NS=1;AN=0 GT:PS ./.:.
+1 650724 . CCATTTAGAACTTAGATGTAGTCAATACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 650777 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 650792 . CCTTTAAATTAGGATGGCAAAGATCGTATATAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 650857 . GCTCCCAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 650911 . TAGATTTTTTTAAAAAGAAAACTGGCCAGGTACTGTGGCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 650968 . CATGTTGGGAGGCCAAGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 651013 . GACCAGCCTGAGAATTTGGCAAAACTCTGTCTCTACAAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 651074 . TGGTGGCATGTGCCTGTAGTACCAGCTACTTGGGAGGCTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 651132 . AGTCTGGGAGGTCAAGGCTGCAATGAGCTGTGATTGCACCACTGCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 651208 . TCTCAAAAAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 651253 . AAGGATCATGTCAAAGGTAAGAAAAATTAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 651344 . TGATTTAGCATTAGAAAATTACATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 651394 . CAAATACTAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 651546 . TTTTGAAATAAAATGTATCTGAGTAGCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 651662 . AGAAGTCAATCAGGAAGAGGGGAGCAGTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 651753 . TATTTTATTTTTTCCCCAACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 651853 . TGCTTATACATATAAACACAGCTGATAATTTATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 651921 . CCAGTTTTTTATTTAAATTGAAGATTAGTATACATTTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 651972 . AAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 652001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=654000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:1.34:.:0:.:0:0.999:218
+1 652011 . TCAGAAAAAAAAAGTCAAAAGCTAGAGTATAGAGAAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 652088 . ACAAGATTTAAATATTTTAATGGAAAATAGAACAGAACTAATTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 652156 . AAAATAAACAGATTATATGGAGGATTTTTAGAAGATAAGTAAATAAATTAATATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 652219 . AAACAAGGGAAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 652264 . TTTGAAATAATGATAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 652309 . AAAGATGCATAAATATATAAATAAATGATAAAAAATGTTGCATACATATATGACTTTTTCAGAATCAAAAAATTTAAATTTCTGTAATAAAATTTAAATGTTTATAAATTTAAAAAACTAGAAGAAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 652457 . CAAATAAATGACAAATATTTGAGGTGATGGATATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 652568 . ATTATTTGTCTCAAAAACAAACAAAAAAAAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 652681 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 652709 . GTGGATTATATATTTAAAATAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 652746 . ATTGAGAAATATATAGCTGGAAAACTTATCCTTCAAAAATGAAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 652803 . ATTTCCGGATTTTTTTTTAAAACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 652981 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 652997 . GTAATTTTGGTTTGTAACTCTGCTTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 653039 . TTAAAAGGCAAATGCATAAAATGTAATTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 653127 . AAAGAGTAGAGCTATATATATAGCAGTAGAATTTTGGTATGTGATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 653235 . TCATAGTAACCAAAAATGAAATATATATAGAATATAAACAAAAGGAAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 653291 . GAAACAAAATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 653336 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 653359 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 653373 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 653377 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 653400 . CATAAATCTGAAAACTCTATTTCACATAAAACTGGAGCTGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 653465 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 653491 . CTAATTTTTTTTAGAAAAAATTATAAAAAGTAAAAATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 653602 . TCCCTCTGTGCCCCCAAAAAACCCTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 653639 . ATTATTACCTAAAAAGTCTATTCTCAAATGCAGCAGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 653746 . CTGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 653774 . AAAGGTCCCTCAGCAATATAACTCACAAACATGTTCAGAAGCAGTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 653834 . TTATCTTTTGAAAGTCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 653862 . CTTTAATGTATGCATATAGCATAGCTAATGTACTATCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 653910 . TTATTCAATGAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 653967 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 654001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=656000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:1.15:.:0:.:0:0.999:218
+1 654004 . GAGTTTCCATTAAAAGACAATTTAGTAAAACTTTTCTTCCCCCAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 654060 . TGTAAGATGATTTAACAACATGTGTAAAAGTCATTGTGGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 654146 . TCACCCAGGCTGGAGTGCAGTGGCACAATCTCTGCTCACTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 654220 . TGCCTCAGCTTTCTGAGTAGCAAGGACTACAGGTGCACACCATCACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 654278 . TTTGTACTATTAGTACAGACGGAGTTTCACCATGTTGGCCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 654348 . ATCCGCCCACCTCGGCCTCCCAAAGTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 654422 . CTTTGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 654583 . CAACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 654605 . TGAAGAAATTATGAGTAGAATTTAAAAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 654642 . GCCTATTAATTAGATTTGTCTTTGTAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 654678 . CTATAATAAATAATATTTTATGCCTATGAGTCCCCAACAAAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 654738 . ATACAAACTGTAAAAGTCACTACTGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 654788 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 654827 . ACATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 654852 . TGGCAAATATTGATTGTCATCTTCGTGTTTGTCTATGTCCTAAGTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 654945 . TCACCCCCTTTTTTTTTTTTTTTTGAGATGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 654999 . TGGAGTGTAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 655028 . TGCAACCTCCACCTCCAGGGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 655056 . ATTCTCCTGCCTCAGCCTCCCAAGTAGCTGGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 655101 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 655119 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 655201 . CTCGGCCTCCCACAGTGCTGAGATTACAGGCATGAGCCACCACGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 655336 . AAAGACAAACTCACAGGAAGATGGGATGTAGAATGATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 655453 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 655470 . TTTCATTTTATAAAAATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 655502 . CCAGTTGTTTTTTCTCTTCCTCGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 655586 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 655639 . TACGCTTTTTTTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 655669 . CAAATCATCACAGTAGAGCACGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 655701 . CAATCTCAAAAACTCAGGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 655784 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 655810 . GTGCGATCTCAGCTCACTGCAACCTCCATCTCCCAGTTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 655887 . CTATAGGCATGCACCACCACTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 655928 . CTGGCTAATTTTTGTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 655959 . GGGTTTTGCCATGATGGCCAGGCTGGTCTCGAACTCCTGACCTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 656001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=658000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.35:.:0:.:0:0.999:218
+1 656031 . AGACTTTTTTTTTTTTTTTTAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 656097 . TCCTGAGCTCAAGTGATCCTCCCACCTCAGCTTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 656164 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 656182 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 656476 . AAAATACAAAAATTAGCCGGGTGTGGTGGCACACACCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 656526 . ACTTAGGAGGCTGAGGCAGGAGAATCGCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 656563 . GGAGGCGGAGGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 656639 . TCTCAAAAAGAAAAAAAAAAGAGACAGAGAAAAGAAAGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 656773 . CAACTGCCTAAATCATGGGAAAGATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 656918 . GAATCGAATAATACATTCAAAGTGCTGAAAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 656999 . AAGGAAAAAGAAATAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 657032 . CAAAGCTGAGGGCATTCAGGACCACTAGGTCTACCTTAAAAAAATGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 657092 . TCAAGTAAAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 657120 . GGTAGCTCATGCCTGTAATCCCATTTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 657240 . ATTAGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 657319 . ACAAAAAACAAAACAAACAAAAAAAACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 657420 . GGGCGAGGAGGTGTGAGCCCCTGCCAGGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 657457 . CCCGGACCAAGTGCTCGGCCCCCAGGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 657501 . TCCCGTGGCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 657581 . CCTGACCCCTCCCTGCAGCCACACGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 657632 . GAGTCTC G . . NS=1;AN=2;AC=2;CGA_FI=100287654|XM_002342011.2|LOC100287654|INTRON|UNKNOWN-INC;CGA_SDO=28 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:12:12,28:12,28:19,6:-28,-12,0:-19,-6,0:1:1,1:0
+1 657676 . AGCCGTTGCTCATGAGCGTCCACCAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 657743 . ACCTGCCCTGAAAGCCCAGGGCCCGCAACCCCACACACTTTGGGGGTGGTGGAACCTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 657819 . CCATGGAGGAGGAGCCCTGGGCCCCTCAGGGGAGTCCCTGCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 657913 . CCTGACACCCAGTTGCCTCTACCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 658001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=660000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:1.42:.:0:.:0:0.999:218
+1 658022 . AATCAGACAAGGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 658058 . GCGCTCATGATCTTCAGCAGGCGGCACCAGGCCCTGGCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 658110 . TCACCCCAACCAGGATAACCGGACCGTCAGCCAGATGCTGAGCGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 658256 . GAGGCCAAGCCCACAAGCCAGGGGCTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 658308 . GAGCGGAGCATATCAGAGACGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 658416 . CTATGGGGCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 658445 . GGGAACCTGGCTCAGCCTGGCCCAAGCCTTCTCCCACAGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 658499 . GGACGGCAGGGAAATAGACCGTCAGGCACTACGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 658628 . GGGAGGTGACCCGTGGGCAGCCCTGCTGCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 658691 . CAGCGAGGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 658726 . TCATCCATGAGGAGGAGGGGGTGATGATGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 658781 . ACCGACACCGATCTCAAGTTCAAGGAGTGGGTGACCGACTGAGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 658835 . CTCTGGGGAGGAGCCAGAGGGCAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 658878 . GTATTTGCACCTGTCATTCCTTCCTCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 658924 . GCTGGATCCTGAGCCCCCAGGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 659080 . AGTCTGGTCAACGCAGCAGAGCGGGCCCCCTACGGCCCCAACCCCTGGGGATGGGGGCCCAGGGACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 659159 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 659199 . AGAGACCTGAAAGTGTGGGTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 659263 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 659300 . CCCGGGGGGCAGAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 659345 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 659354 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 659375 . CCCACACTGGAGGACCCCACCGCGCCCAAATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 659419 . ATGCTCCAGCTGCAGTCCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 659445 . ACACCCCCAAGTGTGCCATGTGTGATGGGGACAGCTTCCCCTTTGCCTGTACAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 659534 . ACCGAGAAGGCGCTGTCCTCTTCACTGCACGTACCCTGGACCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 659632 . GGCCACAGCCGCCCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 659697 . CCCTGACTCCCAGCCCTGTGGGGGTCCTGACCGCACCTCACCTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 659774 . CCACTGCTTCTGCCCGAGAGTCACGTGAGGCTGAGAGTAGGGGCAGGGGCAGCAGTGGTGCCAGTTGGGGGGCGGTCCAGTGGGAGGAGCCTCAGCCTCGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 659887 . GGGACTGATGACTGCATGATCTTCTGGGCACCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 659948 . ATGCTGGTGGTGGGTGCAGGGCCGCTGGGAGCTGCTGCATGGTTCCCAGAGGCTGGACTGAGGCAGGTGCCAACTGAAGCTGCTGGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 660001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=662000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:1.26:.:0:.:0:0.999:218
+1 660074 . AGAAGATGTGTGCATAGCAGGTCCACTGCTGCTGCCCCTGCCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 660193 . TGCCCCAGAGTTGGGGCCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 660220 . TGGTTGGAAGGGGACACCCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 660252 . ACACCTGGGGGTCTCCATAACTACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 660315 . AGTACCCCCTGAGAACATGGACAGTATGTGGGGGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 660409 . TGGGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 660425 . CTTCCTCTTTTTTTTTTTTTTTTTTTTTTTTTTTTTGAGACCGAGTCTTGCTCTGTCGCCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 660502 . GTGCGATCTTGGCTCACTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 660581 . TACAGGTGGACGCTACCACGTCCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 660615 . GTATTTTTAGTACAGACGGGGCTTCATCATCTTGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 660676 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 660697 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 660720 . CGTGAGCCACCACGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 660760 . TATACCCCCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 660787 . GGGGGAAAGCTGGGCAGTTTCCCTCCTCCGAGCCCCTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 660856 . TTTTCACTTTTCGGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 660879 . CCTGCTGGGGCTACAAGATGGAGTGTGAAGAGGGCCTTGGGCCACAGGGAGGCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 660971 . CCA . . . NS=1;AN=0 GT:PS ./.:.
+1 660988 . AGGTGAGTATGGGGGTGGGGGCTCCTGCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 661058 . TGCACTCCCAACTTGAGCTATACTTTTTAAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 661106 . CTTTGCCCCCTTCCCCAGAACAGAACACGTTGATCGTGGGCGATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 661177 . TGACCGTCATTAAACCTGTTTAACACCAAATAATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 661228 . AAATTCGGGCTTGGCGCAGAAACTCACTCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 661269 . CTATCAAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 661295 . AAATATTCCAAAATTCAATATTTTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 661338 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 661376 . AACGGGGCCTGGAATGGCCGACGTGAGGAATGAGCTGGGCCTAAAGAGGCCACTGGCAGGCAGGAGCTGGACCTGCCGAAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 661479 . GACTGGGGAGGCCGCAGTGAGGCGAGAGCTAGCTGGGCGTGGAGAGTCTGCTGTGAGGCCGAGGCCGAGGCCGGGCCCGTGCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 661584 . GGGCCTGCAAAGGCCGACTGGAGATCAAGTTCTGCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 661690 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 661702 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 661828 . GGCCGACTTGAGGACGACTTGGGCCTGCAGAGGCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 661902 . GGACGATTTGGGCCTGCAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 661932 . AGGCCCAAGCTGGGCCTAGAGGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 662001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=664000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.11:.:0:.:0:0.999:218
+1 662026 . ACCGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 662167 . GCCGGGAGGAAGAGCTGGGCCCGGAGGGGGCGCCGGGAGGCTGCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 662380 . ATCCCTTCTCCCAGTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 662425 . GGGCGTCTGCAGACCCCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 662497 . CAGGCCCAAGTCCCTGCCTACCTCCCAGCAGCCCGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 662552 . TCACGGTGGCCTGTTGAGGCAGGGGATCACGCTGACCTCTGTCCGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 662609 . CGGTGTGAGGCAAGGGCTCACATTGACCTCTCTCAGCGTGGGAGGGGCCGGTGTGAGACAAGGGGCTCACGCTGACCTCTGTCCGCGTGGGAGGGGCCGGTGTGAGGCAAGGGCTCACACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 662738 . TCAGCGTGGGAGGAGCCAGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 662766 . GGGGCTCACGCCTCTGGGCAGGGTGCCAGAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 662820 . CACCGTGAGGGAGGAGCTGGGCCGCACGCGGGCTGCTGGGAGGCAGGCAGGGACTTGGCCCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 663048 . CGGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 663094 . GCTGGGAGGCAGGGCCGGGAGAGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 663170 . GCGTGGAGGAGCCCACCGACCGGAGACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 663207 . CTGGAGATGCCATCGGAGGGCAGGAGCTCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 663258 . GCCTGACCTGGGCCTGGGGAGCTTGGCTTGAGGAAGCTGTGGGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 663406 . GAGAACG . . . NS=1;AN=0 GT:PS ./.:.
+1 663447 . TTGAGGAGGTTCTGGGCCCGGAGAGGCCGCCGGAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 663506 . T C . . NS=1;AN=2;AC=2;CGA_FI=100133331|NR_028327.1|LOC100133331|UTR|UNKNOWN-INC&100287654|XM_002342011.2|LOC100287654|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=19 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:18:18,38:18,38:19,15:-38,-18,0:-19,-15,0:4:2,2:2
+1 663641 . TCCGCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 663748 . GGCCGCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 663792 . GGGAGGCAGGAGGAGCTGGGCCTGGAGAGGCTGCCGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 663840 . TTCGCCTGAGGATGCCACAGTGAGACACCATCTGGGTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 663932 . AGTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 664001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=666000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:1.64:.:0:.:0:0.999:218
+1 664007 . TGGACTCACAGTCATGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 664045 . GTGAGGGAGGAGCTGTGCCTGTTGAGGCTGCTGGCAGGCAGGCAGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 664135 . AAAAAGCCCCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 664268 . CCGCCATGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 664301 . AGGCTGTTGTGAGGCAGCAGTTGTGCCTGTAGACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 664375 . AGGCAGAGGTTGGGCCTGTAGACGCTGACAGGAGGCAGGAGCTGGGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 664458 . TAGGCCACCAGGAGGCAGCAGTTGGGACTAGAGAGTCTGACTTGAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 664523 . TGACGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 664574 . CTGGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 664594 . GTGAGGCAAGACCTGGGCCTGTCTAGGCTGCTGGGAGACAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 664674 . TTGGGCCTGGAAAGGCCCTTGTGAAGCATGAGCTTGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 664737 . GCTAGGTGTGTAGAAGCTGCTGAAAGGTTGGGAGCTTGGCTTGGGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 664798 . AGATGCTGGGCGTGAAGAATCTGCTGTGAGGCAGACTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 664927 . TGAGCTGGGCCTGGTGAGGTCGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 664966 . GCCTGGAGAGAAGGCTGGGAGGCAGGAGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 665046 . GTGAGGCAGTAGCCTCATCTGCGGAGGCTGCCGTGACATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 665107 . ATTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 665158 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 665245 . TGTAATATATAATAAAATAATTATAGAACTCACCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 665288 . AATTAGTGGGCGTGTTAAGCTTGTTTTCCTGCAACTGAATGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 665354 . AGTGACAGATCAATAGGTATTAGATTCTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 665391 . AGCGCAACCTCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 665414 . GCACGGTTCACAACAGGGTGCGTTCTCCTATGAGCATCTAATGCTGCTGCTCATC . . . NS=1;AN=0 GT:PS ./.:.
+1 665512 . GCTCTAAATACAGACGAAGCTTCCCTCACTCCCTCACTCGACACCGCTCACCTCCTGCTGTGTGGCTCCTTGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 665610 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 665760 . CTACTGGAAATCACCAGCACCCCATTTCCCACTGGCAAAGAGCTCAGCACTGCCCCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 665843 . CCATCTGTGTGGGTCTACCTCCTGGGACCCTTCCTAACATATTAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 665959 . ATTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 665998 . ACAGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 666001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=668000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.19:1.52:.:0:.:0:0.999:218
+1 666036 . CAGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 666057 . TGCCTGAAATTCCAGCCATTACAGAAGCTAATGCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 666122 . CCGGTCTGGACGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 666179 . TGGGGGTGGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 666212 . ACTCAGAATGCTGAAGTTTGAGCCTGGGAGGTCAAGGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 666268 . TGCCACTACAGTCCAGCCGGATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 666402 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 666441 . ATGGCTTACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 666507 . ATGAGGGGTGGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 666537 . TCCAGGGTAAAGCCTGTCAATTTTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 666571 . GGAGACAGGGTCTCACCATACTGCCATACTGCCTCCTCCAACTCTTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 666709 . GGAGAATGTCCATTCACCATGACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 666751 . GGGGAGACAATTCAATCTAAGCAAAAGGTCATCTGTACACACACAGTAAAAATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 666885 . GGTAAAAGGTCAGTTGATGTTAGCTGCTACTTTTTTGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 667076 . TAATTCATGTATTTTTCTGTAGGGATGGTGACTCCCCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 667225 . TTTATCAATATTATTCTTATTCCACTCAATTAAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 667289 . TGTATCCTACAGCGTAATTGTAAAAACATACACAGTCGTCATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 667374 . TACCAAAATCCATGCTTACTCACGTTTCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 667470 . TGTAGTCTCAGCCACGTGGGAGGTTGAGGTGGGAGGATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 667531 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 667572 . ACAACAGAGGGAGACCCTGTCTCAGAAAAAAAAACAAAATAAAACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 667628 . TGTAATGAGGTCTGCTGGGCAAAATTCCATATAAGCAATGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 667679 . AAAGCAAATCGTGATAAATTAGTACGATTGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 667736 . TAAGGAAAATGCAGAACACAAAGACAGAGAGTAAAAAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 668001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=670000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.32:1.91:.:0:.:0:0.999:218
+1 668020 . GCCTTTTGTTTGTTTGTAAGGAATGTACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 668161 . TTTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 668321 . ATCGTCATGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 668473 . CTGCGGAGGCTGCCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 668510 . TAGGCCATTGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 668682 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 668731 . CTGGATGGTCCCACCTGAGCGTGATGGGAGAAAGTGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 668784 . AGATTCTCATAAGGACAGCGCAACCTAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 668822 . TGCACGGTTCACAACAGGGTGCGTTCTCCTATGAGAATCTAACGCTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 668881 . GAAGGTGGAGCTCAGGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 668932 . AGACGAAGCTTCCCTCACTCCCTCACTCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 668986 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 668989 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 669038 . AAAGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 669159 . GTGAGGTCACCTACTGGAAATCACCAGCATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 669314 . GCATAAACCACTCAAAAGTTTAAAGTGGTAAAATTTAATACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 669367 . ATTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 669462 . TCAGGCCTGAAATTCCAGCAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 669587 . TGGGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 669627 . ATGCGGAAGTTTGAGCCTGGGAGGTCAAGGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 669676 . TGCCACTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 669855 . TACTAAATTATAATACCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 669901 . CACCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 669968 . TTTAAAATAATGGAGACAGGGTCTCACCATACTGCCATACTGCCTCCTCCAACTCTTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 670001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=672000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.21:1.90:.:0:.:0:0.999:218
+1 670117 . GGAGAATGTCCATTCACCATGACTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 670170 . TCAATCTAAGCAAAAGGTCATCTGTACACACACAGTAAAAATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 670293 . GGTAAAAGGTCAGTTGATGTTAGCTGCTACTTTTTTGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 670484 . TAATTCATGTATTTTTCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 670514 . ACTCCCCCTTTGTTTCCAAGGCCTATCGCAAACTCTTGGCCTCAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 670567 . CCTGCCTCAGCCTCCCAAAGTGTTGCGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 670633 . TTTATCAATATTATTCTTATTCCACTCAATTAAAAATTATTATTTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 670701 . TCCCACAGCATAATTGTAAAAACATATAGTCGTCGTCCCTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 670780 . TACCAAAATCCATGCTTACTCACGTTTCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 670819 . TCTGGAATCCACGTATACGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 670850 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 670876 . TGTAGTCTCAGCCACGTGGGAGGTTGAGGTGGGAGGATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 670934 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 670937 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 670978 . ACAACAGAGGGAGACCCTGTCTCAGAAAAAAAAACAAAATAAAACAGGTTAGAAATTGTAATGAGGTCTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 671142 . TAAGGAAAATGCAGAACACAAAGACAGAGAGTAAAAAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 671310 . GAAGCAAAATACTGGTAGCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 671426 . GCCTTTTGTTTGTTTGTAAGGAATGTACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 671563 . GAAATTTTTTTTTTAGAAAATTGAACAAGTGCTCCCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 671737 . ATACCTACAGTCCCAGCTACCTGAACTTACTGAGAAAGTTCAGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 671838 . GAGCTGTGCCTGTGGAGGCTGTTGTGAGGCAGTAGGCTCATCTGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 671922 . ATTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 672001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=674000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.67:1.98:.:0:.:0:0.999:218
+1 672060 . TGTAATATATAATAAAATAATCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 672206 . AGCGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 672246 . GTGCGTTCTCCTATGAGAATCTAACGC . . . NS=1;AN=0 GT:PS ./.:.
+1 672327 . GCTGTAAATACAGACGAAGCTTCCCTCACTCCCTCACTCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 672392 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 672398 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 672660 . ATCTGTGTGGGTCTACCTCCTGGGACCCTTCCTAACATATTAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 672774 . ATTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 672820 . AAACAGAACTCTAGAGAATATGGGACTAGCCCAGGCCAGGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 672908 . GAGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 672945 . GACGACACAGTGAGACCCTGTCTCTATCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 673029 . TCGGAATGCTGAAGTTTGAGCCTGGGAGGTCAAGGCTGCAGTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 673114 . AGATCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 673214 . TAATATGAGTTCTTTTGTCTATGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 673324 . GAGGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 673397 . CTCACCATACTGCCATACTGCCTCCTCCAACTCTTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 673709 . GGTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 673893 . TAATTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 674001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=676000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.29:1.83:.:0:.:0:0.999:218
+1 674048 . AATATTATTCTTATTCCACTCAATTAAAAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 674106 . TGTATCCTACAGCGTAATTGTAAAAACATATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 674244 . ATACGAAAATTCCAAATATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 674287 . TGTAGTCTCAGCCACGTGGGAGGTTGAGGTGGGAGGATCGCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 674410 . TCAGAAAAAAAAAAATAAATAAATAAAACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 674558 . TAAGGAAAATGCAGAACACAAAGACAGAGAGTAAAAAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 674742 . AGCGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 674842 . GCCTTTTGTTTGTTTGTAAGGAATGTATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 674979 . GAAATTTTTTTTTTAGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 675143 . ATCGTCATGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 675253 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 675313 . CGTAGGGTATGGGCCTAAATAGGCCATTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 675527 . GGCGTGTTAAGCTTGTTTTCCTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 675622 . AGCGCAACCTAGATCCCTCACATGCACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 675662 . GTGCGTTCTCCTATGAGAATC . . . NS=1;AN=0 GT:PS ./.:.
+1 675743 . GCTGTAAATACAGACGAAGCTTCCCTCACTCCCTCACTCGACACCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 675813 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 675825 . GCTCAGGGGTTGGGGACCCCTGCTCAAGTGCATCCAAAACGA . . . NS=1;AN=0 GT:PS ./.:.
+1 675943 . TTCGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 676001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=678000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.37:.:0:.:0:0.999:218
+1 676035 . CAGCACTGCCCCCTGGGAAACCAAACCTATGCCCAAATCCCATCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 676147 . GAGACAATCGATTTAGCCCAGGAGTTTGAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 676215 . GGAAGAGGTGGGAGGATCACTTGAGCCCAGGAATTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 676278 . CCCCATCTGGCCAACATGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 676570 . GATAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 676600 . AATATTTGATCTTGGTCCCAGGTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 676748 . ATGCAACCACATGGTAAGAGGCTTGGAACTTTCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 676979 . GCCCCTCCGAACTTAACTTGCCCTGGGTATCTTTCTTTTTTTTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 677049 . GCTGGAGTGCAGTGGCACAATCTCAGCTTACTGTAACCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 677095 . CCCAGTCCCCAGCTCAAGGTATCCTCTCATCTCAGCTTCCCTAGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 677167 . CACCAGTTATTATTATTATTTTTTAATTTTTTATA . . . NS=1;AN=0 GT:PS ./.:.
+1 677220 . GTTGCCCAGGCTGGTCTCAAACTCCTGAGTTTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 677330 . TCATTGACTGTTTCTGAGATGTATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 677449 . ACTTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 677473 . AGCCTGGCCAACACAACAAGACCCCATCTATACAAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 677648 . CAACAAAATAAGACCCTCTCTCTCAGAAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 677746 . TTACGGGAACCCCCGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 677838 . GGACTGAGCCCCTAACTTGTGGGGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 678001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=680000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:1.59:.:0:.:0:0.999:218
+1 678075 . CCAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 678184 . CCTATATGTGATTCTGTGAGAATTAACGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 678309 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 678435 . GGTTTCCCTTCCCGGGCAGTTTGCGCTATCCCATCCCGGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 678486 . CCCTCCACCTCCCCCTTCCCTCCCCACTCTCATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 678671 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 678721 . GTAGATCCAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 678804 . ATGTCATGTGAAAATTAAACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 678871 . CATGAAGGGTTAATTTGTATTTTATTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 678924 . CACCTAGGCTGGAGTGCAGTGGTGCAATCAGGCTCACTGCAGCCTTGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 678984 . AAGTAATCTCACTTAATTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 679066 . GGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 679117 . GGCAATATATTAAGACCCTGCCTCTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 679464 . CAACCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 679509 . GTCGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 679537 . ATACAGATGAAGTTTCCCTTCACTCGCCTGCTGCTCACCTCCAGCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 679601 . AGACCGCTGCTCAAGTGCATTTGAAAGGAACCATCCCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 679655 . CATCTTTACTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 679689 . CCCCAAGCTCGCAGGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 679771 . CCTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 679854 . TCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 679932 . ACTCAAAAGTTTAAACTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 679982 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 679988 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 679993 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 680001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=682000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:1.46:.:0:.:0:0.999:218
+1 680014 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 680093 . TGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 680144 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 680146 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 680149 . CTT . . . NS=1;AN=0 GT:PS ./.:.
+1 680154 . GCC . . . NS=1;AN=0 GT:PS ./.:.
+1 680177 . GAGCAGAGGTTGTGCCACTGTACTCCAGCCTGGGTGACAGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 680243 . AACGTATATATATATATATATGTAAATTTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 680299 . GCAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 680331 . TTGGGAGGCCAAGGCAGACAGATCACCTGAGGTCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 680389 . GCACAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 680407 . TCTACTAAAAATACAAAATTAGCTGGGCATGGTGGCACATGCCTGTAATCCCAACTACTCGGGAGGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 680496 . CCCAGAAGGTGGAGGTTGCGCTGAGCCGAGATAGCACCATTGCACTCCAGCCTGGGCAACAAGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 680571 . TCTCAAAAAAAAAAAAAAAGGTATTAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 680746 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 680792 . TACCAAAAAAAAGAGACATTAGCCAGGTGTGGTGGTGGTGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 680903 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 680938 . GGGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 681053 . CCCACTTTCCTGTATCTTTAACCTATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 681331 . GCCTTGCAAGGCAGCCTCACTGCTTGCCCCTCTCCATTTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 681409 . GAACGCACACTCTTTCTCCTCTGGGAGTCTCTGAAGTGGGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 681547 . TCATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 681573 . GATTCTCAGGAGCATGGCAGGTGAAGTGCTCCTCCCATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 681624 . TTAGGGAGTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 681695 . TGGTGGCCAAAGTAATAACCCCCACCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 681796 . AATGTTGCTCATCAGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 681871 . CTTTAAAACTGGAAGAGGGAGGCAGAAGGTTAAGAACCAGAGACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 682001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=684000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:1.29:.:0:.:0:0.999:218
+1 682044 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 682225 . AGGACCCTCTCCATCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 682264 . CTCTGCAAACGAGTAAACATCACCCTCCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 682389 . AAGCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 682436 . TTCCTTTGGTTCTCAGTAGGCAGGGTAGGGGGCCAGGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 682554 . CCTGGACAACATAGCAAGACCTGGGTGGCATACACC . . . NS=1;AN=0 GT:PS ./.:.
+1 682647 . GTTTCAGGCTGCAGTGAGCCATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 682678 . ACTGCACTTCAGCCTGGGTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 682724 . TTAGAAAAAAAAAAGAGGGAGAGAGACTATACACAGGCACCACCACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 682815 . GTTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 682870 . TTGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 682964 . TATCAAAAATACAAAAATTAGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 683085 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 683120 . CAAAAACAACAACAACAATAACAAAAACAAAAACAACAACAACAAAAAAAACTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 683207 . CAAAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 683276 . GGAAGAAAAAAAAAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 683371 . ATCACTTGAGGCCAGGAGTATGA . . . NS=1;AN=0 GT:PS ./.:.
+1 683411 . TGGTAAAATCCCACCACTACAGAAAAATCTAAAAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 683530 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 683593 . CTAGAAAAAAAAAATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 683638 . ACATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 683711 . TGGATTTTTAAAAAATCAAGACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 683773 . GGCTCAAGCCATCCTCCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 683805 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 683882 . AAAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 683920 . TCACTCCTAGGCATATATCCCAGAAAAATAAAAATATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 684001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=686000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.33:.:0:.:0:0.999:218
+1 684025 . ACATGGAAACAACCCAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 684155 . ACACTGTGCTAAGAGGGAAGAAAAGCCACAAAAGATCACATATTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 684302 . AGTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 684344 . TTCTAAAAGTGACTGTGGTGATCGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 684379 . TGTGAATATTCTAAAACCTACTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 684433 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 684436 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 684460 . TTTAAAATAATAATAATAGGGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 684525 . GGGAGGCTGAGGCAGGAGGATCACTTGAGGTCAGGAGTTTTGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 684659 . AGACTGAAGTGAGAGAACCACTTGAGCCCAGGAGTTTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 684782 . ATGCAAGTTTTTATCACTTTGTGAGTGTAGCCAAGTTGGAGGAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 684859 . ACGGTGAGTGGCTGGTTAGGCTCAGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 685008 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 685046 . ATAGCAGCTGTTTATTAAAGACTACAAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 685368 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 685380 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 685472 . TTACCCAATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 685523 . TTTCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 685707 . TTGAAAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 685761 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 685796 . ACAAGGCCTACATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 685838 . TTTCATTTGTATTTGTATTTTGAGACAGGGTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 686001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=688000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:1.28:.:0:.:0:0.999:218
+1 686003 . GCATTTTTTTCATTTTTGTAGAGAGAGAAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 686055 . GCCTCAAACTCCTAGAATCAAGAGATCTGCCCATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 686144 . TTATTTTATTTTATTAAATTTATTTTTTTTATTTTTGTAGAGAGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 686214 . CTCTCAAACTCATGGCCTTAAAACATACTCCCATCTCTGCCTCTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 686342 . TTGGGAAAAGCAGTAGTGTTTTTTAAAATTACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 686399 . CAACCTTGACCACTGCCTTCTCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 686453 . TACTGACTGACTTCAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 686510 . CCCCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 686535 . CACCGGGATGTTGCCACAGCTTGGCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 686663 . CCTCGGTGGTTCCCATTTTAGTCAGAGTAAAAGCCAAAGCCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 686767 . A T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79616629;CGA_RPT=L2a|L2|37.7;CGA_SDO=20 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:56:56,56:56,56:20,33:-56,0,-56:-20,0,-33:25:3,22:22
+1 686937 . ATTCACTTATGAGGCCAACCCTGACCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 686983 . CTGTCCCCATTCCCACCATGCTCATTTCTTTCTTTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 687092 . ACCTCCCAGGCTTAAACAATCCTCCCGCCTCAGCCACCCTAGGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 687166 . TGGCTTTTTTTTTTTTTTTTTGAGATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 687291 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 687378 . GAACTTCTGACCTCGTGATCCACCCTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 687446 . GCGCCTGGCCTTTAAAAAAATTTTTTTTTAGACATGAGGTCTCATTATGTTGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 687953 . TTGTGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 688001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=690000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:1.34:.:0:.:0:0.999:218
+1 688033 . AGAGTTAGAGATCAGCCTGGGGAAAAAAAGGAAGATCCTGCCTTTACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 688514 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 688525 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 688601 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 688656 . TGCCACAACGCCTAGCTAACTGTTGTTATTTTTAGTAGAAATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 688723 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 688778 . GGATTACAGGAGTGAGCCACCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 688814 . CACATTTTTTGAGGCTTGGAACTTTCAGCCTCACCTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 688923 . CTCCATAAACACCCAAACAGCAGGGTTTGGAGAGCTTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 689029 . CCCTCCCCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 689076 . TGAGATGGAGCCATTACATTGAGCCAGTAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 689140 . CCCGTAATCCCAGCACTTTGGGAGGCAGAGGTGGGCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 689209 . CTGGGCAACATAAGAAGACCCCATCTATACAAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 689374 . AGCTTGGACAACAGAGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 689403 . CTTAAAAAGAAAAGAAAAAAAAACTTGTTTTTCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 689503 . TAACTGGTTGGTCAAAATACAGGTGACAACCTAGGACTTGCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 689765 . CTGAAGCCACAGCAGAAGAACATAAATTGTGAAGATTTCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 689912 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 689962 . CCGGTCATCTTCGTAAGCTGAGGATGAATGTCCCCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 690001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=692000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.36:.:0:.:0:0.999:218
+1 690107 . AACCTTAAACTCTGGCTGCCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 690245 . GGAGGCCTCTGAAATGGCCGCTTTGGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 690292 . ATA . . . NS=1;AN=0 GT:PS ./.:.
+1 690327 . GGCGCTCCCAGCCTTATCAGGACAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 690642 . ACCTGCCGACGTGTGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 690766 . ATCAGGGGTGAATTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 690890 . AACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 690931 . ATCGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 691079 . ACATCAAAAAATTAGAAACTGTAATGAGGTCTCTTGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 691225 . GCTGGTTTCCCTGCCTGGGCAGCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 691279 . CCCTCCACCTCCCCCTTCCCTCCCCACTCTCATACAACTCTTCCTTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 691367 . TCTCTCCCTCTCCAGAAGAGCTTCCGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 691487 . TTCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 691534 . AAAAGACATTTAAAAAAAAAAAAAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 691577 . CATCAGCACTTAAAAGTTTTAAACGATATGTGAAAAACAAAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 691648 . GGAAGGTGTTACTGGGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 691680 . TAATTTTTATTTTATTTTATTTTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 691737 . ACTGCAGTGGTGCAATCACAGTTAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 691821 . GAAATGCAGTCTTGCTCTTAGCAAAGCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 691964 . CTCATTTTTTTTTTTAATTTTTAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 692001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=694000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.68:1.13:.:0:.:0:0.999:218
+1 692037 . CTCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 692067 . AAATGCTGGGATTACAGGTGTGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 692120 . TTAATTATATAAAGAGCTCAAAGCAAATATTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 692170 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 692201 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 692211 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 692214 . TAA . . . NS=1;AN=0 GT:PS ./.:.
+1 692233 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 692285 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 692290 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 692297 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 692308 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 692358 . TTTCCTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 692412 . AAAAGTATTTATCATTTTTATAATTTAATAAAAGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 692562 . TGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 692628 . CTAGTAAAAATAAAAAAATAAAAATAATTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 692790 . ATCTCAAAAAAAAAAAAAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 692912 . GCCAAGGCGGGTAGATCACGAGATCAGGAGTTCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 692989 . ATACAAAAATTAACCAGGCATGGTGGCATATGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 693036 . ACTCAGGAGGCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 693058 . AATCGCTTGAACCTGGGAGGCACAGGTTGCAGTGAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 693267 . CTTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 693321 . GAAATTTTTTTTAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 693525 . TCACCTTTGGATATAATTCAACCTAAACAAAAGGTCATAT . . . NS=1;AN=0 GT:PS ./.:.
+1 693598 . TTTCTCTTTTTTTAAAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 693743 . TGACATAAGGAAGAATTATGGAGAATTTAAAAATCTATGCTATTTATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 693824 . CTACTATTATTATTTTTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 693861 . AAAACTGTCATTAAAAATTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 693945 . GGTATATGGGGGGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 693993 . AATCCACGCATACTCAAGTTTTCGAAGTCAGTCCTGTGGAATCCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 694001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=696000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.29:.:0:.:0:0.999:218
+1 694142 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 694168 . ACTGCACTCCAGTCTGGGCAACACAGTGAGACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 694279 . AAATAACTATCTAATCCAATTAATGCTGGAATTGGGAACAGCAGAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 694344 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 694363 . TGGGGCTCAGGTGTGTTGAGGTCCCCATGCCTGGACTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 694411 . GTGGGATTTACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 694433 . TTTTCTATATTCCAGCACTGGGAAACTAGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 694525 . ATTGAAAAATCGTCGCAGGTCAGGTGAGGTGGCTCATACCTATAATCCCAGCCCACTGGGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 694603 . TCCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 694657 . CTACAAAAAATTAGAAAATGAACTGGGTGCGGTAAAACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 694807 . GAGTGAGACCCTGTCTCAAGACACACACAAACACACACACACACACACACACACACACACACACACACACACCCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 694913 . AGGGCCTTCTGGTTACAGAAGAGGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 695082 . ATGCCTCCTTTGTCAATTAATAAATGGAACATCAGCCTTAAAATCCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 695205 . GCGCGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 695280 . AGATCAGCCTGGCCAACATGGTGAAACCCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 695373 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 695409 . AGGCGGAGGTTGCAGTTACTTCTAGAAGAATTTCCATTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 695752 . AAAATTTGCTTCGGCAAATCTTATGCAGAGCCAACTCCAGGCTCCAGAAACAATAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 695961 . GGGCCCAGCTCCTCACTACTCACCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 696001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=698000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:1.28:.:0:.:0:0.999:218
+1 696011 . AGAGGATGGGGAAACAAGGCTCCTGACTTTTTTTCCCTAATATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 696228 . TGGGATCATTCCAAATTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 696280 . AAGGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 696295 . AAATGGGCCCTGCTGCCAAGCCTTTTTTTTTTTTTTTAACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 696545 . TGCCAAAAATGCACTACAGCCCCCACCCAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 696634 . AACTAAAACAGAAACTCCTGAACTGGGTTCTTTTGAGCCCAGGAAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 696769 . GAGCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 696919 . GTGACTGAAGCAAAAGCTTCAGAACCAGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 697068 . TATATGCAGGGATGCAGGCTGTAGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 697127 . CAAGACCTCAAACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 697154 . AAAGGAATCAAGGTTCCCTAGAGAAACGGCTGACTCCATGTATGGTGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 697222 . TCTTTTTTGCCAGAAAGCAAGGAAGCCATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 697337 . AAAACAATAAAGACTGCAATGGCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 697454 . CTAACTTACTACTCTGAAAAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 697491 . GTAGGGTGGAGAATTAGCTATTTATTCAGTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 697648 . TGAACCCCCTACAAAAAAAGCACAAGACAGAATGTGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 697745 . GTAGTTTTAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 697870 . CTAGGTAGTGGATCTGAGGCTACCTATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 698001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=700000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.09:1.28:.:0:.:0:0.999:218
+1 698016 . AAAGGAAAAAAGGAAGGAAAGAAAAAAGGAAGGAAGGAGGGAAGGAGGGAAAAAGGGAAGGAGGGAAGGAAAGGAAGGAAGGGAAAGAAGGAAAGGAAGGAAGGGAAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 698134 . GGGAAGGAGGAAGGGAGGGAAGGAGGGAGGGAGGGAGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 698218 . GAAGAAAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 698240 . TAAATTTTATTTCTTAACAGTTCTGGATGTTAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 698375 . AAAAGGGGCTGAACTCTGTTTTATAATAAGCCCACTCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 698642 . AAGGAAAAAAGTCACAGTGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 698680 . CTTTATCAAAAGCACCTAAAAAAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 698731 . CACCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 698785 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 698823 . ACCTGACCTCAATAGCTCCAGAACAGCCCTAAAACATTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 698987 . GGGGAAAAAAGGAACAATGAGTAGAGGAGAAACAGACCACTCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 699118 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 699135 . ATGAGCAGGCAAGCTGGCTAGAAAACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 699245 . TCTGAGGATGATGTCAGTATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 699311 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 699352 . TCACGCTGACCCCAGCTCCCTGGATGTTACCATTAGCCAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 699542 . AAAGGGGGAGGAGAAACTAGGAAAATCATATATGGGCTCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 699654 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 699664 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 699699 . GTGGCTCATGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 699807 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 699815 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 699925 . CACGTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 700001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=702000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.21:.:0:.:0:0.999:409
+1 700182 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 700214 . CACGTCTGTAATCTCAGCACTCTGGGAGGCCGAGGCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 700273 . GTCGAGGCGGGAAGATCACTTGACGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 700334 . ACCGCATCTCCACTAAAAATACAAAAATTAGCCTGGTGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 700510 . TCTCAAAAAAAAAAAAAAAAAATTCCTTTGGGAAGGCCTTCTACA . . . NS=1;AN=0 GT:PS ./.:.
+1 700576 . TGGAAAAAAGGGTATGGGATCATCACCGGACCTTTGGCTTTTACAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 700647 . AAGGGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 700774 . AAATAAAAAGAACACCAAAAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 700900 . TCTCCTGAAGTCAGGAGTTCAAGGCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 701063 . CATGAGCTGAGATCATACCACTGCACTCCAGCGTGGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 701120 . CTCCAAAAAAAAAAAAACAGCTAGCAGGTGACATTTGCTATAGGGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 701233 . TTTCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 701390 . GAATGCTTATTCAGTTGACTGGTGTAGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 701439 . TGCATTATGCCAGATGAATCTTGCATCTCAAAAGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 701525 . TAATAAAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 701570 . TTGACAACATGAATTCTCCTGTCCTAGGACATAATTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 701671 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 701709 . CTGACAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 701776 . TGAGAATAAATACTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 701839 . TAACATTTCATCATGAACTGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 701910 . CCAGCAGTTTGGGAGGCCGAGGCAGGCAGATCATGAAGTCAGGAGTTCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 701986 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 702001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=704000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:1.23:.:0:.:0:0.999:409
+1 702045 . AGCGACTCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 702308 . GGAGAAGAGACGTGGCCAGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 702391 . GAGATTTTTGCTTTAAAATGAACCAAAAAAAAACCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 702453 . AAAGTGGGAGAAACACTAAGAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 702709 . CAAGTAAATAGTCACCCAAATAAAAACATCATGTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 702783 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 702796 . GAGAGGATAATAACAAATCGCTAATTTCTTTCATCACTATATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 702928 . CAACAGGGGCACCTTGGTGAGTACTGAACATTTTATTTATTTACTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 703012 . CTAGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 703067 . AGCGATTCTCCTGCCTTGGCCTCCCGAATAGCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 703113 . TGCGCCACCACACCCGTCTAATTTTGTATTTTTAGTAGAGACGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 703342 . CACCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 703381 . GATCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 703424 . GAAACCCCGTCTCTACTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 703492 . GCTCGGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 703523 . ACCCGGGAGGCGGAGGTTGCAGTGAGCCGAGATCGTGCCACTGCACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 703586 . AGTGAGGCTCCGTCTCAAAAAAAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 703804 . GTAAGGATGGCATGACTCGCCGGCAGCCCTGGGCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 703891 . AACATAAGATTACAAGACTTTTCCAGTTTAGACATACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 703973 . CGATACG . . . NS=1;AN=0 GT:PS ./.:.
+1 704001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=706000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:1.31:.:0:.:0:0.999:409
+1 704065 . GATGTCATATATTTTACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 704270 . TCCTAAAACATG . . . NS=1;AN=0 GT:PS ./.:.
+1 704291 . TGTGAAAATAGACTTTACAGCAGCCGGGTGCAGTGGTGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 704345 . GCACTTTGGCAGCAGAGGCAGGTGGATCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 704398 . AAAACCCCCCTCCCCAGCCCCACCCCCACCCCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 704455 . AGGGCATGGTGGCGGGCGCCTGTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 704543 . GAGCCAAGATCACACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 704590 . ACCTCAAAAAAAACAAAAACAAAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 704634 . CCCGACCTTACAGATGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 704685 . ACCCTTTTTCTCCCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 705004 . AACAGATTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 705049 . ACCTTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 705093 . CTGTAAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 705175 . AGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 705205 . TGGGCTCAGTGGCTCATGCCTGTAATCCCAGCACTTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 705317 . ATACAAAAAATACAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 705373 . ACTCAGGAGGCTGAGACAGGAGAATTGTTTGAACCCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 705545 . CACCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 705651 . ACTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 705684 . ATTCTTTTTTTTTTTTTTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 705744 . ACAGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 705917 . TGGTATTACAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 706001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=708000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.19:1.14:.:0:.:0:0.999:409
+1 706131 . CAGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 706177 . TGGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 706222 . CCAGGTTGGCAGGGCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 706355 . GACGTCAGGGGGCAGAGAGGCGCAGTTCCAGGGTGGCTTTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 706582 . CCCTTCTCATGGGTCCTGCTTTCTGGCTTCTCCTTCCTTACCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 706642 . GGAAGAACTGAGACAAAGTTTCTCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 706693 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 706765 . TTTCGCTCCTGTCGCCCAGGCTGGAGTGCAGTGGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 706850 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 706862 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 707216 . CCATCATCCCCTAGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 707246 . GGTATTTGCAGAGCTGAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 707284 . TGTGAAATCGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 707318 . GCTGGGAAGTGAGCGCTGCATCTCCTGCAGCGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 707411 . TTATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 707572 . TATCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 707882 . AAACGATGCAAGGTTTTTTGTTTTTGTTTTGGAGACGGAGTTTCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 708001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=710000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.21:.:0:.:0:0.999:409
+1 708072 . GGCAGGAAGCTAAACTGATACCTAGGGTAATCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 708335 . CAGCTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 708367 . AGCTTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 708434 . GCCCAGAGGGACAGAGGCAGATGAGTTGCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 708504 . GTCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 708673 . CGTGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 708714 . GCAGGCCCATCAGATGCCCAGGCCAGCAGCACAGCCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 708760 . AGGGAAACTTGGGGAGCCTCAGAGCACCCCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 708843 . TTTTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 708894 . AGACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 708930 . AAAGGCTTAACAGATATACAATTGCACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 709007 . TTCAATATATATCCAATCATTGTAACTATGACACAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 709057 . ACTATTTTCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 709137 . GCCACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 709190 . TAGCAGCCGGCAGGCAGTGACACACCGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 709341 . CAATGTCCTATACTTTGGTAAATACAGACTATGTTTAAACAATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 709447 . CATGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 709496 . ACATTGAGAGTCCAGAAGATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 709534 . TTTCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 709571 . TTCCGTTCCCCAGCATTGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 709618 . CTCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 709662 . TTGAAAAAGAAAAAATGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 709884 . TTACTTTTTAGTTTCCTTCATTTGAATCATCATTGTAAGTCTCCCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 710001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=712000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.66:1.03:.:0:.:0:0.999:409
+1 710012 . GCCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 710076 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 710219 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 710248 . TTCAAGACCAGCCTGGCCAACAGAGTGAAACCCTCTCTCTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 710546 . CCATTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 710585 . TCTCAAAAAAAAAAAAAAAGAAAAAATTAGCCAGGCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 710686 . TCTCAAAAAAAAAAAAAAAAGAGAGAGAGAGAGAAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 710905 . TGCAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 710985 . GCCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 711086 . GGTGGCGCGCGCCTGTAATCCCAGCTACTCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 711136 . TCGCTTGAACCCGGGAGGTGGAGGTTGCAGTGAGCCGAGATCTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 711201 . GGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 711217 . TGTCAAAAAAAAAAAAAAAAAAAACAGAAAAAGAAAAAGAAAAAAGAATTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 711377 . AATCTTTTTTTTATTTTGAGACAGAGTTTTGCTCATTGCCCAGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 711453 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 711456 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 711460 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 711466 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 711471 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 711489 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 711510 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 711526 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 711545 . TGTATTTTCAGTTGAGACAGGGTTTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 711591 . CTCGAACTCCTGACCTCAGGTGATCCACTGACCTTGGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 711700 . AGGCGCGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 711816 . AAAAATATATTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 711879 . AGGCTGAGGCAGGAGAACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 711909 . GGAGGTGGAGGTTGCAGTGAGCGGAGATCACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 711985 . AAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 712001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=714000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.91:2:29:=:29:0.999:409
+1 712020 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 712046 . AAAACTAAAACCAAAAACACAACACAAATGTAGTACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 712191 . AAAGAAACAGGCTCAGAGAATGTTATTTGATTGGACCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 712323 . TTGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 712406 . GGGAAAGTGAAAATGCTTCTAGAAGGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 712455 . TTGTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 712622 . GTATTTCAGAAAAACATAATCATATTAACAAATAATAACACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 712683 . AATGCTACTTTAGAAAAACATGCTCAAATCTAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 713162 . CAGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 713362 . GTTCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 713542 . CCGCCTCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 713977 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74512038;CGA_FI=100288069|NR_033908.1|LOC100288069|UTR|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:198:198,198:196,196:43,46:-198,0,-198:-43,0,-46:34:15,19:19
+1 714001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=716000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.82:0.93:2:37:=:37:0.999:409
+1 714202 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 714427 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:714427:PASS:177,.:160,.:0,.:0:0
+1 714439 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74707816&dbsnp.134|rs139182182;CGA_FI=100288069|NR_033908.1|LOC100288069|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:714427:VQLOW:21:116,21:124,28:43,25:-116,0,-21:-43,0,-25:17:10,7:7
+1 715348 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131984;CGA_FI=100288069|NR_033908.1|LOC100288069|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|19.0;CGA_SDO=5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:73:73,229:73,229:48,48:-229,-73,0:-48,-48,0:17:17,17:0
+1 715487 . CTGAGATTACAGGAATGAGCCATCGTGCCTGGCTTTACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 715646 . TTTCTTTTTTGAGATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 715985 . TTTAATTTAAAATTTTTTTTTCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 716001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=718000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.96:2:44:=:44:0.999:409
+1 716137 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 716164 . CTGCAGTGAGCCATGATCGTACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 716215 . AGACCCTGACTCCACAAATAAATAAATCAACGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 716432 . GCATAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 716708 . TCTACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 716908 . CTGCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 716980 . GTGATATATAATCCAACTTGGATTTTTAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 717883 . AGAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 718001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=720000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:1.05:2:49:=:49:0.999:409
+1 718135 . CTGATGCTGCTGGATGGTAGATCACACTTTATAAAGCAAGGGGCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 718206 . AATAGTACAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 718386 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10900602&dbsnp.131|rs77614743;CGA_FI=100287934|XR_108279.1|LOC100287934|TSS-UPSTREAM|UNKNOWN-INC&100288069|NR_033908.1|LOC100288069|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=MER33|hAT-Charlie|30.0;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:39:79,39:79,39:27,28:-79,0,-39:-27,0,-28:23:11,12:12
+1 718449 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 718555 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs10751453;CGA_FI=100287934|XR_108279.1|LOC100287934|TSS-UPSTREAM|UNKNOWN-INC&100288069|NR_033908.1|LOC100288069|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:25:25,133:25,133:25,45:-133,-25,0:-45,-25,0:16:13,13:3
+1 718787 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 719139 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 719195 . ATTATTTACTCATGTTGGGTGTAGTTGTTTTTTTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 719256 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 719408 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 719475 . TTT . . . NS=1;AN=0 GT:PS ./.:.
+1 719481 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 719804 . CCT . . . NS=1;AN=0 GT:PS ./.:.
+1 720001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=722000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:0.94:2:52:=:52:0.999:409
+1 720236 . GAACTACT . . . NS=1;AN=0 GT:PS ./.:.
+1 720447 . TTATAAAAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 720797 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3115846&dbsnp.131|rs75530702;CGA_FI=100287934|XR_108279.1|LOC100287934|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:91:91,91:90,90:22,31:-91,0,-91:-22,0,-31:39:16,23:23
+1 720950 . GTTGTGGGGTAGGGGGAGGGGGGAGGGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 721022 . TGCAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 721677 . T C . . NS=1;AN=2;AC=1;CGA_FI=100287934|XR_108279.1|LOC100287934|UTR|UNKNOWN-INC;CGA_RPT=BLACKJACK|hAT-Blackjack|28.9;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:22:22,22:1,1:0,8:-22,0,-22:0,0,-8:37:6,31:31
+1 721719 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 721722 . TCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 721844 . TCCCAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 721946 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 722001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=724000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.93:2:53:=:53:0.999:409
+1 722025 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 722459 . GACATTTTGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 722715 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 723722 . GGGCAAAAAAAGCAAAACTCTGAAGAAAGAGAGAGAGAGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 723798 . CAG C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34882115&dbsnp.126|rs35182822;CGA_RPT=GA-rich|Low_complexity|16.6;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:189:189,252:227,252:37,39:-189,0,-252:-37,0,-39:34:17,17:17
+1 723882 . TAAGGTGTGGGCCAAAGAAAGTAAGTTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 724001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=726000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:5.71:0.89:.:0:.:0:0.999:409
+1 724013 . TGTGTGGGTTAAATGTAATTAAATTCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 724053 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 724138 . AATGGAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 724153 . AATGGAATGGAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 724176 . GGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 724191 . GGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 724285 . C T . . NS=1;AN=2;AC=1;CGA_RPT=(GAATG)n|Satellite|15.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:724285:PASS:111:111,111:103,103:35,41:-111,0,-111:-35,0,-41:22:6,16:16
+1 724298 . AACGG . . . NS=1;AN=0 GT:PS ./.:.
+1 724421 . G <CGA_NOCALL> . . END=725562;NS=1;AN=0 GT:PS ./.:.
+1 724597 . G <INS:ME:L1> . . IMPRECISE;SVTYPE=INS;END=724597;SVLEN=39;CIPOS=-15,15;MEINFO=L1PA2,653,691,-;NS=1 GT:FT:CGA_IS:CGA_IDC:CGA_IDCL:CGA_IDCR:CGA_RDC:CGA_NBET:CGA_ETS:CGA_KES .:sns95:24:1:1:0:145:L1PA3:0:0.999
+1 725573 . AATGGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 725591 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 725601 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:725601:PASS:67,.:64,.:0,.:0:0
+1 725630 . T A . . NS=1;AN=2;AC=1;CGA_RPT=(GAATG)n|Satellite|13.8;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:725601:PASS:117:145,117:142,123:47,43:-117,0,-145:-43,0,-47:10:5,5:5
+1 725640 . T C . . NS=1;AN=2;AC=1;CGA_RPT=(GAATG)n|Satellite|13.8;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:725601:PASS:94:132,94:131,103:45,40:-94,0,-132:-40,0,-45:13:7,6:7
+1 725665 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 725671 . CTAATGGAATGGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 725689 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 725695 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 725716 . G <CGA_NOCALL> . . END=725994;NS=1;AN=0 GT:PS ./.:.
+1 726001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=728000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:3.31:0.91:.:0:.:0:0.999:409
+1 726014 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 726075 . TGGACTGGAATGTAATGAGTTTGGAATGGACTTGAATGCAATGGAATGGAATGGAATGGAATGGAATGGACTCAAATGGAATAGCATGGAATGGAATGGACTCAAATGCATTGGAATGGAATGGACTCGAATGGAATGGAATGGACTCGAATGGAATGGAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 726293 . ATAACAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 726308 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:726308:PASS:42,.:32,.:0,.:0:0
+1 726311 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 726314 . GAATGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 726324 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 726330 . AATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 726350 . AATGGAATGGAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 726367 . TGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 726376 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 726385 . AATGGAATGGAATGGAATAGAATGGACTCAAATGGAATGGAATATAATGGAATGGGAATGGGAATGGGAATGGAAGGGATGGGATGGGATGGGATGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 726498 . ATGGAATGGACACCTATGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 726563 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 726579 . TGGACTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 726594 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 726606 . GAATGGAATGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 726626 . GACTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 726634 . TGGAATGGAATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 726671 . TAATGGAATGGACTTGAATGAAATGGCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 726706 . ACTCGAATGGAATGGAATGGAATGGAATCGAATGGAATCGAATGGAATGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 726847 . CTCAAATGGAATAGAATGGAATAGAATGGACTCGAATATAATGGAATGAAATTGGCTCGAATGGAATGGAATGGACTTGAATGGAATGGAATGGAATCGAATGGAATGGAATGGAATGGAATGGAATGGAATGGAATGGAATGGAATGGAATGGACTCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 727652 . CGCCGCTGCTCCACCTGCCCCTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 727776 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 727850 . TTACGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 727894 . AAATCTTTTACTTTTACCAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 727959 . CAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 728001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=730000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.15:1.04:.:0:.:0:0.999:409
+1 728010 . TTGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 728086 . TAACTCTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 728153 . TCCACACTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 728239 . TTACTGCCTCCCTGTGATTATAGGTGAGACAGTCAACAAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 728312 . CACTCACACTCTGCTCCATCACCCTCAGCCACACAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 728690 . TCATAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 728858 . GGAACCCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 729676 . GGGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 730001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=732000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:1.00:.:0:.:0:0.999:409
+1 731410 . TCTTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 731883 . TGTTAAAAACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 732001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=734000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.05:.:0:.:0:0.999:409
+1 733639 . TTTTTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 733995 . CAACTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 734001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=736000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:1.08:.:0:.:0:0.999:409
+1 734016 . ATATATGTAAATATATCTTTTTCTGTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 734235 . AAAATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 734462 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 734491 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 734936 . ACACTGTCTTCCACAATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 735149 . CTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 735243 . GATGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 735330 . GCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 735423 . CCCGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 735452 . TAGCTTTTCTTGTAGGGTTTTTATGGTTTTAGGTCTTATGTTTAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 735630 . TTTCTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 735661 . TAGATGTGTGGCGTTATTTCTGAGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 735734 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 735739 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 735810 . ACTATAAAGACACACGCACACGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 735865 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 735879 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 735890 . TCAGTGATAGACTGGATAAAGAAAATGTGGCACATATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 735948 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 736001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=738000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.07:.:0:.:0:0.999:409
+1 736016 . AAACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 736103 . GGGAACATCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 736225 . CCT . . . NS=1;AN=0 GT:PS ./.:.
+1 736253 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 736288 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 736291 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 736522 . AT TC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3094318&dbsnp.103|rs3131974&dbsnp.111|rs4951928&dbsnp.130|rs71490526&dbsnp.131|rs75961312&dbsnp.131|rs76630699;CGA_SDO=2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:29,.:29,.:0,.:-29,0,0:0,0,0:32:32,.:0
+1 736808 . AGGGTCACTCACTAAGCATCTTTCCCATGCGCTGCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 736893 . ATTATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 737044 . CCTCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 737640 . GGCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 737737 . TAGGCTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 738001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=740000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:1.03:.:0:.:0:0.999:409
+1 738020 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 738121 . GGAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 738182 . ACATTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 738527 . ACAGGCTGGGCATGGTGGCTCACACCTGTAATCCCAACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 738811 . ATCTAAAAAAAAAAAAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 738904 . ATCAAGCATTTTTTAAAAATGCTTCTACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 739139 . AAATAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 739207 . GGGATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 739426 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131973&dbsnp.131|rs76643345;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:211,51:211,51:51,44:-211,-51,0:-51,-44,0:23:23,23:0
+1 739528 . GAG . . . NS=1;AN=0 GT:PS ./.:.
+1 739593 . GGGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 740001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=742000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.12:1.16:.:0:.:0:0.999:409
+1 740583 . AAGGTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 740683 . AAATTTCTTCCAAAAGGGTAACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 740896 . CCCCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 741267 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs80183632;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:109:109,109:97,97:24,32:-109,0,-109:-24,0,-32:31:10,21:21
+1 741394 . CTGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 741576 . CGATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 741743 . TGTAACATGGGATTTCTCTCCAGAGCAGCCATGCACTGCCCAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 741903 . CCACGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 741987 . AACCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 742001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=744000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.19:1.15:.:0:.:0:0.999:409
+1 742171 . CAGATTTTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 742231 . AGACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 742433 . TATGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 742573 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 742823 . CAATAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 742941 . TTTTACAA . . . NS=1;AN=0 GT:PS ./.:.
+1 743069 . AATCCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 743151 . AGACGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 743268 . GAAACAGCTTGAAGCTCTCTGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 743449 . ACACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 743510 . TGCGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 743648 . GCATAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 743848 . AAAAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 744001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=746000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.12:1.00:.:0:.:0:0.999:409
+1 744056 . AAGATTTTAAAGGAATTTAAATAATTGGACATATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 744301 . TTCCTATGTACCCTAATAAAAAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 744485 . TACCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 744628 . TCCCTTTGTCAAGCGAGTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 744680 . GTCTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 744739 . TTGCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 744875 . ATACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 744948 . CCCCTGAGGTAAGGAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 745018 . AACGCTATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 745049 . CTCCAGCCTTCCCCTGCTGCCATCTAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 745361 . TTTATTTCATAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 745570 . TATTAAAATATTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 745621 . GGATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 745840 . TA . . . NS=1;AN=0 GT:PS ./.:.
+1 745905 . AATGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 746001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=748000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.17:1.28:.:0:.:0:0.999:409
+1 746099 . TACGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 746203 . ACAGAAAAAAAAAACCTGTAAATAGTAATCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 746365 . ATCCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 746391 . GCCGGTCATCGTTCTTTGACAAGAAAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 746443 . CCACTATTCACTTTTAGTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 746521 . CAGTGCCAACAGTTGTAAATCATCAAGACAAGCAAAGCACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 746691 . TGAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 746819 . GAAGGAGACAGGATTGCTGACTAATCTCATATGTACAGGGAGAACGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 746913 . AAGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 747186 . GAGGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 747295 . AAACAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 747513 . CTGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 747585 . CAACAAGCCCATGAAAAGATGCTCCACATCACGAATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 747688 . AATTAAAAAAAAAAAAAGAAAGTTTAACAAGTATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 747914 . CACATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 748001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=750000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.14:1.02:.:0:.:0:0.999:409
+1 748138 . TATGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 748177 . AAACAGAAAGCCAGTTACCAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 748506 . AACGTTATACAACACAAGCGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 748722 . ATGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 748767 . GAATAGAGAAGAATCAACAAAACAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 748875 . CTAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 749065 . AGGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 749127 . TAAGATA . . . NS=1;AN=0 GT:PS ./.:.
+1 749173 . CATTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 749212 . CTACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 749262 . TGAGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 749393 . AAATTATCTATCTTTTCAAGCTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 749514 . ACTCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 749541 . GCAGACTGTGATAATTTCACCAAAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 749592 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.111|rs4606254&dbsnp.131|rs80161738;CGA_FI=400728|XR_108280.1|FAM87B|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=3 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:749592:PASS:62,.:37,.:4,.:-62,0,0:-4,0,0:57:37,.:20
+1 749602 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 749641 . CGATCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 749680 . GGACGACATTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 749734 . CAATCCCCAGTTGTTGTCTTAAGTCACACT . . . NS=1;AN=0 GT:PS ./.:.
+1 749853 . TTACGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 749899 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 749960 . AGTTAAAAAAAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 749986 . TTTCAGAATTAACATTTCCTTCCTAAACATCTAACACGACACACTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 750001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=752000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:0.99:.:0:.:0:0.999:409
+1 750041 . TAACACACCCTTATTACATGAAGGAGCAGCAGAGCAGAGGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 750110 . CACAGCCAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 750150 . AACTGCACTGTGCACAGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 750342 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3103768&dbsnp.120|rs10793768&dbsnp.129|rs55727401;CGA_FI=400728|XR_108280.1|FAM87B|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:42,91:42,91:39,37:-91,-42,0:-39,-37,0:9:9,9:0
+1 750433 . CTTCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 750501 . ACTCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 750619 . ATCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 750740 . AGATGCAAAGGTGAGCTGCAGGTGGTCTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 750894 . AGGGGCCAACTGGGACTGGGGTGTCCATCAGCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 751000 . GTGGAAAAAAAAGAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 751123 . AAAACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 751169 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 751290 . AAATAATAAAAATAAATTATTTAAACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 751372 . TTACAAATAAAAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 751473 . CTAGGTATCTGATTAGAAAAAAAAAAAAATAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 751530 . AAATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 751562 . AAGAGATAGAGAGAAAAACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 752001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=754000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.18:.:0:.:0:0.999:409
+1 752242 . T A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.120|rs12090487;CGA_FI=400728|XR_108280.1|FAM87B|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L1P4|L1|15.2;CGA_SDO=2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:67,.:67,.:15,.:-67,0,0:-15,0,0:36:16,.:20
+1 752383 . TATTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 752434 . CTAGGAAATACTCTTCTTGACGTTGGCCTTGGCAAAGAATTTTTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 752493 . AACGATTGCAACAAAACAAAATTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 752566 . GT AT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3094315;CGA_FI=400728|XR_108280.1|FAM87B|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L1P4|L1|15.2;CGA_SDO=2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:752566:PASS:725,.:680,.:49,.:-725,0,0:-49,0,0:51:40,.:0
+1 752593 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:752566:PASS:626,.:583,.:0,.:0:0
+1 752721 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3131972;CGA_FI=400728|XR_108280.1|FAM87B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:489:519,489:581,489:48,47:-519,0,-489:-48,0,-47:60:33,27:27
+1 752894 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131971&dbsnp.131|rs77059159;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:34:66,34:66,34:15,27:-66,-34,0:-27,-15,0:31:31,31:0
+1 753269 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3115861&dbsnp.129|rs61770172;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:51,36:51,36:8,28:-51,-36,0:-28,-8,0:44:44,44:0
+1 753376 . CGGCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 753405 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3115860&dbsnp.129|rs61770173;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:54:294,54:294,54:52,34:-294,-54,0:-52,-34,0:32:31,31:1
+1 753425 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131970&dbsnp.130|rs71507459;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_RPT=MER58A|hAT-Charlie|38.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:131,51:131,51:42,44:-131,-51,0:-44,-42,0:28:27,27:1
+1 753471 . GGGCAGCCGTAGACCACACACGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 753537 . AGACGGGCTGGGCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 753648 . AATGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 753845 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 753849 . G T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs56101360&dbsnp.129|rs58324164;CGA_FI=400728|XR_108280.1|FAM87B|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:753849:VQLOW:36,.:26,.:6,.:-36,0,0:-6,0,0:23:17,.:6
+1 753973 . TCTGTGGACACTTTTTCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 754001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=756000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.94:2:46:=:46:0.999:409
+1 754182 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3131969;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:754182:PASS:68,.:70,.:15,.:-68,0,0:-15,0,0:33:19,.:14
+1 754192 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL .|0:754182:PASS:.,43:.,50:.,0:0:0
+1 754334 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3131967;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:30:30,74:70,70:9,26:-30,0,-74:-9,0,-26:51:17,34:34
+1 754379 . TTCCTCCAGACACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 754444 . CAGCCAATGGAACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 754503 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3115859;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_RPT=MLT1I|ERVL-MaLR|41.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:280:339,280:339,280:53,52:-339,0,-280:-53,0,-52:37:19,18:18
+1 754629 . AGTGATT . . . NS=1;AN=0 GT:PS ./.:.
+1 754758 . CAGTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 754964 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3131966&dbsnp.131|rs74918077;CGA_FI=400728|XR_108280.1|FAM87B|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:89:148,89:148,89:36,30:-148,0,-89:-36,0,-30:39:17,22:22
+1 755153 . AGTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 755274 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs78408995;CGA_RPT=MER44D|TcMar-Tigger|23.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:55:55,55:50,50:17,31:-55,0,-55:-17,0,-31:26:9,17:17
+1 755482 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 755486 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 755638 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 755648 . CCAGCAGAACCACCCTGTCTATACTACCTGCCTGTCCAGCAGATCCACCCTGTCTACACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 755714 . CTGGCCAGCAGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 755736 . CTATACTACCTGCCGCTCCAGCAGATCCACCCTGTCTACACTACCTGCCTGTCCAGCAGACCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 755828 . ATATCCACCCTATCTACACTACCTGCCTGGCCAGCATATCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 755883 . ACCTCCCAGCCCAGCAGATCCGCCCTGTCTACACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 755924 . CTGGCCAGTAGATCCACGCTATCTACACTACCTGCCTGGCCAGCAGATCCACCCTGTCAACACTACCTGCTTGTCCAGCAGGTCCACACTGTCTACACTACCTGCCTGTCCAGCAGGTGCACCCTATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 756001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=758000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.41:1.54:2:38:=:38:0.999:409
+1 756068 . CCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 756080 . CCCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 756108 . AGATCCACCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 756167 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 756176 . GCAGATCC GCAGGTCC . . NS=1;AN=1;AC=1;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:756176:PASS:42,.:42,.:17,.:-42,0,0:-17,0,0:2:2,.:0
+1 756238 . CTTGTCCAGCAGGTCCACCATGTCTACACTGCCTGCCTGGCCAGCAGATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 756299 . CACTACCTGCTTGTCCAGCAGGTCCACCCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 756341 . TGCCTGCAAAGCAGATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 756366 . CTACACTACCTGGCTGGCCAGTAGATCCACGCTATCTACACTACCTTCCTGTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 756427 . CCAACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 756448 . CCTGTCGAGCAGATCCACCCTGTCTATACTACCTGCCTGTCCAGCAGGTCCACCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 756513 . ACCTGCGTGCCCAGCTGATCCGCCCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 756551 . TGCTTGTCGAGCAGATCTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 756586 . TGCCTGTCCAGCAGATCCACCCTGTCTATACTCCGTACCTGGCCAGCAGATCCACGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 756650 . ACTACCTGCCTGTCCAGCAGATCCACACTGTCTACACTACTTGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 756781 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75189095 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:73:74,73:74,73:26,36:-74,0,-73:-26,0,-36:27:14,13:13
+1 756879 . GGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 756944 . CAGTAGATCCACGGTATCTACACTACCTCCCTGGCCAGCAGATTCACCCAGTCTACACTAACTGCTTGTCCAGCAGGTCCACCCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 757047 . CCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 757103 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs76779543 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:757103:PASS:290:311,290:311,300:53,53:-311,0,-290:-53,0,-53:32:16,16:16
+1 757120 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77598327 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:757103:PASS:230:408,230:408,229:53,49:-408,0,-230:-53,0,-49:33:20,13:13
+1 757253 . TCCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 757520 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:757520:VQLOW:37,.:37,.:0,.:0:0
+1 757527 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 757532 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:757520:VQLOW:37,.:37,.:0,.:0:0
+1 757535 . C T . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:757520:VQLOW:37:37,37:37,37:27,14:-37,0,-37:-14,0,-27:14:6,8:6
+1 757595 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 757640 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3115853 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:65:65,65:65,65:28,35:-65,0,-65:-28,0,-35:13:6,7:7
+1 757734 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4951929 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:75:620,75:620,75:52,48:-620,-75,0:-52,-48,0:29:29,29:0
+1 757803 . ACCTCCCTGGCCAGCAGATCCACCCTGTCTATACTACCTGCCTGGCCAGCAGATCCACCCTGTCTATACTACCTGACTGGCCAGCAGATCCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 757936 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4951862 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:21:21,58:21,58:22,26:-58,-21,0:-26,-22,0:17:17,17:0
+1 757976 . CTAGCTGCCTGTCCAGCATGTCCACCCTATCTACACTACCTGCCTGTCCAGCAGATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 758001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=760000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.54:1.09:2:42:=:42:0.999:409
+1 758052 . GCCTATCCAGCAGATCTACCCTGTCTACACTACCTGCCTGCTCAGCAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 758138 . CACGCTATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 758202 . CAGGTCCACCCTATCTACACTACCTGCCTGCCCAGCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 758262 . GCCTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 758298 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 758302 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 758305 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 758309 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 758315 . CCCTGTCCATACTACC CCCTGTCCACACTACC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3131955;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:758315:PASS:47,.:38,.:15,.:-47,0,0:-15,0,0:16:3,.:0
+1 758333 . CCT . . . NS=1;AN=0 GT:PS ./.:.
+1 758339 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 758375 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 758378 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 758482 . CAGATCCACCCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 758585 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 758607 . TACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 758626 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131954;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:11:98,11:98,11:39,15:-98,-11,0:-39,-15,0:6:6,6:0
+1 758863 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 758866 . GCAGGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 758881 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 758896 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:758896:PASS:48,.:21,.:0,.:0:0
+1 758901 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:758896:PASS:48,.:21,.:0,.:0:0
+1 758905 . ATC . . . NS=1;AN=0 GT:PS ./.:.
+1 759146 . AAATTTTAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 759435 . GTTATATAAATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 759663 . CATCAAAAATGAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 759686 . AAATTCATTTTACATATATGTCTATAAAATAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 759837 . T A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3115851&dbsnp.132|rs114111569;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:34:96,34:96,34:33,32:-96,-34,0:-33,-32,0:28:28,28:0
+1 760001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=762000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:1.08:2:46:=:46:0.999:409
+1 760354 . GCACAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 760418 . TTGCCTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 760575 . TGGGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 760806 . ACATCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 760912 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.86|rs1048488;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:760912:PASS:833,.:794,.:49,.:-833,0,0:-49,0,0:50:35,.:15
+1 760913 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:760912:PASS:841,.:801,.:0,.:0:0
+1 760954 . TTCATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 761147 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3115850;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:39,270:39,270:29,34:-270,-39,0:-34,-29,0:54:45,45:9
+1 761257 . CCTATCCACTGCCTTGTGTCAGTATGTGTGTGTCTTGGGGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 761732 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 761752 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.86|rs1057213;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:97,48:96,67:23,38:-97,-48,0:-38,-23,0:38:38,38:0
+1 761800 . A T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.130|rs71507460&dbsnp.86|rs1064272;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:70,.:70,.:9,.:-70,0,0:-9,0,0:57:57,.:0
+1 761811 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs55884753&dbsnp.86|rs1057212;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:31,.:31,.:4,.:-31,0,0:-4,0,0:59:59,.:0
+1 761957 . A AT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs59038458&dbsnp.130|rs70949521;CGA_FI=643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:40,.:40,.:5,.:-40,0,0:-5,0,0:28:28,.:0
+1 762001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=764000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:1.01:.:0:.:0:0.999:409
+1 762269 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 762273 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3115849;CGA_FI=100506327|XR_108281.1|LOC100506327|TSS-UPSTREAM|UNKNOWN-INC&643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:29:56,29:56,29:12,26:-56,-29,0:-26,-12,0:34:34,34:0
+1 762317 . AGTCACCGCTAGTGGGAGGCGATTGTGCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 762589 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 762592 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 762601 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.103|rs3131949&dbsnp.130|rs71507463;CGA_FI=100506327|XR_108281.1|LOC100506327|TSS-UPSTREAM|UNKNOWN-INC&643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:24,.:24,.:3,.:-24,0,0:-3,0,0:19:19,.:0
+1 762628 . AGGGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 762696 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 762784 . AGC . . . NS=1;AN=0 GT:PS ./.:.
+1 762818 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.135|rs188947985;CGA_FI=100506327|XR_108281.1|LOC100506327|TSS-UPSTREAM|UNKNOWN-INC&643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:762818:PASS:167:204,167:205,168:50,49:-204,0,-167:-50,0,-49:20:11,9:9
+1 762856 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs142969342;CGA_FI=100506327|XR_108281.1|LOC100506327|TSS-UPSTREAM|UNKNOWN-INC&643837|NR_015368.1|LOC643837|TSS-UPSTREAM|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:762818:PASS:151:151,189:151,189:45,50:-151,0,-189:-45,0,-50:21:8,13:13
+1 762932 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 762964 . GGCGGGGCCGGGCCGGGCCGGGGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 763012 . CTTACCGACCTCCCGCCCCCGCTGCGCGCGTTTCTGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 763185 . GGGCGGTGCTGCTCCCGAGTCGGCGCGCGGCGGGGACGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 763254 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 763263 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 763267 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 763289 . TGGGTAGCAGCCTCTTCGGCCCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 763319 . GTGACGCGCGCTCGGGCTCCGCGTTCGCGTCGAGGCAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 763387 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 763394 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.103|rs3115847&dbsnp.126|rs35782521;CGA_FI=100506327|XR_108281.1|LOC100506327|TSS-UPSTREAM|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:763394:VQLOW:21:21,21:16,16:0,21:-21,0,-21:0,0,-21:5:2,3:3
+1 763410 . CCGCCCTGGACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 763459 . CCTATTTTCAACCTGTCCTGCTCCGCACCTGAGATGATTTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 763532 . ACATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 763583 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 763629 . TA . . . NS=1;AN=0 GT:PS ./.:.
+1 763633 . TAT . . . NS=1;AN=0 GT:PS ./.:.
+1 763652 . TTTGTCTCCAGTACATATAATGAGGCTTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 763765 . AAGGATTTTTTTTTTTAAATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 763869 . TCATTTTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 764001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=766000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.22:1.05:.:0:.:0:0.999:409
+1 764259 . AGTCACAGAGTATAGTGAAGTTAAGATAGTTTATTAGCAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 764646 . TAATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 764847 . ATGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 764902 . TGTATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 764941 . TATATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 764993 . TGTGCAGCATCTGAGCTCCGTCTTCACCTTAATCCCGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 765079 . CACTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 765161 . AACAGTTTCTCTGGTTATAGAATATATTTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 765345 . TTCATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 765496 . CAAGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 765933 . TTATACT . . . NS=1;AN=0 GT:PS ./.:.
+1 766001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=768000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:0.69:.:0:.:0:0.999:409
+1 766102 . CAGTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 766203 . CCCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 766294 . CAGTCTCCCTTGGCAGCTCTCAGCTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 766593 . CTGCTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 766972 . TCTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 767038 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 767076 . TGTGTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 767348 . TGTCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 767780 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2905042;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC&79854|NR_024321.1|NCRNA00115|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:39,252:39,252:36,52:-252,-39,0:-52,-36,0:21:20,20:1
+1 768001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=770000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:0.80:.:0:.:0:0.999:409
+1 768113 . GTAAGTTTTGTTTTGTTTTGTTTTGTTTTGTTTTGTTTTGTTTTGTTTTAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 768241 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 768253 . A C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.101|rs2977608;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=AluSx|Alu|16.1;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:768253:PASS:118,.:107,.:37,.:-118,0,0:-37,0,0:23:14,.:9
+1 768349 . GGTGTCCAACTCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 768553 . CCCACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 768592 . CCATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 768630 . TTTTCTACATTCCTAACTTACTTTCCAGGGGATCGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 768726 . TGCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 768796 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 768910 . ATTTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 769237 . CAGGCCCTCATGTACGTCCAGGATGCGGTGACAGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 769308 . TGAAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 769357 . ATCCGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 769548 . GTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 769632 . GCACCAGCAGCAAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 769829 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2977607;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:107:470,107:470,107:32,36:-470,-107,0:-36,-32,0:80:78,78:2
+1 769866 . GCAAGATGTTCCTGCTTCCGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 769959 . TGACGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 770001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=772000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.00:1.25:.:0:.:0:0.999:409
+1 770075 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2977606&dbsnp.131|rs77324267;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:63:63,303:63,303:37,53:-303,-63,0:-53,-37,0:35:35,35:0
+1 770125 . TTACGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 770297 . ACCTATCCATCCACCCATCTATCCAACCCTCCCTCCCTCCATCCACCCATCC . . . NS=1;AN=0 GT:PS ./.:.
+1 770412 . TCTACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 770568 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131943&dbsnp.131|rs78979757;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:36,126:36,126:34,41:-126,-36,0:-41,-34,0:25:25,25:0
+1 770735 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs139531586;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:60:73,60:72,59:11,24:-73,0,-60:-11,0,-24:40:18,22:22
+1 770847 . GCAGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 770897 . CCACGGCCACGATGGTGTGCCTCCGAAGCCACGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 771083 . GAGTTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 771150 . TGAGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 771316 . CAGTCCAATCCATGGGGAAGGTGTGAATTAAAGACCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 771410 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2519006&dbsnp.134|rs142008205;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:33,.:33,.:4,.:-33,0,0:-4,0,0:58:22,.:36
+1 771665 . CCAATAAGTCCCTGGCTCCATGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 771823 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2977605&dbsnp.131|rs74599385;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=LTR16C|ERVL|52.2;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:33:33,102:33,102:31,35:-102,-33,0:-35,-31,0:25:24,24:1
+1 771854 . CCAGGGGCCAGTGCCCCCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 771947 . TCCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 772001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=774000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:1.13:2:49:=:49:0.999:409
+1 772043 . GCACTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 772430 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79479693&dbsnp.134|rs145794890;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=FLAM_C|Alu|17.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:101:179,101:179,101:33,33:-179,0,-101:-33,0,-33:40:18,22:22
+1 772734 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:772734:PASS:308,.:274,.:0,.:0:0
+1 772755 . A C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.101|rs2905039;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=MER58A|hAT-Charlie|26.3;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:772734:PASS:224,.:211,.:45,.:-224,0,0:-45,0,0:33:23,.:0
+1 772764 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 772782 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 772978 . CTAGAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 773885 . AGCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 774001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=776000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:1.07:2:49:=:49:0.999:409
+1 774363 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.120|rs12563150&dbsnp.131|rs76662252;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=L1M4b|L1|50.3;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:56,.:56,.:7,.:-56,0,0:-7,0,0:56:20,.:36
+1 774419 . GAAAGAACCAGCAGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 774568 . AAAGTTCCAAGCAGAAAAAATGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 774715 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 774722 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 774729 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 774736 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 774785 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 774801 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 774839 . CTGGACACACACACCTAGACACACACACCTGGACAAACACACCTGGACACACACACCTAGACACACACACCTGGACACACACACGTAGACACACACACCTAGAGACACACACCTGGACACACACACCTAGACACACACACCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 775092 . CTCACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 775252 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 775255 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 775278 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 775423 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 775426 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2905037;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=L1M4b|L1|50.3;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:775426:PASS:27:27,77:34,51:27,8:-77,-27,0:-27,-8,0:40:40,40:0
+1 775429 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 775570 . GGAGTAAGGAAATGGCCACATATTGTATAATCCCATTTATATGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 775637 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 775648 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 775659 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 775683 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 775779 . ATT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:775779:PASS:77,.:7,.:0,.:0:0
+1 775790 . AAA AA . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs55687311&dbsnp.134|rs145511843;CGA_FI=100506327|XR_108281.1|LOC100506327|INTRON|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=(A)n|Simple_repeat|12.9;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:775779:VQLOW:31,.:7,.:0,.:-31,0,0:0,0,0:44:13,.:1
+1 776001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=778000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:1.10:2:49:=:49:0.999:409
+1 776170 . TTTTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 776232 . AACGCGTTGGTGAGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 776288 . GTAGGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 776455 . TGCCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 776804 . GCCAAGGCCACCGTCAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 776854 . TCTCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 777122 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2980319;CGA_FI=100506327|XR_108281.1|LOC100506327|UTR|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=MIRb|MIR|36.1;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:127:127,681:94,641:48,49:-681,-127,0:-49,-48,0:53:53,53:0
+1 777126 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:495,.:455,.:0,.:0:0
+1 777202 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 777856 . GCATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 777992 . CAGGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 778001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=780000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.01:1.14:.:0:.:0:0.999:409
+1 778067 . CTGATCCTTCCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 778124 . ACACCTCCTGCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 778204 . TGTGTCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 778302 . C CCT . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs58115377&dbsnp.132|rs112119688;CGA_FI=100506327|XR_108281.1|LOC100506327|UTR|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:16:16,32:16,34:18,5:-32,-16,0:-18,-5,0:19:18,18:1
+1 778569 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2977615;CGA_FI=100506327|XR_108281.1|LOC100506327|UTR|UNKNOWN-INC&643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:43:43,156:43,156:39,46:-156,-43,0:-46,-39,0:29:29,29:0
+1 778836 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 779310 . TGA T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs80302052;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:264:264,342:302,342:35,42:-264,0,-342:-35,0,-42:41:20,21:21
+1 779356 . C T . . NS=1;AN=2;AC=1;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:103:238,103:238,103:22,24:-238,0,-103:-22,0,-24:75:34,41:41
+1 779911 . G GT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs112437059;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:96:195,96:195,96:39,33:-195,0,-96:-39,0,-33:28:14,14:14
+1 780001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=782000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:1.02:2:50:=:50:0.999:409
+1 780027 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2977613;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:95:95,787:95,787:48,53:-787,-95,0:-53,-48,0:35:35,35:0
+1 780229 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12565032;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:163:186,163:184,161:42,42:-186,0,-163:-42,0,-42:32:22,10:10
+1 780559 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 780785 . T A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2977612;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:427:427,427:426,426:50,54:-427,0,-427:-50,0,-54:46:22,24:24
+1 781640 . ACCAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 781686 . CAGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 781929 . TACATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 782001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=784000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.21:1.09:2:49:=:49:0.999:409
+1 782051 . TGATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 782470 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs76135949;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=MER103C|hAT-Charlie|32.0;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:197:197,197:196,196:20,30:-197,0,-197:-20,0,-30:60:25,35:35
+1 782528 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 782553 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 782677 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs140041012;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:107:240,107:240,107:31,34:-240,0,-107:-31,0,-34:54:24,30:30
+1 782830 . GCCCGGCACAAGGTAGGAACTGAGTGTGAGTGCGGGATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 782981 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6594026;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:123:123,123:123,123:13,25:-123,0,-123:-13,0,-25:67:38,29:29
+1 783304 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2980295&dbsnp.131|rs75216674;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=MIR|MIR|37.7;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:92:627,92:627,92:50,47:-627,-92,0:-50,-47,0:42:41,41:1
+1 784001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=786000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.08:2:49:=:49:0.999:409
+1 784020 . CAGATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 785050 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2905062;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_RPT=MSTD|ERVL-MaLR|31.4;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:286:396,286:396,285:53,52:-396,0,-286:-53,0,-52:36:19,17:17
+1 785340 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 785440 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 785734 . AAGTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 785989 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2980300;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:177:177,177:174,174:41,44:-177,0,-177:-41,0,-44:31:12,19:19
+1 786001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=788000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.82:1.04:.:0:.:0:0.999:409
+1 786038 . TAT . . . NS=1;AN=0 GT:PS ./.:.
+1 786069 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 786096 . TACATGTGCCATGCTGGTGCGCTGCACCCACTAACTCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 786163 . CTATCCCTCCCCCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 786185 . CCCCACAACAGTCCCCAGAGTGTGATGTTCCCCTTCCTGTGTCCATGTGTTCTCATTGTTCAGTTCCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 786346 . GGACATGAACTCATCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 786435 . ATTCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 786730 . CTCAGCCAGCAGGGTTGCCCAGTGCCCCTTGTCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 786787 . CTGCGGTTACTCTGGGTCTGTGCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 787019 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2905060&dbsnp.129|rs56289866;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:34:206,34:206,34:44,27:-206,-34,0:-44,-27,0:30:27,27:3
+1 787066 . AAACAGGGAAAATGTCTTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 787121 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 787135 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28753393;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:787135:PASS:36:100,36:115,34:19,27:-100,-36,0:-27,-19,0:42:42,42:0
+1 787151 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 787185 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 787205 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.101|rs2905057&dbsnp.129|rs55693913;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:105,.:105,.:13,.:-105,0,0:-13,0,0:54:54,.:0
+1 787262 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2905056&dbsnp.129|rs56108613;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:62:62,184:62,184:37,22:-184,-62,0:-37,-22,0:58:58,58:0
+1 787399 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 787675 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 787680 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 787685 . G T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.101|rs2905054;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:318,.:318,.:38,.:-318,0,0:-38,0,0:55:29,.:26
+1 787717 . CGTTGGTTCCCAGTTGGCTTCCGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 787765 . CCAACAGGCTGGTGTTGAATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 787844 . C T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.101|rs2905053;CGA_FI=643837|NR_015368.1|LOC643837|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:191,.:180,.:33,.:-191,0,0:-33,0,0:48:32,.:16
+1 788001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=790000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.82:0.96:.:0:.:0:0.999:409
+1 788380 . TCCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 788560 . CTTGGAACTCCTGACCTCAAGTGATCTGCCCGCCTCGGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 788709 . TTGATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 788837 . CCCTGACCCTGATCAACATGAGATGACCG . . . NS=1;AN=0 GT:PS ./.:.
+1 788886 . CCCCGACCCTGATGAACGTGAGATGACCGCCGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 788931 . TGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 788984 . CCCCGACCCTGATGAACGTGAGATGACCGCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 789033 . CCCCGACCCTGATGAACGTGAGATGACCG . . . NS=1;AN=0 GT:PS ./.:.
+1 789086 . GACCCTGATCAACGTGAGATGACCG . . . NS=1;AN=0 GT:PS ./.:.
+1 789138 . CCTGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 789239 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 789242 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 789256 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3131939;CGA_FI=643837|NR_015368.1|LOC643837|UTR|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:25:117,25:117,25:42,25:-117,-25,0:-42,-25,0:19:18,18:1
+1 789848 . ATGCGTGCAGATGAGTGACGCAGTGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 789932 . CAGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 790001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=792000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:1.14:.:0:.:0:0.999:409
+1 790033 . CACGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 790215 . ACACCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 790278 . AGGAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 790381 . GGCATCA . . . NS=1;AN=0 GT:PS ./.:.
+1 790696 . C CAT . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs56224346&dbsnp.131|rs76089329&dbsnp.134|rs146955212;CGA_RPT=(TATG)n|Simple_repeat|37.8;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:29:29,398:29,398:22,39:-398,-29,0:-39,-22,0:31:30,30:1
+1 790753 . CAT C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74609473;CGA_RPT=(TG)n|Simple_repeat|35.6;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:790753:PASS:222:460,222:459,225:38,39:-460,0,-222:-38,0,-39:40:21,17:17
+1 790758 . GTA G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.102|rs3039191&dbsnp.130|rs72558500&dbsnp.134|rs137938478;CGA_RPT=(TG)n|Simple_repeat|35.6;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:790753:PASS:309:415,309:424,317:42,36:-309,0,-415:-36,0,-42:41:21,17:21
+1 790904 . GCACGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 790948 . GGATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 790958 . AGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 791315 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77384149&dbsnp.132|rs116928318;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:791315:PASS:347:557,347:555,356:49,54:-557,0,-347:-49,0,-54:56:30,26:26
+1 791328 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74877941;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:791315:PASS:271:521,271:518,278:49,52:-521,0,-271:-49,0,-52:50:29,21:21
+1 791389 . ACTCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 791675 . GTGTTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 791984 . TCTTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 792001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=794000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:1.13:.:0:.:0:0.999:409
+1 792263 . A G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.131|rs76190315&dbsnp.86|rs1044922;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:46,.:46,.:6,.:-46,0,0:-6,0,0:67:24,.:43
+1 792297 . GCTCACCTTCCTGCCTCAAGCCCCTCTCCCACGT . . . NS=1;AN=0 GT:PS ./.:.
+1 792360 . CCCTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 792480 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2905036;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:58:236,58:236,58:40,35:-236,-58,0:-40,-35,0:43:43,43:0
+1 793062 . GGGGAAATGAGAGGCCTGAGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 793145 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2905030;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:38:38,66:57,57:7,24:-38,0,-66:-7,0,-24:55:43,12:12
+1 793205 . TGTCAGTTTAATTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 793325 . TTTCTTTTTTTTTTTCTTAAGGGGTCAGTTTTAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 793408 . TTTAACATAGAAATATCTGCCTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 793466 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 793907 . ACTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 793915 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 793922 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs145538528;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:793922:PASS:69:69,69:63,63:28,34:-69,0,-69:-28,0,-34:17:5,12:12
+1 794001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=796000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:1.05:.:0:.:0:0.999:409
+1 794319 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:113,.:0,.:0,.:0:0
+1 794332 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:51,.:0,.:0,.:0:0
+1 794575 . GAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 794613 . TACTGGTAACGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 794874 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs146966553;CGA_RPT=L1ME4a|L1|37.6;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:55:157,55:157,55:48,33:-157,0,-55:-48,0,-33:18:9,9:9
+1 794901 . CATATTTTCAATAATTTCCATTATAATAATAATGTCATGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 795555 . CATCTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 795985 . TCCCTCCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 796001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=798000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.30:0.91:.:0:.:0:0.999:409
+1 796111 . ACTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 796373 . GGTGGGGGGGTTAATCTTTTAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 796531 . GACTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 796714 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 796727 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2909612&dbsnp.132|rs115637794;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:95:255,95:254,124:42,52:-255,-95,0:-52,-42,0:43:43,43:0
+1 796767 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.120|rs12076540&dbsnp.131|rs75932129;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:796767:PASS:197,.:175,.:18,.:-197,0,0:-18,0,0:61:43,.:18
+1 796771 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:796767:PASS:177,.:167,.:0,.:0:0
+1 797039 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 797297 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 797474 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:797474:PASS:156,.:72,.:0,.:0:0
+1 797502 . GATAGAT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:797474:PASS:156,.:72,.:0,.:0:0
+1 797541 . GAGATAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 797810 . GGCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 798001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=800000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.25:1.07:.:0:.:0:0.999:409
+1 798026 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 798045 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 798245 . TGTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 798400 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10900604;CGA_RPT=MLT1K|ERVL-MaLR|41.9;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:91:91,91:86,86:21,30:-91,0,-91:-21,0,-30:33:13,20:20
+1 798616 . GGGAGGGCAGATTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 798730 . TTGGATAAAGAGCAAGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 798754 . GAGCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 798798 . TGCGAGAGAAAGATGTGAGCAAATATCCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 798872 . A G . . NS=1;AN=1;AC=1;CGA_SDO=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:38,.:38,.:6,.:-38,0,0:-6,0,0:41:19,.:22
+1 798959 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:VQLOW:30,.:29,.:0,.:0:0
+1 798999 . GACGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 799303 . ACAAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 799447 . CCA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:428,.:363,.:0,.:0:0
+1 799463 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4245756;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:115:360,115:363,169:31,44:-360,-115,0:-44,-31,0:79:79,79:0
+1 799496 . AAATCTCCCCTAAGGAGGAGATACGACGTGTGCAGATTGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 799625 . GTGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 799668 . CACCCTCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 799791 . AGAATTGGTATGCCTTACCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 799883 . GAGGTGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 800001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=802000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.29:1.03:.:0:.:0:0.999:406
+1 800001 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:63,.:15,.:0,.:0:0
+1 800007 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 800102 . CTAACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 800191 . CTATCCCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 800342 . CTGGCATCGATGACATGGGAACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 800383 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4951931&dbsnp.131|rs76291278;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:67:179,67:179,67:22,26:-179,0,-67:-22,0,-26:54:27,27:27
+1 800691 . CCTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 800974 . GTGCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 801706 . TGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 801943 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7516866;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:283:507,283:502,237:43,33:-507,0,-283:-43,0,-33:66:35,31:31
+1 801957 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:547,.:502,.:0,.:0:0
+1 801983 . CCGTGCCCTCACGTGGTCCTCCCTCTGCACTCACATCCCTGACGTCCTCCCGTGCCCTCACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 802001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=804000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:0.97:.:0:.:0:0.999:406
+1 802090 . CACGTGGTCCTCCCTCTGCACTCACATCCCTGACGTCCTCCCGTGCCCTCACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 802170 . GACGTCCTCCCGAGCCCTCACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 802219 . GACGTCCTCCCGAGCCCTCACGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 802268 . GACGTCCTCCCGAGCCCTCACGTGGTCCTCCCTCTGCACTCACATCCCTGACGTCCTCCCGAGCCCTCACGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 802366 . GACATCCTCCCGTGCTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 802415 . GACGTCCTCCCGTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 802846 . AAAAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 803045 . CAGCAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 803062 . GTGTGCTCAGCAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 803081 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 803085 . CATCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 803417 . AGACGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 803544 . GGCACATATGGCATAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 803859 . CAGATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 803896 . AAGTCTGGGATTCTTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 804001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=806000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:1.26:.:0:.:0:0.999:406
+1 804115 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9725068&dbsnp.131|rs79338660;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=23 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:113:113,113:113,113:28,35:-113,0,-113:-28,0,-35:36:14,22:22
+1 804527 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 804537 . TTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 804593 . TA T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75244191;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=22 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:83:83,83:83,83:25,34:-83,0,-83:-25,0,-34:24:12,12:12
+1 804975 . AATTAAAAAAAAAAATCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 805202 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 805226 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 805230 . CGG . . . NS=1;AN=0 GT:PS ./.:.
+1 805251 . CCGC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:805251:PASS:59,.:33,.:0,.:0:0
+1 805436 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 805470 . C A . . NS=1;AN=2;AC=1;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:805470:VQLOW:20:20,20:0,0:0,11:-20,0,-20:0,0,-11:11:1,10:10
+1 805477 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 805485 . GC G . . NS=1;AN=2;AC=1;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:805470:VQLOW:20:20,20:0,0:0,3:-20,0,-20:0,0,-3:15:1,14:14
+1 805491 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 805494 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 806001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=808000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.01:.:0:.:0:0.999:406
+1 806367 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 806466 . TCTATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 806805 . CTTATCTAACATTTTTATGTGTTGCTTCTTCCAGTTTACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 807299 . AAGATTTTTTTTTTTTTTTTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 807512 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs10751454;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:57:57,559:57,559:46,52:-559,-57,0:-52,-46,0:26:26,26:0
+1 807652 . ATAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 807761 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4951932;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:46,470:46,470:41,48:-470,-46,0:-48,-41,0:19:19,19:0
+1 807960 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 807983 . GAAGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 808001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=810000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:1.08:.:0:.:0:0.999:406
+1 808010 . AGGGGGTTTCATTTGCTCCACCTGCAGCGAGGTTAGCCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 808073 . ATTCCTAACAGGGGAAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 808125 . TTGCTCCACCTGCAGTGAGGTCTGTTAGCCCATCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 808220 . AGGGGGTTTCATTTGCTCCACCTGCAGCGAGGTTAGCCCATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 808283 . ATTCCTAACAGGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 808309 . TGTGACTCTGGAGAAGGGGGTTTCATTTGCTCCACCTGCAGCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 808430 . AGGGGGTTTCATTTGCTCCACCTGCAGCGAGGTTAGCCCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 808517 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 808631 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11240779;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:123:409,123:403,117:53,35:-409,0,-123:-53,0,-35:35:22,13:13
+1 808812 . GGTTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 808922 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6594027;CGA_FI=284593|NR_027055.1|FAM41C|INTRON|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:32:32,145:53,166:34,39:-145,-32,0:-39,-34,0:31:27,27:4
+1 808928 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 809107 . CGTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 809164 . TGGTCCCGTTTCCCCGGCTGCATTTCTTCATGCCCGGCTTTGCCCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 809236 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 809298 . GCCCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 809350 . GTCGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 809378 . ACAAATGTTCAACATTCAAAATA . . . NS=1;AN=0 GT:PS ./.:.
+1 809436 . AACTTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 809624 . GGCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 809729 . GAATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 810001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=812000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.99:.:0:.:0:0.999:406
+1 810286 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28410559;CGA_FI=284593|NR_027055.1|FAM41C|UTR|UNKNOWN-INC;CGA_SDO=7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:420:420,420:420,420:50,54:-420,0,-420:-50,0,-54:44:21,23:23
+1 811242 . C <CGA_NOCALL> . . END=811812;NS=1;AN=0 GT:PS ./.:.
+1 811897 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 811919 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 812001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=814000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.22:0.93:.:0:.:0:0.999:406
+1 812267 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7541694;CGA_FI=284593|NR_027055.1|FAM41C|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:78:783,78:783,78:53,42:-783,-78,0:-53,-42,0:34:34,34:0
+1 812284 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7545373;CGA_FI=284593|NR_027055.1|FAM41C|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:92:682,92:682,92:53,47:-682,-92,0:-53,-47,0:36:36,36:0
+1 812732 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:812732:PASS:502,.:454,.:0,.:0:0
+1 812750 . CT CC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.111|rs4246500;CGA_FI=284593|NR_027055.1|FAM41C|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:812732:PASS:548,.:518,.:50,.:-548,0,0:-50,0,0:45:34,.:0
+1 812762 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:812732:PASS:360,.:323,.:0,.:0:0
+1 813219 . TGGTTGTTGGTGGCTCAGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 813285 . TGAGAAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 813314 . ATCCGTTGAAAAGGTGAGTAATGCTGGCAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 813374 . ACTGGTGTTCAGAGGTGGATTTGGTTCCTTCCCAGCCTTTCCCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 813436 . CATGTACTTTTGTGTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 813566 . GGGCATCAGGTACTTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 813613 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:813613:PASS:55,.:17,.:0,.:0:0
+1 813624 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:813613:PASS:83,.:17,.:0,.:0:0
+1 813639 . CA . . . NS=1;AN=0 GT:PS .|.:813613
+1 813649 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:813613:PASS:83,.:17,.:0,.:0:0
+1 813747 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.111|rs4970388;CGA_FI=284593|NR_027055.1|FAM41C|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:49,.:49,.:8,.:-49,0,0:-8,0,0:45:45,.:0
+1 813816 . AACATTTGGGCCCTGAGAAACGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 813861 . TATGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 813944 . TCAGAGAGAAAGCGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 813971 . TAATGACAGCCTGAAATTATTTGAGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 814001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=816000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.33:2.65:.:0:.:0:0.999:406
+1 814024 . TTCTGTCTCCTTGTGCATCACCTGCGCAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 814074 . TGTATACAGGTAGCTGTGTTACCCTCCTAGCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 814144 . GCAGTTAAGGTCTAGGCTCATGGGAGGACAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 814212 . CATACCTGCCTTCCTCCATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 814241 . AGTAGGGGATGGGAGGTCTCACACTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 814294 . CTCTGCTTTCCCAGACAGCCCCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 814368 . TGGGTGTTTGTTTATAAATTATTCCCCTGGAGGGGAATAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 814439 . AAACATCTAAGCCTGGCCCTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 814487 . AAACTTACCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 814606 . CCAAAACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 814685 . ATGCCTCAAAGTCAAAAGTCAACAGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 814728 . CAATATA . . . NS=1;AN=0 GT:PS ./.:.
+1 814767 . ATATACACACACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 814784 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 814788 . TACAT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:814788:PASS:47,.:34,.:0,.:0:0
+1 814795 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 814800 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 814813 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:814788:PASS:47,.:34,.:0,.:0:0
+1 814822 . ACCATTTAAAAAATACCATCCTTTCCCCATTGAATAGTGTTGACTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 814885 . ACCGTGTTTTTCGGTTCTTTATTTCTATCGCATTGGTCTTTATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 814998 . GTTTGTCCTCTAACT . . . NS=1;AN=0 GT:PS ./.:.
+1 815036 . GGCTACTTAGGATTTTTTTAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 815074 . GAATAGGTTTTTCTATTTTTGAATATATTGGAATTTTTATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 815135 . GATCGCTATAGATAACAATGGCATCTTGACAAGGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 815178 . CCAGTCCATAAACACATGATGTCTTTTCATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 815226 . ATACTTTTCTGCCATGTTTATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 815278 . AAGTATTTTATTCTTTTGATGCTATCATACATGATACTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 815334 . TCAGATAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 815370 . TTTCTGTGTATTGATTTTGTATCCTGCAACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 815417 . ATTGTATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 815449 . AAAGATTTTTAATATATAAGGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 815489 . ATAATTTTACTTTTTAAAAAATTGGAATATCCTTTATTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 815538 . CTTATTGTTTTAACTAACTAGAACCTTCAGTACTACATTAAATAGAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 815603 . CTTGTTTTCGCTCTGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 815661 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 815668 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 815676 . TTGGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 815695 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 815704 . TAAGGTGTTTTCTTTCTTTTTATAATTTATTAAGTACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 815751 . GAATGTGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 815778 . TTTTTCTTTAAGATGATCACATGAGGTTTTTTCCTTCATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 815833 . TACACTGATTTTCATGTGTTAGAACATACTTTTATTTCAGGAGTCAGTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 815889 . TTCATAGTGTATAATCCTTTTAATGTACTGCTAAATTTGAATTG . . . NS=1;AN=0 GT:PS ./.:.
+1 815959 . TCAACATTTGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 815989 . TTTTCTTATGGTGCCTTTGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 816001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=818000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:3.93:2.74:.:0:.:0:0.999:406
+1 816039 . ATATAATAAGTTAGAAAATGTTACCTCCTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 816098 . CTGGGTTAATTCTGCTTTAAACGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 816138 . GTGTAGCCATCTGGTCCAGGCTTTTCTTTGTTGCTGGGTTTTTTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 816200 . CTGCTGAATCTCCTTGCTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 816289 . TTAGGTTATTCAATTTTTTTAGTGTATAATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 816331 . TCTTCTACATCCTTTTTTTACTCCAAAAGTTTGTTAGTTATG . . . NS=1;AN=0 GT:PS ./.:.
+1 816380 . TTTATTTTTGAGTTTGGTAATTTGAGTATTCCCTTTTTTTCTTAGTCAATCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 816449 . TTTTTATCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 816471 . AACTGGGTTTTGTTGGTTTTTGATATCTTTTTCTATTCTCTATTTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 816539 . ATCATTTTTAAAATTTTGCTAGCTTTTAGTTGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 816594 . TTCCCTCTGTTTTTCTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 816632 . ATTCGGTATCTTATTTTTTATAATCATTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 816674 . TTTCCCGTGTGGTACTGTTTTTGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 816713 . TTGGTATTTCATATTTTTAATTTGTCTCTAGATATTTTCTATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 816784 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 816789 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 816796 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 816800 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 816831 . ACAAAATTTTCTTGATTTGTTACAGTTTTATTTGTTGTAAGTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 816889 . AATATGTGTATCAACATTTGTTGTGTTCTCATAAACTTTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 816941 . ATTTCTGGTCCACATATGTAAGTCTCTACATTAATATTATTTTGAAGCATTTAAACTTCTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 817033 . AGATTTTTGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 817060 . TGGTAGGTGACTGAGAAATGCTTAAAAATTAGCCAAAACTTAAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 817118 . TACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 817146 . AAAAAATAAGTCTTTAATGGTATAAAAGCAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 817185 . GAATGTTTTCTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 817209 . CATCTAAATTAAAAGCTGGAAAAAAATTTTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 817266 . ATTTCAACTTTTTCTGGTTAAAATTTTTCCAAACAGATTCCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 817323 . AAA . . . NS=1;AN=0 GT:PS ./.:.
+1 817341 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 817368 . AAACTTTCTTGAACCTGTGGGAATCCATGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 817405 . TGACATTATGTTCTATTCTCTTGGAAGGTAGAAATATCGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 817457 . TTGCTGACAAGAAATATGGTCCTGAGCAAGGCTCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 817518 . AACGCTGGAGCCCATCTGTCTCCAATCTGCTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 817566 . GAACTTCAGTTTTCCCTTTGATACTCTGTATTTCTACCAACCACAACGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 817642 . AATGACAAATATAGGCCTGAAGGAAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 817677 . TGGCATTCCCAGCTTACTACCACTCCTTGGGTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 817722 . TACGTGGATTCAACTCATAGACTCAGGTGGGTGAGGATCTATTGTTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 817780 . AAGTGACTGCTTAAGACTCTGGTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 817828 . CTCAATGCAGTGTTAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 817860 . TAATTACCATCTTACTATCACTAAATCATAGCTAAAATAAGGAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 817921 . AGAGATGTAATCTTATGAAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 817951 . AGAGATTTGTGGAGAGCCCTTCATAATTTCATGGTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 818001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=820000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.62:2.36:.:0:.:0:0.999:406
+1 818012 . GACATTTCATTATAATATATTAGCTATTC . . . NS=1;AN=0 GT:PS ./.:.
+1 818057 . ATGTAAAGTTTTCTTTGTTGCACTTTAAGTTCTGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 818103 . AGAGCATGCAGGTTTGTTACGTAAGTATACACGTGTCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 818152 . TGCACCCATCAACCCATCATCTACATTAAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 818205 . CCCCAGCCTCTCACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 818241 . GATGTTCCTCTCCCTGTGTCCATGTGTTCTCATTGTTCAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 818292 . GAGTGAGAACATGCAGTGTTTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 818379 . TTGTCCAGGTTGGAGTGCAATGGCGCGATCTCGGCTCACTGCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 818431 . TCCCGGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 818483 . AGGCATGTGCCAACATGCCTGGCTAATTGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 818528 . AGACGGGGTTTCTCCATGTTGGTCAGGCTGGTCTCAACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 818581 . CCCGAAGATCTGAGACTACAGGTGTGAGCCAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 818646 . TTTGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 818676 . CTACGTCCCTGGAAAGGACATAAATGCGTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 818728 . CACCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 818775 . CCAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 818798 . GTGCTGAAATAAACATACAGTGCATGTGTCTTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 818864 . ACCCCGTAATGGGATTGCTAGGTCAAATTGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 818905 . CTAGATCCTTGAGGAATTGTCACACTGTCTTCCATAATGACTGAACTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 818962 . CCTACCAACAGTATGAAAGCATTCCTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 819012 . CTGTTGTTTCCTGACTTTTAATAATAGCCATTCTAACTGGCTTGAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 819086 . ATTTATC . . . NS=1;AN=0 GT:PS ./.:.
+1 819103 . TGACGATGAGCTTTTTTTCATGTTTGTTGGCCACATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 819161 . TCTGTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 819191 . GGGGTTGTTTTTTCTTGTAAATTTGTTTAAGTTATTTGTAGATTCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 819262 . TAGATTGCAAAAATTTTCTCCCAATCTATAGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 819323 . TTGCTGTGCAGAAGCTCTTTAGTTTAATTAGATCCCATTCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 819394 . TTTGGTGTTTTAGTCATAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 819461 . CTAGGGTTTTTATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 819504 . CCATCTTCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 819530 . GGTGTAAGGAAGATGTCCAGTTTCAATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 819597 . AAATAAGGAATCCTTTCCCCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 819650 . GATGGTTGTATGTGTATGCTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 819683 . TCTATATTCTGGTTCATTGGTCTATGTGTCTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 819731 . ATGCTGTTTTGGTTACTGTAGCCTTATAGTATATTTTGAAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 819798 . GTTATTTTTGCTTAGAATTGTCTTGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 819837 . TTTTTGGTTCATGAGAATTTTTAAATAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 819873 . AATTCTGTGAAGAATGTCATTGGTAGTTTAATGGGAATACCATTAAATT . . . NS=1;AN=0 GT:PS ./.:.
+1 819949 . GCTATTTTCACGAATTAATTCTTCCGTATCCATGAGCATGGAATGC . . . NS=1;AN=0 GT:PS ./.:.
+1 820001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=822000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:2.16:2.62:.:0:.:0:0.999:406
+1 820013 . CCTGTCTGATTTCTCTGAGCAGTGGTTTGTAGTCCTCCTTGAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 820084 . ATTCTGATGTATTTTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 820123 . GAATTTCATTCATGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 820161 . TTGATGTATAGAAATACTAGCAATTTTTGCACATTGGTTTTGTATACTG . . . NS=1;AN=0 GT:PS ./.:.
+1 820239 . AGAAGCTTTTGGGCTGAGATGATGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 820274 . ATACAGGATCATGTCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 820313 . TTCCTCTCTTCCTATTTAAATACCTTTATTTCTTTCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 820359 . TTGCCCTGGCCAGAAATTCCAGCACTATATTGAATAGGAGTGGTAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 820416 . CTTGTCTTGTGCCAGTTTTCAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 820460 . TCATTCAGTATGATATTGGCTGTGGGTTTGTCATTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 820521 . ATCCTTCAATAGCTATTTTATTGAGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 820572 . TTTCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 820604 . ATAGTCGTGTTGTTTTATGTTTAGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 820653 . TATAGATTTGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 820698 . GCCATCTTGATCGTGGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 820729 . TTAACAGTCTTAAGTTCAGTCTTTTTACATAATCCCACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 820787 . TTCATTCTTTTTTGTTCTTTTTTCTCTATTCTTTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 820845 . TTTCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 820931 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 820935 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:820935:PASS:268,.:245,.:0,.:0:0
+1 820943 . AGGTCAGTTA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:820935:PASS:356,.:320,.:0,.:0:0
+1 820955 . TTCC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:820935:PASS:356,.:320,.:0,.:0:0
+1 820967 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:820935:PASS:356,.:320,.:0,.:0:0
+1 820979 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs72890746;CGA_RPT=L1PA7|L1|26.4;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:820935:PASS:113:113,121:194,222:30,16:-121,0,-113:-16,0,-30:109:56,53:56
+1 821001 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:821001:PASS:435,.:0,.:0,.:0:0
+1 821034 . GTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 821051 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 821054 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:821001:PASS:435,.:0,.:0,.:0:0
+1 821056 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:435,.:0,.:0,.:0:0
+1 821060 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:435,.:0,.:0,.:0:0
+1 821069 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:435,.:0,.:0,.:0:0
+1 821108 . CCTTGGGTCAGTTCTGTGCCCTTGCTGGGGAGGTGGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 821187 . TTCAGCGTTTTTGTGTTGATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 821246 . GACGTTGCTGACCTTTGAATGGGGTTTTTGTGGGGTCTTTTTTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 821305 . TTGCTTTCTGTTTGTTCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 821339 . CTTTCCTAGGGCTGCTGTGGTTTTCTGGGGGTCCACTCTGGACCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 821418 . CACCGGTGAAGGCTGCAAAACAGCAAAGATGGCAGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 821470 . GAGTACT . . . NS=1;AN=0 GT:PS ./.:.
+1 821509 . AACGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 821530 . TGGAGACCCCTGTTGGGAGGCCTCACCCAGTCAGGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 821577 . AGGAACTGCTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 821602 . GCTGCCCTTTGGCAGAGCAGGTGTGCTGTGCTGTGCTGATCCCCGGGAGTCTCCAGAGCCAGCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 821697 . AGATAGCAGCTACCCCTCTCCCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 821757 . ATAGAACTCTGGCTGGAGTTGCTAAAATTCCAATGGGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 821817 . ATGTGTTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 821852 . CACAATCAGGCACAGCAGCTGTGCTGTGTTATGGGAAACTCCTCCTGGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 821942 . ATATAGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 822001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=824000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:3.07:3.18:.:0:.:0:0.999:406
+1 822002 . GCCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 822032 . GGGTTTTGTGAGGTGCTGTGGGAGTGGGGCCTCAGAATGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 822086 . CTGGATTCAGCCCCCTTCCTAGGGGAATGCACAGATGTATCTCCCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 822154 . TAGGATGCAAAACTCCTGGGTTTCCACGCATGCCCCAGTGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 822203 . GCATTCCGCCGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 822247 . GCCATGGTGGCTGAGCTCACCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 822353 . TTGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 822397 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 822400 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 822469 . CTGGATACCTCAACTCAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 822493 . GAAGTCACTTGTAGTTTTCATTGCTCTCCATGAGAGCCATGGGCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 822545 . CTTCTAATCGGCCAGCTTGGCCCCATCTAAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 822607 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:822607:PASS:411,.:0,.:0,.:0:0
+1 822613 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:822607:VQLOW:24,.:0,.:0,.:0:0
+1 822633 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 822638 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:822638:PASS:672,.:0,.:0,.:0:0
+1 822646 . ATC . . . NS=1;AN=0 GT:PS ./.:.
+1 822654 . TTCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 822662 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:822638:PASS:484,.:0,.:0,.:0:0
+1 822673 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75183112;CGA_SDO=8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:148:148,606:0,0:0,11:-148,0,-606:0,0,-11:93:51,42:42
+1 822683 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 822694 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:698,.:0,.:0,.:0:0
+1 822716 . AACATCTGCCATTTATAAAATTTCTGTAGAGTTAATAGAACTTTTCCTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 822837 . CATCTTGTTACTGGGATTATAAATTCGTACAAACTTTATCTAATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 822906 . TATCAGTGGAAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 822938 . TCATTTCCAGAAATTTACCCTACAGACATACTCATGATGCCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 823009 . ATACCACCTTTTTCATTAACAAAAGTCTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 823057 . TCAGTCACATTAATGGTTATCACCTTACTGTGTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 823112 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 823118 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 823135 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 823139 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 823164 . AAAATGATGTCCAATATATGTTACAAAGTAAAAACAACCCTGGGTGCAGAGCCATGTGTCTGATATGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 823257 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823260 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823270 . GAA . . . NS=1;AN=0 GT:PS ./.:.
+1 823279 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 823294 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823321 . AAGACCTTTCTTTTTTAATGCTAACACTCCAAGGAACTGCACAACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 823374 . ATTCCCAACACATGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 823405 . CTATAGATTTTAGGAAATGAGGCTCTTGAAAAATCAAGCATTAATATTCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 823469 . TGGGACATA . . . NS=1;AN=0 GT:PS ./.:.
+1 823496 . TTGAAAGTTTGTAAAATTTGCTCATACAACCATCTGGGCCTAGTCCTTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 823558 . TTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 823570 . GAGATTTTATTTTCTTCACATTATAGTACTTCTTAGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 823618 . TTCTTGATTCTTGAGTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 823648 . CTTTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 823661 . AGGGTGCTTGGGCTCAGAACACATCAGTCAAAGAAAGAGAGAAGAAAGGAAGGCAGGAAGGCAGGGTGAAAGGAAGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 823758 . GAC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:823758:PASS:274,.:268,.:0,.:0:0
+1 823763 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823767 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 823769 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 823774 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823777 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 823812 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 823831 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:456,.:314,.:0,.:0:0
+1 823865 . GG GAGGGAAAGA . . NS=1;AN=2;AC=1;CGA_RPT=GA-rich|Low_complexity|23.3;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:177:177,456:35,314:4,34:-177,0,-456:-4,0,-34:161:95,161:66
+1 823893 . GGGAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 823906 . AGAGGAAGGGAGAGGAAGGAAGGAGGGAGAGAAAGAGGGAAAGGGAAAGAAGGAAGGAAAGAAACAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 823994 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:823994:PASS:203,.:190,.:0,.:0:0
+1 824001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=826000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:3.82:3.29:.:0:.:0:0.999:406
+1 824002 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:823994:PASS:203,.:190,.:0,.:0:0
+1 824004 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:823994:PASS:203,.:190,.:0,.:0:0
+1 824036 . GAGGGAGAGAGGGAGGGAGGGAGGAAAAGAAAGAGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 824091 . CGGGAAGGGAAGAGGAGCCAGCCAAAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 824139 . CTTCTACTTGACCCAAGCAGTTCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 824201 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 824215 . T C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.131|rs72890762;CGA_SDO=4 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:244,.:22,.:3,.:-244,0,0:-3,0,0:123:74,.:48
+1 824254 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:824254:PASS:629,.:221,.:0,.:0:0
+1 824258 . CC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:824254:PASS:715,.:293,.:0,.:0:0
+1 824269 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:240,.:0,.:0,.:0:0
+1 824284 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:415,.:35,.:0,.:0:0
+1 824300 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:715,.:293,.:0,.:0:0
+1 824308 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 824332 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 824338 . GC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:110,.:0,.:0,.:0:0
+1 824357 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:189,.:0,.:0,.:0:0
+1 824370 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 824376 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 824379 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:188,.:0,.:0,.:0:0
+1 824393 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 824407 . AGGCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 824428 . ATTAGGTTATAAGATTCTGCCCCCATGTATGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 824467 . TCATTATCTCAGGAGTGGGTTAGTTATCTTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 824518 . GCATGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 824547 . TTCCCAAACTCTTGCTCTTCTGCCTTCAGCCATGAGATGACACAGCCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 824604 . CACGAGATGCAGACCCCTCATCCTTGGATTTCCTAGCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 824663 . TTTCTTTTCTTTATAGAT . . . NS=1;AN=0 GT:PS ./.:.
+1 824691 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.131|rs75026264;CGA_RPT=MSTD|ERVL-MaLR|17.5;CGA_SDO=4 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:824691:PASS:88,.:69,.:8,.:-88,0,0:-8,0,0:65:36,.:26
+1 824694 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL .|0:824691:PASS:.,67:.,28:.,0:0:0
+1 824709 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:206,.:144,.:0,.:0:0
+1 824727 . ATGCCCCATTACATGGAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 824764 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 824816 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 824838 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 824845 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 824857 . TTC . . . NS=1;AN=0 GT:PS ./.:.
+1 824874 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 824901 . GCCGACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 824921 . ATCATGGCAGGTTTGATGTGCTCACTTCTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 824970 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 824983 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825016 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 825037 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 825051 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 825063 . GCCCCTGTAGGCAGAGCCTAGACAAGAGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 825104 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825140 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 825144 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 825197 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825207 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61768257;CGA_RPT=MIRb|MIR|33.0;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:314:314,377:296,307:27,37:-314,0,-377:-27,0,-37:77:34,42:42
+1 825250 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61768258;CGA_RPT=MIRb|MIR|33.0;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825250:PASS:288:379,288:304,216:45,48:-379,0,-288:-45,0,-48:49:31,18:18
+1 825266 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs72890768;CGA_RPT=MIRb|MIR|33.0;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825250:PASS:138:138,241:71,179:24,49:-138,0,-241:-24,0,-49:27:14,13:13
+1 825288 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:889,.:811,.:0,.:0:0
+1 825292 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825307 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77548197;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825307:VQLOW:28:28,100:0,42:0,29:-28,0,-100:0,0,-29:22:10,12:12
+1 825323 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:403,.:334,.:0,.:0:0
+1 825333 . TGT TATAACA . . NS=1;AN=2;AC=1;CGA_RPT=MLT1I|ERVL-MaLR|36.7;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825333:PASS:287:287,889:226,811:28,43:-287,0,-889:-28,0,-43:37:11,26:26
+1 825346 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74765147;CGA_RPT=MLT1I|ERVL-MaLR|36.7;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825333:PASS:164:164,506:111,438:27,54:-164,0,-506:-27,0,-54:39:15,24:24
+1 825360 . GTC TTA . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74559290&dbsnp.134|rs150390413;CGA_RPT=MLT1I|ERVL-MaLR|36.7;CGA_SDO=4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:825333:PASS:363:363,889:288,811:18,43:-363,0,-889:-18,0,-43:54:24,30:30
+1 825382 . TTTCTTTTTTCTATTTTTTCTTTTGTTGGGGGGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 825431 . AGTCTCACTCTGTCACCCGGGCTGGAGTGCAGTGGTGCAATCTCAGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 825500 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825508 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 825513 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 825518 . G <INS:ME:ALU> . . IMPRECISE;SVTYPE=INS;END=825518;SVLEN=208;CIPOS=-241,242;MEINFO=AluY,9,216,-;NS=1 GT:FT:CGA_IS:CGA_IDC:CGA_IDCL:CGA_IDCR:CGA_RDC:CGA_NBET:CGA_ETS:CGA_KES .:sns95:257:11:10:1:.:AluYa1:0:0.979
+1 825542 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 825549 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 825552 . TCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 825562 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 825566 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 825595 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 825604 . ACCATGTTAGTCAGGATGGTCTCGATCTCCTGACCTCATGATCTGCCTGCCTCAGCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 825681 . AGGCGTGAGCCACTGCACCCGGCCTTGACTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 825748 . GGGTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 825767 GS000016676-ASM_3618_L C ]1:5726936]C . . NS=1;SVTYPE=BND;MATEID=GS000016676-ASM_3618_R;CGA_BF=0.92 GT:FT:CGA_BNDMPC:CGA_BNDPOS:CGA_BNDDEF:CGA_BNDP 1:MPCBT:8:825767:]5726936]C:PRECISE
+1 825826 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 825851 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:825851:PASS:77,.:64,.:0,.:0:0
+1 826001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=828000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.23:1.18:.:0:.:0:0.999:406
+1 826023 . GCTTCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 826057 . TAAACTGGGAAGCACTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 826113 . TAGATAGAGGATGGAGAGACTGCAGGGGGCAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 826162 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 826182 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 826194 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 826214 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 826225 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 826240 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 826255 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 826259 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 826272 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 826277 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 826282 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 826301 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 826458 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 826610 . GGAAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 826643 . AAGTTTTTATAGTAACAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 826676 . GTCAATATTTATTACTTCATTAAGAGCAAATAAATACTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 826748 . AGTTTTGTATCACTATGTTTTTAATATTATACCTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 826802 . ACAATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 826831 . CCTCGAGGTAAGATTTACGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 826885 . TTTTCCAACTTTTTATACACATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 826936 . AAAATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 827052 . CCACAATATTAATACTTAGTAACCTTTATTTTAATAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 827105 . GAAATCTTGAATTGTCATATAGCAGTATCTTACAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 827162 . GAAATATGTGTTCCTAAAACATTTTTTTTTAAGATGGAGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 827232 . TGGAACAATCTCGGCTCACTGCAACTTCCGCCCCCCGGGTTCAAGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 827323 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970386;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:148:148,187:173,187:48,50:-148,0,-187:-48,0,-50:19:10,9:9
+1 827597 . ACACTTTAGGAGGCCAAGGCAGGAGTATCATGAGACC . . . NS=1;AN=0 GT:PS ./.:.
+1 827639 . GAGCAAAATAGTGAGATGCTAACTCTACAAAAAAAATAAAAATTAGCTGAGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 827709 . AATTACAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 827731 . GAGGTGGGAGGATCCCTTGAGGGCAGGAGGTCAAGGTTGCAGTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 827783 . TCATACCACTGTACTTCAGCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 827828 . TCAAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 827869 . CATGTATCCATTTACATTTACTTATTTTTAACAGTTTATCTAGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 827966 . TTTCTTGTTAACCATGTTATAGCCTGTGAATATCAGGTGTTCACGTAA . . . NS=1;AN=0 GT:PS ./.:.
+1 828001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=830000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:1.18:.:0:.:0:0.999:406
+1 828054 . AACTCAGAAAATTCCATTACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828112 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 828154 . TTATAGCTTTATAACC . . . NS=1;AN=0 GT:PS ./.:.
+1 828264 . TTTCCTTTTATCCATACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828338 . CCTATATCCCAGCACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828362 . CTGAGGCAAGAGGATCACTTGAGCTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 828415 . TAGCGAGACCGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 828447 . AAAAAATTGCCAGGCATGGTGGTGCATGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 828583 . AGAGTGAGACCTTAGAGAGAGACCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828618 . AGAGAAAAAAATAAAGAATTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828682 . TAATACC . . . NS=1;AN=0 GT:PS ./.:.
+1 828722 . CAAGTCAGCAGATTCAAAATAGGCAGGGAAAAAAAAATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 828778 . AGATTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 828899 . GTAATCCCAACACTTTGGGAGGCTGAGGCAGGTGGATCACTT . . . NS=1;AN=0 GT:PS ./.:.
+1 829037 . GTCCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 829105 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 829159 . AAAGAAAAAAAAAAAAAAAAAAAATATATATATATATATATATATATATATATATATATATAGGCCTGTGCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 829267 . GCCGAGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 829298 . GTTCGAGACCAGCCTGGCCAACATGGTGAAACCTCGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 829361 . GGGCATGGTGGCACACA . . . NS=1;AN=0 GT:PS ./.:.
+1 829397 . AGGAGGCTGAGACAGGAGAATCACTAGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 829442 . TGCAGTCAGCCAAGATCACACCACTGCACTCCAGCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 829515 . TGTGTGTGTGACTAGAATGGTCTATAATATATAGCCAGCTCGAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 829593 . TCTCATTTTGGCCTTTTCAAGATTAAATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 829631 . AGGCCCCTCCCCTCTAGGGAAGTACTTGCCGGAGCGCTGCCTAAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 829710 . AATGGTTTATTTCTCATTATAAGGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 829767 . ACAGGAAGAATTTTTAAAATCGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 829816 . GGTCCCCAGAAAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 829858 . CAGGAGGAGGAGTGAGGGAGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 829912 . TCAGTTTTTCACGTGTGCCATTTTCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 829969 . TTCAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 830001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=832000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.41:1.32:.:0:.:0:0.999:406
+1 830051 . AATATTTTAGACCAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 830104 . ATTGAGTATCTCAGTGGCTGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 830151 . ATTTGAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 830172 . AGGGTTCTAACCCAACCAGGATCCCCCTGGGGTGAAACTGAAACCCACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 830244 . TTTTTATTTTTATTTTTTCATTTTTTTTGAGACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 830324 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77038580;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:830324:PASS:662:798,662:798,668:35,53:-798,0,-662:-35,0,-53:117:68,49:49
+1 830340 . CGC TGT . . NS=1;AN=2;AC=1;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:830324:PASS:830:1376,830:1387,861:25,43:-1376,0,-830:-25,0,-43:124:81,43:43
+1 830348 . G A . . NS=1;AN=2;AC=1;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:830324:PASS:328:639,328:640,330:41,39:-639,0,-328:-41,0,-39:84:52,32:32
+1 830357 . C A . . NS=1;AN=2;AC=1;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:830324:PASS:439:439,570:439,580:30,51:-439,0,-570:-30,0,-51:80:42,38:38
+1 830361 . C T . . NS=1;AN=2;AC=1;CGA_RPT=AluSp|Alu|10.6;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:830324:PASS:284:284,455:283,459:26,46:-284,0,-455:-26,0,-46:72:36,36:36
+1 830374 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL .|0:830324:PASS:.,451:.,459:.,0:0:0
+1 830440 . GACAGGGTTTCTCCATGTTGGTCAAGCTGGTCTCGAACTCCCAACCTCAGGTGATCCGCCTGCCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 830520 . TTGGGTTTACAGGCGTGAGCCACCACGCCCGGCCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 830578 . GATGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 830602 . TGAGTTGGTGTTCACTAACAAGCACGGAAGCTTTGTTACATT . . . NS=1;AN=0 GT:PS ./.:.
+1 830658 . TGGCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 830681 . CCCGGGTTCACGCCATTCTCCTGCCTCAGCCTCCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 830778 . CAC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:830778:VQLOW:28,.:22,.:0,.:0:0
+1 830783 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 830807 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 830830 . TCCGCCCACCTCGGCCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 830862 . TTACAGGCGTGAGCCACGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 830953 . AAAGTTTCAGGTGTCTTGATGGTATCTAAATCAGTTGTTGATTCGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 831009 . ACACAAGCATAACCTCTACGCCAAGTTATTATT . . . NS=1;AN=0 GT:PS ./.:.
+1 831072 . TCTGTCCACCAAACCAGTTGTTCTGCTTGTGTCTTTGCAGCTGGTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 831132 . TCAGCTGCTGGTAACATCTGGCCTTTGGGAAGGCTCGAAAAATGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 831233 . CCCCTTTTTTGTTTTTATTA . . . NS=1;AN=0 GT:PS ./.:.
+1 831269 . ATGAAATCGTAACTGAGCATTTTCAATTAACTGTGTGGAATGAACCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 831340 . TAATAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 831362 . CAATACCTCAATTACAGCTACAAGCTCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 831402 . GAAATATAGGGCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 831461 . ACTGGACCCATCTGTGAAACAATGAAAACGCTTAGCAGGCTGCAGGTTGTTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 831533 . AAATCGTTCACAGTCTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 831606 . CAATTTTTTGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 831648 . TATATTTCCATAGGTTGTATAACTGAATTGATGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 831724 . TAATTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 831787 . GTTCAGTAACTAATTTCTCTAAAGCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 831848 . CCATATT . . . NS=1;AN=0 GT:PS ./.:.
+1 831902 . CAACGGCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 831938 . GCAGGAACTTTGTCTTTCCACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 832001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=834000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.77:1.04:.:0:.:0:0.999:406
+1 832060 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9697378;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:832060:VQLOW:32,.:0,.:0,.:-32,0,0:0,0,0:25:13,.:12
+1 832066 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 832089 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:832089:PASS:62,.:17,.:0,.:0:0
+1 832092 . TG . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:832089:PASS:80,.:35,.:0,.:0:0
+1 832109 . CCATAAATTTATAGGTACAGAAGTTATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 832178 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13302934;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:97:97,140:134,165:43,49:-97,0,-140:-43,0,-49:21:11,10:10
+1 832297 . CTG C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34017275;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832297:PASS:164:224,164:223,168:38,39:-224,0,-164:-38,0,-39:26:13,13:13
+1 832318 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4500250;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832297:PASS:130:237,130:237,131:52,45:-237,0,-130:-52,0,-45:25:14,11:11
+1 832359 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:832359:PASS:50,.:16,.:0,.:0:0
+1 832368 . GTAATTTGATTTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 832398 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:832359:PASS:50,.:16,.:0,.:0:0
+1 832603 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 832606 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 832619 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 832756 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28833197;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:186:186,213:211,211:45,48:-186,0,-213:-45,0,-48:30:12,18:18
+1 832918 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28765502;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:66:66,66:52,52:18,32:-66,0,-66:-18,0,-32:21:5,16:16
+1 832960 . AT A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs34260203&dbsnp.131|rs77120786&dbsnp.134|rs146934973;CGA_RPT=HERVK9-int|ERVK|6.4;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:77:77,465:77,496:38,40:-465,-77,0:-40,-38,0:33:30,30:3
+1 833172 . T TCGAA . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs57423329&dbsnp.132|rs113551494;CGA_RPT=HERVK9-int|ERVK|8.0;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:114:114,152:124,152:35,38:-114,0,-152:-35,0,-38:19:8,11:11
+1 833223 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303211;CGA_RPT=HERVK9-int|ERVK|8.0;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:195:195,195:191,191:43,46:-195,0,-195:-43,0,-46:32:13,19:19
+1 833302 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28752186&dbsnp.131|rs75132976;CGA_RPT=HERVK9-int|ERVK|8.0;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:162:162,162:161,161:46,48:-162,0,-162:-46,0,-48:27:12,15:15
+1 833641 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28594623;CGA_RPT=MER9a2|ERVK|8.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:330:330,396:331,395:53,54:-330,0,-396:-53,0,-54:32:14,18:18
+1 833659 . T A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28522102;CGA_RPT=MER9a2|ERVK|8.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:398:398,435:397,434:53,54:-398,0,-435:-53,0,-54:33:15,18:18
+1 833663 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28416910;CGA_RPT=MER9a2|ERVK|8.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:832918:PASS:303:471,303:472,302:53,53:-471,0,-303:-53,0,-53:33:18,15:15
+1 833824 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28484835;CGA_RPT=MER9a2|ERVK|8.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:833824:PASS:234:234,234:229,229:46,49:-234,0,-234:-46,0,-49:39:15,24:24
+1 833927 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28593608;CGA_RPT=MER9a2|ERVK|8.2;CGA_SDO=11 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:73:73,73:66,66:23,35:-73,0,-73:-23,0,-35:20:6,14:14
+1 833977 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 834001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=836000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:0.92:2:27:=:27:0.999:406
+1 834198 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28385272 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:833824:PASS:244:244,244:244,244:52,50:-244,0,-244:-52,0,-50:27:13,14:14
+1 834401 . CTTCTTTTTTTTTTTTTTTTTTTTTAGACGG . . . NS=1;AN=0 GT:PS ./.:.
+1 834832 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4411087 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:115:115,115:114,114:42,43:-115,0,-115:-42,0,-43:18:7,11:11
+1 834928 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4422949 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:335:335,335:331,331:46,54:-335,0,-335:-46,0,-54:48:19,29:29
+1 834999 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28570054 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:347:347,347:347,347:53,54:-347,0,-347:-53,0,-54:39:19,20:20
+1 835499 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4422948 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:287:287,287:286,286:52,52:-287,0,-287:-52,0,-52:39:17,22:22
+1 836001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=838000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:0.98:2:37:=:37:0.999:406
+1 836529 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28731045;CGA_RPT=AluJb|Alu|17.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:836529:PASS:191:191,191:191,191:49,50:-191,0,-191:-49,0,-50:28:13,15:15
+1 836896 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28705752;CGA_RPT=MER67A|ERV1|43.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:836529:PASS:264:264,648:265,650:33,54:-264,0,-648:-33,0,-54:50:17,33:33
+1 836924 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.130|rs72890788;CGA_RPT=MER67A|ERV1|43.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:836529:PASS:351:440,351:470,359:50,54:-440,0,-351:-50,0,-54:43:24,19:19
+1 838001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=840000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:0.99:2:43:=:43:0.999:406
+1 838150 . TCTCAAAAAAAAAAAAAAATCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 838329 . G GC . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:46:46,46:44,44:12,23:-46,0,-46:-12,0,-23:16:6,10:10
+1 838387 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970384;CGA_RPT=MLT1B|ERVL-MaLR|29.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:100:160,100:159,99:48,40:-160,0,-100:-48,0,-40:16:9,7:7
+1 838555 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970383;CGA_RPT=MLT1B|ERVL-MaLR|29.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:447:473,447:473,447:50,54:-473,0,-447:-50,0,-54:47:24,23:23
+1 838931 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7523690;CGA_RPT=MLT1A|ERVL-MaLR|29.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:196:196,196:196,196:49,50:-196,0,-196:-49,0,-50:26:12,14:14
+1 839103 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28562941;CGA_RPT=MER2|TcMar-Tigger|17.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:289:441,289:439,287:53,52:-441,0,-289:-53,0,-52:37:21,16:16
+1 839301 . CAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 839670 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 839676 . AGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 839688 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 839695 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 839853 . GCCGCCGCCTCCTCCGAACGCGGCCGCCTCCTCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 839894 . GTGGCCTCCTCCGAACGCGGCCGCCTCCTCCTCCGAACGCGGCCGCCTCCTCCTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 839958 . CCT . . . NS=1;AN=0 GT:PS ./.:.
+1 839980 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 839982 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 839985 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 840001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=842000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.69:0.95:2:48:=:48:0.999:406
+1 840006 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 840017 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 840020 . TCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 840027 . AAC . . . NS=1;AN=0 GT:PS ./.:.
+1 840036 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.135|rs186698170;CGA_RPT=(CCG)n|Simple_repeat|30.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:840036:VQLOW:29:29,33:22,26:1,25:-29,0,-33:-1,0,-25:4:2,2:2
+1 840056 . CCG . . . NS=1;AN=0 GT:PS ./.:.
+1 840061 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 840114 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 840123 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 840134 . GCG . . . NS=1;AN=0 GT:PS ./.:.
+1 840139 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 840153 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 840165 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 840180 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 840197 . ACAAAGAGCCGCGCGGCCACGACGGCCGCGTGCCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 840303 . CGGTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 840327 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28625089 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:840327:PASS:43:45,43:44,43:18,29:-45,0,-43:-18,0,-29:4:2,2:2
+1 840331 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 840336 . AGGCGGGTGAGGGGAGGGGGCCGGAGGGTCGGGGGTGCCGGGGGGTGCGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 840409 . GGGACGTTCGTGGCGGGGGAGGCTGTTGGGGACGTTCGTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 840472 . TCGTGGCGGGGGAGGCTGTTGGGTCCCCTCCCCGCCCCACCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 840561 . TCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 840576 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 840577 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 840580 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 840587 . GCCTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 840753 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970382;CGA_RPT=L1MB5|L1|41.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:44:44,349:44,349:40,52:-349,-44,0:-52,-40,0:21:21,21:0
+1 841085 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.88|rs1574243;CGA_RPT=L1MB5|L1|26.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:337:348,337:348,337:53,54:-348,0,-337:-53,0,-54:35:18,17:17
+1 841666 . TTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 841678 . C T . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:841678:VQLOW:24:24,24:5,5:0,13:-24,0,-24:0,0,-13:13:1,12:12
+1 842001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=844000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:1.03:2:49:=:49:0.999:406
+1 842013 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7419119;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:124:124,124:124,124:43,44:-124,0,-124:-43,0,-44:14:7,7:7
+1 842057 . A AAACTCAGCTGCCTCTCCCCTTC . . NS=1;AN=2;AC=1;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:38:38,101:37,95:7,33:-38,0,-101:-7,0,-33:18:13,5:5
+1 842291 . ACCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 842359 . ACCCGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 842616 . TA T . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs55764002&dbsnp.132|rs111389427;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz6|Alu|20.0 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:20,.:20,.:1,.:-20,0,0:-1,0,0:11:6,.:5
+1 842825 . AA GA . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs7519340;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz|Alu|13.3 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:283,.:283,.:48,.:-283,0,0:-48,0,0:15:15,.:0
+1 842919 . AA . . . NS=1;AN=0 GT:PS ./.:.
+1 843215 . C CCTGCCCGGTCCTTCTGACCAGCCGAGAGTA . . NS=1;AN=2;AC=1;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:843215:PASS:129:129,336:126,328:35,40:-129,0,-336:-35,0,-40:16:7,9:9
+1 843249 . AGACCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 843405 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11516185;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:148:148,148:141,141:44,46:-148,0,-148:-44,0,-46:28:9,19:19
+1 843558 . GGGCAGCACA . . . NS=1;AN=0 GT:PS ./.:.
+1 843579 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 843595 . GT . . . NS=1;AN=0 GT:PS ./.:.
+1 843607 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 843639 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 843642 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 843676 . AGGCTGGTGGGGGCAGCAGCTGGAAGGGGAAGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 843913 . G <CGA_NOCALL> . . END=844215;NS=1;AN=0 GT:PS ./.:.
+1 844001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=846000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.76:0.84:.:0:.:0:0.999:406
+1 844300 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61769713;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:53:112,53:111,51:41,31:-112,0,-53:-41,0,-31:8:5,3:3
+1 844356 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 844383 . CGGGGGTCGGGGTCAGGCCCCCGGGCGCACCGTTGCTGGTATATGCGGGGGTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 844443 . GGCCCCCGGGCGCACCGTTGCTGGTATATGCGGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 844516 . ATGCGGTGGTCGGGGTCAGGCCCCCGGGCGCACCTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 844680 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 844691 . ATGCGGTGGTCGGGGTCAGGCCCCCGGGCGCACCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 844736 . ATGCGGGGGTCGGGGTCAGGCCCCCGGGCGCACCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 844781 . ATGCGGTGGTCGGGGTCAGGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 844810 . GCATCTTTGCTGGTATATGCGGTGGTCGGGGTCAGGCCCCCCGGGCGCACTGTTGCTGGTATATGCGGTGGTCGGGGTCAGGCCCCCGGGCGCACCTTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 844924 . GGTCGGGGTCAGGCCCCCGGGCGCACCGTTGCTGGTATATGCGGGGGTCGGGGTCAGGCCCCCGGGCGTTGCTGGTATATGCGGTGGTCGGGGTCAGGCCCCCGGGCGCACCGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 845047 . ATGCGGGGGTCGGGGTCAGGCCCCCGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 845092 . ATGCGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 845108 . CAGGCCCCCGGGCGCACCGTTGCTGGTATATGCGGGGGTCGGGGTCAGGCCCCCGGGCGTTGCTGGTATATGCGGGGGTCGGGGTCAGGCCCCCGGGCGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 845273 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs117039017;CGA_FI=284600|XR_108282.1|LOC284600|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:37:65,37:65,37:28,27:-65,0,-37:-28,0,-27:16:7,9:9
+1 845337 . GGCGCGGGTGGGGGTCCTGGGCAGGGGCGGCGGCGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 845398 . CCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 845589 . GGACGCCCTGGCGCCTCTGCTGCCCACGGCGGCCCCGAGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 845654 . CATCTCCTGTCAGCCGCAGTGACTGCGGGTGCCTGACGGCGCCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 845708 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 845711 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 846001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=848000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:1.08:2:48:=:48:0.999:406
+1 847220 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 847223 . ACAATCGTGTTTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 847250 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7416129;CGA_FI=284600|XR_108282.1|LOC284600|INTRON|UNKNOWN-INC;CGA_RPT=MIRb|MIR|38.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:847250:PASS:117:880,117:775,12:53,20:-880,-117,0:-53,-20,0:37:37,37:0
+1 848001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=850000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:1.10:2:48:=:48:0.999:406
+1 848828 . GA G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3841265&dbsnp.134|rs150504890;CGA_FI=284600|XR_108282.1|LOC284600|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:333:333,339:363,337:38,42:-333,0,-339:-38,0,-42:41:23,18:18
+1 849440 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79376265;CGA_FI=284600|XR_108282.1|LOC284600|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:148:336,148:331,143:52,47:-336,0,-148:-52,0,-47:27:17,10:10
+1 849998 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303222;CGA_FI=284600|XR_108282.1|LOC284600|UTR|UNKNOWN-INC;CGA_RPT=L1MEc|L1|46.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:95:95,95:84,84:29,38:-95,0,-95:-29,0,-38:23:6,17:17
+1 850001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=852000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:1.10:2:48:=:48:0.999:406
+1 850328 . CA CAA . . NS=1;AN=1;AC=1;CGA_FI=284600|XR_108282.1|LOC284600|UTR|UNKNOWN-INC;CGA_RPT=AluSp|Alu|10.6 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:850328:VQLOW:26,.:0,.:0,.:-26,0,0:0,0,0:12:4,.:4
+1 850352 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:850328:VQLOW:26,.:0,.:0,.:0:0
+1 850528 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs114889920;CGA_FI=284600|XR_108282.1|LOC284600|UTR|UNKNOWN-INC;CGA_RPT=L1MEc|L1|46.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:850528:PASS:113:113,113:111,111:38,42:-113,0,-113:-38,0,-42:21:8,13:13
+1 850542 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 850780 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6657440 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:333:333,333:331,331:46,54:-333,0,-333:-46,0,-54:43:18,25:25
+1 851030 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79709025 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:323:356,323:354,322:48,54:-356,0,-323:-48,0,-54:43:24,19:19
+1 851499 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970465 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:219:332,219:331,219:52,50:-332,0,-219:-52,0,-50:29:15,14:14
+1 851757 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970464&dbsnp.129|rs62677860 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:30:312,30:312,30:48,29:-312,-30,0:-48,-29,0:16:16,16:0
+1 852001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=854000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:0.92:2:49:=:49:0.999:406
+1 852016 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 852020 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 852964 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970461;CGA_FI=100130417|NR_026874.1|FLJ39609|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:368:383,368:383,368:53,54:-383,0,-368:-53,0,-54:39:20,19:19
+1 853488 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77644389;CGA_FI=100130417|NR_026874.1|FLJ39609|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:267:267,267:265,265:50,51:-267,0,-267:-50,0,-51:32:13,19:19
+1 854001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=856000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.10:0.97:2:44:=:44:0.999:406
+1 854168 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79188446;CGA_FI=100130417|NR_026874.1|FLJ39609|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:226:226,226:226,226:46,49:-226,0,-226:-46,0,-49:31:16,15:15
+1 854777 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303019;CGA_FI=100130417|NR_026874.1|FLJ39609|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:854777:PASS:196:196,196:195,195:49,50:-196,0,-196:-49,0,-50:27:13,14:14
+1 855075 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6673914;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:854777:PASS:285:285,285:285,285:51,52:-285,0,-285:-51,0,-52:34:15,19:19
+1 855168 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs78128413;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:99:99,99:98,98:39,40:-99,0,-99:-39,0,-40:15:6,9:9
+1 855635 . CCTT C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77712898;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:855635:PASS:174:174,272:189,256:30,39:-174,0,-272:-30,0,-39:46:14,32:32
+1 855774 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77595185;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:855635:PASS:171:298,171:296,169:52,49:-298,0,-171:-52,0,-49:26:15,11:11
+1 855878 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs117282503;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:855635:PASS:207:207,207:204,204:44,47:-207,0,-207:-44,0,-47:35:14,21:21
+1 856001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=858000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.28:0.98:2:37:=:37:0.999:406
+1 856041 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61769717;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:163:260,163:257,161:52,48:-260,0,-163:-52,0,-48:27:16,11:11
+1 856099 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28534711;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:855635:PASS:109:193,109:183,104:50,36:-109,0,-193:-36,0,-50:28:21,7:21
+1 856108 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28742275;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:855635:PASS:129:193,129:183,138:50,43:-129,0,-193:-43,0,-50:22:15,7:15
+1 856329 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7414599;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:284:355,284:354,282:53,52:-355,0,-284:-53,0,-52:35:20,15:15
+1 856476 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4040605;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:310,36:310,36:48,34:-310,-36,0:-48,-34,0:17:16,16:1
+1 856628 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 857177 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28409649;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:857177:PASS:454:454,454:454,454:47,54:-454,0,-454:-47,0,-54:54:26,28:28
+1 857318 . G GGAGT . . NS=1;AN=2;AC=1;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:270:270,351:287,345:31,41:-270,0,-351:-31,0,-41:52:21,31:31
+1 857443 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs56663360;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:857177:PASS:371:639,371:635,366:49,54:-639,0,-371:-49,0,-54:57:34,23:23
+1 857728 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6689107;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:59:542,59:542,59:52,46:-542,-59,0:-52,-46,0:27:27,27:0
+1 858001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=860000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.36:2.02:2:25:=:25:0.999:406
+1 858149 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77520260;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:389:472,389:472,389:50,54:-472,0,-389:-50,0,-54:49:24,25:25
+1 858691 . TG T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34628185&dbsnp.134|rs149702675;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:228:322,228:320,227:40,39:-322,0,-228:-40,0,-39:34:19,15:15
+1 858801 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7418179;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=MIR|MIR|30.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:56:56,56:40,40:12,28:-56,0,-56:-12,0,-28:25:6,19:19
+1 859222 . CCGTCACGCACCCCCCGCGGGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 859332 . GCGGGCGCGCGCCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 859353 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 859359 . CACGACTGA . . . NS=1;AN=0 GT:PS ./.:.
+1 859404 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.130|rs71509444;CGA_FI=100130417|NR_026874.1|FLJ39609|TSS-UPSTREAM|UNKNOWN-INC&148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:859404:VQLOW:36:74,36:73,35:31,27:-74,0,-36:-31,0,-27:9:5,4:4
+1 859436 . GCCGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 859465 . GAGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 859473 . CGTGGGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 859490 . CGCGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 859506 . CGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 859520 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 859620 . CGGCCGGCTGGGCAGTCCGGGGAGGCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 859656 . GCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 859664 . GCGGCGGCTGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 859689 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 859701 . CGT . . . NS=1;AN=0 GT:PS ./.:.
+1 859709 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:859709:VQLOW:21,.:17,.:0,.:0:0
+1 859713 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 859716 . AGGCGCCTCCCCGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 859820 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 859894 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 859908 . CTGCCA CTGCCG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.132|rs112703963;CGA_FI=148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=(CCG)n|Simple_repeat|27.9 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:859908:PASS:61,.:61,.:27,.:-61,0,0:-27,0,0:6:6,.:0
+1 859967 . GCCCGCCTCGGCCGCCGGTTACGAGGCTCTGCTGGCCCCGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 860001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=862000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.05:2:36:=:36:0.999:406
+1 860100 . GCCACCGCGGCCGCGGCCCCGGATTTCCAGCCGCTGCTGGACAACGGCGAGCCGTGCATCGAGGTGGAGTGCGGCGCCAACCGCGCGCTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 860254 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 860280 . AGC . . . NS=1;AN=0 GT:PS ./.:.
+1 860288 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 860296 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 860302 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 860307 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 860310 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 860363 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 860366 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 860371 . GACC . . . NS=1;AN=0 GT:PS ./.:.
+1 860383 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:860383:VQLOW:22,.:21,.:0,.:0:0
+1 860391 . TCGGACACCCGGGAGCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 860413 . CTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 860428 . GCTGCAGCTCCAGGGCTGCGCGGGGACACCCCCGCCGCGCGCGGAGGCCTCGGTGAACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 860504 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs78033073;CGA_FI=148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:860504:VQLOW:30:30,64:32,64:10,35:-30,0,-64:-10,0,-35:7:2,5:5
+1 860521 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs57924093;CGA_FI=148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:860504:PASS:49:49,60:49,60:31,27:-60,0,-49:-27,0,-31:8:3,5:3
+1 860583 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 860598 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 860601 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 860655 . CGGAGCCCCGGGTTCGGGGGAGACTGGAGGGGCGCACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 860730 . GGGCGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 860761 . CGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 860778 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:860778:VQLOW:28,.:23,.:0,.:0:0
+1 860789 . TGGCGCCTGCGGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 860810 . CCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 860854 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs57816555;CGA_FI=148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:36,146:30,139:29,45:-146,-36,0:-45,-29,0:11:11,11:0
+1 860862 . GCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 860929 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 860932 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 861008 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28521172;CGA_FI=148398|NM_152486.2|SAMD11|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:21:189,21:189,21:48,22:-189,-21,0:-48,-22,0:8:8,8:0
+1 861115 . AGCCCAGCAGATCCCTGCGGCGTTCGCGAGGGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 861630 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.101|rs2879816;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:135:279,135:277,132:52,45:-279,0,-135:-52,0,-45:21:13,8:8
+1 861808 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13302982;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:861808:PASS:345:345,345:345,345:47,54:-345,0,-345:-47,0,-54:41:20,21:21
+1 862001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=864000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.14:0.92:2:44:=:44:0.999:406
+1 862093 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303291;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:861808:PASS:239:319,239:321,240:53,50:-319,0,-239:-53,0,-50:35:19,16:16
+1 862124 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303101;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:861808:PASS:393:393,530:393,531:50,54:-393,0,-530:-50,0,-54:46:20,26:26
+1 862186 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74442310;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:194:194,194:194,194:49,50:-194,0,-194:-49,0,-50:21:11,10:10
+1 862383 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6680268;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:861808:PASS:415:422,415:421,419:53,54:-422,0,-415:-53,0,-54:38:19,19:19
+1 862389 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6693546;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:861808:PASS:330:441,330:440,331:53,54:-441,0,-330:-53,0,-54:36:19,17:17
+1 862866 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs3892970;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:260:260,260:259,259:49,51:-260,0,-260:-49,0,-51:33:15,18:18
+1 863124 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4040604;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:200:200,200:197,197:43,47:-200,0,-200:-43,0,-47:33:12,21:21
+1 863499 . TGT CGC . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28649395&dbsnp.125|rs28718350;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:367:456,367:456,366:33,35:-456,0,-367:-33,0,-35:41:23,18:18
+1 863508 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7410984;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:165:165,780:181,783:52,50:-780,-165,0:-52,-50,0:40:40,40:0
+1 863511 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28626846;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:117:117,576:124,577:48,52:-576,-117,0:-52,-48,0:29:29,29:0
+1 863556 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7410998;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:274:274,274:272,272:50,52:-274,0,-274:-50,0,-52:36:15,21:21
+1 863632 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28403979;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:95:281,95:278,93:52,39:-281,0,-95:-52,0,-39:24:14,10:10
+1 863641 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28569249;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:61:61,476:95,477:48,52:-476,-61,0:-52,-48,0:24:23,23:0
+1 863644 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28739566;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:74,465:98,462:48,52:-465,-74,0:-52,-48,0:22:22,22:0
+1 863689 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7417994;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:863689:PASS:250:250,251:254,251:52,50:-250,0,-251:-52,0,-50:20:10,10:10
+1 863696 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35717056;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:863689:PASS:181:181,289:181,289:50,52:-289,0,-181:-52,0,-50:23:9,14:9
+1 863776 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35485427;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|6.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:323:349,323:348,322:53,54:-349,0,-323:-53,0,-54:35:20,15:15
+1 863843 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35599603;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:863843:PASS:248:356,248:355,253:53,51:-356,0,-248:-53,0,-51:32:18,14:14
+1 863863 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35854196;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:863843:PASS:168:323,168:322,169:52,49:-323,0,-168:-52,0,-49:24:15,9:9
+1 863978 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.130|rs74047403;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:863978:PASS:102:268,102:265,98:48,40:-268,0,-102:-48,0,-40:19:12,7:7
+1 863999 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 864001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=866000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.16:0.88:2:50:=:50:0.999:406
+1 864278 . G A . . NS=1;AN=2;AC=1;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:240:436,240:434,238:50,50:-436,0,-240:-50,0,-50:41:23,18:18
+1 864726 . T A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340590;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:135:1145,135:1145,135:49,52:-1145,-135,0:-52,-49,0:50:50,50:0
+1 864755 . AGG GGA . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340588&dbsnp.100|rs2340589;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:109:109,1078:109,1078:37,33:-1078,-109,0:-37,-33,0:49:49,49:0
+1 864938 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2340587&dbsnp.130|rs78370858;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:864938:PASS:435:435,435:435,435:46,54:-435,0,-435:-46,0,-54:52:26,26:26
+1 865367 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75294478;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:864938:PASS:269:269,269:267,267:51,50:-269,0,-269:-50,0,-51:34:20,14:20
+1 865663 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 865694 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9988179;CGA_FI=148398|NM_152486.2|SAMD11|CDS|MISSENSE GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:865694:PASS:214:214,214:212,212:51,50:-214,0,-214:-51,0,-50:29:12,17:17
+1 865948 . GCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 866001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=868000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:0.90:2:52:=:52:0.999:406
+1 866319 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9988021;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:37:240,37:240,37:48,35:-240,-37,0:-48,-35,0:14:14,14:0
+1 866511 . C CCCCT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs60722469;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:66:66,101:63,98:25,33:-66,0,-101:-25,0,-33:18:11,7:7
+1 866920 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341361;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:72,632:72,632:48,52:-632,-72,0:-52,-48,0:29:29,29:0
+1 867584 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341360;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:132:132,1154:132,1154:52,50:-1154,-132,0:-52,-50,0:47:47,47:0
+1 867993 . GTTTC G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.114|rs5772025&dbsnp.134|rs138211850;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:101:101,764:101,787:39,38:-764,-101,0:-39,-38,0:42:42,42:0
+1 868001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=870000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.71:0.81:2:50:=:50:0.999:406
+1 868329 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341359;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:75:75,658:75,658:41,53:-658,-75,0:-53,-41,0:31:31,31:0
+1 868404 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302914;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=C-rich|Low_complexity|28.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:94:94,753:94,753:48,53:-753,-94,0:-53,-48,0:33:33,33:0
+1 868791 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303003;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:35:35,374:35,374:33,48:-374,-35,0:-48,-33,0:16:16,16:0
+1 868840 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28532704;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:166:185,166:185,166:48,49:-185,0,-166:-48,0,-49:19:9,10:10
+1 868891 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303066;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:71:613,71:613,71:53,40:-613,-71,0:-53,-40,0:30:30,30:0
+1 868928 . A AG . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs56367715&dbsnp.131|rs74724223;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:60:60,135:85,123:29,36:-60,0,-135:-29,0,-36:29:8,21:21
+1 868981 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303037;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:120:120,120:117,117:42,43:-120,0,-120:-42,0,-43:17:6,11:11
+1 869121 . T TG . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs56190854&dbsnp.131|rs75384481&dbsnp.132|rs111626358;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:40:40,70:51,69:18,29:-40,0,-70:-18,0,-29:14:5,9:9
+1 869244 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 869323 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303207;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:12:103,12:103,12:40,16:-103,-12,0:-40,-16,0:8:8,8:0
+1 869354 . G <CGA_NOCALL> . . END=870152;NS=1;AN=0 GT:PS ./.:.
+1 870001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=872000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.77:2:50:=:50:0.999:406
+1 870166 . GTGGGCAGGGGAGGCGGCTGCGTTACAGGTGGGCGGGGGAGGCGGCTCCGTTACAGGTGGGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 870233 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 870248 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 870252 . GGGCGGGG GTGCAGGG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.125|rs28491190&dbsnp.135|rs186226871;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:870252:PASS:137,.:105,.:23,.:-137,0,0:-23,0,0:18:14,.:4
+1 870269 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 870284 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.125|rs28621383;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:870252:PASS:83,.:85,.:35,.:-83,0,0:-35,0,0:11:7,.:4
+1 870290 . GGCGGCTGCGTTACAGGTGGGCGGGGGGGG GG . . NS=1;AN=1;AC=1;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:870252:PASS:60,.:28,.:6,.:-60,0,0:-6,0,0:13:6,.:4
+1 870331 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 870340 . GGGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 870360 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 870373 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 870388 . GCCCCTTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 870806 . C CGGAGCTCCTCT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs59561572&dbsnp.132|rs113002845;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:226:226,373:222,369:33,41:-226,0,-373:-33,0,-41:40:25,15:15
+1 870903 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303094;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:93:753,93:753,93:53,47:-753,-93,0:-53,-47,0:31:31,31:0
+1 871098 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 871215 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28419423;CGA_FI=148398|NM_152486.2|SAMD11|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:143:143,181:155,178:45,49:-143,0,-181:-45,0,-49:26:10,16:16
+1 871280 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 871334 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4072383;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:103:265,103:261,99:48,40:-265,0,-103:-48,0,-40:19:12,7:7
+1 871683 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4504834;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:16:41,16:41,16:17,18:-41,-16,0:-18,-17,0:2:1,1:1
+1 871958 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 871966 . G A . . NS=1;AN=2;AC=1;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:871966:VQLOW:27:27,27:26,26:5,25:-27,0,-27:-5,0,-25:6:2,4:4
+1 872001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=874000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.00:1.02:2:52:=:52:0.999:406
+1 872087 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs78308511;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:872087:PASS:205:205,263:210,262:51,50:-205,0,-263:-51,0,-50:29:12,17:17
+1 872091 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77614634;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:872087:PASS:206:206,263:213,262:45,51:-206,0,-263:-45,0,-51:30:13,17:17
+1 872352 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.92|rs1806780;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:872087:PASS:114:114,114:103,103:35,41:-114,0,-114:-35,0,-41:28:8,20:20
+1 872776 . GCCC G,GC . . NS=1;AN=2;AC=1,1;CGA_XR=.,dbsnp.114|rs5772023&dbsnp.126|rs34936020;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC,148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/2:.:PASS:64:64,64:63,64:19,13:-64,-64,-64,-64,0,-64:-19,-13,-13,-19,0,-19:12:6,5:0
+1 872817 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 872964 . GCCTTGGCCCACCCCCTCCCAGCCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 873394 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.86|rs1110051;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:150:150,150:146,146:44,47:-150,0,-150:-44,0,-47:27:10,17:17
+1 873558 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.86|rs1110052;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:873558:PASS:389:389,389:387,387:43,54:-389,0,-389:-43,0,-54:51:22,29:29
+1 874001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=876000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.15:1.04:2:52:=:52:0.999:406
+1 874073 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28450942;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:873558:PASS:408:408,419:408,419:54,50:-419,0,-408:-50,0,-54:47:24,23:24
+1 874659 . GCT . . . NS=1;AN=0 GT:PS ./.:.
+1 874925 . ACA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:874925:PASS:206,.:202,.:0,.:0:0
+1 874950 . T TCCCTGGAGGACC . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.114|rs6143081&dbsnp.131|rs79212057;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:874925:PASS:112:206,112:202,124:40,35:-112,0,-206:-35,0,-40:22:12,10:12
+1 875605 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 875612 . GCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 875623 . GGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 875667 . CGCGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 875676 . GCC G . . NS=1;AN=1;AC=1;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC;CGA_RPT=GC_rich|Low_complexity|2.2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:875676:PASS:49,.:48,.:14,.:-49,0,0:-14,0,0:10:3,.:0
+1 875741 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 876001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=878000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.70:2:48:=:48:0.999:406
+1 876402 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 876499 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4372192;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:29:237,29:237,29:52,28:-237,-29,0:-52,-28,0:20:20,20:0
+1 876949 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.130|rs72902600;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:69:69,127:70,127:24,45:-69,0,-127:-24,0,-45:20:10,10:10
+1 877003 . G T . . NS=1;AN=2;AC=1;CGA_FI=148398|NM_152486.2|SAMD11|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:88:88,88:87,87:36,38:-88,0,-88:-36,0,-38:18:8,10:10
+1 877176 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 877235 . CGCTCGGGTCCGCAGGGGAGGGGAGCAGGCGGGGCCGGCGCCCCGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 877334 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 877337 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 877353 . GCC . . . NS=1;AN=0 GT:PS ./.:.
+1 877358 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 877361 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 877375 . GCGCGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 877472 . AACGGGGGCGGGGGGGACGCCGCTCATTGCGCTGCCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 877522 . G <CGA_NOCALL> . . END=877723;NS=1;AN=0 GT:PS ./.:.
+1 877746 . CCGCCTCGGACCCCCCGACCCCGCGTTGTCCCCCTCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 877800 . C <CGA_NOCALL> . . END=878115;NS=1;AN=0 GT:PS ./.:.
+1 878001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=880000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.77:0.85:2:50:=:50:0.999:406
+1 878121 . CGGGCTCCGGACCCCCCACCCCGTCCCGGGACTCTGCCCGGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 878302 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 878326 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 878361 . CCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 878369 . AAGGGGCTTTTCCCAGGGTCCACACTGCCCCTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 878440 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 878445 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 878903 . CACCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 879013 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 879118 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 879317 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7523549;CGA_FI=148398|NM_152486.2|SAMD11|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:104:104,104:100,100:35,40:-104,0,-104:-35,0,-40:20:7,13:13
+1 879676 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605067;CGA_FI=148398|NM_152486.2|SAMD11|UTR3|UNKNOWN-INC&26155|NM_015658.3|NOC2L|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:126:126,803:126,803:52,49:-803,-126,0:-52,-49,0:52:51,51:1
+1 879687 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.36|rs2839;CGA_FI=148398|NM_152486.2|SAMD11|UTR3|UNKNOWN-INC&26155|NM_015658.3|NOC2L|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:78:653,78:653,78:53,42:-653,-78,0:-53,-42,0:39:39,39:0
+1 879776 . CTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 879786 . CCC . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:879786:PASS:41,.:14,.:0,.:0:0
+1 879792 . GGAA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:879786:PASS:41,.:14,.:0,.:0:0
+1 879817 . TGACG . . . NS=1;AN=0 GT:PS ./.:.
+1 879824 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 879830 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 879840 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 879858 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 879863 . AGCTGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 879874 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:879874:PASS:92,.:91,.:0,.:0:0
+1 879878 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 879881 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 880001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=882000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.10:1.15:2:50:=:50:0.999:406
+1 880128 . CAGCTGCTGCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 880238 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3748592;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:111:1006,111:1006,111:53,52:-1006,-111,0:-53,-52,0:39:39,39:0
+1 880390 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3748593;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:233:332,233:331,231:53,49:-332,0,-233:-53,0,-49:31:18,13:13
+1 881377 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs74360597;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:390:452,390:452,390:50,54:-452,0,-390:-50,0,-54:49:25,24:24
+1 882001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=884000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.97:2:51:=:51:0.999:406
+1 882275 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 882280 . TAC . . . NS=1;AN=0 GT:PS ./.:.
+1 882803 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340582;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:94:94,1002:94,1002:48,50:-1002,-94,0:-50,-48,0:45:45,45:0
+1 883625 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970378;CGA_FI=26155|NM_015658.3|NOC2L|ACCEPTOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:63:427,63:427,63:52,48:-427,-63,0:-52,-48,0:25:25,25:0
+1 883948 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 883961 . TTGAAGTCGACCTGCTGGAACATCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 884001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=886000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:1.10:2:50:=:50:0.999:406
+1 884038 . TCACCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 884091 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 884426 . A AACAGCAAAG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.114|rs5772022;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:36,379:36,379:25,39:-379,-36,0:-39,-25,0:33:33,33:0
+1 884815 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4246503;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:76:793,76:793,76:53,41:-793,-76,0:-53,-41,0:36:36,36:0
+1 885654 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 885657 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs116767636;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|38.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:885657:PASS:80:230,80:183,34:48,27:-230,0,-80:-48,0,-27:20:14,6:6
+1 885676 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970377;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|38.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:885676:PASS:46:46,561:75,517:48,52:-561,-46,0:-52,-48,0:28:28,28:0
+1 885689 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970452;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|38.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:885676:PASS:45:45,697:79,651:48,52:-697,-45,0:-52,-48,0:29:29,29:0
+1 885699 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970376;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|38.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:885676:VQLOW:17:17,577:17,531:19,52:-577,-17,0:-52,-19,0:26:26,26:0
+1 885704 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 885945 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28535998;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|34.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:127:127,127:126,126:41,44:-127,0,-127:-41,0,-44:20:9,11:11
+1 886001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=888000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:0.96:2:52:=:52:0.999:406
+1 886006 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970375;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|34.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:26:182,26:182,26:48,26:-182,-26,0:-48,-26,0:11:11,11:0
+1 886049 . ACAG A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs146799731;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC;CGA_RPT=(CA)n|Simple_repeat|34.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:885657:PASS:387:387,435:387,435:42,40:-435,0,-387:-40,0,-42:39:18,21:18
+1 886180 . ACTGTTC CTTTCAG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs34767445&dbsnp.130|rs71490527;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:83:723,83:723,83:32,35:-723,-83,0:-35,-32,0:53:53,53:0
+1 886309 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 886384 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3748594;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:885657:PASS:69:69,69:54,54:32,19:-69,0,-69:-19,0,-32:23:17,6:17
+1 886654 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 886786 . CAGGTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 886802 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 886817 . C CATTTT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs111748052;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:886817:PASS:166:166,180:175,175:39,40:-166,0,-180:-39,0,-40:23:8,15:15
+1 886982 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 887059 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs76456117;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:299:395,299:393,297:53,53:-395,0,-299:-53,0,-53:38:22,16:16
+1 887560 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3748595;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:76:778,76:778,76:53,41:-778,-76,0:-53,-41,0:38:38,38:0
+1 887801 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3828047;CGA_FI=26155|NM_015658.3|NOC2L|CDS|NO-CHANGE;CGA_PFAM=PFAM|PF03715|Noc2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:69:69,583:69,583:39,53:-583,-69,0:-53,-39,0:31:31,31:0
+1 888001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=890000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.10:0.92:2:53:=:53:0.999:406
+1 888220 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28711536;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:327:327,327:326,326:46,54:-327,0,-327:-46,0,-54:41:19,22:22
+1 888639 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3748596;CGA_FI=26155|NM_015658.3|NOC2L|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:92:689,92:689,92:50,47:-689,-92,0:-50,-47,0:42:42,42:0
+1 888659 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3748597;CGA_FI=26155|NM_015658.3|NOC2L|CDS|NO-CHANGE GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:73:632,73:632,73:52,48:-632,-73,0:-52,-48,0:27:27,27:0
+1 888740 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 888743 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 888747 . GCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 889131 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3828048;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:218:218,285:218,285:52,50:-218,0,-285:-52,0,-50:26:13,13:13
+1 889158 . GA CC . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302945&dbsnp.121|rs13303056&dbsnp.129|rs56262069;CGA_FI=26155|NM_015658.3|NOC2L|DONOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:96:789,96:789,124:35,37:-789,-96,0:-37,-35,0:34:34,34:0
+1 889638 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303206;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:113:1027,113:1027,113:50,52:-1027,-113,0:-52,-50,0:42:42,42:0
+1 889713 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303051;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:84,855:84,855:44,53:-855,-84,0:-53,-44,0:37:37,37:0
+1 890001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=892000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.94:2:53:=:53:0.999:406
+1 890295 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6661531;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:242:412,242:410,240:53,50:-412,0,-242:-53,0,-50:36:21,15:15
+1 890447 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 891021 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13302957;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:301:301,301:298,298:45,53:-301,0,-301:-45,0,-53:40:16,24:24
+1 892001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=894000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.09:0.97:2:52:=:52:0.999:406
+1 892460 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.127|rs41285802;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:295:295,295:295,295:53,53:-295,0,-295:-53,0,-53:34:17,17:17
+1 892745 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303227;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:112:1023,112:1023,112:50,52:-1023,-112,0:-52,-50,0:44:44,44:0
+1 893280 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970371;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L2a|L2|63.0;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:74,739:74,739:41,53:-739,-74,0:-53,-41,0:31:30,30:1
+1 893461 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 893615 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75254714;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx|Alu|9.3;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:511:511,511:511,511:49,54:-511,0,-511:-49,0,-54:54:27,26:26
+1 893631 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605069;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx|Alu|11.8;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:249:1359,249:1359,300:49,52:-1359,-249,0:-52,-49,0:54:54,54:0
+1 893719 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970445;CGA_FI=26155|NM_015658.3|NOC2L|INTRON|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx|Alu|11.8;CGA_SDO=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:174:351,174:347,170:52,49:-351,0,-174:-52,0,-49:29:18,11:11
+1 894001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=896000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:0.93:2:51:=:51:0.999:406
+1 894673 . T C . . NS=1;AN=2;AC=1;CGA_FI=26155|NM_015658.3|NOC2L|UTR5|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:117:119,117:118,117:43,43:-119,0,-117:-43,0,-43:18:10,8:8
+1 894890 . A AAGAC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.130|rs70949531&dbsnp.132|rs111643007;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|TSS-UPSTREAM|UNKNOWN-INC;CGA_SDO=2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:134,.:134,.:35,.:-134,0,0:-35,0,0:11:11,.:0
+1 895115 . GATCCGCGGAGACCGGGCCAGCGCCACGAACACCACGCAGGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 895181 . GTGCCGACCGCGGCTCTTCCCGGGGACGCCGCACGGGACGAAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 895293 . AGACCCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 895324 . GGCACGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 895427 . GCTCCCCGGAGGAGAGCAAGTTAGGGGGTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 895507 . CTCCTCGGAGGAGGAAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 895545 . CCTTGGAGGAGGAGGAGGGCGAGGCTTAGGAGGGCTCTTCGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 895610 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 895613 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 895616 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 895619 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 895622 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 895629 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 895641 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 895652 . CTTCCCGGAGGAGGAGGAGGAGGAGGGCTAGGCCGGGGGCTTCCCAGAGGAGGAGGATGGCGGGGCCTGGGGGGCTTCTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 895767 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 895845 . GGAGGAGGCGGACCCGGGGCGCAGCGCTGGAAGAATCCGCGTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 895895 . TAGTCCCCGAAGCCTCTCGGGAGGCGGGGCGGGCGGCGCCGAGAAACA . . . NS=1;AN=0 GT:PS ./.:.
+1 895972 . GCGACACAGAGCGGGCCGCCACCGCCGAGCAGCCCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 896001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=898000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:1.34:2:45:=:45:0.999:406
+1 896041 . CCTCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 896061 . GTCCGGCAGCCGAATGCAGCCCCGCAGCGAGCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 896103 . CAGGACGCAGAGCCCGGAGCACGGCAGCCCGGGGCCCGGGCCCGAGGCGCCGCCGCCTCCACCGCCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 896190 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 896202 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 896225 . GCGCGGCCCCCGCCCTCGCGTCCGCTCGCAGAAGGGGCGGGGGCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 896333 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs144174542;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:896333:PASS:49:93,49:92,49:37,31:-93,0,-49:-37,0,-31:11:6,5:5
+1 896339 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 896348 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 896350 . CGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 896426 . GGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 896476 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28393498;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:21:208,21:208,21:48,22:-208,-21,0:-48,-22,0:17:17,17:0
+1 896554 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 896563 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 896610 . AGCGGCTCCAGGGCGGGCGGGCGGCTCCAGCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 896731 . GCCGTGCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 897325 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970441;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|CDS|SYNONYMOUS&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC;CGA_PFAM=PFAM|PF07707|BACK GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:589,84:589,84:53,44:-589,-84,0:-53,-44,0:31:31,31:0
+1 897564 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303229;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:39,330:39,330:36,52:-330,-39,0:-52,-36,0:20:20,20:0
+1 897718 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 897730 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7549631;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|ACCEPTOR|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:897730:PASS:125:193,125:186,119:48,43:-193,0,-125:-48,0,-43:16:9,7:7
+1 897879 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79008338;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:101:215,101:211,97:48,40:-215,0,-101:-48,0,-40:18:12,6:6
+1 898001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=900000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:0.86:2:50:=:50:0.999:406
+1 898323 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605071;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:644,84:644,84:52,48:-644,-84,0:-52,-48,0:29:29,29:0
+1 899000 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3813184;CGA_FI=26155|NM_015658.3|NOC2L|TSS-UPSTREAM|UNKNOWN-INC&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:234:283,234:282,234:51,49:-283,0,-234:-51,0,-49:30:16,14:14
+1 899462 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 899667 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 899692 . TGGCGGGTCTGCGTCCAGCCCACGCCCTCGCCCCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 899739 . GGTCGCCCGTGGCGTCCATGCTGAGCCGACGCAGCTCAGCGGGCGTGGCCGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 899803 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 899809 . CGTGGCAGGGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 899829 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 899832 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 899911 . GTCCGCAGTGGGGCTGCGGGGAGGGGGG GTCCGCAGTGGGGCTGCCGGGAGGGGTC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6677386&dbsnp.130|rs71490529&dbsnp.134|rs143296006&dbsnp.134|rs147467971;CGA_FI=339451|NM_198317.1|KLHL17|DONOR|NO-CHANGE&339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:26,.:25,.:0,.:-26,0,0:0,0,0:6:6,.:0
+1 899942 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.130|rs71509448;CGA_FI=339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:17:17,75:17,75:19,32:-75,-17,0:-32,-19,0:5:5,5:0
+1 899946 . CCGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 899970 . CCGCGCGTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 899986 . G <CGA_NOCALL> . . END=900204;NS=1;AN=0 GT:PS ./.:.
+1 900001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=902000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.89:0.77:2:49:=:49:0.999:303
+1 900285 . CA TG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970434&dbsnp.111|rs4970435&dbsnp.130|rs71490530;CGA_FI=339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:28:411,28:411,28:30,33:-411,-28,0:-33,-30,0:17:17,17:0
+1 900319 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs80351873;CGA_FI=339451|NM_198317.1|KLHL17|INTRON|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:96:96,96:95,95:38,40:-96,0,-96:-38,0,-40:16:6,10:10
+1 900540 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 900553 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 900972 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9697711;CGA_FI=339451|NM_198317.1|KLHL17|UTR3|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:72,578:72,578:48,52:-578,-72,0:-52,-48,0:24:24,24:0
+1 901023 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303351;CGA_FI=339451|NM_198317.1|KLHL17|UTR3|UNKNOWN-INC&84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:71:529,71:529,71:52,48:-529,-71,0:-52,-48,0:24:24,24:0
+1 901607 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302996;CGA_FI=84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:74,447:74,447:41,50:-447,-74,0:-50,-41,0:45:45,45:0
+1 901652 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2879814;CGA_FI=84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:82:721,82:721,82:53,44:-721,-82,0:-53,-44,0:36:36,36:0
+1 901806 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.101|rs2879815;CGA_FI=84069|NM_001160184.1|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:53:431,53:431,53:52,44:-431,-53,0:-52,-44,0:24:24,24:0
+1 902001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=904000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:0.88:2:49:=:49:0.999:303
+1 902069 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs116147894;CGA_FI=84069|NM_001160184.1|PLEKHN1|ACCEPTOR|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|ACCEPTOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:91:91,91:90,90:37,39:-91,0,-91:-37,0,-39:12:5,7:7
+1 902128 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28499371;CGA_FI=84069|NM_001160184.1|PLEKHN1|CDS|MISSENSE&84069|NM_032129.2|PLEKHN1|CDS|MISSENSE GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:95:95,95:94,94:38,40:-95,0,-95:-38,0,-40:14:6,8:8
+1 902749 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs113967711;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:176:257,176:217,134:52,46:-257,0,-176:-52,0,-46:27:15,12:12
+1 902756 . ACC AC,A . . NS=1;AN=2;AC=1,1;CGA_XR=dbsnp.126|rs33930611&dbsnp.126|rs34363514&dbsnp.129|rs55994311&dbsnp.134|rs141099258,dbsnp.126|rs33930611&dbsnp.126|rs34363514&dbsnp.129|rs55994311&dbsnp.134|rs141099258;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC,84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/2:.:PASS:45:189,45:145,0:28,0:-189,-45,-45,-189,0,-189:-28,0,0,-28,0,-28:29:15,5:0
+1 903104 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6696281;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:41:351,41:351,41:52,38:-351,-41,0:-52,-38,0:28:28,28:0
+1 903321 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6669800;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:191:249,191:247,190:52,50:-249,0,-191:-52,0,-50:23:14,9:9
+1 903426 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6696609;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:271:271,271:270,270:50,51:-271,0,-271:-50,0,-51:36:17,19:19
+1 904001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=906000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.87:.:0:.:0:0.999:303
+1 904165 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28391282;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=L2a|L2|47.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:440:440,440:440,440:50,54:-440,0,-440:-50,0,-54:45:22,23:23
+1 904628 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28562326;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:904628:PASS:291:291,291:290,290:44,52:-291,0,-291:-44,0,-52:42:19,23:23
+1 904752 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35241590;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|23.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:97:97,207:93,203:38,50:-97,0,-207:-38,0,-50:16:5,11:11
+1 904757 . A ATG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs56411007&dbsnp.131|rs77716369;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|23.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:27:27,179:52,177:37,35:-179,-27,0:-37,-35,0:18:15,15:1
+1 904942 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6667868;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|27.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:904628:PASS:82:259,82:253,75:48,36:-259,0,-82:-48,0,-36:19:13,6:6
+1 905017 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs59766802;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|29.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:904628:PASS:152:152,225:154,221:45,50:-152,0,-225:-45,0,-50:25:10,15:15
+1 905029 . CGTGT C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs111304798;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|29.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:904628:PASS:154:154,225:175,221:36,40:-154,0,-225:-36,0,-40:26:8,18:18
+1 905275 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 906001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=908000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:0.74:2:47:=:47:0.999:303
+1 906272 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28507236;CGA_FI=84069|NM_001160184.1|PLEKHN1|CDS|SYNONYMOUS&84069|NM_032129.2|PLEKHN1|CDS|SYNONYMOUS;CGA_PFAM=PFAM|PF00169|PH-like GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:41:41,356:41,356:38,52:-356,-41,0:-52,-38,0:20:20,20:0
+1 906695 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 907082 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 907170 . AG A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs113797882&dbsnp.134|rs137960056;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:143:143,213:176,187:27,38:-143,0,-213:-27,0,-38:54:14,40:40
+1 907609 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs79890672;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:907609:PASS:86:86,86:61,61:27,34:-86,0,-86:-27,0,-34:18:4,14:14
+1 907622 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs59928984;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:907609:PASS:64:64,86:55,61:19,34:-64,0,-86:-19,0,-34:21:4,17:17
+1 907884 . GCCTGTAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 907920 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28430926;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=AluSz|Alu|13.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:907920:VQLOW:33:33,35:33,35:11,27:-33,0,-35:-11,0,-27:12:5,7:7
+1 907992 . AAA AC . . NS=1;AN=2;AC=1;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=AluSz|Alu|13.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:907609:PASS:177:177,216:177,221:29,30:-177,0,-216:-29,0,-30:19:12,7:7
+1 907998 . A T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs77098784;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC;CGA_RPT=AluSz|Alu|13.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:907609:PASS:206:255,206:255,208:52,50:-255,0,-206:-52,0,-50:23:12,11:11
+1 908001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=910000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.01:0.88:2:49:=:49:0.999:303
+1 908151 . GAT . . . NS=1;AN=0 GT:PS ./.:.
+1 908170 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28542142;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:908170:PASS:41:41,41:6,6:0,14:-41,0,-41:0,0,-14:13:3,10:10
+1 908414 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28504611;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:232:478,232:471,225:53,49:-478,0,-232:-53,0,-49:37:24,13:13
+1 909073 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs3892467;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:909073:PASS:206:478,206:472,200:50,47:-478,0,-206:-50,0,-47:41:26,15:15
+1 909238 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3829740;CGA_FI=84069|NM_001160184.1|PLEKHN1|CDS|MISSENSE&84069|NM_032129.2|PLEKHN1|CDS|MISSENSE GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:78:78,612:78,612:48,52:-612,-78,0:-52,-48,0:28:28,28:0
+1 909309 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3829738;CGA_FI=84069|NM_001160184.1|PLEKHN1|CDS|MISSENSE&84069|NM_032129.2|PLEKHN1|CDS|MISSENSE GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:909073:PASS:166:439,166:428,155:53,42:-439,0,-166:-53,0,-42:32:22,10:10
+1 909363 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs62639990;CGA_FI=84069|NM_001160184.1|PLEKHN1|CDS|SYNONYMOUS&84069|NM_032129.2|PLEKHN1|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:909073:PASS:124:207,124:205,122:48,44:-207,0,-124:-48,0,-44:17:10,7:7
+1 909555 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340594;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:270,46:270,46:52,41:-270,-46,0:-52,-41,0:20:20,20:0
+1 909768 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340593;CGA_FI=84069|NM_001160184.1|PLEKHN1|INTRON|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:51,487:51,487:44,48:-487,-51,0:-48,-44,0:19:19,19:0
+1 909880 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 910001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=912000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.12:0.87:2:48:=:48:0.999:303
+1 910394 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28477686;CGA_FI=84069|NM_001160184.1|PLEKHN1|UTR3|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:153:153,153:152,152:45,48:-153,0,-153:-45,0,-48:25:11,14:14
+1 910438 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6685581;CGA_FI=84069|NM_001160184.1|PLEKHN1|UTR3|UNKNOWN-INC&84069|NM_032129.2|PLEKHN1|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:19:172,19:172,19:48,20:-172,-19,0:-48,-20,0:16:16,16:0
+1 910512 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs74424855;CGA_RPT=GA-rich|Low_complexity|21.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:910512:PASS:413:413,810:405,801:34,57:-413,0,-810:-34,0,-57:73:32,41:41
+1 910527 . GAGAGA G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs141996759;CGA_RPT=GA-rich|Low_complexity|21.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:910512:PASS:147:147,709:184,701:28,45:-147,0,-709:-28,0,-45:73:26,47:47
+1 910903 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970429;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:77:77,687:77,687:42,53:-687,-77,0:-53,-42,0:34:33,33:1
+1 910935 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2340592;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:910512:PASS:207:207,207:192,192:35,46:-207,0,-207:-35,0,-46:45:14,31:31
+1 911595 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7417106;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:65:65,469:65,469:38,53:-469,-65,0:-53,-38,0:31:29,29:2
+1 911768 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 911934 . AAAAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 912001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=914000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.26:0.67:2:45:=:45:0.999:303
+1 912049 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9803103;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:73:73,73:73,73:31,36:-73,0,-73:-31,0,-36:11:5,6:6
+1 912074 . AACCGCCTGCCCCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 912099 . C <CGA_NOCALL> . . END=913448;NS=1;AN=0 GT:PS ./.:.
+1 913464 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 913497 . CGCACACGGCCGCCCCGGGAACCGCCTGCCTCCCCCTCCAACCCCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 913606 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 913889 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2340596;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:173:455,173:448,166:53,43:-455,0,-173:-53,0,-43:39:25,14:14
+1 914001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=916000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.22:0.85:2:49:=:49:0.999:303
+1 914192 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2340595;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:65:545,65:545,65:52,48:-545,-65,0:-52,-48,0:26:26,26:0
+1 914333 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302979;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:63:63,447:63,447:37,53:-447,-63,0:-53,-37,0:30:30,30:0
+1 914852 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303368;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:914852:PASS:187:187,275:186,274:49,50:-187,0,-275:-49,0,-50:29:13,16:16
+1 914876 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302983;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:69:69,561:118,561:48,52:-561,-69,0:-52,-48,0:26:26,26:0
+1 914940 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303033;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:914852:PASS:167:167,167:166,166:47,49:-167,0,-167:-47,0,-49:25:11,14:14
+1 915227 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303355;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:30:30,615:74,615:48,52:-615,-30,0:-52,-48,0:25:25,25:0
+1 915264 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs113243246;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:914852:PASS:136:429,136:422,129:54,32:-136,0,-429:-32,0,-54:33:21,12:21
+1 916001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=918000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:1.05:2:52:=:52:0.999:303
+1 916071 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.135|rs190879197;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:916071:PASS:118:118,118:118,118:39,43:-118,0,-118:-39,0,-43:20:9,11:11
+1 916108 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 916144 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 916160 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 916164 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 916239 . TGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 916246 . CTG . . . NS=1;AN=0 GT:PS ./.:.
+1 916257 . GCTGCCACTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 916549 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6660139;CGA_FI=84808|NR_027693.1|C1orf170|UTR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:306,42:306,42:48,39:-306,-42,0:-48,-39,0:14:14,14:0
+1 916834 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6694632;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:86,698:86,698:45,53:-698,-86,0:-53,-45,0:31:31,31:0
+1 917060 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605058;CGA_FI=84808|NR_027693.1|C1orf170|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:38:260,38:260,38:48,36:-260,-38,0:-48,-36,0:16:16,16:0
+1 918001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=920000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:0.93:2:53:=:53:0.999:303
+1 918384 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13303118;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:378:440,378:440,378:50,54:-440,0,-378:-50,0,-54:48:25,23:23
+1 918573 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341354;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:111:111,918:111,918:52,53:-918,-111,0:-53,-52,0:39:38,38:1
+1 919419 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605059;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:32:32,290:32,290:31,48:-290,-32,0:-48,-31,0:12:12,12:0
+1 919501 . GGA TGA . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.111|rs4970414;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:919501:PASS:441,.:412,.:52,.:-441,0,0:-52,0,0:20:18,.:0
+1 919507 . TGGCTGTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 919521 . GGAATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 919555 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 919645 . GGGGAGGTCGGGCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 919695 . A <CGA_NOCALL> . . END=919922;NS=1;AN=0 GT:PS ./.:.
+1 919946 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 919960 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 919963 . TCTTTTTTCTTTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 919979 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 919983 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 919987 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 919991 . CTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 919998 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341357;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:919998:VQLOW:15:159,15:158,15:48,17:-159,-15,0:-48,-17,0:7:7,7:0
+1 920001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=922000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.80:0.88:2:53:=:53:0.999:303
+1 920002 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs55746336;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:919998:VQLOW:15:150,15:152,15:47,17:-150,-15,0:-47,-17,0:6:6,6:0
+1 920006 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4039719;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:919998:PASS:33:150,33:152,33:47,31:-150,-33,0:-47,-31,0:5:5,5:0
+1 920010 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4039720;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:919998:PASS:47:47,150:47,151:42,47:-150,-47,0:-47,-42,0:7:7,7:0
+1 920014 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs142966470;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:919998:VQLOW:14:14,106:14,106:17,41:-106,-14,0:-41,-17,0:5:5,5:0
+1 920018 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:919998:VQLOW:24,.:24,.:0,.:0:0
+1 920022 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 920109 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 920128 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 920133 . CCTCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 920149 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 920166 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 920648 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6677020;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L1ME1|L1|39.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:88:758,88:758,88:53,46:-758,-88,0:-53,-46,0:36:36,36:0
+1 920733 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6677131;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L1ME1|L1|39.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:70:70,682:70,682:39,53:-682,-70,0:-53,-39,0:32:32,32:0
+1 921570 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6662128;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:96:836,96:836,96:53,48:-836,-96,0:-53,-48,0:35:35,35:0
+1 921716 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303278;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:54:516,54:516,54:48,45:-516,-54,0:-48,-45,0:19:19,19:0
+1 922001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=924000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.88:2:52:=:52:0.999:303
+1 922207 . TGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 922305 . G GC . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs34505725&dbsnp.130|rs70954422&dbsnp.131|rs79100974;CGA_FI=84808|NR_027693.1|C1orf170|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz|Alu|14.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:37:37,403:37,403:32,35:-403,-37,0:-35,-32,0:19:19,19:0
+1 922538 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 922576 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 922643 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 922646 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:922646:PASS:124,.:64,.:0,.:0:0
+1 922664 . TTCTTTTTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 922741 . A C . . NS=1;AN=2;AC=1;CGA_RPT=AluSq|Alu|10.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:922741:VQLOW:20:20,25:8,13:0,19:-20,0,-25:0,0,-19:5:1,4:4
+1 922745 . TCCGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 922760 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 922873 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 922877 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 922882 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 922905 . ATCCACCCAACTCGGCCTCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 922937 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 923050 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:922646:PASS:363,.:328,.:0,.:0:0
+1 923061 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 923074 . CCA CCG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs6605060;CGA_RPT=AluSx1|Alu|12.1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:922646:PASS:399,.:364,.:48,.:-399,0,0:-48,0,0:17:17,.:0
+1 923459 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442609;CGA_RPT=L1ME1|L1|40.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:48,371:48,371:42,48:-371,-48,0:-48,-42,0:19:19,19:0
+1 923522 . AGATTTTTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 923749 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442610;CGA_RPT=AluSg|Alu|10.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:68:68,653:68,653:48,52:-653,-68,0:-52,-48,0:25:25,25:0
+1 923978 . A AG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.130|rs70949537;CGA_RPT=AluSg|Alu|10.7 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:66,.:66,.:13,.:-66,0,0:-13,0,0:38:36,.:2
+1 924001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=926000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.95:.:0:.:0:0.999:303
+1 924095 . GGAGTTCGGGGTTGATTGTTTCTGGCGTTTAGGGTTGATTGTTTCTGGCGTTCAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 924177 . GATAGTTTCTGGAGTTCTGGGTGGAGTGTTTCGGGAGTTCTGGGTTGATTGTTTCTGGGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 924245 . TGATTGTTTCTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 924267 . TTGATTATTTCTGGTGTTCTGGGTTAATTGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 924314 . TGATTGTTTCTGGAGTTTGGGGTTGACTGTTTCTGGAGTTCTGGGTTGATTGTTCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 924383 . TGATTGTTTCTGGAGTTTGGGGTTGACTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 924440 . AGTTTGGGGTTGATTGTTTCTGGAGTTCGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 924485 . GAGTTCTGGGTTGACTGTTTCTGGAGTTCAGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 924525 . TTTCTGGAATTTGGGGTTGATTGTTTCTGGAGTTCTGGGTTGATTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 924583 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 924603 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 924616 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 924629 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 924802 . CACTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 924898 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6665000;CGA_RPT=MSTD|ERVL-MaLR|32.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:118:118,1011:118,1011:52,50:-1011,-118,0:-52,-50,0:41:41,41:0
+1 925551 . AT A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs35621903&dbsnp.132|rs111393411;CGA_RPT=AluSx1|Alu|9.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:73:73,344:38,310:35,35:-344,-73,0:-35,-35,0:16:16,16:0
+1 925575 . ACCATGTTGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 925684 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6605061;CGA_RPT=AluSq2|Alu|8.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:446,66:446,66:53,38:-446,-66,0:-53,-38,0:38:38,38:0
+1 925798 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 925880 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 926001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=928000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.88:.:0:.:0:0.999:303
+1 926166 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 926169 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 926351 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6671243;CGA_RPT=AluSx1|Alu|16.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:59:59,551:59,551:46,52:-551,-59,0:-52,-46,0:25:24,24:1
+1 926431 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970403;CGA_RPT=L1MEc|L1|32.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:903,79:903,79:53,42:-903,-79,0:-53,-42,0:38:37,37:1
+1 926621 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970351;CGA_RPT=AluJb|Alu|21.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,477:52,477:44,48:-477,-52,0:-48,-44,0:17:17,17:0
+1 927309 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341362 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:83:752,83:752,83:53,44:-752,-83,0:-53,-44,0:35:35,35:0
+1 927535 . TTTTG T . . NS=1;AN=2;AC=1;CGA_RPT=AluSx|Alu|16.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:25:25,25:0,0:0,3:-25,0,-25:0,0,-3:26:2,24:24
+1 928001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=930000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.82:0.89:.:0:.:0:0.999:303
+1 928346 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 928364 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 928373 . AGG . . . NS=1;AN=0 GT:PS ./.:.
+1 928441 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:928441:PASS:51,.:48,.:0,.:0:0
+1 928520 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 928578 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28394749 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:151:151,1329:151,1329:52,49:-1329,-151,0:-52,-49,0:59:58,58:1
+1 928836 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9777703 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:82:82,925:82,925:44,53:-925,-82,0:-53,-44,0:38:37,37:1
+1 928979 . GGGG GGGGC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs35805387&dbsnp.131|rs79780781 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:928979:PASS:117,.:117,.:35,.:-117,0,0:-35,0,0:16:16,.:0
+1 928993 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 929190 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9777939;CGA_RPT=MSTA|ERVL-MaLR|19.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:63:490,63:490,63:52,48:-490,-63,0:-52,-48,0:28:28,28:0
+1 929316 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9777893;CGA_RPT=MSTA|ERVL-MaLR|19.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:50,606:50,606:43,52:-606,-50,0:-52,-43,0:25:24,24:1
+1 929321 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302916;CGA_RPT=MSTA|ERVL-MaLR|19.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:31:31,619:31,619:30,52:-619,-31,0:-52,-30,0:26:25,25:1
+1 929327 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13302924;CGA_RPT=MSTA|ERVL-MaLR|19.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:62:62,536:62,536:47,52:-536,-62,0:-52,-47,0:23:23,23:0
+1 929459 . TTTGGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 929701 . C CA . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs35561765;CGA_RPT=AluSz|Alu|12.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:14:14,106:14,106:16,35:-106,-14,0:-35,-16,0:12:11,11:1
+1 929872 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 930001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=932000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:0.92:.:0:.:0:0.999:303
+1 930162 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 930888 . ATTTTT A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs55952878&dbsnp.134|rs139994710;CGA_RPT=L1MEc|L1|22.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:54:250,54:250,54:38,38:-250,-54,0:-38,-38,0:29:29,29:0
+1 930923 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710882 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:22:165,22:165,22:48,22:-165,-22,0:-48,-22,0:7:7,7:0
+1 931014 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710881 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:72,735:72,735:40,53:-735,-72,0:-53,-40,0:39:37,37:2
+1 931096 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 931327 . CGTTCCCCCCGCGGCAGACAAGCCCAGACACACACGGCCCAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 931497 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 931500 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 931504 . CTATTCAGCAG CTATACAGCAG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2799061 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:931504:PASS:261,.:257,.:48,.:-261,0,0:-48,0,0:12:10,.:0
+1 931518 . ACC . . . NS=1;AN=0 GT:PS ./.:.
+1 931582 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 931589 . CAGCCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 931659 . TGAGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 931730 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 931769 . AGGGCAGAAAGGACCCCCCGCTGGAGGGGGCACCCCACGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 931835 . AGGACAGAAAGGACCCCCCGCTGGAGGGGGCACCCCACAT . . . NS=1;AN=0 GT:PS ./.:.
+1 931901 . AGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 931928 . GGGCACCTCACGTCTGGGGCCACAGGATGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 931967 . AGGACAGAAAGGACCCCCCGCTGGAGGGGGCACCCCACATCT . . . NS=1;AN=0 GT:PS ./.:.
+1 932001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=934000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.30:0.75:2:48:=:48:0.999:303
+1 932018 . GGATGCAGGGTGGGGAGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 932050 . CCGCGGGAGGGGGCACCTCACGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 932084 . GGATGCAGGGTGGGGAGGGCAGAAAGAACCCCCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 932126 . GGCACCCCACATCTGGGGCCACAGGATGCAGGGTGGGGAGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 932182 . CGCGGGAGGGGGCACCTCACGTCT . . . NS=1;AN=0 GT:PS ./.:.
+1 932215 . GGATGCAGGGTGGGGAGGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 932618 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128112 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:170:1385,170:1385,170:48,44:-1385,-170,0:-48,-44,0:62:62,62:0
+1 933790 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442392 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:58:532,58:532,58:52,46:-532,-58,0:-52,-46,0:25:25,25:0
+1 933854 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 933858 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 933866 . CTC . . . NS=1;AN=0 GT:PS ./.:.
+1 934001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=936000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.23:1.45:2:40:=:40:0.999:303
+1 934099 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710867;CGA_RPT=G-rich|Low_complexity|26.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:934099:VQLOW:14:144,14:142,14:46,17:-144,-14,0:-46,-17,0:7:7,7:0
+1 934102 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 934106 . AT GT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2799063;CGA_RPT=G-rich|Low_complexity|26.3 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:934099:PASS:94,.:91,.:37,.:-94,0,0:-37,0,0:7:7,.:0
+1 934126 . GGAGGGGAGGGCGCGGAGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 934154 . GGAGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 934164 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 934180 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 934183 . TCCCGCCCCCAGCCCGCCCCCCCGGGCCCGCCCGACGCCCCCAGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 934248 . GGGAGGAGGGCGGGACCCCGGCGCGGCGTGGCTGCGGGGCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 934339 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 934402 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 934435 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 934459 . AGGGCCCACCCGGGCCCTGCGGCCCCGCCCTGGGGGCGGCGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 934514 . CAGACCCGGCAGCAGCGGCGGCGCGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 934565 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 935020 . C T . . NS=1;AN=2;AC=1;CGA_FI=57801|NM_001142467.1|HES4|INTRON|UNKNOWN-INC&57801|NM_021170.3|HES4|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:92:244,92:238,86:48,38:-244,0,-92:-48,0,-38:19:13,6:6
+1 935222 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2298214;CGA_FI=57801|NM_001142467.1|HES4|CDS|MISSENSE&57801|NM_021170.3|HES4|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:36:196,36:196,36:48,34:-196,-36,0:-48,-34,0:12:12,12:0
+1 935416 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 935420 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 935423 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 935428 . GAGCGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 935437 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 935459 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128113;CGA_FI=57801|NM_001142467.1|HES4|UTR5|UNKNOWN-INC&57801|NM_021170.3|HES4|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:14:14,73:14,73:17,31:-73,-14,0:-31,-17,0:5:5,5:0
+1 935492 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121571;CGA_FI=57801|NM_001142467.1|HES4|UTR5|UNKNOWN-INC&57801|NM_021170.3|HES4|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:45:45,341:45,341:41,48:-341,-45,0:-48,-41,0:14:14,14:0
+1 935600 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 935633 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 935650 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:935650:VQLOW:20,.:15,.:0,.:0:0
+1 935656 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 935673 . TCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 935703 . TCCCGCGCTCCCCGATGCAGCCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 935792 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 935833 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128115;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:32:32,300:32,300:31,48:-300,-32,0:-48,-31,0:13:13,13:0
+1 935893 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 935896 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 935899 . AGG . . . NS=1;AN=0 GT:PS ./.:.
+1 935910 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 936001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=938000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:1.21:2:43:=:43:0.999:303
+1 936003 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 936111 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.92|rs1936360;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:58:574,58:574,58:52,46:-574,-58,0:-52,-46,0:25:25,25:0
+1 936194 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121570;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:417,46:417,46:48,41:-417,-46,0:-48,-41,0:18:18,18:0
+1 936210 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121569;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:37:372,37:372,37:48,35:-372,-37,0:-48,-35,0:19:18,18:1
+1 936265 . GGCACCCGGGGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 936317 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 936335 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 936338 . CCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 936520 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 936525 . ACGCGTCCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 936614 . CAGGAGGCAGATGGCAGACTCAGCAGTCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 936675 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 936724 . GT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:936724:PASS:41,.:21,.:0,.:0:0
+1 936734 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 936827 . G A . . NS=1;AN=2;AC=1;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:24:24,24:11,11:0,17:-24,0,-24:0,0,-17:17:4,13:13
+1 936957 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 937032 . TTCG . . . NS=1;AN=0 GT:PS ./.:.
+1 937038 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 937057 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 937069 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 937651 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 937680 . GGC . . . NS=1;AN=0 GT:PS ./.:.
+1 937688 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2489000;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=FLAM_C|Alu|13.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:937688:PASS:42:322,42:322,42:48,39:-322,-42,0:-48,-39,0:17:17,17:0
+1 937795 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 938001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=940000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:0.89:2:49:=:49:0.999:303
+1 938116 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710869;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSp|Alu|10.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:64:676,64:676,64:52,48:-676,-64,0:-52,-48,0:26:26,26:0
+1 938213 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799058;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSp|Alu|10.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:86,994:86,994:45,50:-994,-86,0:-50,-45,0:41:40,40:1
+1 940001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=942000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:1.01:2:51:=:51:0.999:303
+1 940096 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4503294;CGA_FI=57801|NM_001142467.1|HES4|TSS-UPSTREAM|UNKNOWN-INC&57801|NM_021170.3|HES4|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L2|L2|43.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:137:137,1223:137,1223:52,50:-1223,-137,0:-52,-50,0:49:48,48:1
+1 940413 . CTGCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 940725 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 940728 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 940735 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 940739 . CCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 941146 . A C . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:941146:VQLOW:34:34,48:20,34:0,27:-34,0,-48:0,0,-27:9:2,7:7
+1 941150 . GTG GTTC . . NS=1;AN=1;AC=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP .|1:941146:PASS:.,48:.,34:.,1:-48,0,0:-1,0,0:8:.,8:0
+1 941161 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 941163 . AGTTCCCCCCGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 941182 . CCC . . . NS=1;AN=0 GT:PS ./.:.
+1 941471 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 941490 . GGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 941644 . GGACCCTCCGCCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 941665 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 941672 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 941680 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 942001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=944000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.84:2:52:=:52:0.999:303
+1 942849 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 942865 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 943127 . TTTTTTTTTTTTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 943250 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121568;CGA_RPT=AluYa5|Alu|0.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:42,303:42,303:39,48:-303,-42,0:-48,-39,0:14:14,14:0
+1 943468 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121567 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:73:73,687:73,687:48,52:-687,-73,0:-52,-48,0:29:29,29:0
+1 943907 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488992;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSq2|Alu|9.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:51,461:51,461:33,50:-461,-51,0:-50,-33,0:42:42,42:0
+1 943968 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303313;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSq2|Alu|9.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:57:486,57:486,57:52,46:-486,-57,0:-52,-46,0:21:21,21:0
+1 944001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=946000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.58:0.80:2:51:=:51:0.999:303
+1 945058 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 945096 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3135457;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:87:87,901:87,901:45,50:-901,-87,0:-50,-45,0:41:41,41:0
+1 945111 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303172;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:69:733,69:733,69:53,39:-733,-69,0:-53,-39,0:33:33,33:0
+1 945214 . GAGTGCAGTGGTGCGATCACAGCTCACTGCAGCCTCCACCTCCCGGGTTCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 945474 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121566;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz|Alu|10.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:79,762:79,762:42,53:-762,-79,0:-53,-42,0:32:31,31:1
+1 945612 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121565;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz|Alu|10.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:26:26,243:26,243:26,48:-243,-26,0:-48,-26,0:9:9,9:0
+1 945669 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 945720 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 945862 . TTATTTATTT TTATTT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.108|rs4039717&dbsnp.126|rs35187710&dbsnp.129|rs61025563&dbsnp.134|rs141880858&dbsnp.134|rs150242145;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=(TTTA)n|Simple_repeat|13.6 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:945862:PASS:53,.:53,.:16,.:-53,0,0:-16,0,0:6:6,.:0
+1 945876 . ATTTGAATCTTAT . . . NS=1;AN=0 GT:PS ./.:.
+1 946001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=948000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.70:1.09:2:51:=:51:0.999:303
+1 946134 . TTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 946180 . TGCAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 946195 . TCGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 946204 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 946207 . AGCCTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 946331 . TTGGCCAGGCTGGTCTCAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 947034 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465126;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=L1MB5|L1|34.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:74,662:74,662:48,52:-662,-74,0:-52,-48,0:28:28,28:0
+1 947538 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465125;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx|Alu|12.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:50,517:50,517:43,52:-517,-50,0:-52,-43,0:22:22,22:0
+1 947617 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 947761 . TTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 947769 . TATTATTACAGTCATAC . . . NS=1;AN=0 GT:PS ./.:.
+1 947791 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:947791:VQLOW:30,.:0,.:0,.:0:0
+1 947838 . AATGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 947934 . TTTATTTTTATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 948001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=950000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:0.86:2:52:=:52:0.999:303
+1 948134 . CAGTGGCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 948421 . A AAAC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs60637872&dbsnp.132|rs113047134;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJb|Alu|14.5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:230,.:199,.:36,.:-230,0,0:-36,0,0:34:21,.:0
+1 948692 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2341365;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:78:78,722:78,722:42,53:-722,-78,0:-53,-42,0:35:35,35:0
+1 948846 . T TA . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3841266&dbsnp.114|rs5772027;CGA_FI=9636|NM_005101.3|ISG15|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:93:93,854:93,854:39,37:-854,-93,0:-39,-37,0:44:44,44:0
+1 948870 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4615788;CGA_FI=9636|NM_005101.3|ISG15|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:93:816,93:816,93:53,47:-816,-93,0:-53,-47,0:39:39,39:0
+1 948921 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.52|rs15842;CGA_FI=9636|NM_005101.3|ISG15|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:84,679:84,679:44,53:-679,-84,0:-53,-44,0:30:30,30:0
+1 949235 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465124;CGA_FI=9636|NM_005101.3|ISG15|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:389,52:389,52:52,44:-389,-52,0:-52,-44,0:21:21,21:0
+1 949654 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.52|rs8997;CGA_FI=9636|NM_005101.3|ISG15|CDS|SYNONYMOUS;CGA_PFAM=PFAM|PF00240|ubiquitin,PFAM|PF00788|UBQ GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:20:150,20:150,20:47,21:-150,-20,0:-47,-21,0:9:9,9:0
+1 949925 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799070 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:108:108,904:108,904:52,50:-904,-108,0:-52,-50,0:42:42,42:0
+1 950001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=952000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.91:2:53:=:53:0.999:303
+1 950113 . GAAGT G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.130|rs70949541&dbsnp.134|rs140258289 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:844,86:867,86:38,39:-844,-86,0:-39,-38,0:42:42,42:0
+1 950677 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9331223;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:140:1162,140:1162,140:49,52:-1162,-140,0:-52,-49,0:51:51,51:0
+1 950716 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799069;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:70:254,70:254,70:52,48:-254,-70,0:-52,-48,0:29:29,29:0
+1 950837 . GCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 951283 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442363;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:91:765,91:765,91:53,47:-765,-91,0:-53,-47,0:35:35,35:0
+1 951295 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442388;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:93:847,93:847,93:53,47:-847,-93,0:-53,-47,0:34:34,34:0
+1 951322 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9697362&dbsnp.131|rs75970800;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:43:411,43:411,43:52,39:-411,-43,0:-52,-39,0:25:25,25:0
+1 951330 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9697717;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:49:49,432:49,432:43,52:-432,-49,0:-52,-43,0:20:20,20:0
+1 951441 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 951824 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 952001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=954000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.61:0.88:.:0:.:0:0.999:303
+1 952428 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442611;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx|Alu|12.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:27:27,215:27,215:27,48:-215,-27,0:-48,-27,0:11:11,11:0
+1 952554 . TCCAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 952712 . GCT . . . NS=1;AN=0 GT:PS ./.:.
+1 952839 . CTTGGCGGTGAGTGTTACAGCTCATAAAAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 953033 . GCCTAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 953168 . AGT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:116,.:89,.:0,.:0:0
+1 953183 . A AGT . . NS=1;AN=2;AC=2;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=LTR12C|ERV1|19.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:61:222,61:195,40:39,34:-222,-61,0:-39,-34,0:29:29,29:0
+1 953215 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:953215:PASS:153,.:120,.:0,.:0:0
+1 953223 . GCGCT GTGCT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs139816136;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=LTR12C|ERV1|19.5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:953215:PASS:112,.:79,.:27,.:-112,0,0:-27,0,0:27:18,.:1
+1 953238 . TTT . . . NS=1;AN=0 GT:PS ./.:.
+1 953351 . GGCGCAT . . . NS=1;AN=0 GT:PS ./.:.
+1 953428 . CGATTGGTGTATTTACAATCCCTGAGCTAGACATAAAGGTT . . . NS=1;AN=0 GT:PS ./.:.
+1 953678 . TGCGC CGCGC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs142441326;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=LTR12C|ERV1|19.5 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:71,.:71,.:31,.:-71,0,0:-31,0,0:6:6,.:0
+1 953761 . GCTTGGGCCGCACAGGAGCCCACGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 953875 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 953893 . CTGCTGGGGGACCCACTACACCCTCCGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 953952 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442612;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=LTR12C|ERV1|19.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:15:15,124:15,124:17,43:-124,-15,0:-43,-17,0:14:12,12:2
+1 954001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=956000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.51:1.01:2:52:=:52:0.999:303
+1 954619 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 954625 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 954777 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs61766299;CGA_FI=375790|NM_198576.2|AGRN|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:954777:VQLOW:13:99,13:99,13:39,16:-99,-13,0:-39,-16,0:6:6,6:0
+1 954794 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 954800 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 954813 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 954824 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 954832 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 954860 . GGGGGGGCGGGGCCGGGAGGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 954908 . CAGCGCCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 954924 . ACCGGGACCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 954951 . C <CGA_NOCALL> . . END=955175;NS=1;AN=0 GT:PS ./.:.
+1 955185 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 955203 . C <CGA_NOCALL> . . END=955678;NS=1;AN=0 GT:PS ./.:.
+1 955686 . TGGTGCTCACCGGGACGGTGGAGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 955728 . C <CGA_NOCALL> . . END=955977;NS=1;AN=0 GT:PS ./.:.
+1 955985 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 955987 . ACAAG . . . NS=1;AN=0 GT:PS ./.:.
+1 955994 . CACCTCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 956001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=958000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.72:0.79:2:50:=:50:0.999:303
+1 956007 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:956007:VQLOW:23,.:21,.:0,.:0:0
+1 956015 . CTG . . . NS=1;AN=0 GT:PS ./.:.
+1 956022 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 956024 . GCCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 956051 . GGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 956071 . TTTGCCCGGGCGGGGAGCGGGGGCTGGGCCTGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 956109 . GCTTGTTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 956129 . GCCTTTCCCGGGGCGAGAGGGGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 956162 . AGAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 956178 . AGCCTGAGCTCCCAACCCCGGGAGCCAGGTGGGGGGTGCCGCAGTGGTGCGGGGGGGGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 956241 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 956308 . AT . . . NS=1;AN=0 GT:PS ./.:.
+1 956315 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 956327 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 956333 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 956455 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 956467 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 956818 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 956852 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9777931;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:64:457,64:459,64:52,48:-457,-64,0:-52,-48,0:27:27,27:0
+1 957633 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 957967 . T TTGTAGTCTGACCTGTGGTCTGAC . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.114|rs6143083;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:14:14,346:14,346:16,37:-346,-14,0:-37,-16,0:40:39,39:1
+1 957980 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 958001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=960000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.32:0.89:2:52:=:52:0.999:303
+1 958067 . T C . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:958067:VQLOW:27:27,27:14,14:0,19:-27,0,-27:0,0,-19:17:4,13:13
+1 958070 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 958905 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2710890;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:406:406,406:406,406:50,54:-406,0,-406:-50,0,-54:41:20,21:21
+1 959155 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3845291;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:111:1019,111:1019,111:50,52:-1019,-111,0:-52,-50,0:44:44,44:0
+1 959169 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs3845292;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:86,881:86,881:45,50:-881,-86,0:-50,-45,0:43:43,43:0
+1 959231 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4039721;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:85:85,698:85,698:45,53:-698,-85,0:-53,-45,0:30:30,30:0
+1 959509 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28591569;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:208:288,208:288,208:52,50:-288,0,-208:-52,0,-50:28:15,13:13
+1 960001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=962000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.10:0.80:2:51:=:51:0.999:303
+1 960409 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970392;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:77:77,761:77,761:42,53:-761,-77,0:-53,-42,0:31:31,31:0
+1 960734 . CA . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:VQLOW:22,.:8,.:0,.:0:0
+1 960790 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 961008 . CCC . . . NS=1;AN=0 GT:PS ./.:.
+1 961054 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 961057 . TCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 961252 . GT . . . NS=1;AN=0 GT:PS ./.:.
+1 961257 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 961364 . TGGGGGGGGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 961827 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121556;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:671,79:671,79:52,48:-671,-79,0:-52,-48,0:28:28,28:0
+1 962001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=964000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:0.91:.:0:.:0:0.999:303
+1 962236 . GGC . . . NS=1;AN=0 GT:PS ./.:.
+1 962241 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 962266 . GGGTGAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 962606 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970393;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|21.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:91:773,91:773,91:53,47:-773,-91,0:-53,-47,0:32:32,32:0
+1 962891 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970394;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:101:790,101:790,101:50,50:-790,-101,0:-50,-50,0:40:40,40:0
+1 963013 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442389;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:91:736,91:736,91:53,47:-736,-91,0:-53,-47,0:34:34,34:0
+1 963143 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 963249 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710870;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:65:524,65:524,65:52,48:-524,-65,0:-52,-48,0:25:25,25:0
+1 963701 . CCCCCCCC CACCCC . . NS=1;AN=1;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=GC_rich|Low_complexity|0.0 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:40,.:38,.:2,.:-40,0,0:-2,0,0:19:10,.:0
+1 963721 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465127;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:66,288:64,290:48,52:-288,-66,0:-52,-48,0:25:25,25:0
+1 963835 . AGCCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 964001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=966000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.66:0.79:.:0:.:0:0.999:303
+1 964051 . ACAGCCACGCCACCCTCTCCCAAGGAACCGAGCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 964115 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 964128 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4246501;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:144,46:144,46:44,41:-144,-46,0:-44,-41,0:28:26,26:2
+1 964267 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 964298 . C G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.111|rs4970396;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:47,.:47,.:20,.:-47,0,0:-20,0,0:11:7,.:4
+1 964385 . GAGCCCCAGCCCCTCGTGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 964447 . AGCCCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 964766 . G GGAC . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:48:48,282:55,282:7,38:-48,0,-282:-7,0,-38:36:17,19:19
+1 964815 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 964840 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970343;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:15:195,15:199,15:48,17:-195,-15,0:-48,-17,0:9:9,9:0
+1 964848 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970344;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:20:261,20:268,37:48,35:-261,-20,0:-48,-35,0:12:12,12:0
+1 964891 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 964896 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 964978 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 964984 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 964992 . GGTGCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 965009 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 965020 . TGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 965051 . ATGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 965097 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 965102 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 965122 . CAGCGTGTGTGTGTGCAGTGCATGGTGCTGTGAGTGTGAGATCGTGTGTGTGTATGCAGTGCATGGTGCTGTGTGAGATCAGCGTGTGTGTGTGTGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 965240 . AGCATGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGTGTGTGTGTGTGTGCAGTGCATGGTGCTGTGTGAGATCAGCATGTGTGTGTGTGTGCAGCGCATGGTGCTGTGTGAGATCAGCATGTGTGTGTGTGTGTGCAGTGCATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 965400 . AGCATGTGTGTGTGCAGTGCATGGTGCTGTGTGAGATCAGCATGTGTGTGTGTGTGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 965477 . CAGCATGTGTGTGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGCATGTGTGTGTGTGTGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 965563 . CAGCATGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGCGCGTGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGCGTGTGTGTGTGTGCAGTGCATGGTGCTGTGTGAGATCAGCATGTGTGTGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 965730 . CAGCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 965744 . GTGTGAGATCAGCATGTGTGTGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGCGTGTGTGTGTGTGCAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 965826 . GTGTGAGATCAGCATGTGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGCGTGTGTGTGCAGCGCATGGTGCTGTGTGAGATCAGCGTGTGTGTGTGCAGCGCATGGTGCTGAGAGATCAGCATGTGTGTGTGCAGTGCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 965981 . CAGCGTGTGTGTGTGTGCAGTGCATGGTGCTGAGTGTGAGATCAGCATGTGTGTGTGTGCAGTGCATGGTGCTGTGAGATCAGTGTGTGTGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 966001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=968000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:0.91:.:0:.:0:0.999:303
+1 966090 . GTGTGAGATCAGCATGTGTGTGTGTGTGTGTGCAGCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 966144 . AGCATGTGTGTGTGTGTGTGTGTGCAGTGCATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 966190 . AGCATGTGTGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 966225 . CAGCGTGTGTGTGTGTGCAGTGCATGGTGCTGTGTGAGATCAGCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 966291 . GTGCTGTGAGATCAGCGTGTGTGTGTGTGCAGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 966374 . CTGAGTGTGAGATCAGCATGTGTGTGTGCAGTGCATGGTGCTGTGAGTGTATCAGCATGTGTGTGTGTGCAGTGCATGGTGCTGTGAGTGTGATTGTGTGTGTGTGTGTGCGGTGCATGGTGCTGTGTGAGATG . . . NS=1;AN=0 GT:PS ./.:.
+1 966577 . AGCATGTGTGTGTGCAGTGCATGGTGCTGTGAGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 966725 . AGTACTC . . . NS=1;AN=0 GT:PS ./.:.
+1 967658 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970349;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:99:99,897:99,897:49,53:-897,-99,0:-53,-49,0:38:38,38:0
+1 968001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=970000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.34:1.10:2:51:=:51:0.999:303
+1 968275 . TCTAGGGGATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 968468 . TTC . . . NS=1;AN=0 GT:PS ./.:.
+1 968475 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 968709 . GGGCGGGGCCGCGGGCGCAGACACTCGCGGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 968767 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 968772 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 968775 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 968785 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 968797 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 968820 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 968925 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 968928 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 969009 . C <CGA_NOCALL> . . END=969242;NS=1;AN=0 GT:PS ./.:.
+1 969293 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 969419 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 969448 . GCT . . . NS=1;AN=0 GT:PS ./.:.
+1 969452 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 969459 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 969461 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 969513 . CTGCCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 969532 . CAGACCCTCGGCGCCCGGCCCCCGCGCACCTGCCGCGCGCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 969593 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 969809 . GG CT . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:969809:VQLOW:25:25,25:9,9:0,18:-25,0,-25:0,0,-18:8:1,7:7
+1 969815 . AGCTGCTGTCCGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 969892 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 969899 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 969947 . GCCGCTGGCGCGGGACACCCGGCAGCCGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 969996 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 970001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=972000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.85:2:52:=:52:0.999:303
+1 970215 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442364;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:361,50:368,50:48,43:-361,-50,0:-48,-43,0:17:17,17:0
+1 970312 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 970320 . GGCTA . . . NS=1;AN=0 GT:PS ./.:.
+1 970348 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 970354 . AGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 970388 . CCTGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 970399 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 970404 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 970409 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 970499 . ATGGGGTCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 970516 . CCCTTCAGCAGCCTGGATCCCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 970550 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 970622 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 970625 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 971224 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799055;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:361,52:361,52:53,33:-361,-52,0:-53,-33,0:34:34,34:0
+1 971367 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710883;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:90:90,743:90,743:46,50:-743,-90,0:-50,-46,0:40:40,40:0
+1 972001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=974000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.05:0.98:2:53:=:53:0.999:303
+1 972134 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121575;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,480:52,480:44,52:-480,-52,0:-52,-44,0:27:27,27:0
+1 972180 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970350;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:579,66:579,66:52,48:-579,-66,0:-52,-48,0:26:26,26:0
+1 973105 . TGT . . . NS=1;AN=0 GT:PS ./.:.
+1 973336 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488993;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:30:217,30:217,30:48,29:-217,-30,0:-48,-29,0:19:19,19:0
+1 973377 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465129;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:66,650:66,650:38,53:-650,-66,0:-53,-38,0:30:30,30:0
+1 973458 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465130;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:110:110,812:110,812:52,50:-812,-110,0:-52,-50,0:42:42,42:0
+1 973668 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488994;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:64:64,479:64,479:48,52:-479,-64,0:-52,-48,0:28:28,28:0
+1 974001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=976000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:0.97:2:52:=:52:0.999:303
+1 974180 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121577;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:56:471,56:471,56:52,45:-471,-56,0:-52,-45,0:23:23,23:0
+1 974199 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465131;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:703,79:703,79:53,42:-703,-79,0:-53,-42,0:31:31,31:0
+1 974225 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488995;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=G-rich|Low_complexity|27.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:110:110,937:110,937:52,50:-937,-110,0:-52,-50,0:42:42,42:0
+1 974296 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488996;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:86,815:86,815:45,53:-815,-86,0:-53,-45,0:31:31,31:0
+1 974355 . GT AC . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465132&dbsnp.100|rs2488997&dbsnp.126|rs35487305;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:47:47,414:47,414:34,33:-414,-47,0:-34,-33,0:21:21,21:0
+1 974494 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465133;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=MIR|MIR|35.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:678,84:678,84:53,44:-678,-84,0:-53,-44,0:32:32,32:0
+1 974570 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465134;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:471,66:471,66:53,38:-471,-66,0:-53,-38,0:30:30,30:0
+1 974662 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2465135;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:72,553:72,553:48,52:-553,-72,0:-52,-48,0:28:27,27:1
+1 974791 . T TGG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs36095704&dbsnp.130|rs71576591;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:28,.:28,.:1,.:-28,0,0:-1,0,0:11:3,.:0
+1 974894 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121578;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:44:395,44:395,44:52,40:-395,-44,0:-52,-40,0:20:20,20:0
+1 975133 . T A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121579;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:40:40,351:40,351:37,48:-351,-40,0:-48,-37,0:16:16,16:0
+1 975419 . GGCGCCAGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 975537 . TCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 975551 . CGCTGCAGCAGCGCGG . . . NS=1;AN=0 GT:PS ./.:.
+1 975702 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9331224;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:62:457,62:457,62:52,47:-457,-62,0:-52,-47,0:29:29,29:0
+1 975862 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 975900 . ACG . . . NS=1;AN=0 GT:PS ./.:.
+1 976001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=978000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.28:0.70:2:47:=:47:0.999:303
+1 976047 . TGCCGGGGAATG . . . NS=1;AN=0 GT:PS ./.:.
+1 976075 . CCGTGTGCGAGCCCAACGCGGAGGGGCCGGGCCGGGCGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 976200 . C <CGA_NOCALL> . . END=976517;NS=1;AN=0 GT:PS ./.:.
+1 976620 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 976623 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 976788 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 976793 . GGC . . . NS=1;AN=0 GT:PS ./.:.
+1 976805 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 976937 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:976937:VQLOW:24,.:0,.:0,.:0:0
+1 976940 . GCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 976956 . GCCCGGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 977056 . TGCGCTCCGGCCAGTGCCAGGGTCGAGGTGAGCGGCTCCCCCGGGGGAGGGCTCCGGCCAGTGCCAGGGTCGAGGTGGGCGGCTCCCCCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 977178 . TGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 977187 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 977203 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3121552;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:977203:PASS:20:20,53:20,53:21,23:-53,-20,0:-23,-21,0:15:15,15:0
+1 977330 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799066;CGA_FI=375790|NM_198576.2|AGRN|ACCEPTOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:551,74:551,74:53,41:-551,-74,0:-53,-41,0:32:31,31:1
+1 977516 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 977522 . GCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 977570 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710876;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:77:737,77:737,77:53,42:-737,-77,0:-53,-42,0:31:31,31:0
+1 977780 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710875;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:89:735,89:735,89:53,46:-735,-89,0:-53,-46,0:33:33,33:0
+1 978001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=980000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:0.95:2:50:=:50:0.999:303
+1 978387 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 978477 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 978480 . G C . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:978480:VQLOW:26:26,26:0,0:0,11:-26,0,-26:0,0,-11:22:2,20:20
+1 978580 . GGGGCTTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 978592 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 978598 . GAGCCCCT GAGCCC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.134|rs138543546;CGA_FI=375790|NM_198576.2|AGRN|ACCEPTOR|UNKNOWN-INC&375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:978598:PASS:331,.:289,.:40,.:-331,0,0:-40,0,0:39:25,.:0
+1 978747 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 979121 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 979384 . GGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 980001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=982000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.77:2:48:=:48:0.999:303
+1 980460 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128097;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:74:74,665:74,665:48,52:-665,-74,0:-52,-48,0:28:27,27:1
+1 981087 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128098;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:101:101,818:101,818:50,53:-818,-101,0:-53,-50,0:39:39,39:0
+1 981639 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 981776 . GAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 981828 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 981891 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 981895 . TACC . . . NS=1;AN=0 GT:PS ./.:.
+1 981905 . AGC . . . NS=1;AN=0 GT:PS ./.:.
+1 981917 . CCTCCGCCCTCATCACGAC CCTCCGCCCTCATCGCGAC . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2465128;CGA_FI=375790|NM_198576.2|AGRN|CDS|NO-CHANGE&375790|NM_198576.2|AGRN|CDS|SYNONYMOUS GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:981917:PASS:116,.:109,.:41,.:-116,0,0:-41,0,0:9:5,.:0
+1 981948 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 981951 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 981958 . CAGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 981970 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 981974 . CCCGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 981997 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 982001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=984000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:0.86:2:47:=:47:0.999:303
+1 982001 . CCC . . . NS=1;AN=0 GT:PS ./.:.
+1 982013 . CCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 982249 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 982258 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 982268 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 982274 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 982278 . TGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 982413 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 982444 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128099;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:31:31,269:31,269:30,52:-269,-31,0:-52,-30,0:20:20,20:0
+1 982455 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 982462 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128100;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:75:75,635:75,635:48,52:-635,-75,0:-52,-48,0:26:26,26:0
+1 982513 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128101;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:363,48:363,48:48,42:-363,-48,0:-48,-42,0:18:18,18:0
+1 982941 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.103|rs3128102;CGA_FI=375790|NM_198576.2|AGRN|ACCEPTOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:67:576,67:576,67:53,38:-576,-67,0:-53,-38,0:33:33,33:0
+1 982994 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.52|rs10267;CGA_FI=375790|NM_198576.2|AGRN|CDS|SYNONYMOUS;CGA_PFAM=PFAM|PF01390|SEA GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:80:579,80:579,80:53,43:-579,-80,0:-53,-43,0:32:32,32:0
+1 983149 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 983422 . CGTCAGGAGCCATT . . . NS=1;AN=0 GT:PS ./.:.
+1 983444 . AGCCACGG . . . NS=1;AN=0 GT:PS ./.:.
+1 983473 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 983502 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 983506 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 983521 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 983566 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 983589 . CGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 983601 . CGTCGGCCCCCGGCCCCCCAGCAGCCTCCAAAGCCCTGTGACTCACAGCCCTGCTTCCACGGGGGGACCTGCCAGGACTGGGCATTGGGCGGGGGCTTCACCTGCAGCTGCCCGGCAGGCAGGGGAGGCGCCGTCTGTGAGAAGGGTAAGGATGTCCACTGCAGAGGAGGGCGGGGAGGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 984001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=986000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.89:0.63:2:43:=:43:0.999:303
+1 984103 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 984148 . AAAAAAAAAAAA A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.129|rs57668569;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=AluSp|Alu|7.6 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:54,.:54,.:16,.:-54,0,0:-16,0,0:7:7,.:0
+1 984171 . CAG C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.134|rs140904842;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:15:208,15:208,15:35,21:-208,-15,0:-35,-21,0:12:12,12:0
+1 984249 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 984252 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 984262 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 984302 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442391;CGA_FI=375790|NM_198576.2|AGRN|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:42,402:42,402:39,48:-402,-42,0:-48,-39,0:19:19,19:0
+1 984386 . CCG . . . NS=1;AN=0 GT:PS ./.:.
+1 984469 . GCTCAGGTGGGCGGGGAGGGGACGGGCGGGGGAGGGGGGGCCGGGGCAGCTCAGGTGGGTGGGGTGGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 984543 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 984635 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 985202 . A G . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:32:32,32:29,29:7,25:-32,0,-32:-7,0,-25:10:3,7:7
+1 985266 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2275813;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:45:45,453:45,453:41,52:-453,-45,0:-52,-41,0:20:20,20:0
+1 985379 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:985379:VQLOW:20,.:14,.:0,.:0:0
+1 985396 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 985424 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 985445 . GGG GT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2275812&dbsnp.100|rs2799067&dbsnp.130|rs71576592;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:985445:PASS:241:325,241:323,269:33,32:-325,0,-241:-33,0,-32:27:16,11:11
+1 985449 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2799067&dbsnp.129|rs56255212;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:985445:PASS:136:231,136:230,143:50,44:-136,0,-231:-44,0,-50:23:16,7:16
+1 985450 . G A . . NS=1;AN=2;AC=1;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:985445:PASS:136:231,136:230,143:52,47:-231,0,-136:-52,0,-47:20:13,7:7
+1 985460 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2275811;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:985445:PASS:273:508,273:505,278:53,52:-508,0,-273:-53,0,-52:35:22,13:13
+1 986001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=988000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.89:2:48:=:48:0.999:303
+1 986060 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 986064 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:986064:VQLOW:24,.:0,.:0,.:0:0
+1 986443 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710887;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:675,79:675,79:53,42:-675,-79,0:-53,-42,0:31:31,31:0
+1 986579 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 986684 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 986687 . GACT . . . NS=1;AN=0 GT:PS ./.:.
+1 986816 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 986840 . TCGGAGGCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 986855 . TGCTGACCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 987200 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9803031;CGA_FI=375790|NM_198576.2|AGRN|DONOR|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:32:32,295:32,295:31,48:-295,-32,0:-48,-31,0:11:11,11:0
+1 987670 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303287;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|33.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:70:70,582:70,582:48,52:-582,-70,0:-52,-48,0:25:25,25:0
+1 987894 . C CGT . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs34235844;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC;CGA_RPT=(TG)n|Simple_repeat|23.6 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:151,.:169,.:35,.:-151,0,0:-35,0,0:15:11,.:4
+1 988001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=990000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:0.84:2:50:=:50:0.999:303
+1 988503 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799071;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:126:126,1072:126,1072:52,50:-1072,-126,0:-52,-50,0:47:46,46:1
+1 988742 . TGTGGGTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 988900 . GAC . . . NS=1;AN=0 GT:PS ./.:.
+1 988932 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710871;CGA_FI=375790|NM_198576.2|AGRN|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:32:564,32:564,75:52,48:-564,-32,0:-52,-48,0:23:23,23:0
+1 990001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=992000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:1.03:2:52:=:52:0.999:303
+1 990126 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 990280 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4275402;CGA_FI=375790|NM_198576.2|AGRN|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:41:339,41:339,41:48,38:-339,-41,0:-48,-38,0:19:19,19:0
+1 990320 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 990465 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 990517 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2710872;CGA_FI=375790|NM_198576.2|AGRN|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:476,50:476,50:52,43:-476,-50,0:-52,-43,0:23:23,23:0
+1 990602 . AGC . . . NS=1;AN=0 GT:PS ./.:.
+1 990773 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799072;CGA_FI=375790|NM_198576.2|AGRN|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:57:57,416:57,416:46,52:-416,-57,0:-52,-46,0:25:23,23:2
+1 990806 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2799073;CGA_FI=375790|NM_198576.2|AGRN|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:91:762,91:762,91:52,48:-762,-91,0:-52,-48,0:29:29,29:0
+1 990898 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 990905 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 990984 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.52|rs8014;CGA_FI=375790|NM_198576.2|AGRN|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:69:604,69:604,69:52,48:-604,-69,0:-52,-48,0:27:27,27:0
+1 991233 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 991249 . GCCAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 991347 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 991364 . CCCCCAGCCCCAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 991392 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 991397 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 991401 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 991500 . TTGCTTTTGTCCATCCTCACCAGCGCGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 991548 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 991604 . CAG . . . NS=1;AN=0 GT:PS ./.:.
+1 991611 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 991658 . AC A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs60286592 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:24:24,207:24,207:28,35:-207,-24,0:-35,-28,0:13:13,13:0
+1 991721 . GTGCGTGTACGTGTGGGGGTGTGTGTGTGTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 991793 . GCACGTGTGGGTGCGGGTACGTGTGGGTGCGGGTACGTGTGGGTGCATGT . . . NS=1;AN=0 GT:PS ./.:.
+1 991884 . GGTACGTGTGGGTGTGGGTGCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 991925 . GCATGTATGGGTGCATGTACGTGTGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 991974 . GGCGTGTATGTGTGGGTGCGTGTGCGTGTGGGTGCGTGTGCTTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 992001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=994000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:0.90:2:53:=:53:0.999:303
+1 992038 . GGTGCATGTACGTGTGTGGGTGCGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 992076 . GGCGTGTATGTGTGGGTGTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 992115 . GCACGTGTGTGTGTGTGTGTGTGTGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 992327 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2245754 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:667,72:667,72:53,40:-667,-72,0:-53,-40,0:35:35,35:0
+1 992622 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 992635 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 992651 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 992819 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9331226 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:33:367,33:367,33:48,31:-367,-33,0:-48,-31,0:15:15,15:0
+1 992840 . ATT A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs34537778&dbsnp.129|rs55929714&dbsnp.134|rs150915126 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:416,51:422,51:38,38:-416,-51,0:-38,-38,0:27:27,27:0
+1 992852 . AAG A . . NS=1;AN=2;AC=2;CGA_RPT=AluSx1|Alu|12.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:22:22,165:22,165:27,36:-165,-22,0:-36,-27,0:20:20,20:0
+1 993360 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs13303240;CGA_RPT=AluSx1|Alu|11.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:455,66:455,66:52,48:-455,-66,0:-52,-48,0:25:25,25:0
+1 993402 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:993402:VQLOW:21,.:0,.:0,.:0:0
+1 993405 . TTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 993916 . GCA . . . NS=1;AN=0 GT:PS ./.:.
+1 994001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=996000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.98:2:53:=:53:0.999:303
+1 994391 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2488991 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:394,50:394,50:52,43:-394,-50,0:-52,-43,0:27:27,27:0
+1 994519 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 994569 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 994573 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 994674 . GGGAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 994768 . CCCGCCCCCCAGCCTGGAGCGCCCCCCTCCGGCCCCGGTCCGCAGTGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 994874 . AA . . . NS=1;AN=0 GT:PS ./.:.
+1 994878 . CCCGCGACCCGCGACCCGGCTGCCCGCGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 994915 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 994924 . CGGGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 994933 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 994939 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 994958 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 994966 . CGCCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 994991 . ACCCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 995481 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442393;CGA_RPT=AluSg|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:82:637,82:637,82:53,44:-637,-82,0:-53,-44,0:38:37,37:1
+1 996001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=998000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.62:0.92:2:53:=:53:0.999:303
+1 996354 . TTAT . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:VQLOW:28,.:0,.:0,.:0:0
+1 997263 . GTGGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 997275 . TCCTAGAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 997290 . GCCCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 997309 . TA . . . NS=1;AN=0 GT:PS ./.:.
+1 997408 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.125|rs28397086 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:104:799,104:799,104:50,51:-799,-104,0:-51,-50,0:43:43,43:0
+1 997436 . CTCCCTCCCTTGTCCCCGTTCCCTCCG C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs34515215&dbsnp.129|rs59725750&dbsnp.134|rs145846158;CGA_RPT=C-rich|Low_complexity|18.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:325,46:325,46:36,31:-325,-46,0:-36,-31,0:43:43,43:0
+1 997524 . CCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 997585 . AGT . . . NS=1;AN=0 GT:PS ./.:.
+1 997602 . GCTCTTTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 997618 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 997623 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 997636 . GCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 997666 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 997709 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 997715 . GCTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 997723 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 997727 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 997731 . GACCTCATGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 997747 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 997751 . ACTAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 997765 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 997768 . ACGGGGGGGCTGTTCACCAGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 997797 . GGGCTGCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 997828 . CGGCAGCACCAGCCTCTGCCTGCATGGGGCCGCGAGGTTTGCAGTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 997886 . CTTCCTGACCTGCCCCGGACACGGAGCACGGCTCC . . . NS=1;AN=0 GT:PS ./.:.
+1 998001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1000000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.09:0.88:2:51:=:51:0.999:303
+1 998055 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 998062 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 998185 . GAGGCTGGGAGGGGTGGAGTGGACGGGGCTGCTGACAGCCTCGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 998395 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7526076 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:136:1183,136:1183,136:49,52:-1183,-136,0:-52,-49,0:50:50,50:0
+1 998582 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3813194 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:77:637,77:637,77:53,42:-637,-77,0:-53,-42,0:33:33,33:0
+1 999041 . ATGTG A,ATG . . NS=1;AN=2;AC=1,1;CGA_XR=dbsnp.114|rs5772030&dbsnp.114|rs5772031&dbsnp.114|rs5772032&dbsnp.114|rs5772033&dbsnp.129|rs59276350&dbsnp.130|rs63618161&dbsnp.132|rs112424583&dbsnp.134|rs150552617,dbsnp.114|rs5772030&dbsnp.114|rs5772031&dbsnp.114|rs5772032&dbsnp.114|rs5772033&dbsnp.129|rs59276350&dbsnp.130|rs63618161&dbsnp.134|rs150552617;CGA_RPT=(TG)n|Simple_repeat|19.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/2:.:PASS:89:89,252:84,247:26,30:-252,-252,-252,-89,0,-89:-30,-30,-30,-26,0,-26:19:7,9:1
+1 999840 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1000001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1002000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.62:0.70:2:47:=:47:0.999:312
+1 1000156 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs11584349 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:43:43,228:43,228:39,48:-228,-43,0:-48,-39,0:18:18,18:0
+1 1000164 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 1000204 . CCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1000212 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1000231 . G <CGA_NOCALL> . . END=1000696;NS=1;AN=0 GT:PS ./.:.
+1 1001177 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970401;CGA_RPT=LTR13A|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,501:52,501:44,52:-501,-52,0:-52,-44,0:25:24,24:1
+1 1001562 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1002001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1004000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:1.02:2:51:=:51:0.999:312
+1 1002434 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs11260596 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:86:711,86:711,86:53,45:-711,-86,0:-53,-45,0:33:33,33:0
+1 1002932 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4246502 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:61:61,396:61,396:47,52:-396,-61,0:-52,-47,0:29:29,29:0
+1 1003053 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4074992 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:100:847,100:847,100:53,50:-847,-100,0:-53,-50,0:36:36,36:0
+1 1003629 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4075116 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:111:111,1064:111,1064:52,50:-1064,-111,0:-52,-50,0:44:43,43:1
+1 1003678 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1003683 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 1003962 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1004001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1006000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:0.73:2:48:=:48:0.999:312
+1 1004017 . CGGGCCGATCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1004034 . CGTCCTCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1004051 . ACG . . . NS=1;AN=0 GT:PS ./.:.
+1 1004090 . G <CGA_NOCALL> . . END=1004535;NS=1;AN=0 GT:PS ./.:.
+1 1004581 . GTCCGCAGGGCTGGACTGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1004687 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 1004721 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1004736 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 1004740 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1004754 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1004957 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4073176 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:33:463,33:463,33:52,31:-463,-33,0:-52,-31,0:23:23,23:0
+1 1004980 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.108|rs4073177 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:72:607,72:607,72:52,48:-607,-72,0:-52,-48,0:29:29,29:0
+1 1005251 . CACCGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1006001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1008000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.88:2:51:=:51:0.999:312
+1 1006223 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442394 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:57:57,455:57,455:46,52:-455,-57,0:-52,-46,0:22:22,22:0
+1 1006947 . GACTGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1006990 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4326571 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:79:79,548:79,548:42,53:-548,-79,0:-53,-42,0:31:31,31:0
+1 1007120 . GTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1007158 . CTGGGCGAGGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1007203 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4633229;CGA_FI=401934|XM_002342025.2|LOC401934|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:22:225,22:225,22:48,22:-225,-22,0:-48,-22,0:13:13,13:0
+1 1007360 . CGCCTCCAGTCCCTGCAGCGCGCCCAGCAGCGGGCCAGGCGGCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1007432 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4333796;CGA_FI=401934|XM_002342025.2|LOC401934|CDS|MISSENSE;CGA_RPT=GC_rich|Low_complexity|5.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:13:200,13:200,13:48,16:-200,-13,0:-48,-16,0:12:12,12:0
+1 1007439 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1008001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1010000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:0.92:2:53:=:53:0.999:312
+1 1009231 . CTGTGA CTGCGA . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9442366;CGA_FI=401934|XM_002342025.2|LOC401934|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:99,.:89,.:37,.:-99,0,0:-37,0,0:12:9,.:0
+1 1009478 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442367;CGA_FI=401934|XM_002342025.2|LOC401934|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:40:40,483:40,483:37,52:-483,-40,0:-52,-37,0:25:25,25:0
+1 1009554 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1009558 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1010001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1012000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:0.72:.:0:.:0:0.999:312
+1 1010717 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442368;CGA_FI=401934|XM_002342025.2|LOC401934|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:83:690,83:690,83:53,44:-690,-83,0:-53,-44,0:31:31,31:0
+1 1010997 . GTCGGGGGTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1011041 . GTCAGGGGTTGGGGGGACCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1011088 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1011092 . GTTA . . . NS=1;AN=0 GT:PS ./.:.
+1 1011303 . GGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1011310 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 1011318 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1011331 . GTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1011342 . CTGAGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1011373 . TGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1011401 . GCTGAGGTTATGGGGACTCCGTGCTGGGAGGCTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1011471 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1011485 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1011492 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1011522 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1011538 . TC . . . NS=1;AN=0 GT:PS ./.:.
+1 1011543 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1011561 . GGTGACTCCGTGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1011582 . GA . . . NS=1;AN=0 GT:PS ./.:.
+1 1011595 . CT . . . NS=1;AN=0 GT:PS ./.:.
+1 1011616 . C <CGA_NOCALL> . . END=1014753;NS=1;AN=0 GT:PS ./.:.
+1 1012001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1014000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.01:0.19:.:0:.:0:0.999:312
+1 1014001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1016000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.61:0.82:.:0:.:0:0.999:312
+1 1014836 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12401605 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:317,39:317,39:48,36:-317,-39,0:-48,-36,0:14:14,14:0
+1 1014864 . T A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12411041 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,396:52,396:44,48:-396,-52,0:-48,-44,0:19:19,19:0
+1 1015026 . GTGTGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1015126 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs36027499 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:51:498,51:498,51:52,44:-498,-51,0:-52,-44,0:22:22,22:0
+1 1015257 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442369 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:35:35,349:35,349:33,52:-349,-35,0:-52,-33,0:21:21,21:0
+1 1015551 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442370 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:71:467,71:467,71:53,40:-467,-71,0:-53,-40,0:33:33,33:0
+1 1015618 . TAGGCCCAGCTC T . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:469:469,469:468,468:40,42:-469,0,-469:-40,0,-42:37:17,20:20
+1 1015817 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.121|rs12746483 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:12:209,12:209,12:48,16:-209,-12,0:-48,-16,0:10:10,10:0
+1 1015855 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1016001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1018000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.96:0.85:2:52:=:52:0.999:312
+1 1016058 . TCCGCCCCCACCTCGGTCCCTGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1016088 . CCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTCGGTCCCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1016278 . TCCCTCCGCCCCCACCTCGGTCCCTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1016310 . TCCCTCCGCCCCCACCTCGGTCCCTGTCTCCTTCCCTCCGCCCCCACCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1016429 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1017028 . GC GG,G . . NS=1;AN=2;AC=1,1;CGA_XR=dbsnp.111|rs4970352,. GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/2:.:VQLOW:31:127,31:124,27:41,5:-127,-31,-31,-127,0,-127:-41,-5,-5,-41,0,-41:21:9,4:0
+1 1017170 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3766193 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:70:70,721:70,721:48,52:-721,-70,0:-52,-48,0:28:28,28:0
+1 1017197 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3766192 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:139:139,1236:139,1236:52,49:-1236,-139,0:-52,-49,0:55:55,55:0
+1 1017341 . G T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.86|rs1133647;CGA_FI=54991|NM_017891.4|C1orf159|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:59:530,59:530,59:52,46:-530,-59,0:-52,-46,0:23:23,23:0
+1 1018001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1020000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.10:0.97:2:53:=:53:0.999:312
+1 1018144 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442395;CGA_FI=54991|NM_017891.4|C1orf159|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:90:90,859:90,859:46,53:-859,-90,0:-53,-46,0:34:34,34:0
+1 1018562 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442371;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:89:89,720:89,720:46,53:-720,-89,0:-53,-46,0:32:32,32:0
+1 1018678 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:504,.:457,.:0,.:0:0
+1 1018704 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442372;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=MLT1C|ERVL-MaLR|17.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:91:811,91:764,44:50,31:-811,-91,0:-50,-31,0:44:43,43:0
+1 1019106 . GGACGGGGCACG . . . NS=1;AN=0 GT:PS ./.:.
+1 1019175 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2298215;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:83:717,83:717,83:53,44:-717,-83,0:-53,-44,0:30:30,30:0
+1 1019180 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442396;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:84:84,727:84,727:44,53:-727,-84,0:-53,-44,0:36:36,36:0
+1 1019962 . CAGCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1020001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1022000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.20:0.92:2:53:=:53:0.999:312
+1 1020007 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1020177 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1020406 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442397;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:76:618,76:618,76:52,48:-618,-76,0:-52,-48,0:28:28,28:0
+1 1020581 . ACCTGGGAGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1020885 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1020923 . A AG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.130|rs70949550;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:67:67,512:67,512:37,39:-512,-67,0:-39,-37,0:29:29,29:0
+1 1021415 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.107|rs3737728;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:26:343,26:343,26:52,26:-343,-26,0:-52,-26,0:20:20,20:0
+1 1021695 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442398;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:76:76,713:76,713:41,53:-713,-76,0:-53,-41,0:32:31,31:1
+1 1021873 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1022001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1024000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.85:0.96:2:53:=:53:0.999:312
+1 1022037 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6701114;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:110:110,984:110,984:52,50:-984,-110,0:-52,-50,0:41:41,41:0
+1 1022457 . GCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1022502 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1023139 . C G . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:254:338,254:338,253:53,51:-338,0,-254:-53,0,-51:31:17,14:14
+1 1023441 . CTGCAGG CTGGAGG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.119|rs9442399;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:1023441:PASS:267,.:274,.:52,.:-267,0,0:-52,0,0:21:14,.:0
+1 1023453 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1023456 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1023531 . A <CGA_NOCALL> . . END=1023734;NS=1;AN=0 GT:PS ./.:.
+1 1023853 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1023910 . CCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1023920 . CAAGCCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1023930 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1023934 . CCGCCAGGCCGACGCTGCGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1024001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1026000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:0.88:2:53:=:53:0.999:312
+1 1024095 . GGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1024194 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1024897 . GGCTCCCCAACCCCCACGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1025301 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442400;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:228:228,228:227,227:46,49:-228,0,-228:-46,0,-49:37:17,20:20
+1 1025797 . GGGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1026001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1028000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:0.92:2:53:=:53:0.999:312
+1 1026657 . CCCAGCAGGCAGCGGAGGGCTCCCCTCTGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1026702 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1026707 . CC AC,AT . . NS=1;AN=2;AC=1,1;CGA_XR=dbsnp.108|rs4074137,.;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC,54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|2:1026707:VQLOW:20:186,20:182,17:30,0:-186,-20,-20,-186,0,-186:-30,0,0,-30,0,-30:11:8,1:0
+1 1026712 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 1026726 . AGCACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1026801 . T A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4562563;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:46:377,46:377,46:48,41:-377,-46,0:-48,-41,0:16:16,16:0
+1 1026898 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1026902 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1026905 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 1026924 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 1027094 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1027097 . TGGAGGGTGGGGCCAAATGGAAGTGGGCGGGGCTGTGGTGGAGGGTGGGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1027155 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1027155:VQLOW:24,.:20,.:0,.:0:0
+1 1027199 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1027208 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1028001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1030000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.94:2:53:=:53:0.999:312
+1 1030001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1032000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.88:2:51:=:51:0.999:312
+1 1030347 . TGCTAAAAAGACT . . . NS=1;AN=0 GT:PS ./.:.
+1 1030592 . TCCATTT . . . NS=1;AN=0 GT:PS ./.:.
+1 1031540 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9651273;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=L1MD3|L1|40.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:109:109,1043:109,1043:52,50:-1043,-109,0:-52,-50,0:40:40,40:0
+1 1031719 . ATAATTTTTTTTGTAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1032001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1034000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.90:0.92:2:48:=:48:0.999:312
+1 1032184 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9651272;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=AluJo|Alu|22.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:55:55,572:55,572:45,52:-572,-55,0:-52,-45,0:25:25,25:0
+1 1032579 . C T . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|10.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:32:32,32:12,12:0,18:-32,0,-32:0,0,-18:16:2,14:14
+1 1032965 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1033999 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970353;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=L1MD3|L1|40.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:38:288,38:288,38:52,36:-288,-38,0:-52,-36,0:26:26,26:0
+1 1034001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1036000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:0.58:2:42:=:42:0.999:312
+1 1035053 . C T . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=L1MD|L1|23.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:20:20,20:0,0:0,11:-20,0,-20:0,0,-11:20:3,17:17
+1 1036001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1038000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.94:2:49:=:49:0.999:312
+1 1036533 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1036533:PASS:40,.:5,.:0,.:0:0
+1 1036536 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1037452 . AAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1037995 . AAAAAAAAAAAAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1038001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1040000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:0.86:2:51:=:51:0.999:312
+1 1038106 . AGGCAGAAGTTGCAGTGAGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1038130 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1038134 . TGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1040001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1042000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.67:0.95:2:51:=:51:0.999:312
+1 1040195 . AAGGAAAAAAAAAAAGCTTT . . . NS=1;AN=0 GT:PS ./.:.
+1 1040418 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs77147003;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=MIR|MIR|47.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:171:543,171:528,156:53,42:-543,0,-171:-53,0,-42:33:24,9:9
+1 1040548 . AACAAATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 1040918 . ATTTAAAAAACAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 1041007 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1041026 . CCGAGGTGGGTGGATCACGAGGTCAGGAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1041088 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1041129 . CTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1041154 . TGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1041165 . AATAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1041173 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1041176 . CCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1041261 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1041265 . AAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 1041451 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1041455 . AACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 1041462 . AAGTCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1042001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1044000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:1.25:2:47:=:47:0.999:312
+1 1043612 . CAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1043673 . TCTCGGCACCGTTCACCACAGCCACCATGTCTCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1043854 . CTCAGCAGCACCGTCCACCACAGCCACCATGTCTCGGCAGCACCGTCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1044001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1046000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.03:.:0:.:0:0.999:312
+1 1044008 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1044036 . TCTCGGCAGCACCGTTCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 1044098 . CTCAGCAGCACCGTCCACCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1044137 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1044143 . TT . . . NS=1;AN=0 GT:PS ./.:.
+1 1044171 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1044242 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1044251 . GT . . . NS=1;AN=0 GT:PS ./.:.
+1 1044256 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1044448 . ACCGTTCACCACAGCCACCATGTCTGCAGCAGCATTGTTCACCACAGCCAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1046001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1048000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.92:1.02:2:50:=:50:0.999:312
+1 1046320 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 1046746 . AAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1047077 . CAGTA . . . NS=1;AN=0 GT:PS ./.:.
+1 1047094 . AGG AT . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1047094:VQLOW:21:21,21:4,4:0,17:-21,0,-21:0,0,-17:8:1,7:7
+1 1047106 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 1047422 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1048001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1050000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.79:0.90:2:53:=:53:0.999:312
+1 1048572 . TCTCAAAAAAAAAAAAAAAACTTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1048862 . TCCGGGGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1049294 . TTT . . . NS=1;AN=0 GT:PS ./.:.
+1 1050001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1052000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.69:0.87:2:52:=:52:0.999:312
+1 1050391 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs149580770;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC;CGA_RPT=L1PB1|L1|21.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:514:726,514:724,512:49,54:-726,0,-514:-49,0,-54:56:32,24:24
+1 1050892 . A <CGA_NOCALL> . . END=1051143;NS=1;AN=0 GT:PS ./.:.
+1 1051157 . GACGCCCCTTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051187 . TCAGCCCGGGCGCCTTTCCCCCATAGGACCGCGGCCAGGCTCGTTGGGAGGCGGCGACGAGGACGCGGGCCCAGGCGCTGGCGGCTCCTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1051297 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051303 . AGGGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1051312 . C T . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1051312:VQLOW:21:21,25:20,24:0,24:-21,0,-25:0,0,-24:4:1,3:3
+1 1051319 . GCCGGGCCAGGGCCGCCGACCTTTGTCTGCCTCTCGCACTCCCTGCGCCGACCCGGCCGCCCAGACGGACCCCAGCGCCCCAACCCGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1051417 . CCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1051439 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1051444 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1051447 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1051450 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1051496 . GCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051514 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1051526 . CCCGGCCCCGGCGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1051547 . CGCAGCTCCCAAAGAAAACTACAACTCCCGGCGGCCCGCGCGAGAGCCGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051617 . GCGCGCGGCCGTGGGTGGGGCGCCGGGGCGGGGCGCGAAGCGCCCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1051748 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1051751 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1051757 . CGTTGCCGGGAGACGGGGCGGGGCGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1051785 . TCGGGGTCTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051927 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 1051931 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1051968 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1052001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1054000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:0.83:2:52:=:52:0.999:312
+1 1052368 . CTTCAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 1052397 . CTCCTAGCTGGGCCAGCGCGCAGGGTGGGGGGGCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1052511 . CCCCCCATATACCCCCAACCCCTCAGACCCCCCAACCCCCCAGACCCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1052574 . CCCAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1052842 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1052845 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1054001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1056000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.79:0.95:2:53:=:53:0.999:312
+1 1054179 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1055427 . GA G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs34088984&dbsnp.130|rs71576602;CGA_FI=54991|NM_017891.4|C1orf159|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSz|Alu|16.2 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:VQLOW:22,.:17,.:0,.:-22,0,0:0,0,0:18:5,.:13
+1 1055654 . T C . . NS=1;AN=2;AC=1;CGA_FI=54991|NM_017891.4|C1orf159|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluJo|Alu|28.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:84:84,84:64,64:22,35:-84,0,-84:-22,0,-35:28:6,22:22
+1 1056001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1058000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:0.87:2:53:=:53:0.999:312
+1 1056050 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1056053 . GCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1056058 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1056874 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1056928 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1056938 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1057024 . CCCCATTGCCCCCTGGGATTGCCCCCCCTCCCCCACCCCACCCCATCTGGGACTCCTGCCCCTACAGCAGCCGCAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1057321 . CTCTGTGTGGGGGGCGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1058001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1060000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.97:0.87:2:53:=:53:0.999:312
+1 1060001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1062000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.84:1.06:2:52:=:52:0.999:312
+1 1061395 . CCATAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1061483 . C G . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:20:20,20:3,3:0,12:-20,0,-20:0,0,-12:14:2,12:12
+1 1061537 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1061691 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1061931 . TCCGCCCGCCTTGGGGGAGGCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1062001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1064000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.80:2:51:=:51:0.999:312
+1 1062171 . CCCTCCGCGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1062229 . CCGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1062638 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442373 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:87:87,87:78,78:27,37:-87,0,-87:-27,0,-37:25:7,18:18
+1 1063044 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7545801 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:66:148,66:145,63:46,34:-148,0,-66:-46,0,-34:11:7,4:4
+1 1063241 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970413 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:180:304,180:301,177:53,44:-304,0,-180:-53,0,-44:30:18,12:12
+1 1064001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1066000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:0.86:2:52:=:52:0.999:312
+1 1064535 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6682475 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:144:171,144:171,144:48,47:-171,0,-144:-48,0,-47:17:9,8:8
+1 1064670 . C G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs7547403 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:53:53,395:53,395:34,53:-395,-53,0:-53,-34,0:35:35,35:0
+1 1064802 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2298216 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:420:520,420:520,420:50,54:-520,0,-420:-50,0,-54:49:25,24:24
+1 1064979 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2298217 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1064979:PASS:101:374,101:365,92:53,31:-374,0,-101:-53,0,-31:31:21,10:10
+1 1065296 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4072537 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1064979:PASS:308:308,310:308,310:53,53:-310,0,-308:-53,0,-53:34:18,16:18
+1 1065567 . AAAAAAAAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1065592 . AAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1065824 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1065842 . TCTCTGCTGTCACT . . . NS=1;AN=0 GT:PS ./.:.
+1 1066001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1068000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:1.09:2:50:=:50:0.999:312
+1 1066259 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4072496;CGA_RPT=HAL1|L1|45.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066259:PASS:264:264,407:263,407:43,54:-264,0,-407:-43,0,-54:40:17,23:23
+1 1066282 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907181 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066259:PASS:236:284,236:287,236:52,50:-284,0,-236:-52,0,-50:29:16,13:13
+1 1066388 . C CT . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34287831&dbsnp.134|rs140634183;CGA_RPT=HAL1|L1|43.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066388:PASS:204:204,205:218,205:39,40:-204,0,-205:-39,0,-40:27:17,10:10
+1 1066403 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907182;CGA_RPT=HAL1|L1|43.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066388:PASS:258:258,329:258,333:52,50:-258,0,-329:-52,0,-50:27:12,15:15
+1 1066816 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1066819 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7513297;CGA_RPT=AluY|Alu|6.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066819:PASS:388:450,388:451,392:50,54:-450,0,-388:-50,0,-54:40:19,21:21
+1 1066828 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7553878;CGA_RPT=AluY|Alu|6.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066819:PASS:228:594,228:597,229:53,49:-594,0,-228:-53,0,-49:37:23,14:14
+1 1066946 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7513404;CGA_RPT=AluY|Alu|6.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066946:PASS:335:335,551:335,549:47,54:-335,0,-551:-47,0,-54:45:18,27:27
+1 1066952 . AT GC . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7513405&dbsnp.116|rs7516160&dbsnp.126|rs34955020;CGA_RPT=AluY|Alu|6.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1066946:PASS:460:460,568:478,567:33,39:-460,0,-568:-33,0,-39:43:19,24:24
+1 1067596 . CAG C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs141257782 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:250:250,250:250,250:38,40:-250,0,-250:-38,0,-40:28:13,15:15
+1 1067674 . T TGG . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs35574593&dbsnp.131|rs79280895 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:160:160,162:160,162:39,39:-160,0,-162:-39,0,-39:21:10,11:11
+1 1067862 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442374 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1067862:PASS:287:287,360:323,356:53,54:-287,0,-360:-53,0,-54:33:15,18:18
+1 1067865 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442358 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1067862:PASS:245:245,360:245,356:48,54:-245,0,-360:-48,0,-54:31:11,20:20
+1 1068001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1070000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.95:1.15:2:49:=:49:0.999:312
+1 1068441 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1068450 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1068459 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1068669 . GT G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34990026&dbsnp.134|rs148642912&dbsnp.134|rs149201820 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:157:157,184:183,183:35,38:-157,0,-184:-35,0,-38:35:16,19:19
+1 1068720 . GGTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1068836 . GCCTGCCTGCCCGGCCTCCTCAGCAGATGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1069425 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442375;CGA_RPT=AluY|Alu|3.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1069425:PASS:233:323,233:326,234:52,50:-323,0,-233:-52,0,-50:28:16,12:12
+1 1069443 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442376;CGA_RPT=AluY|Alu|3.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1069425:PASS:224:377,224:379,223:53,49:-377,0,-224:-53,0,-49:30:18,12:12
+1 1069451 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442377;CGA_RPT=AluY|Alu|3.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1069425:PASS:282:282,443:281,445:51,54:-282,0,-443:-51,0,-54:36:16,20:20
+1 1069475 . AAAAAAAG A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs57309011;CGA_RPT=AluY|Alu|3.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1069425:PASS:146:146,292:150,292:34,40:-146,0,-292:-34,0,-40:25:11,14:14
+1 1070001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1072000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.99:0.86:2:52:=:52:0.999:312
+1 1070128 . T G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442378 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:337,42:337,42:48,39:-337,-42,0:-48,-39,0:17:17,17:0
+1 1070202 . GGAGGGGGACAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1070441 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442379 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:169:340,169:335,165:52,49:-340,0,-169:-52,0,-49:26:17,9:9
+1 1071118 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907183;CGA_RPT=MER41B|ERV1|22.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:423:423,423:423,423:45,54:-423,0,-423:-45,0,-54:50:25,25:25
+1 1071192 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6604971;CGA_RPT=MER41B|ERV1|22.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:392:392,392:392,392:50,54:-392,0,-392:-50,0,-54:49:22,27:27
+1 1072001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1074000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.73:0.87:2:52:=:52:0.999:312
+1 1072409 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1072458 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1072498 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442360 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:115:115,115:115,115:42,43:-115,0,-115:-42,0,-43:13:6,7:7
+1 1072536 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1072542 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1072613 . CGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1072636 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1072645 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 1072732 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1072744 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1072764 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1072810 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1072814 . GAGAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1072830 . GCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1072847 . GCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1073099 . AGCCTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1073949 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1074001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1076000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.64:0.99:.:0:.:0:0.999:312
+1 1074125 . ACAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1074606 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1074671 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1074715 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1074763 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1074824 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1074885 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1074946 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075007 . GGGGACCTGGGTCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075030 . TCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCGAACCGGGGGACCTGGGTCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075130 . GGGGCCTGGGTCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1075152 . TCCTGGGGTCAGAAGGTAGGGGTGTCAACGTCGAACCGGGGGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1075208 . GAGCTTCCTGGGTTCAGAAGGTGGGGGTGTCAGCATCGAACCGGGGGACCTGGGTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1075276 . CTGAGGTCAGAAGGTGGGGGTGTCAGCATCGAACCGGGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCGAACCGGGGGACCTGGGTCCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075423 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1075434 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAGCATCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075495 . GGGGACCTGGGTCCTGGGGAGCTTCCTGGGGTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1075539 . GTGTCAGCATCGAACCGGGGGACCTGGGTCATGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075585 . GGTCAGAAGGTGGGGGTGTCAACGTCGAACCGGGGGGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075640 . TCCTGGGGTCAGAAGGTAGGGGTGTCAACGTCGAACCGGGGGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1075696 . GAGCTTCCTGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075725 . TCAACGTCGAACCGGGGGACCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1075757 . GAGCTTCCTGGGGTCAGAAGGTGGGGGTGTCAACGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1075801 . GGGACCTGGGTCCTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1075925 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs75969607 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:66:66,66:66,66:15,26:-66,0,-66:-15,0,-26:30:14,16:16
+1 1076001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1078000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.75:.:0:.:0:0.999:312
+1 1076491 . C <CGA_NOCALL> . . END=1076943;NS=1;AN=0 GT:PS ./.:.
+1 1077010 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1077064 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970357 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1077064:PASS:102:102,133:108,130:41,45:-102,0,-133:-41,0,-45:14:7,7:7
+1 1077100 . AGGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1077432 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1077435 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1077962 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs55750860 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:210:210,210:209,209:45,48:-210,0,-210:-45,0,-48:30:13,17:17
+1 1078001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1080000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.07:0.91:.:0:.:0:0.999:312
+1 1078279 . GACACCCTTGTGTCTTCGGAAAATGCCAGGTCCCCCCCCAGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1080001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1082000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.24:0.78:.:0:.:0:0.999:312
+1 1080286 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.88|rs1539638 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:70:306,70:323,63:52,34:-306,0,-70:-52,0,-34:26:17,9:9
+1 1080511 . CAACCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1080927 . T TCTGACCTCATGGCCGACCCCAC . . NS=1;AN=2;AC=1;CGA_RPT=(ACCTG)n|Simple_repeat|35.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:238:238,411:244,408:39,41:-238,0,-411:-39,0,-41:39:23,16:16
+1 1081648 . CAGCCCCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1081975 . GCCCAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1082001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1084000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.18:0.97:.:0:.:0:0.999:312
+1 1083613 . CAGCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1084001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1086000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.20:0.96:.:0:.:0:0.999:312
+1 1085043 . GCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1086001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1088000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.02:0.99:.:0:.:0:0.999:312
+1 1086293 . TGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1086305 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1086313 . TCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1086320 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1086325 . GTGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1086338 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1086678 . GT . . . NS=1;AN=0 GT:PS ./.:.
+1 1086896 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1087683 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442380;CGA_RPT=L1MC3|L1|39.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:604:604,604:603,603:48,51:-604,0,-604:-48,0,-51:62:30,32:32
+1 1088001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1090000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.29:0.88:.:0:.:0:0.999:312
+1 1089167 . A AT . . NS=1;AN=2;AC=1;CGA_RPT=AluJb|Alu|41.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:31:31,31:2,2:0,5:-31,0,-31:0,0,-5:31:5,26:26
+1 1089262 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4970358;CGA_RPT=AluJb|Alu|41.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:101:887,101:887,101:53,50:-887,-101,0:-53,-50,0:39:39,39:0
+1 1090001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1092000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:0.86:2:50:=:50:0.999:312
+1 1090010 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442361;CGA_RPT=AluSp|Alu|10.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:192:192,192:192,192:49,50:-192,0,-192:-49,0,-50:21:9,12:12
+1 1090038 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1090055 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1090170 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1090577 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6604972;CGA_RPT=L1MEg|L1|42.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:64:64,640:64,640:48,52:-640,-64,0:-52,-48,0:26:26,26:0
+1 1092001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1094000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.86:0.73:2:48:=:48:0.999:312
+1 1092153 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1092157 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1092269 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1092297 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1092300 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 1092331 . AGGTCGCACAGCAGGACCAGGACCCAGGACCTCGGGCTGGGGACAGAGTGACCTTC . . . NS=1;AN=0 GT:PS ./.:.
+1 1092396 . CCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1092411 . CTTGCAGTGGCCAACAGGTGCCTGGGGTCTT . . . NS=1;AN=0 GT:PS ./.:.
+1 1092459 . CCGGGACAAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1092474 . AGAAAAGGTAGCTAGCTGGAAGAGGGTGCAGGAGGCCCCCCGCTCTGTGCAGCATTAAATCATGGTGGGGTCACCTGCCTTGTCTGGCAGCATGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1092577 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1092599 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs56863140 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1092599:VQLOW:39:39,59:35,55:13,33:-39,0,-59:-13,0,-33:8:2,6:6
+1 1092613 . A <CGA_NOCALL> . . END=1092863;NS=1;AN=0 GT:PS ./.:.
+1 1092982 . CGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1092989 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1092989:VQLOW:29,.:27,.:0,.:0:0
+1 1092991 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1092994 . CGAGGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1093009 . GCTGGGAGGGGCCTCCCTCCGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1094001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1096000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.03:0.89:2:51:=:51:0.999:312
+1 1094157 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 1094190 . C G . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1094190:VQLOW:25:25,25:0,0:0,11:-25,0,-25:0,0,-11:13:2,11:11
+1 1094199 . TG . . . NS=1;AN=0 GT:PS ./.:.
+1 1094485 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970360;CGA_RPT=AluSx|Alu|12.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:90:382,90:372,80:52,37:-382,0,-90:-52,0,-37:22:16,6:6
+1 1094672 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970361;CGA_RPT=AluSx|Alu|12.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:190:190,190:184,184:48,50:-190,0,-190:-48,0,-50:29:10,19:19
+1 1094738 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970362 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:442:442,442:441,441:46,54:-442,0,-442:-46,0,-54:51:23,28:28
+1 1094979 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7538773 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:163:339,163:335,159:52,48:-339,0,-163:-52,0,-48:29:17,12:12
+1 1095383 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6604973;CGA_RPT=AluJb|Alu|18.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:203:426,203:422,199:53,47:-426,0,-203:-53,0,-47:34:21,13:13
+1 1095619 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970419 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:186:263,186:263,185:52,50:-263,0,-186:-52,0,-50:27:14,13:13
+1 1096001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1098000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.18:0.97:2:53:=:53:0.999:312
+1 1096011 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442381 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:70:70,70:57,57:20,33:-70,0,-70:-20,0,-33:24:6,18:18
+1 1096198 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442382 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:216:349,216:348,214:52,50:-349,0,-216:-52,0,-50:28:16,12:12
+1 1096908 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.88|rs1539636 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:145:145,145:117,117:39,43:-145,0,-145:-39,0,-43:26:12,14:14
+1 1096928 . G GT . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:28:28,145:2,117:0,27:-28,0,-145:0,0,-27:34:3,29:29
+1 1097092 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.88|rs1539635 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1097092:PASS:132:184,132:183,136:48,46:-184,0,-132:-48,0,-46:18:10,8:8
+1 1097100 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.88|rs1539634 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1097092:PASS:150:272,150:273,151:52,47:-272,0,-150:-52,0,-47:26:15,11:11
+1 1097287 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442384 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1097287:PASS:56:378,56:359,37:52,27:-378,0,-56:-52,0,-27:22:18,4:4
+1 1097301 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1097335 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9442385 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:192:192,192:185,185:42,45:-192,0,-192:-42,0,-45:39:14,25:25
+1 1097407 . CCCCA C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.134|rs145121017 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:286:358,286:358,286:40,40:-358,0,-286:-40,0,-40:33:17,16:16
+1 1097618 . CGGAGCCTCTTGCCCATTGGGGTGGGACTGGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1097937 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.92|rs1891907;CGA_FI=406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:421,48:421,48:48,42:-421,-48,0:-48,-42,0:17:17,17:0
+1 1098001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1100000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.11:0.87:2:53:=:53:0.999:312
+1 1098088 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1098421 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12135382;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:125:125,125:124,124:43,44:-125,0,-125:-43,0,-44:19:8,11:11
+1 1098610 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 1098618 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 1098645 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1098714 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4379629;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:51:127,51:126,50:44,31:-127,0,-51:-44,0,-31:12:7,5:5
+1 1098820 . TC T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.114|rs5772039&dbsnp.130|rs71578305&dbsnp.134|rs148367913;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:88:88,124:111,123:31,38:-88,0,-124:-31,0,-38:27:16,11:11
+1 1099020 . CTTCTAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1099342 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9660710;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:332:332,332:332,332:39,54:-332,0,-332:-39,0,-54:52:25,27:27
+1 1100001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1102000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.08:0.94:2:53:=:53:0.999:315
+1 1100217 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.92|rs1891905;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC&554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:475:619,475:618,474:49,54:-619,0,-475:-49,0,-54:53:29,24:24
+1 1100319 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.92|rs1891904;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC&554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:144:144,144:143,143:44,47:-144,0,-144:-44,0,-47:23:10,13:13
+1 1101003 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7549819;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC&554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:50:182,50:176,44:48,29:-182,0,-50:-48,0,-29:16:11,5:5
+1 1101393 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1101397 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1101689 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1101858 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1101899 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1101910 . CACCCCCACCCCCAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1101949 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1102001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1104000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:0.91:2:53:=:53:0.999:315
+1 1102069 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442386;CGA_FI=406983|NR_029834.1|MIR200A|TSS-UPSTREAM|UNKNOWN-INC&406984|NR_029639.1|MIR200B|TSS-UPSTREAM|UNKNOWN-INC&554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:55:530,55:530,55:52,45:-530,-55,0:-52,-45,0:26:26,26:0
+1 1102116 . CAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1102162 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1102306 . TGCCCACCCCAGGACCCAAAGCTGGTGGCTGCTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1102573 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1102593 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1102841 . GCCCACAGCGCCTGGGCGGGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1103053 . TCAGCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1103068 . T . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1103068:VQLOW:38,.:16,.:0,.:0:0
+1 1103071 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1103387 . CTGGGGCGGAGGGCCGAGCGGGGCCAGCAGACGGGTGAGGGCGGAGGGCCGAGCGGGGCCAGCAGACGGGTGAGGGCGGAGGGCCGAGCGGGGCCAGCAGACGGGTGAGGGCGGAGGGCTGAGCGGGCGGCAGAGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1103534 . CTCCGAAGTCCAGCCCCCAGGGGAGGGGCCGGCCTCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1103690 . T TCA . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.126|rs34866926&dbsnp.126|rs36015232&dbsnp.131|rs77069227;CGA_FI=554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:128:226,128:222,124:39,36:-226,0,-128:-39,0,-36:23:15,8:8
+1 1103717 . TTTAGGGTGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1103812 . CC . . . NS=1;AN=0 GT:PS ./.:.
+1 1103958 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7521584;CGA_FI=554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:325:325,325:325,325:53,54:-325,0,-325:-53,0,-54:34:16,18:18
+1 1104001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1106000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.04:0.86:2:52:=:52:0.999:315
+1 1104117 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.132|rs117987012;CGA_FI=554210|NR_029957.1|MIR429|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:273:273,273:272,272:50,52:-273,0,-273:-50,0,-52:30:14,16:16
+1 1105002 . C T . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:181:195,181:195,181:49,50:-195,0,-181:-49,0,-50:23:11,12:12
+1 1105773 . GC . . . NS=1;AN=0 GT:PS ./.:.
+1 1105788 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1105814 . CGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1105821 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1106001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1108000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.98:0.91:2:53:=:53:0.999:315
+1 1106061 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6656236;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1106061:PASS:148:398,148:393,143:52,47:-398,0,-148:-52,0,-47:29:18,11:11
+1 1106473 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4970420;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1106061:PASS:331:331,331:328,328:54,46:-331,0,-331:-46,0,-54:45:27,18:27
+1 1106784 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4442317;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:39:377,39:377,39:48,36:-377,-39,0:-48,-36,0:18:18,18:0
+1 1106950 . GAACCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1107294 . G GC . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1107294:PASS:181:181,293:181,291:36,39:-181,0,-293:-36,0,-39:33:12,21:21
+1 1107303 . ACACA GCCCG . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1107294:PASS:394:394,428:409,428:35,37:-394,0,-428:-35,0,-37:31:14,17:17
+1 1107879 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1107990 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1107994 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1108001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1110000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.88:0.99:2:51:=:51:0.999:315
+1 1108015 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1108044 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 1108117 . AATAAATTAATAATAAAATAATTATAATAATATAGAAAATAATATATAAAATAATAAATATATAAAATATAAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1108203 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1108207 . AATAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1108218 . TATTAAAAATACA . . . NS=1;AN=0 GT:PS ./.:.
+1 1108277 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12136529;CGA_FI=254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|13.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:1108277:PASS:21:21,227:21,227:22,48:-227,-21,0:-48,-22,0:12:12,12:0
+1 1108286 . AG . . . NS=1;AN=0 GT:PS ./.:.
+1 1108315 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1108368 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1109154 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12041521;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:53:53,394:53,394:44,52:-394,-53,0:-52,-44,0:22:22,22:0
+1 1109252 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.113|rs5010607;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:108:108,900:108,900:52,50:-900,-108,0:-52,-50,0:41:41,41:0
+1 1109437 . G A . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:319:461,319:460,318:50,53:-461,0,-319:-50,0,-53:49:26,23:23
+1 1109476 . C A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12141369;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:67:455,67:455,67:52,48:-455,-67,0:-52,-48,0:24:24,24:0
+1 1109782 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.130|rs72894004;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:92:793,92:793,92:53,47:-793,-92,0:-53,-47,0:34:34,34:0
+1 1110001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1112000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:1.20:2:48:=:48:0.999:315
+1 1110374 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1110377 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1110586 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9442387;CGA_FI=100506376|XM_003118493.1|LOC100506376|UTR3|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:102:829,102:829,102:50,50:-829,-102,0:-50,-50,0:44:44,44:0
+1 1111114 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1111116 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1111130 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1111173 . GCTGTGCAGGTGGAGAGAGGCTGCCACCGTGCAGGTGGAGAGAGGCTGCCGCTGTGCCGGTGGAGAGGCTGCTGCCGTGCAGGTGGAGAGAGGCTGCCGCTGTGCCGGTGGAGAGGCTGCTGCTCCCAGCCGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1111455 . CTGCCGCTGTGCAGGTGGAGAGACTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1111586 . CTGCTGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1111621 . CTGACGCTGTGCAGGTGGAGAGAGGCTGACGCTGTGCAGGTGGAGAGGCTGCTGCTCCCAGCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1111955 . AATTCCCGGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1112001 . C <CGA_CNVWIN> . . NS=1;CGA_WINEND=1114000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.85:2:52:=:52:0.999:315
+1 1112309 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9724957;CGA_FI=100506376|XM_003118493.1|LOC100506376|INTRON|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:294:479,294:477,293:50,52:-479,0,-294:-50,0,-52:43:24,19:19
+1 1112405 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1112408 . CTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1112422 . GTCTGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1112434 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1112699 . T TC . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|INTRON|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|8.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:93:93,262:94,261:26,37:-93,0,-262:-26,0,-37:37:17,20:20
+1 1112982 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6671609;CGA_FI=100506376|XM_003118493.1|LOC100506376|INTRON|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|TSS-UPSTREAM|UNKNOWN-INC;CGA_RPT=AluY|Alu|8.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:110:353,110:346,103:52,41:-353,0,-110:-52,0,-41:23:16,7:7
+1 1113087 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1113091 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1113094 . C T . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|INTRON|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|TSS-UPSTREAM|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1113094:VQLOW:22:22,22:0,0:0,11:-22,0,-22:0,0,-11:25:1,24:24
+1 1114001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1116000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.14:0.87:2:53:=:53:0.999:315
+1 1114297 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1115210 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260546;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1115210:PASS:208:411,208:411,220:53,49:-411,0,-208:-53,0,-49:30:17,13:13
+1 1115213 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260547;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|UTR5|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1115210:PASS:224:379,224:378,240:53,50:-379,0,-224:-53,0,-50:31:17,14:14
+1 1115427 . GAGGCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1115776 . G T . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:175:216,175:215,175:52,49:-216,0,-175:-52,0,-49:23:12,11:11
+1 1115908 . CAGCCGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1115922 . ACTA . . . NS=1;AN=0 GT:PS ./.:.
+1 1115976 . GGGAAGGTAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1115994 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1116001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1118000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.91:0.99:2:53:=:53:0.999:315
+1 1116004 . GCGGGCAGCCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1116018 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1116021 . GCCTCCTGGGCCGGAGCACAGGGCAGGTTGGAGGGGTGGGGACCGAGGTCCTGCGCTCCCTCCACACGAGCCCTGGCCTCTGACCTCCAGGAGAGCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1116123 . TGTACC . . . NS=1;AN=0 GT:PS ./.:.
+1 1116136 . CAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1116142 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 1116183 . GGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1116195 . GGGCCATG . . . NS=1;AN=0 GT:PS ./.:.
+1 1116208 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 1116223 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1116284 . TACGCCTGCCCCTGCCCCTGCCCCTGCACCCGCCCCACCCCTGCCCCTGCGCCCGCCCCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1116370 . AGGCTCCCAGGCTGGCTCCAGCCCCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1116553 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.130|rs72631898;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:315:383,315:382,315:53,53:-383,0,-315:-53,0,-53:37:19,18:18
+1 1116601 . T C . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:22:22,22:0,0:0,11:-22,0,-22:0,0,-11:29:4,25:25
+1 1116683 . G A . . NS=1;AN=2;AC=1;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:126:325,126:322,124:52,44:-325,0,-126:-52,0,-44:22:13,9:9
+1 1117398 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12097586;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:80:80,670:80,670:48,52:-670,-80,0:-52,-48,0:28:28,28:0
+1 1117486 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.121|rs13376670;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:370:449,370:449,370:50,54:-449,0,-370:-50,0,-54:41:21,20:20
+1 1118001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1120000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.15:0.93:2:53:=:53:0.999:315
+1 1118212 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907171;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:142:142,142:142,142:44,47:-142,0,-142:-44,0,-47:22:11,11:11
+1 1119657 . G C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.111|rs4560982;CGA_FI=100506376|XM_003118493.1|LOC100506376|TSS-UPSTREAM|UNKNOWN-INC&254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|INTRON|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,469:52,469:44,52:-469,-52,0:-52,-44,0:28:28,28:0
+1 1120001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1122000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:0.97:.:0:.:0:0.999:315
+1 1120032 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1120035 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1120035:PASS:69,.:43,.:0,.:0:0
+1 1120069 . C . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1120035:VQLOW:36,.:29,.:0,.:0:0
+1 1120173 . GGGACCCCCGTGAGGACAGGCCCTCCGGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1120307 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1120035:PASS:83,.:79,.:0,.:0:0
+1 1121014 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.107|rs3813204;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC&254173|NM_153254.2|TTLL10|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:344:344,344:344,344:47,54:-344,0,-344:-47,0,-54:40:20,20:20
+1 1121341 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4297230;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|56.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1121341:PASS:340:340,417:343,416:53,54:-340,0,-417:-53,0,-54:35:17,18:18
+1 1121358 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.131|rs80057011;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|56.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1121341:PASS:345:345,521:343,520:54,53:-521,0,-345:-53,0,-54:39:17,22:17
+1 1121472 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1121480 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12063663;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|9.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1121480:PASS:162:259,162:241,143:47,40:-259,0,-162:-47,0,-40:32:24,8:8
+1 1121625 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4081334;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|9.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:137:137,251:136,250:43,50:-137,0,-251:-43,0,-50:22:9,13:13
+1 1121657 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs11260548;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|9.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:VQLOW:19:301,19:301,43:48,39:-301,-19,0:-48,-39,0:18:17,17:1
+1 1121715 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.108|rs4081333;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|9.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:46:46,46:32,32:10,26:-46,0,-46:-10,0,-26:16:3,13:13
+1 1121794 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260549;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|44.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:300:308,300:308,300:53,53:-308,0,-300:-53,0,-53:34:17,17:17
+1 1121835 . C CTG . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs57346441;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|44.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:61:654,61:654,61:39,33:-654,-61,0:-39,-33,0:36:36,36:0
+1 1122001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1124000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.86:2:52:=:52:0.999:315
+1 1122021 . CACTTGTTATTTTCCTTTCTTTATTAAAGAATCACCATCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1122196 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4634847;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|44.0 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:203:203,203:193,193:43,46:-203,0,-203:-43,0,-46:39:12,27:27
+1 1122230 . GT G . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs35158481;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:51,.:42,.:12,.:-51,0,0:-12,0,0:15:9,.:1
+1 1122283 . T G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12064046;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:217:292,217:292,217:52,48:-292,0,-217:-52,0,-48:31:17,14:14
+1 1122319 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs7415847;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:86:86,86:86,86:36,38:-86,0,-86:-36,0,-38:18:9,9:9
+1 1122388 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260550;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1122388:PASS:235:461,235:457,235:53,50:-461,0,-235:-53,0,-50:30:19,11:11
+1 1122395 . TC TTT . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1122388:PASS:332:349,332:345,346:34,35:-349,0,-332:-34,0,-35:31:19,12:12
+1 1122468 . TG CG . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.116|rs7545694&dbsnp.131|rs80083461;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:1122468:PASS:81,.:81,.:34,.:-81,0,0:-34,0,0:8:8,.:0
+1 1122473 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1122478 . GGCCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1122485 . CA . . . NS=1;AN=0 GT:PS ./.:.
+1 1122516 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260551;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|4.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1122516:PASS:89:157,89:157,90:48,39:-157,0,-89:-48,0,-39:13:8,5:5
+1 1122539 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12063897;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1122516:PASS:198:308,198:308,199:53,44:-198,0,-308:-44,0,-53:30:16,14:16
+1 1122771 . A G . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|12.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:VQLOW:22:22,22:16,16:0,21:-22,0,-22:0,0,-21:9:2,7:7
+1 1122844 . TACTAAAAATATG . . . NS=1;AN=0 GT:PS ./.:.
+1 1122915 . AA GA,GG . . NS=1;AN=2;AC=1,1;CGA_XR=dbsnp.125|rs28595293,dbsnp.125|rs28460227&dbsnp.125|rs28595293;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC,254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|12.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|2:1122915:PASS:123:126,123:126,124:25,25:-126,-123,-123,-126,0,-126:-25,-25,-25,-25,0,-25:13:6,7:0
+1 1122937 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.125|rs28648687;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx1|Alu|12.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1122915:PASS:139:191,139:191,139:50,45:-139,0,-191:-45,0,-50:19:11,8:11
+1 1123106 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12401472;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx|Alu|12.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:184:186,184:186,184:48,50:-186,0,-184:-48,0,-50:17:8,9:9
+1 1123284 . CA C . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.126|rs35247267;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSx|Alu|12.1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/.:.:PASS:53,.:53,.:16,.:-53,0,0:-16,0,0:8:6,.:2
+1 1123434 . T A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12066716;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:264:264,264:263,263:49,51:-264,0,-264:-49,0,-51:31:14,17:17
+1 1123785 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1124001 . T <CGA_CNVWIN> . . NS=1;CGA_WINEND=1126000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.87:0.92:2:53:=:53:0.999:315
+1 1124257 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907172;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:219:219,219:217,217:45,48:-219,0,-219:-45,0,-48:31:13,18:18
+1 1124399 . C G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907173;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:166:166,166:160,160:46,48:-166,0,-166:-46,0,-48:27:9,18:18
+1 1124663 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6684820;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:76:76,823:76,823:41,53:-823,-76,0:-53,-41,0:35:35,35:0
+1 1124750 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.116|rs6702156;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:103:103,1038:103,1038:51,50:-1038,-103,0:-51,-50,0:41:41,41:0
+1 1124819 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.116|rs6694487;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:340:473,340:471,339:50,54:-473,0,-340:-50,0,-54:42:24,18:18
+1 1124891 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61768485;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:260:260,260:259,259:52,50:-260,0,-260:-52,0,-50:29:13,16:16
+1 1125110 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12124436;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1125110:VQLOW:24:24,71:28,70:6,36:-24,0,-71:-6,0,-36:10:4,6:6
+1 1125119 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1125122 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1125220 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12065129;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:215:315,215:313,213:53,48:-315,0,-215:-53,0,-48:30:17,13:13
+1 1125348 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12029885;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|5.7 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:186:186,213:198,213:50,50:-186,0,-213:-50,0,-50:22:11,11:11
+1 1125488 . TCTCAAAAAAAAAAAAAAAAGAAT . . . NS=1;AN=0 GT:PS ./.:.
+1 1125553 . A T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs10907174;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:80:80,788:80,788:43,53:-788,-80,0:-53,-43,0:33:33,33:0
+1 1125811 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1126001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1128000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.93:0.88:2:53:=:53:0.999:315
+1 1126236 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4578157;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSq2|Alu|13.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1126236:PASS:414:414,540:414,537:50,54:-414,0,-540:-50,0,-54:40:17,23:23
+1 1126255 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.111|rs4449971;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSq2|Alu|13.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1126236:PASS:361:361,540:360,537:53,54:-361,0,-540:-53,0,-54:38:17,21:21
+1 1126455 . CTGCCAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1126723 . GCTACTCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1126968 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1126994 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.96|rs2094830;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1126994:PASS:98:98,98:88,88:31,39:-98,0,-98:-31,0,-39:25:7,18:18
+1 1127101 . C A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9659458;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1127101:PASS:222:222,459:220,457:46,54:-222,0,-459:-46,0,-54:36:13,23:23
+1 1127137 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12062271;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1127101:PASS:281:281,325:298,326:53,54:-281,0,-325:-53,0,-54:34:19,15:15
+1 1127322 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260552;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1127322:PASS:214:246,214:246,214:52,50:-246,0,-214:-52,0,-50:28:14,13:13
+1 1127330 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12061357;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1127322:PASS:246:254,246:254,248:51,48:-246,0,-254:-48,0,-51:35:19,14:19
+1 1127380 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260553;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=MER9a1|ERVK|6.3 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:267:267,267:267,267:50,51:-267,0,-267:-50,0,-51:35:17,18:18
+1 1127507 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12021582;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1127507:PASS:299:415,299:415,299:53,53:-415,0,-299:-53,0,-53:39:21,18:18
+1 1127523 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12024296;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1127507:PASS:382:440,382:440,385:50,54:-440,0,-382:-50,0,-54:42:21,21:21
+1 1127608 . G GT . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:139:253,139:277,136:39,37:-253,0,-139:-39,0,-37:29:17,12:12
+1 1127681 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61768486;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|34.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:113:113,113:106,106:36,41:-113,0,-113:-36,0,-41:20:6,14:14
+1 1127739 . TGATATGAGTGTAGACACTCCAGTTGGTATAAGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1127786 . TTGATATGAATGTAGACACTCCAGTTGGTATGAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1127838 . TATGAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1127857 . GTTCGTATCAGTGTAGACACTCCAGTTGATATCAGTGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1127939 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1127948 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1127948:VQLOW:37,.:37,.:0,.:0:0
+1 1127977 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1128001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1130000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.75:0.89:2:53:=:53:0.999:315
+1 1128122 . TTGGTATGAGTGTAGACACTCCAGTTGTTCATA . . . NS=1;AN=0 GT:PS ./.:.
+1 1128196 . GTTGGTATGAGTGTAGACA . . . NS=1;AN=0 GT:PS ./.:.
+1 1128331 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61766176;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|30.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:260:260,260:260,260:49,51:-260,0,-260:-49,0,-51:31:14,17:17
+1 1128429 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.129|rs61766177;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|8.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:134:241,134:239,132:48,45:-241,0,-134:-48,0,-45:19:11,8:8
+1 1128521 . C T . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.129|rs61766178;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluY|Alu|8.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:52:52,391:52,391:44,52:-391,-52,0:-52,-44,0:25:25,25:0
+1 1128605 . TGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1128709 . AAT . . . NS=1;AN=0 GT:PS ./.:.
+1 1128717 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1128778 . C CTTA . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.131|rs78174849;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|45.8 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:50:50,338:50,341:37,39:-338,-50,0:-39,-37,0:28:28,28:0
+1 1128849 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1128861 . GAAAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1128886 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1129009 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1129122 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9659213;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSq|Alu|7.4 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:77:111,77:111,76:41,37:-111,0,-77:-41,0,-37:10:6,4:4
+1 1129263 . TGTGGTGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1129672 . G T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260554;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1129672:PASS:247:594,247:589,242:50,50:-594,0,-247:-50,0,-50:43:28,15:15
+1 1129707 . A G . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260555;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1129672:PASS:210:294,210:294,213:52,50:-294,0,-210:-52,0,-50:27:16,11:11
+1 1129789 . G C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12060374;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1129789:PASS:207:451,207:447,202:52,50:-451,0,-207:-52,0,-50:29:18,11:11
+1 1129920 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12060422;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:150:494,150:481,137:53,39:-494,0,-150:-53,0,-39:32:23,9:9
+1 1130001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1132000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.81:0.96:2:53:=:53:0.999:315
+1 1130093 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12026524;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1129789:PASS:183:183,189:183,189:50,49:-189,0,-183:-49,0,-50:20:10,10:10
+1 1130206 . G A . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs11260556;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|39.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:118:1037,118:1037,118:50,52:-1037,-118,0:-52,-50,0:44:44,44:0
+1 1130480 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9659772;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluSz|Alu|16.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1130480:PASS:185:185,185:185,185:48,50:-185,0,-185:-48,0,-50:22:11,11:11
+1 1130727 . A C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907175;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|36.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:118:123,118:125,116:44,43:-123,0,-118:-44,0,-43:15:9,6:6
+1 1130843 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9727857;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|36.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:58:664,58:653,98:53,49:-664,-58,0:-53,-49,0:34:34,34:0
+1 1130855 . T C . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs10907176;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|36.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1130480:PASS:148:148,491:136,479:39,53:-491,0,-148:-53,0,-39:33:10,23:10
+1 1130881 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1130954 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1130960 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1130968 . AC . . . NS=1;AN=0 GT:PS ./.:.
+1 1131052 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12066103;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluJo|Alu|17.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:42:42,42:39,39:15,28:-42,0,-42:-15,0,-28:10:3,7:7
+1 1131233 . GCTGATCTCAGACTCCTGAGCTCAAGCGATC . . . NS=1;AN=0 GT:PS ./.:.
+1 1131310 . C T . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluJo|Alu|17.5 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1131310:PASS:122:122,341:127,302:32,53:-122,0,-341:-32,0,-53:36:10,26:26
+1 1131323 . C T . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluJb|Alu|34.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1131310:PASS:88:88,341:89,302:22,53:-88,0,-341:-22,0,-53:36:5,31:31
+1 1131334 . C T . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluJb|Alu|34.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1131310:PASS:221:221,341:224,302:29,53:-221,0,-341:-29,0,-53:51:13,38:38
+1 1131394 . A . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0/.:.:PASS:103,.:98,.:0,.:0:0
+1 1131419 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1131441 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12403745;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=AluJb|Alu|34.1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:50:50,50:50,50:22,31:-50,0,-50:-22,0,-31:7:3,4:4
+1 1131581 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9329409;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=L1MB8|L1|47.6 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:66:493,66:493,66:52,48:-493,-66,0:-52,-48,0:24:24,24:0
+1 1131825 . GTGAGGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1132001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1134000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.94:0.92:2:53:=:53:0.999:315
+1 1132010 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1132013 . CTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1132744 . CCCTCACACCCTCCCCACC . . . NS=1;AN=0 GT:PS ./.:.
+1 1132785 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.100|rs2274792;CGA_FI=254173|NM_001130045.1|TTLL10|INTRON|UNKNOWN-INC;CGA_RPT=C-rich|Low_complexity|30.9 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1132785:PASS:64:66,64:66,64:29,35:-66,0,-64:-29,0,-35:8:4,4:4
+1 1132797 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1132869 . CTCTGCTGTCCCAGCGCCGCTTCGTGCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1132907 . GGTGAGGCCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1132938 . G A . . NS=1;AN=1;AC=1;CGA_XR=dbsnp.100|rs2274791;CGA_FI=254173|NM_001130045.1|TTLL10|CDS|MISSENSE GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|.:1132938:PASS:66,.:65,.:28,.:-66,0,0:-28,0,0:6:4,.:2
+1 1132947 . GCCTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1132960 . GCCGCCCCTGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1133005 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1133008 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1133023 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1133033 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 1133038 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1133042 . GCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1133047 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1133065 . GCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1133074 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1133077 . A G . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.100|rs2274789;CGA_FI=254173|NM_001130045.1|TTLL10|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|1:1133077:VQLOW:12:107,12:107,12:41,16:-107,-12,0:-41,-16,0:7:7,7:0
+1 1133084 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1133110 . G A . . NS=1;AN=2;AC=1;CGA_FI=254173|NM_001130045.1|TTLL10|CDS|SYNONYMOUS GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 0|1:1132785:PASS:47:47,47:47,47:30,20:-47,0,-47:-20,0,-30:11:6,5:6
+1 1133254 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs12026794;CGA_FI=254173|NM_001130045.1|TTLL10|UTR3|UNKNOWN-INC;CGA_RPT=GC_rich|Low_complexity|3.2 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:72:174,72:173,70:48,36:-174,0,-72:-48,0,-36:17:10,7:7
+1 1133273 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12031928;CGA_FI=254173|NM_001130045.1|TTLL10|UTR3|UNKNOWN-INC GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:48:449,48:449,73:48,48:-449,-48,0:-48,-48,0:19:19,19:0
+1 1133787 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.120|rs12405246 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:42:42,268:42,268:39,48:-268,-42,0:-48,-39,0:15:15,15:0
+1 1133815 . T C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.126|rs35009659 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:63:512,63:512,63:52,48:-512,-63,0:-52,-48,0:22:22,22:0
+1 1134001 . A <CGA_CNVWIN> . . NS=1;CGA_WINEND=1136000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:1.06:1.00:2:53:=:53:0.999:315
+1 1134295 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1134633 . C T . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.120|rs11260558 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1134633:PASS:242:264,242:254,232:52,50:-264,0,-242:-52,0,-50:26:14,12:12
+1 1134642 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1134659 . G . . . NS=1;AN=1 GT:PS:FT:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL 0|.:1134633:PASS:249,.:239,.:0,.:0:0
+1 1134707 . G A . . NS=1;AN=2;AC=1;CGA_XR=dbsnp.119|rs9727747 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/0:.:PASS:135:135,171:134,168:43,49:-135,0,-171:-43,0,-49:26:10,16:16
+1 1135242 . A C . . NS=1;AN=2;AC=2;CGA_XR=dbsnp.119|rs9729550 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1/1:.:PASS:80:80,690:80,690:48,52:-690,-80,0:-52,-48,0:29:28,28:1
+1 1135760 . TCTCACCAGCAGCGGGGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1135789 . CTGAGGGCCAG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136001 . G <CGA_CNVWIN> . . NS=1;CGA_WINEND=1138000 GT:CGA_GP:CGA_NP:CGA_CP:CGA_PS:CGA_CT:CGA_TS:CGA_CL:CGA_LS .:0.78:0.75:.:0:.:0:0.999:315
+1 1136124 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1136148 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1136477 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1136480 . GC G . . NS=1;AN=2;AC=1 GT:PS:FT:GQ:HQ:EHQ:CGA_CEHQ:GL:CGA_CEGL:DP:AD:CGA_RDP 1|0:1136480:VQLOW:22:22,22:4,4:0,3:-22,0,-22:0,0,-3:10:1,9:9
+1 1136485 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1136488 . CTCGGG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136513 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1136515 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1136540 . ACTGCGCCCCGAGTGGGAGGGGGCGGCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136574 . CCGCCGCGAC . . . NS=1;AN=0 GT:PS ./.:.
+1 1136586 . CG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136644 . CCATGGCGGGT . . . NS=1;AN=0 GT:PS ./.:.
+1 1136659 . GCT . . . NS=1;AN=0 GT:PS ./.:.
+1 1136664 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1136668 . GG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136672 . TCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1136692 . A . . . NS=1;AN=0 GT:PS ./.:.
+1 1136695 . CGCCCCCGCCCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1136713 . C . . . NS=1;AN=0 GT:PS ./.:.
+1 1136715 . AGAACCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136726 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1136729 . T . . . NS=1;AN=0 GT:PS ./.:.
+1 1136746 . CGCACGCTG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136757 . G . . . NS=1;AN=0 GT:PS ./.:.
+1 1136761 . CCGCCAGGCCCGGAGGGTCGCGCTCCAGGTAAA . . . NS=1;AN=0 GT:PS ./.:.
+1 1136797 . CGCGGGGCGGGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1136828 . CCGGAGACCCCGCCCAGAGCCCGCTCCGCCGCCCGCGGAATCCCCCGCCCGTCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1136897 . GGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1136930 . CCCACGCGAGGCCGCCCACGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1136960 . GCGCTGAGTCAGCCCCGCGGGACCCGCGCTACGCGGGCCGCCG . . . NS=1;AN=0 GT:PS ./.:.
+1 1137014 . GGCCGGTGCGGGACAGCCCCGGTGTGGGGGGCGCGTGGGGAGA . . . NS=1;AN=0 GT:PS ./.:.
+1 1137097 . TGCGCGGGGCAGGGCCCGACCGCTCAGCCTCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137169 . GCCCCGAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCGAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCGAGCCCAGTACCCAGCCTCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137283 . GCCCCGAGCCCAGTACCCAGCCTCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137323 . CCCGAGCCCAGTACCCAGCCTCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137361 . CCCGAGCCCAGTACCCAGCCTCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137399 . CCCGAGCCCAGTACCCAGCCTCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137437 . CCCGAGCCCAGTACCCAGCCTCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137470 . CCAGCCCCGAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCGAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCGAGCCCAGTACCCAGCCTCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137584 . CCAGCCCCGAGCCCAGTACCCAGCCTCCAGCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137622 . CCAGCCCCGAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCCAGC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137679 . CCAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137693 . AGTACCCAGCCCCCAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCCCGAGCCCAGTACCCAGCCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1137774 . CCAGCCCCGAGCCCAGTACCCAGCCCCCAGCCCAGTACCCAGCCTCCAGCCCAGTACCCAGCCTCCA . . . NS=1;AN=0 GT:PS ./.:.
+1 1137850 . CCAGCCCCGAGCCCAGTACCCAGCCCCCAGCCCAGTACCCAGCCC . . . NS=1;AN=0 GT:PS ./.:.
+1 1137907 . CCAGCCCCGAGCCCAGTACCCAGCCCCGAGCCCAGCACCCAGCCTCCAGCCCAGTACCCATCCC . . . NS=1;AN=0 GT:PS ./.:.
\ No newline at end of file
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf.gz b/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf.gz
new file mode 100644
index 0000000..e2d0fed
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf.gz
Binary files differ
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf
new file mode 100644
index 0000000..c42d71c
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf
@@ -0,0 +1,42 @@
+##fileformat=VCFv4.2
+##fileDate=20090805
+##source=myImputationProgramV3.1
+##phasing=partial
+##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
+##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
+##INFO=<ID=AF,Number=A,Type=Float,Description="Allele Frequency">
+##INFO=<ID=AA,Number=1,Type=String,Description="Ancestral Allele">
+##INFO=<ID=DB,Number=0,Type=Flag,Description="dbSNP membership, build 129">
+##INFO=<ID=H2,Number=0,Type=Flag,Description="HapMap2 membership">
+##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant (with unïcodé)">
+##INFO=<ID=END,Number=1,Type=Integer,Description="End position of variant">
+##FILTER=<ID=q10,Description="Quality below 10">
+##FILTER=<ID=s50,Description="Less than 50% of samples have data (with \\ backslash)">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Read Depth">
+##FORMAT=<ID=HQ,Number=2,Type=Integer,Description="Haplotype Quality">
+##FORMAT=<ID=GL,Number=G,Type=Integer,Description="Genotype Likelihood">
+##reference=file:/lustre/scratch105/projects/g1k/ref/main_project/human_g1k_v37.fasta
+##contig=<ID=19,length=59128983,md5=1aacd71f30db8e561810913e0b72636d,species="Homo Sapiens">
+##contig=<ID=20,length=63025520,md5=0dec9660ec1efaaf33281c0d5ea2560f,species="Homo Sapiens">
+##contig=<ID=Y,length=63025520,md5=0dec9660ec1efaaf33281c0d5ea2560f,species="Homo Sapiens">
+##SAMPLE=<ID=Blood,Genomes=Germline,Mixture=1.,Description="Patient germline genome">
+##SAMPLE=<ID=TissueSample,Genomes=Germline;Tumor,Mixture=.3;.7,Description="Patient germline genome;Patient tumor genome">
+##PEDIGREE=<Derived=ID2,Original=ID1>
+##PEDIGREE=<Child=CHILD-GENOME-ID,Mother=MOTHER-GENOME-ID,Father=FATHER-GENOME-ID>
+##pedigreeDB=url
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA00001 NA00002 NA00003
+19 14370 rs6054257 G A 29 PASS NS=3;DP=14;AF=0.5;DB;H2 GT:GQ:DP:HQ 0|0:48:1:51,51 1|0:48:8:51,51 1/1:43:5:.,.
+20 17330 . T A 3 q10 NS=3;DP=11;AF=0.017 GT:GQ:DP:HQ 0|0:49:3:58,50 0|1:3:5:65,3 0/0:41:3
+20 1110696 rs6040355 A G,T 67 PASS NS=2;DP=10;AF=0.333,0.667;AA=T;DB GT:GQ:DP:HQ 1|2:21:6:23,27 2|1:2:0:18,2 2/2:35:4
+20 1230237 . T . 47 PASS NS=3;DP=13;AA=T GT:GQ:DP:HQ 0|0:54:7:56,60 0|0:48:4:51,51 0/0:61:2
+20 1234567 microsat1 GTC G,GTCTC 50 PASS NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/2:17:2 1/1:40:3
+20 2234567 . C [13:123457[ACGC 50 PASS SVTYPE=BÑD;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3
+20 2234568 . C .TC 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3
+20 2234569 . C CT. 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3
+20 3234569 . C <SYMBOLIC> 50 PASS END=3235677;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3
+20 4234569 . N .[13:123457[ 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 ./.:40:3
+20 5234569 . N [13:123457[. 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3
+Y 17330 . T A 3 q10 NS=3;DP=11;AF=0.017 GT:GL 0:0,49 0:0,3 1:41,0
+HLA-A*01:01:01:01 1 . N T 50 PASS END=1;NS=3;DP=9;AA=G GT:GQ:DP:HQ 0|0:48:1:51,51 1|0:48:8:51,51 1/1:43:5:.,.
\ No newline at end of file
diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz
new file mode 100644
index 0000000..4208e3e
--- /dev/null
+++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz
Binary files differ
diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py
index 883343a..c421e25 100644
--- a/sdks/python/apache_beam/testing/pipeline_verifiers.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py
@@ -32,7 +32,6 @@
from apache_beam.testing import test_utils as utils
from apache_beam.utils import retry
-
__all__ = [
'PipelineStateMatcher',
'FileChecksumMatcher',
diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py
index 15e0a04..3b02431 100644
--- a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py
@@ -22,13 +22,14 @@
import unittest
from hamcrest import assert_that as hc_assert_that
-from mock import Mock, patch
+from mock import Mock
+from mock import patch
from apache_beam.io.localfilesystem import LocalFileSystem
from apache_beam.runners.runner import PipelineResult
from apache_beam.runners.runner import PipelineState
-from apache_beam.testing.test_utils import patch_retry
from apache_beam.testing import pipeline_verifiers as verifiers
+from apache_beam.testing.test_utils import patch_retry
try:
# pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
index 13b1639..46eeb75 100644
--- a/sdks/python/apache_beam/testing/test_pipeline.py
+++ b/sdks/python/apache_beam/testing/test_pipeline.py
@@ -20,12 +20,12 @@
import argparse
import shlex
-from apache_beam.internal import pickler
-from apache_beam.pipeline import Pipeline
-from apache_beam.runners.runner import PipelineState
-from apache_beam.options.pipeline_options import PipelineOptions
from nose.plugins.skip import SkipTest
+from apache_beam.internal import pickler
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.pipeline import Pipeline
+from apache_beam.runners.runner import PipelineState
__all__ = [
'TestPipeline',
@@ -33,23 +33,23 @@
class TestPipeline(Pipeline):
- """TestPipeline class is used inside of Beam tests that can be configured to
- run against pipeline runner.
+ """:class:`TestPipeline` class is used inside of Beam tests that can be
+ configured to run against pipeline runner.
It has a functionality to parse arguments from command line and build pipeline
options for tests who runs against a pipeline runner and utilizes resources
of the pipeline runner. Those test functions are recommended to be tagged by
- @attr("ValidatesRunner") annotation.
+ ``@attr("ValidatesRunner")`` annotation.
In order to configure the test with customized pipeline options from command
- line, system argument 'test-pipeline-options' can be used to obtains a list
- of pipeline options. If no options specified, default value will be used.
+ line, system argument ``--test-pipeline-options`` can be used to obtains a
+ list of pipeline options. If no options specified, default value will be used.
For example, use following command line to execute all ValidatesRunner tests::
- python setup.py nosetests -a ValidatesRunner \
- --test-pipeline-options="--runner=DirectRunner \
- --job_name=myJobName \
+ python setup.py nosetests -a ValidatesRunner \\
+ --test-pipeline-options="--runner=DirectRunner \\
+ --job_name=myJobName \\
--num_workers=1"
For example, use assert_that for test validation::
@@ -69,21 +69,27 @@
"""Initialize a pipeline object for test.
Args:
- runner: An object of type 'PipelineRunner' that will be used to execute
- the pipeline. For registered runners, the runner name can be specified,
- otherwise a runner object must be supplied.
- options: A configured 'PipelineOptions' object containing arguments
- that should be used for running the pipeline job.
- argv: A list of arguments (such as sys.argv) to be used for building a
- 'PipelineOptions' object. This will only be used if argument 'options'
- is None.
- is_integration_test: True if the test is an integration test, False
- otherwise.
- blocking: Run method will wait until pipeline execution is completed.
+ runner (~apache_beam.runners.runner.PipelineRunner): An object of type
+ :class:`~apache_beam.runners.runner.PipelineRunner` that will be used
+ to execute the pipeline. For registered runners, the runner name can be
+ specified, otherwise a runner object must be supplied.
+ options (~apache_beam.options.pipeline_options.PipelineOptions):
+ A configured
+ :class:`~apache_beam.options.pipeline_options.PipelineOptions`
+ object containing arguments that should be used for running the
+ pipeline job.
+ argv (List[str]): A list of arguments (such as :data:`sys.argv`) to be
+ used for building a
+ :class:`~apache_beam.options.pipeline_options.PipelineOptions` object.
+ This will only be used if argument **options** is :data:`None`.
+ is_integration_test (bool): :data:`True` if the test is an integration
+ test, :data:`False` otherwise.
+ blocking (bool): Run method will wait until pipeline execution is
+ completed.
Raises:
- ValueError: if either the runner or options argument is not of the
- expected type.
+ ~exceptions.ValueError: if either the runner or options argument is not
+ of the expected type.
"""
self.is_integration_test = is_integration_test
self.options_list = self._parse_test_option_args(argv)
diff --git a/sdks/python/apache_beam/testing/test_pipeline_test.py b/sdks/python/apache_beam/testing/test_pipeline_test.py
index 747d64c7..c642c65 100644
--- a/sdks/python/apache_beam/testing/test_pipeline_test.py
+++ b/sdks/python/apache_beam/testing/test_pipeline_test.py
@@ -20,12 +20,12 @@
import logging
import unittest
-from hamcrest.core.base_matcher import BaseMatcher
from hamcrest.core.assert_that import assert_that as hc_assert_that
+from hamcrest.core.base_matcher import BaseMatcher
from apache_beam.internal import pickler
-from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.testing.test_pipeline import TestPipeline
# A simple matcher that is ued for testing extra options appending.
diff --git a/sdks/python/apache_beam/testing/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py
index a06bcd0..8a63e7b 100644
--- a/sdks/python/apache_beam/testing/test_stream.py
+++ b/sdks/python/apache_beam/testing/test_stream.py
@@ -24,13 +24,14 @@
from abc import abstractmethod
from apache_beam import coders
+from apache_beam import core
from apache_beam import pvalue
from apache_beam.transforms import PTransform
+from apache_beam.transforms import window
from apache_beam.transforms.window import TimestampedValue
from apache_beam.utils import timestamp
from apache_beam.utils.windowed_value import WindowedValue
-
__all__ = [
'Event',
'ElementEvent',
@@ -99,6 +100,9 @@
self.current_watermark = timestamp.MIN_TIMESTAMP
self.events = []
+ def get_windowing(self, unused_inputs):
+ return core.Windowing(window.GlobalWindows())
+
def expand(self, pbegin):
assert isinstance(pbegin, pvalue.PBegin)
self.pipeline = pbegin.pipeline
diff --git a/sdks/python/apache_beam/testing/test_stream_test.py b/sdks/python/apache_beam/testing/test_stream_test.py
index e32dda2..0f6691f 100644
--- a/sdks/python/apache_beam/testing/test_stream_test.py
+++ b/sdks/python/apache_beam/testing/test_stream_test.py
@@ -19,10 +19,17 @@
import unittest
+import apache_beam as beam
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.test_stream import ElementEvent
from apache_beam.testing.test_stream import ProcessingTimeEvent
from apache_beam.testing.test_stream import TestStream
from apache_beam.testing.test_stream import WatermarkEvent
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.window import TimestampedValue
from apache_beam.utils import timestamp
from apache_beam.utils.windowed_value import WindowedValue
@@ -78,6 +85,68 @@
TimestampedValue('a', timestamp.MAX_TIMESTAMP)
]))
+ def test_basic_execution(self):
+ test_stream = (TestStream()
+ .advance_watermark_to(10)
+ .add_elements(['a', 'b', 'c'])
+ .advance_watermark_to(20)
+ .add_elements(['d'])
+ .add_elements(['e'])
+ .advance_processing_time(10)
+ .advance_watermark_to(300)
+ .add_elements([TimestampedValue('late', 12)])
+ .add_elements([TimestampedValue('last', 310)]))
+
+ class RecordFn(beam.DoFn):
+ def process(self, element=beam.DoFn.ElementParam,
+ timestamp=beam.DoFn.TimestampParam):
+ yield (element, timestamp)
+
+ options = PipelineOptions()
+ options.view_as(StandardOptions).streaming = True
+ p = TestPipeline(options=options)
+ my_record_fn = RecordFn()
+ records = p | test_stream | beam.ParDo(my_record_fn)
+ assert_that(records, equal_to([
+ ('a', timestamp.Timestamp(10)),
+ ('b', timestamp.Timestamp(10)),
+ ('c', timestamp.Timestamp(10)),
+ ('d', timestamp.Timestamp(20)),
+ ('e', timestamp.Timestamp(20)),
+ ('late', timestamp.Timestamp(12)),
+ ('last', timestamp.Timestamp(310)),]))
+ p.run()
+
+ def test_gbk_execution(self):
+ test_stream = (TestStream()
+ .advance_watermark_to(10)
+ .add_elements(['a', 'b', 'c'])
+ .advance_watermark_to(20)
+ .add_elements(['d'])
+ .add_elements(['e'])
+ .advance_processing_time(10)
+ .advance_watermark_to(300)
+ .add_elements([TimestampedValue('late', 12)])
+ .add_elements([TimestampedValue('last', 310)]))
+
+ options = PipelineOptions()
+ options.view_as(StandardOptions).streaming = True
+ p = TestPipeline(options=options)
+ records = (p
+ | test_stream
+ | beam.WindowInto(FixedWindows(15))
+ | beam.Map(lambda x: ('k', x))
+ | beam.GroupByKey())
+ # TODO(BEAM-2519): timestamp assignment for elements from a GBK should
+ # respect the TimestampCombiner. The test below should also verify the
+ # timestamps of the outputted elements once this is implemented.
+ assert_that(records, equal_to([
+ ('k', ['a', 'b', 'c']),
+ ('k', ['d', 'e']),
+ ('k', ['late']),
+ ('k', ['last'])]))
+ p.run()
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py
index 9feb80e..c28b692 100644
--- a/sdks/python/apache_beam/testing/test_utils.py
+++ b/sdks/python/apache_beam/testing/test_utils.py
@@ -22,14 +22,56 @@
import hashlib
import imp
-from mock import Mock, patch
+import os
+import shutil
+import tempfile
+from mock import Mock
+from mock import patch
+
+from apache_beam.io.filesystems import FileSystems
from apache_beam.utils import retry
-
DEFAULT_HASHING_ALG = 'sha1'
+class TempDir(object):
+ """Context Manager to create and clean-up a temporary directory."""
+
+ def __init__(self):
+ self._tempdir = tempfile.mkdtemp()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ if os.path.exists(self._tempdir):
+ shutil.rmtree(self._tempdir)
+
+ def get_path(self):
+ """Returns the path to the temporary directory."""
+ return self._tempdir
+
+ def create_temp_file(self, suffix='', lines=None):
+ """Creates a temporary file in the temporary directory.
+
+ Args:
+ suffix (str): The filename suffix of the temporary file (e.g. '.txt')
+ lines (List[str]): A list of lines that will be written to the temporary
+ file.
+ Returns:
+ The name of the temporary file created.
+ """
+ f = tempfile.NamedTemporaryFile(delete=False,
+ dir=self._tempdir,
+ suffix=suffix)
+ if lines:
+ for line in lines:
+ f.write(line)
+
+ return f.name
+
+
def compute_hash(content, hashing_alg=DEFAULT_HASHING_ALG):
"""Compute a hash value from a list of string."""
content.sort()
@@ -71,3 +113,20 @@
imp.reload(module)
testcase.addCleanup(remove_patches)
+
+
+@retry.with_exponential_backoff(
+ num_retries=3,
+ retry_filter=retry.retry_on_beam_io_error_filter)
+def delete_files(file_paths):
+ """A function to clean up files or directories using ``FileSystems``.
+
+ Glob is supported in file path and directories will be deleted recursively.
+
+ Args:
+ file_paths: A list of strings contains file paths or directories.
+ """
+ if len(file_paths) == 0:
+ raise RuntimeError('Clean up failed. Invalid file path: %s.' %
+ file_paths)
+ FileSystems.delete(file_paths)
diff --git a/sdks/python/apache_beam/testing/test_utils_test.py b/sdks/python/apache_beam/testing/test_utils_test.py
new file mode 100644
index 0000000..877ee39
--- /dev/null
+++ b/sdks/python/apache_beam/testing/test_utils_test.py
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unittest for testing utilities,"""
+
+import logging
+import os
+import tempfile
+import unittest
+
+from apache_beam.io.filesystem import BeamIOError
+from apache_beam.io.filesystems import FileSystems
+from apache_beam.testing import test_utils as utils
+
+
+class TestUtilsTest(unittest.TestCase):
+
+ def setUp(self):
+ utils.patch_retry(self, utils)
+ self.tmpdir = tempfile.mkdtemp()
+
+ def test_delete_files_succeeds(self):
+ path = os.path.join(self.tmpdir, 'f1')
+
+ with open(path, 'a') as f:
+ f.write('test')
+
+ assert FileSystems.exists(path)
+ utils.delete_files([path])
+ assert not FileSystems.exists(path)
+
+ def test_delete_files_fails_with_io_error(self):
+ path = os.path.join(self.tmpdir, 'f2')
+
+ with self.assertRaises(BeamIOError) as error:
+ utils.delete_files([path])
+ self.assertTrue(
+ error.exception.message.startswith('Delete operation failed'))
+ self.assertEqual(error.exception.exception_details.keys(), [path])
+
+ def test_delete_files_fails_with_invalid_arg(self):
+ with self.assertRaises(RuntimeError):
+ utils.delete_files([])
+
+ def test_temp_dir_removes_files(self):
+ dir_path = ''
+ file_path = ''
+ with utils.TempDir() as tempdir:
+ dir_path = tempdir.get_path()
+ file_path = tempdir.create_temp_file()
+ self.assertTrue(os.path.exists(dir_path))
+ self.assertTrue(os.path.exists(file_path))
+
+ self.assertFalse(os.path.exists(dir_path))
+ self.assertFalse(os.path.exists(file_path))
+
+ def test_temp_file_field_correct(self):
+ with utils.TempDir() as tempdir:
+ filename = tempdir.create_temp_file(
+ suffix='.txt',
+ lines=['line1\n', 'line2\n', 'line3\n'])
+ self.assertTrue(filename.endswith('.txt'))
+
+ with open(filename, 'rb') as f:
+ self.assertEqual(f.readline(), 'line1\n')
+ self.assertEqual(f.readline(), 'line2\n')
+ self.assertEqual(f.readline(), 'line3\n')
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py
index 60a6b21..34c15f9 100644
--- a/sdks/python/apache_beam/testing/util.py
+++ b/sdks/python/apache_beam/testing/util.py
@@ -19,19 +19,24 @@
from __future__ import absolute_import
+import glob
+import tempfile
+
from apache_beam import pvalue
from apache_beam.transforms import window
from apache_beam.transforms.core import Create
from apache_beam.transforms.core import Map
from apache_beam.transforms.core import WindowInto
-from apache_beam.transforms.util import CoGroupByKey
from apache_beam.transforms.ptransform import PTransform
-
+from apache_beam.transforms.util import CoGroupByKey
+from apache_beam.utils.annotations import experimental
__all__ = [
'assert_that',
'equal_to',
'is_empty',
+ # open_shards is internal and has no backwards compatibility guarantees.
+ 'open_shards',
]
@@ -98,10 +103,20 @@
| "ToVoidKey" >> Map(lambda v: (None, v)))
_ = ((keyed_singleton, keyed_actual)
| "Group" >> CoGroupByKey()
- | "Unkey" >> Map(lambda (k, (_, actual_values)): actual_values)
+ | "Unkey" >> Map(lambda k___actual_values: k___actual_values[1][1])
| "Match" >> Map(matcher))
def default_label(self):
return label
actual | AssertThat() # pylint: disable=expression-not-assigned
+
+
+@experimental()
+def open_shards(glob_pattern):
+ """Returns a composite file of all shards matching the given glob pattern."""
+ with tempfile.NamedTemporaryFile(delete=False) as f:
+ for shard in glob.glob(glob_pattern):
+ f.write(file(shard).read())
+ concatenated_file_name = f.name
+ return file(concatenated_file_name, 'rb')
diff --git a/sdks/python/apache_beam/testing/util_test.py b/sdks/python/apache_beam/testing/util_test.py
index 1acebb6..9d38693 100644
--- a/sdks/python/apache_beam/testing/util_test.py
+++ b/sdks/python/apache_beam/testing/util_test.py
@@ -21,7 +21,9 @@
from apache_beam import Create
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to, is_empty
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+from apache_beam.testing.util import is_empty
class UtilTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py
index fa0742d..ce5e942 100644
--- a/sdks/python/apache_beam/transforms/combiners.py
+++ b/sdks/python/apache_beam/transforms/combiners.py
@@ -26,9 +26,9 @@
from apache_beam.transforms import cy_combiners
from apache_beam.transforms import ptransform
from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.typehints import KV
from apache_beam.typehints import Any
from apache_beam.typehints import Dict
-from apache_beam.typehints import KV
from apache_beam.typehints import List
from apache_beam.typehints import Tuple
from apache_beam.typehints import TypeVariable
@@ -36,7 +36,6 @@
from apache_beam.typehints import with_input_types
from apache_beam.typehints import with_output_types
-
__all__ = [
'Count',
'Mean',
@@ -78,14 +77,16 @@
def create_accumulator(self):
return (0, 0)
- def add_input(self, (sum_, count), element):
+ def add_input(self, sum_count, element):
+ (sum_, count) = sum_count
return sum_ + element, count + 1
def merge_accumulators(self, accumulators):
sums, counts = zip(*accumulators)
return sum(sums), sum(counts)
- def extract_output(self, (sum_, count)):
+ def extract_output(self, sum_count):
+ (sum_, count) = sum_count
if count == 0:
return float('NaN')
return sum_ / float(count)
@@ -149,6 +150,7 @@
"""Combiners for obtaining extremal elements."""
# pylint: disable=no-self-argument
+ @staticmethod
@ptransform.ptransform_fn
def Of(pcoll, n, compare=None, *args, **kwargs):
"""Obtain a list of the compare-most N elements in a PCollection.
@@ -177,6 +179,7 @@
return pcoll | core.CombineGlobally(
TopCombineFn(n, compare, key, reverse), *args, **kwargs)
+ @staticmethod
@ptransform.ptransform_fn
def PerKey(pcoll, n, compare=None, *args, **kwargs):
"""Identifies the compare-most N elements associated with each key.
@@ -210,21 +213,25 @@
return pcoll | core.CombinePerKey(
TopCombineFn(n, compare, key, reverse), *args, **kwargs)
+ @staticmethod
@ptransform.ptransform_fn
def Largest(pcoll, n):
"""Obtain a list of the greatest N elements in a PCollection."""
return pcoll | Top.Of(n)
+ @staticmethod
@ptransform.ptransform_fn
def Smallest(pcoll, n):
"""Obtain a list of the least N elements in a PCollection."""
return pcoll | Top.Of(n, reverse=True)
+ @staticmethod
@ptransform.ptransform_fn
def LargestPerKey(pcoll, n):
"""Identifies the N greatest elements associated with each key."""
return pcoll | Top.PerKey(n)
+ @staticmethod
@ptransform.ptransform_fn
def SmallestPerKey(pcoll, n, reverse=True):
"""Identifies the N least elements associated with each key."""
@@ -369,10 +376,12 @@
"""Combiners for sampling n elements without replacement."""
# pylint: disable=no-self-argument
+ @staticmethod
@ptransform.ptransform_fn
def FixedSizeGlobally(pcoll, n):
return pcoll | core.CombineGlobally(SampleCombineFn(n))
+ @staticmethod
@ptransform.ptransform_fn
def FixedSizePerKey(pcoll, n):
return pcoll | core.CombinePerKey(SampleCombineFn(n))
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index 946a60a..8885d27 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -22,9 +22,10 @@
import hamcrest as hc
import apache_beam as beam
-from apache_beam.testing.test_pipeline import TestPipeline
import apache_beam.transforms.combiners as combine
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.transforms.core import CombineGlobally
from apache_beam.transforms.core import Create
from apache_beam.transforms.core import Map
@@ -156,14 +157,11 @@
def test_combine_sample_display_data(self):
def individual_test_per_key_dd(sampleFn, args, kwargs):
- trs = [beam.CombinePerKey(sampleFn(*args, **kwargs)),
- beam.CombineGlobally(sampleFn(*args, **kwargs))]
+ trs = [sampleFn(*args, **kwargs)]
for transform in trs:
dd = DisplayData.create_from(transform)
expected_items = [
- DisplayDataItemMatcher('fn', sampleFn.fn.__name__),
- DisplayDataItemMatcher('combine_fn',
- transform.fn.__class__)]
+ DisplayDataItemMatcher('fn', transform._fn.__name__)]
if args:
expected_items.append(
DisplayDataItemMatcher('args', str(args)))
@@ -247,26 +245,23 @@
pipeline.run()
def test_tuple_combine_fn(self):
- p = TestPipeline()
- result = (
- p
- | Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
- | beam.CombineGlobally(combine.TupleCombineFn(max,
- combine.MeanCombineFn(),
- sum)).without_defaults())
- assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
- p.run()
+ with TestPipeline() as p:
+ result = (
+ p
+ | Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
+ | beam.CombineGlobally(combine.TupleCombineFn(
+ max, combine.MeanCombineFn(), sum)).without_defaults())
+ assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
def test_tuple_combine_fn_without_defaults(self):
- p = TestPipeline()
- result = (
- p
- | Create([1, 1, 2, 3])
- | beam.CombineGlobally(
- combine.TupleCombineFn(min, combine.MeanCombineFn(), max)
- .with_common_input()).without_defaults())
- assert_that(result, equal_to([(1, 7.0 / 4, 3)]))
- p.run()
+ with TestPipeline() as p:
+ result = (
+ p
+ | Create([1, 1, 2, 3])
+ | beam.CombineGlobally(
+ combine.TupleCombineFn(min, combine.MeanCombineFn(), max)
+ .with_common_input()).without_defaults())
+ assert_that(result, equal_to([(1, 7.0 / 4, 3)]))
def test_to_list_and_to_dict(self):
pipeline = TestPipeline()
@@ -295,29 +290,26 @@
pipeline.run()
def test_combine_globally_with_default(self):
- p = TestPipeline()
- assert_that(p | Create([]) | CombineGlobally(sum), equal_to([0]))
- p.run()
+ with TestPipeline() as p:
+ assert_that(p | Create([]) | CombineGlobally(sum), equal_to([0]))
def test_combine_globally_without_default(self):
- p = TestPipeline()
- result = p | Create([]) | CombineGlobally(sum).without_defaults()
- assert_that(result, equal_to([]))
- p.run()
+ with TestPipeline() as p:
+ result = p | Create([]) | CombineGlobally(sum).without_defaults()
+ assert_that(result, equal_to([]))
def test_combine_globally_with_default_side_input(self):
- class CombineWithSideInput(PTransform):
+ class SideInputCombine(PTransform):
def expand(self, pcoll):
side = pcoll | CombineGlobally(sum).as_singleton_view()
main = pcoll.pipeline | Create([None])
return main | Map(lambda _, s: s, side)
- p = TestPipeline()
- result1 = p | 'i1' >> Create([]) | 'c1' >> CombineWithSideInput()
- result2 = p | 'i2' >> Create([1, 2, 3, 4]) | 'c2' >> CombineWithSideInput()
- assert_that(result1, equal_to([0]), label='r1')
- assert_that(result2, equal_to([10]), label='r2')
- p.run()
+ with TestPipeline() as p:
+ result1 = p | 'i1' >> Create([]) | 'c1' >> SideInputCombine()
+ result2 = p | 'i2' >> Create([1, 2, 3, 4]) | 'c2' >> SideInputCombine()
+ assert_that(result1, equal_to([0]), label='r1')
+ assert_that(result2, equal_to([10]), label='r2')
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 0e497f9..1c05e97 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -23,35 +23,37 @@
import inspect
import types
+from google.protobuf import wrappers_pb2
+
+from apache_beam import coders
from apache_beam import pvalue
from apache_beam import typehints
from apache_beam.coders import typecoders
+from apache_beam.internal import pickler
from apache_beam.internal import util
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.options.pipeline_options import TypeOptions
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.transforms import ptransform
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.ptransform import PTransform
from apache_beam.transforms.ptransform import PTransformWithSideInputs
-from apache_beam.transforms.window import MIN_TIMESTAMP
-from apache_beam.transforms.window import TimestampCombiner
-from apache_beam.transforms.window import WindowedValue
-from apache_beam.transforms.window import TimestampedValue
from apache_beam.transforms.window import GlobalWindows
+from apache_beam.transforms.window import TimestampCombiner
+from apache_beam.transforms.window import TimestampedValue
+from apache_beam.transforms.window import WindowedValue
from apache_beam.transforms.window import WindowFn
+from apache_beam.typehints import KV
from apache_beam.typehints import Any
from apache_beam.typehints import Iterable
-from apache_beam.typehints import KV
-from apache_beam.typehints import trivial_inference
from apache_beam.typehints import Union
-from apache_beam.typehints.decorators import get_type_hints
+from apache_beam.typehints import trivial_inference
from apache_beam.typehints.decorators import TypeCheckError
from apache_beam.typehints.decorators import WithTypeHints
+from apache_beam.typehints.decorators import get_type_hints
from apache_beam.typehints.trivial_inference import element_type
from apache_beam.typehints.typehints import is_consistent_with
from apache_beam.utils import urns
-from apache_beam.options.pipeline_options import TypeOptions
-
__all__ = [
'DoFn',
@@ -87,6 +89,8 @@
class DoFnProcessContext(DoFnContext):
"""A processing context passed to DoFn process() during execution.
+ Experimental; no backwards-compatibility guarantees.
+
Most importantly, a DoFn.process method will access context.element
to get the element it is supposed to process.
@@ -135,7 +139,127 @@
self.windows = windowed_value.windows
-class DoFn(WithTypeHints, HasDisplayData):
+class ProcessContinuation(object):
+ """An object that may be produced as the last element of a process method
+ invocation.
+
+ Experimental; no backwards-compatibility guarantees.
+
+ If produced, indicates that there is more work to be done for the current
+ input element.
+ """
+
+ def __init__(self, resume_delay=0):
+ """Initializes a ProcessContinuation object.
+
+ Args:
+ resume_delay: indicates the minimum time, in seconds, that should elapse
+ before re-invoking process() method for resuming the invocation of the
+ current element.
+ """
+ self.resume_delay = resume_delay
+
+ @staticmethod
+ def resume(resume_delay=0):
+ """A convenient method that produces a ``ProcessContinuation``.
+
+ Args:
+ resume_delay: delay after which processing current element should be
+ resumed.
+ Returns: a ``ProcessContinuation`` for signalling the runner that current
+ input element has not been fully processed and should be resumed later.
+ """
+ return ProcessContinuation(resume_delay=resume_delay)
+
+
+class RestrictionProvider(object):
+ """Provides methods for generating and manipulating restrictions.
+
+ This class should be implemented to support Splittable ``DoFn``s in Python
+ SDK. See https://s.apache.org/splittable-do-fn for more details about
+ Splittable ``DoFn``s.
+
+ To denote a ``DoFn`` class to be Splittable ``DoFn``, ``DoFn.process()``
+ method of that class should have exactly one parameter whose default value is
+ an instance of ``RestrictionProvider``.
+
+ The provided ``RestrictionProvider`` instance must provide suitable overrides
+ for the following methods.
+ * create_tracker()
+ * initial_restriction()
+
+ Optionally, ``RestrictionProvider`` may override default implementations of
+ following methods.
+ * restriction_coder()
+ * split()
+
+ ** Pausing and resuming processing of an element **
+
+ As the last element produced by the iterator returned by the
+ ``DoFn.process()`` method, a Splittable ``DoFn`` may return an object of type
+ ``ProcessContinuation``.
+
+ If provided, ``ProcessContinuation`` object specifies that runner should
+ later re-invoke ``DoFn.process()`` method to resume processing the current
+ element and the manner in which the re-invocation should be performed. A
+ ``ProcessContinuation`` object must only be specified as the last element of
+ the iterator. If a ``ProcessContinuation`` object is not provided the runner
+ will assume that the current input element has been fully processed.
+
+ ** Updating output watermark **
+
+ ``DoFn.process()`` method of Splittable ``DoFn``s could contain a parameter
+ with default value ``DoFn.WatermarkReporterParam``. If specified this asks the
+ runner to provide a function that can be used to give the runner a
+ (best-effort) lower bound about the timestamps of future output associated
+ with the current element processed by the ``DoFn``. If the ``DoFn`` has
+ multiple outputs, the watermark applies to all of them. Provided function must
+ be invoked with a single parameter of type ``Timestamp`` or as an integer that
+ gives the watermark in number of seconds.
+ """
+
+ def create_tracker(self, restriction):
+ """Produces a new ``RestrictionTracker`` for the given restriction.
+
+ Args:
+ restriction: an object that defines a restriction as identified by a
+ Splittable ``DoFn`` that utilizes the current ``RestrictionProvider``.
+ For example, a tuple that gives a range of positions for a Splittable
+ ``DoFn`` that reads files based on byte positions.
+ Returns: an object of type ``RestrictionTracker``.
+ """
+ raise NotImplementedError
+
+ def initial_restriction(self, element):
+ """Produces an initial restriction for the given element."""
+ raise NotImplementedError
+
+ def split(self, element, restriction):
+ """Splits the given element and restriction.
+
+ Returns an iterator of restrictions. The total set of elements produced by
+ reading input element for each of the returned restrictions should be the
+ same as the total set of elements produced by reading the input element for
+ the input restriction.
+
+ TODO(chamikara): give suitable hints for performing splitting, for example
+ number of parts or size in bytes.
+ """
+ yield restriction
+
+ def restriction_coder(self):
+ """Returns a ``Coder`` for restrictions.
+
+ Returned``Coder`` will be used for the restrictions produced by the current
+ ``RestrictionProvider``.
+
+ Returns:
+ an object of type ``Coder``.
+ """
+ return coders.registry.get_coder(object)
+
+
+class DoFn(WithTypeHints, HasDisplayData, urns.RunnerApiFn):
"""A function object used by a transform with custom processing.
The ParDo transform is such a transform. The ParDo.apply
@@ -151,6 +275,7 @@
SideInputParam = 'SideInputParam'
TimestampParam = 'TimestampParam'
WindowParam = 'WindowParam'
+ WatermarkReporterParam = 'WatermarkReporterParam'
DoFnParams = [ElementParam, SideInputParam, TimestampParam, WindowParam]
@@ -162,13 +287,27 @@
return self.__class__.__name__
def process(self, element, *args, **kwargs):
- """Called for each element of a pipeline. The default arguments are needed
- for the DoFnRunner to be able to pass the parameters correctly.
+ """Method to use for processing elements.
+
+ This is invoked by ``DoFnRunner`` for each element of a input
+ ``PCollection``.
+
+ If specified, following default arguments are used by the ``DoFnRunner`` to
+ be able to pass the parameters correctly.
+
+ ``DoFn.ElementParam``: element to be processed.
+ ``DoFn.SideInputParam``: a side input that may be used when processing.
+ ``DoFn.TimestampParam``: timestamp of the input element.
+ ``DoFn.WindowParam``: ``Window`` the input element belongs to.
+ A ``RestrictionProvider`` instance: an ``iobase.RestrictionTracker`` will be
+ provided here to allow treatment as a Splittable `DoFn``.
+ ``DoFn.WatermarkReporterParam``: a function that can be used to report
+ output watermark of Splittable ``DoFn`` implementations.
Args:
element: The element to be processed
*args: side inputs
- **kwargs: keyword side inputs
+ **kwargs: other keyword arguments.
"""
raise NotImplementedError
@@ -198,7 +337,7 @@
f = getattr(self, func)
return inspect.getargspec(f)
- # TODO(sourabhbajaj): Do we want to remove the responsiblity of these from
+ # TODO(sourabhbajaj): Do we want to remove the responsibility of these from
# the DoFn or maybe the runner
def infer_output_type(self, input_type):
# TODO(robertwb): Side inputs types.
@@ -228,13 +367,15 @@
"""Checks if an object is a bound method on an instance."""
if not isinstance(self.process, types.MethodType):
return False # Not a method
- if self.process.im_self is None:
+ if self.process.__self__ is None:
return False # Method is not bound
- if issubclass(self.process.im_class, type) or \
- self.process.im_class is types.ClassType:
+ if issubclass(self.process.__self__.__class__, type) or \
+ self.process.__self__.__class__ is type:
return False # Method is a classmethod
return True
+ urns.RunnerApiFn.register_pickle_urn(urns.PICKLED_DO_FN)
+
def _fn_takes_side_inputs(fn):
try:
@@ -242,7 +383,7 @@
except TypeError:
# We can't tell; maybe it does.
return True
- is_bound = isinstance(fn, types.MethodType) and fn.im_self is not None
+ is_bound = isinstance(fn, types.MethodType) and fn.__self__ is not None
return len(argspec.args) > 1 + is_bound or argspec.varargs or argspec.keywords
@@ -311,7 +452,7 @@
return getattr(self._fn, '_argspec_fn', self._fn)
-class CombineFn(WithTypeHints, HasDisplayData):
+class CombineFn(WithTypeHints, HasDisplayData, urns.RunnerApiFn):
"""A function object used by a Combine transform with custom processing.
A CombineFn specifies how multiple values in all or part of a PCollection can
@@ -332,6 +473,10 @@
accumulator value left.
5. The extract_output operation is invoked on the final accumulator to get
the output value.
+
+ Note: If this **CombineFn** is used with a transform that has defaults,
+ **apply** will be called with an empty list at expansion time to get the
+ default value.
"""
def default_label(self):
@@ -430,6 +575,11 @@
def maybe_from_callable(fn):
return fn if isinstance(fn, CombineFn) else CallableWrapperCombineFn(fn)
+ def get_accumulator_coder(self):
+ return coders.registry.get_coder(object)
+
+ urns.RunnerApiFn.register_pickle_urn(urns.PICKLED_COMBINE_FN)
+
class CallableWrapperCombineFn(CombineFn):
"""For internal use only; no backwards-compatibility guarantees.
@@ -589,31 +739,35 @@
class ParDo(PTransformWithSideInputs):
- """A ParDo transform.
+ """A :class:`ParDo` transform.
- Processes an input PCollection by applying a DoFn to each element and
- returning the accumulated results into an output PCollection. The type of the
- elements is not fixed as long as the DoFn can deal with it. In reality
- the type is restrained to some extent because the elements sometimes must be
- persisted to external storage. See the expand() method comments for a detailed
- description of all possible arguments.
+ Processes an input :class:`~apache_beam.pvalue.PCollection` by applying a
+ :class:`DoFn` to each element and returning the accumulated results into an
+ output :class:`~apache_beam.pvalue.PCollection`. The type of the elements is
+ not fixed as long as the :class:`DoFn` can deal with it. In reality the type
+ is restrained to some extent because the elements sometimes must be persisted
+ to external storage. See the :meth:`.expand()` method comments for a
+ detailed description of all possible arguments.
- Note that the DoFn must return an iterable for each element of the input
- PCollection. An easy way to do this is to use the yield keyword in the
- process method.
+ Note that the :class:`DoFn` must return an iterable for each element of the
+ input :class:`~apache_beam.pvalue.PCollection`. An easy way to do this is to
+ use the ``yield`` keyword in the process method.
Args:
- pcoll: a PCollection to be processed.
- fn: a DoFn object to be applied to each element of pcoll argument.
- *args: positional arguments passed to the dofn object.
- **kwargs: keyword arguments passed to the dofn object.
+ pcoll (~apache_beam.pvalue.PCollection):
+ a :class:`~apache_beam.pvalue.PCollection` to be processed.
+ fn (DoFn): a :class:`DoFn` object to be applied to each element
+ of **pcoll** argument.
+ *args: positional arguments passed to the :class:`DoFn` object.
+ **kwargs: keyword arguments passed to the :class:`DoFn` object.
Note that the positional and keyword arguments will be processed in order
- to detect PCollections that will be computed as side inputs to the
- transform. During pipeline execution whenever the DoFn object gets executed
- (its apply() method gets called) the PCollection arguments will be replaced
- by values from the PCollection in the exact positions where they appear in
- the argument lists.
+ to detect :class:`~apache_beam.pvalue.PCollection` s that will be computed as
+ side inputs to the transform. During pipeline execution whenever the
+ :class:`DoFn` object gets executed (its :meth:`DoFn.process()` method gets
+ called) the :class:`~apache_beam.pvalue.PCollection` arguments will be
+ replaced by values from the :class:`~apache_beam.pvalue.PCollection` in the
+ exact positions where they appear in the argument lists.
"""
def __init__(self, fn, *args, **kwargs):
@@ -653,33 +807,84 @@
return pvalue.PCollection(pcoll.pipeline)
def with_outputs(self, *tags, **main_kw):
- """Returns a tagged tuple allowing access to the outputs of a ParDo.
+ """Returns a tagged tuple allowing access to the outputs of a
+ :class:`ParDo`.
The resulting object supports access to the
- PCollection associated with a tag (e.g., o.tag, o[tag]) and iterating over
- the available tags (e.g., for tag in o: ...).
+ :class:`~apache_beam.pvalue.PCollection` associated with a tag
+ (e.g. ``o.tag``, ``o[tag]``) and iterating over the available tags
+ (e.g. ``for tag in o: ...``).
Args:
*tags: if non-empty, list of valid tags. If a list of valid tags is given,
it will be an error to use an undeclared tag later in the pipeline.
- **main_kw: dictionary empty or with one key 'main' defining the tag to be
- used for the main output (which will not have a tag associated with it).
+ **main_kw: dictionary empty or with one key ``'main'`` defining the tag to
+ be used for the main output (which will not have a tag associated with
+ it).
Returns:
- An object of type DoOutputsTuple that bundles together all the outputs
- of a ParDo transform and allows accessing the individual
- PCollections for each output using an object.tag syntax.
+ ~apache_beam.pvalue.DoOutputsTuple: An object of type
+ :class:`~apache_beam.pvalue.DoOutputsTuple` that bundles together all
+ the outputs of a :class:`ParDo` transform and allows accessing the
+ individual :class:`~apache_beam.pvalue.PCollection` s for each output
+ using an ``object.tag`` syntax.
Raises:
- TypeError: if the self object is not a PCollection that is the result of
- a ParDo transform.
- ValueError: if main_kw contains any key other than 'main'.
+ ~exceptions.TypeError: if the **self** object is not a
+ :class:`~apache_beam.pvalue.PCollection` that is the result of a
+ :class:`ParDo` transform.
+ ~exceptions.ValueError: if **main_kw** contains any key other than
+ ``'main'``.
"""
main_tag = main_kw.pop('main', None)
if main_kw:
raise ValueError('Unexpected keyword arguments: %s' % main_kw.keys())
return _MultiParDo(self, tags, main_tag)
+ def _pardo_fn_data(self):
+ si_tags_and_types = None
+ windowing = None
+ return self.fn, self.args, self.kwargs, si_tags_and_types, windowing
+
+ def to_runner_api_parameter(self, context):
+ assert isinstance(self, ParDo), \
+ "expected instance of ParDo, but got %s" % self.__class__
+ picked_pardo_fn_data = pickler.dumps(self._pardo_fn_data())
+ return (
+ urns.PARDO_TRANSFORM,
+ beam_runner_api_pb2.ParDoPayload(
+ do_fn=beam_runner_api_pb2.SdkFunctionSpec(
+ spec=beam_runner_api_pb2.FunctionSpec(
+ urn=urns.PICKLED_DO_FN_INFO,
+ payload=picked_pardo_fn_data)),
+ # It'd be nice to name these according to their actual
+ # names/positions in the orignal argument list, but such a
+ # transformation is currently irreversible given how
+ # remove_objects_from_args and insert_values_in_args
+ # are currently implemented.
+ side_inputs={
+ "side%s" % ix: si.to_runner_api(context)
+ for ix, si in enumerate(self.side_inputs)}))
+
+ @PTransform.register_urn(
+ urns.PARDO_TRANSFORM, beam_runner_api_pb2.ParDoPayload)
+ def from_runner_api_parameter(pardo_payload, context):
+ assert pardo_payload.do_fn.spec.urn == urns.PICKLED_DO_FN_INFO
+ fn, args, kwargs, si_tags_and_types, windowing = pickler.loads(
+ pardo_payload.do_fn.spec.payload)
+ if si_tags_and_types:
+ raise NotImplementedError('explicit side input data')
+ elif windowing:
+ raise NotImplementedError('explicit windowing')
+ result = ParDo(fn, *args, **kwargs)
+ # This is an ordered list stored as a dict (see the comments in
+ # to_runner_api_parameter above).
+ indexed_side_inputs = [
+ (int(ix[4:]), pvalue.AsSideInput.from_runner_api(si, context))
+ for ix, si in pardo_payload.side_inputs.items()]
+ result.side_inputs = [si for _, si in sorted(indexed_side_inputs)]
+ return result
+
class _MultiParDo(PTransform):
@@ -696,24 +901,27 @@
def FlatMap(fn, *args, **kwargs): # pylint: disable=invalid-name
- """FlatMap is like ParDo except it takes a callable to specify the
- transformation.
+ """:func:`FlatMap` is like :class:`ParDo` except it takes a callable to
+ specify the transformation.
The callable must return an iterable for each element of the input
- PCollection. The elements of these iterables will be flattened into
- the output PCollection.
+ :class:`~apache_beam.pvalue.PCollection`. The elements of these iterables will
+ be flattened into the output :class:`~apache_beam.pvalue.PCollection`.
Args:
- fn: a callable object.
+ fn (callable): a callable object.
*args: positional arguments passed to the transform callable.
**kwargs: keyword arguments passed to the transform callable.
Returns:
- A PCollection containing the Map outputs.
+ ~apache_beam.pvalue.PCollection:
+ A :class:`~apache_beam.pvalue.PCollection` containing the
+ :func:`FlatMap` outputs.
Raises:
- TypeError: If the fn passed as argument is not a callable. Typical error
- is to pass a DoFn instance which is supported only for ParDo.
+ ~exceptions.TypeError: If the **fn** passed as argument is not a callable.
+ Typical error is to pass a :class:`DoFn` instance which is supported only
+ for :class:`ParDo`.
"""
label = 'FlatMap(%s)' % ptransform.label_from_callable(fn)
if not callable(fn):
@@ -727,19 +935,23 @@
def Map(fn, *args, **kwargs): # pylint: disable=invalid-name
- """Map is like FlatMap except its callable returns only a single element.
+ """:func:`Map` is like :func:`FlatMap` except its callable returns only a
+ single element.
Args:
- fn: a callable object.
+ fn (callable): a callable object.
*args: positional arguments passed to the transform callable.
**kwargs: keyword arguments passed to the transform callable.
Returns:
- A PCollection containing the Map outputs.
+ ~apache_beam.pvalue.PCollection:
+ A :class:`~apache_beam.pvalue.PCollection` containing the
+ :func:`Map` outputs.
Raises:
- TypeError: If the fn passed as argument is not a callable. Typical error
- is to pass a DoFn instance which is supported only for ParDo.
+ ~exceptions.TypeError: If the **fn** passed as argument is not a callable.
+ Typical error is to pass a :class:`DoFn` instance which is supported only
+ for :class:`ParDo`.
"""
if not callable(fn):
raise TypeError(
@@ -772,19 +984,23 @@
def Filter(fn, *args, **kwargs): # pylint: disable=invalid-name
- """Filter is a FlatMap with its callable filtering out elements.
+ """:func:`Filter` is a :func:`FlatMap` with its callable filtering out
+ elements.
Args:
- fn: a callable object.
+ fn (callable): a callable object.
*args: positional arguments passed to the transform callable.
**kwargs: keyword arguments passed to the transform callable.
Returns:
- A PCollection containing the Filter outputs.
+ ~apache_beam.pvalue.PCollection:
+ A :class:`~apache_beam.pvalue.PCollection` containing the
+ :func:`Filter` outputs.
Raises:
- TypeError: If the fn passed as argument is not a callable. Typical error
- is to pass a DoFn instance which is supported only for FlatMap.
+ ~exceptions.TypeError: If the **fn** passed as argument is not a callable.
+ Typical error is to pass a :class:`DoFn` instance which is supported only
+ for :class:`ParDo`.
"""
if not callable(fn):
raise TypeError(
@@ -816,36 +1032,50 @@
return pardo
-class CombineGlobally(PTransform):
- """A CombineGlobally transform.
+def _combine_payload(combine_fn, context):
+ return beam_runner_api_pb2.CombinePayload(
+ combine_fn=combine_fn.to_runner_api(context),
+ accumulator_coder_id=context.coders.get_id(
+ combine_fn.get_accumulator_coder()))
- Reduces a PCollection to a single value by progressively applying a CombineFn
- to portions of the PCollection (and to intermediate values created thereby).
- See documentation in CombineFn for details on the specifics on how CombineFns
- are applied.
+
+class CombineGlobally(PTransform):
+ """A :class:`CombineGlobally` transform.
+
+ Reduces a :class:`~apache_beam.pvalue.PCollection` to a single value by
+ progressively applying a :class:`CombineFn` to portions of the
+ :class:`~apache_beam.pvalue.PCollection` (and to intermediate values created
+ thereby). See documentation in :class:`CombineFn` for details on the specifics
+ on how :class:`CombineFn` s are applied.
Args:
- pcoll: a PCollection to be reduced into a single value.
- fn: a CombineFn object that will be called to progressively reduce the
- PCollection into single values, or a callable suitable for wrapping
- by CallableWrapperCombineFn.
- *args: positional arguments passed to the CombineFn object.
- **kwargs: keyword arguments passed to the CombineFn object.
+ pcoll (~apache_beam.pvalue.PCollection):
+ a :class:`~apache_beam.pvalue.PCollection` to be reduced into a single
+ value.
+ fn (callable): a :class:`CombineFn` object that will be called to
+ progressively reduce the :class:`~apache_beam.pvalue.PCollection` into
+ single values, or a callable suitable for wrapping by
+ :class:`~apache_beam.transforms.core.CallableWrapperCombineFn`.
+ *args: positional arguments passed to the :class:`CombineFn` object.
+ **kwargs: keyword arguments passed to the :class:`CombineFn` object.
Raises:
- TypeError: If the output type of the input PCollection is not compatible
- with Iterable[A].
+ ~exceptions.TypeError: If the output type of the input
+ :class:`~apache_beam.pvalue.PCollection` is not compatible
+ with ``Iterable[A]``.
Returns:
- A single-element PCollection containing the main output of the Combine
- transform.
+ ~apache_beam.pvalue.PCollection: A single-element
+ :class:`~apache_beam.pvalue.PCollection` containing the main output of
+ the :class:`CombineGlobally` transform.
Note that the positional and keyword arguments will be processed in order
- to detect PObjects that will be computed as side inputs to the transform.
- During pipeline execution whenever the CombineFn object gets executed (i.e.,
- any of the CombineFn methods get called), the PObject arguments will be
- replaced by their actual value in the exact position where they appear in
- the argument lists.
+ to detect :class:`~apache_beam.pvalue.PValue` s that will be computed as side
+ inputs to the transform.
+ During pipeline execution whenever the :class:`CombineFn` object gets executed
+ (i.e. any of the :class:`CombineFn` methods get called), the
+ :class:`~apache_beam.pvalue.PValue` arguments will be replaced by their
+ actual value in the exact position where they appear in the argument lists.
"""
has_defaults = True
as_view = False
@@ -897,7 +1127,7 @@
KV[None, pcoll.element_type]))
| 'CombinePerKey' >> CombinePerKey(
self.fn, *self.args, **self.kwargs)
- | 'UnKey' >> Map(lambda (k, v): v))
+ | 'UnKey' >> Map(lambda k_v: k_v[1]))
if not self.has_defaults and not self.as_view:
return combined
@@ -973,6 +1203,17 @@
return pcoll | GroupByKey() | 'Combine' >> CombineValues(
self.fn, *args, **kwargs)
+ def to_runner_api_parameter(self, context):
+ return (
+ urns.COMBINE_PER_KEY_TRANSFORM,
+ _combine_payload(self.fn, context))
+
+ @PTransform.register_urn(
+ urns.COMBINE_PER_KEY_TRANSFORM, beam_runner_api_pb2.CombinePayload)
+ def from_runner_api_parameter(combine_payload, context):
+ return CombinePerKey(
+ CombineFn.from_runner_api(combine_payload.combine_fn, context))
+
# TODO(robertwb): Rename to CombineGroupedValues?
class CombineValues(PTransformWithSideInputs):
@@ -995,6 +1236,17 @@
CombineValuesDoFn(key_type, self.fn, runtime_type_check),
*args, **kwargs)
+ def to_runner_api_parameter(self, context):
+ return (
+ urns.COMBINE_GROUPED_VALUES_TRANSFORM,
+ _combine_payload(self.fn, context))
+
+ @PTransform.register_urn(
+ urns.COMBINE_GROUPED_VALUES_TRANSFORM, beam_runner_api_pb2.CombinePayload)
+ def from_runner_api_parameter(combine_payload, context):
+ return CombineValues(
+ CombineFn.from_runner_api(combine_payload.combine_fn, context))
+
class CombineValuesDoFn(DoFn):
"""DoFn for performing per-key Combine transforms."""
@@ -1017,7 +1269,7 @@
self.combinefn.apply(element[1], *args, **kwargs))]
# Add the elements into three accumulators (for testing of merge).
- elements = element[1]
+ elements = list(element[1])
accumulators = []
for k in range(3):
if len(elements) <= k:
@@ -1078,40 +1330,6 @@
key_type, value_type = trivial_inference.key_value_types(input_type)
return Iterable[KV[key_type, typehints.WindowedValue[value_type]]]
- class GroupAlsoByWindow(DoFn):
- # TODO(robertwb): Support combiner lifting.
-
- def __init__(self, windowing):
- super(GroupByKey.GroupAlsoByWindow, self).__init__()
- self.windowing = windowing
-
- def infer_output_type(self, input_type):
- key_type, windowed_value_iter_type = trivial_inference.key_value_types(
- input_type)
- value_type = windowed_value_iter_type.inner_type.inner_type
- return Iterable[KV[key_type, Iterable[value_type]]]
-
- def start_bundle(self):
- # pylint: disable=wrong-import-order, wrong-import-position
- from apache_beam.transforms.trigger import InMemoryUnmergedState
- from apache_beam.transforms.trigger import create_trigger_driver
- # pylint: enable=wrong-import-order, wrong-import-position
- self.driver = create_trigger_driver(self.windowing, True)
- self.state_type = InMemoryUnmergedState
-
- def process(self, element):
- k, vs = element
- state = self.state_type()
- # TODO(robertwb): Conditionally process in smaller chunks.
- for wvalue in self.driver.process_elements(state, vs, MIN_TIMESTAMP):
- yield wvalue.with_value((k, wvalue.value))
- while state.timers:
- fired = state.get_and_clear_timers()
- for timer_window, (name, time_domain, fire_time) in fired:
- for wvalue in self.driver.process_timer(
- timer_window, name, time_domain, fire_time, state):
- yield wvalue.with_value((k, wvalue.value))
-
def expand(self, pcoll):
# This code path is only used in the local direct runner. For Dataflow
# runner execution, the GroupByKey transform is expanded on the service.
@@ -1120,6 +1338,8 @@
# Initialize type-hints used below to enforce type-checking and to pass
# downstream to further PTransforms.
key_type, value_type = trivial_inference.key_value_types(input_type)
+ # Enforce the input to a GBK has a KV element type.
+ pcoll.element_type = KV[key_type, value_type]
typecoders.registry.verify_deterministic(
typecoders.registry.get_coder(key_type),
'GroupByKey operation "%s"' % self.label)
@@ -1136,8 +1356,7 @@
| 'GroupByKey' >> (_GroupByKeyOnly()
.with_input_types(reify_output_type)
.with_output_types(gbk_input_type))
- | ('GroupByWindow' >> ParDo(
- self.GroupAlsoByWindow(pcoll.windowing))
+ | ('GroupByWindow' >> _GroupAlsoByWindow(pcoll.windowing)
.with_input_types(gbk_input_type)
.with_output_types(gbk_output_type)))
else:
@@ -1145,8 +1364,14 @@
return (pcoll
| 'ReifyWindows' >> ParDo(self.ReifyWindows())
| 'GroupByKey' >> _GroupByKeyOnly()
- | 'GroupByWindow' >> ParDo(
- self.GroupAlsoByWindow(pcoll.windowing)))
+ | 'GroupByWindow' >> _GroupAlsoByWindow(pcoll.windowing))
+
+ def to_runner_api_parameter(self, unused_context):
+ return urns.GROUP_BY_KEY_TRANSFORM, None
+
+ @PTransform.register_urn(urns.GROUP_BY_KEY_TRANSFORM, None)
+ def from_runner_api_parameter(unused_payload, unused_context):
+ return GroupByKey()
@typehints.with_input_types(typehints.KV[K, V])
@@ -1161,6 +1386,63 @@
self._check_pcollection(pcoll)
return pvalue.PCollection(pcoll.pipeline)
+ def to_runner_api_parameter(self, unused_context):
+ return urns.GROUP_BY_KEY_ONLY_TRANSFORM, None
+
+ @PTransform.register_urn(urns.GROUP_BY_KEY_ONLY_TRANSFORM, None)
+ def from_runner_api_parameter(unused_payload, unused_context):
+ return _GroupByKeyOnly()
+
+
+@typehints.with_input_types(typehints.KV[K, typehints.Iterable[V]])
+@typehints.with_output_types(typehints.KV[K, typehints.Iterable[V]])
+class _GroupAlsoByWindow(ParDo):
+ """The GroupAlsoByWindow transform."""
+ def __init__(self, windowing):
+ super(_GroupAlsoByWindow, self).__init__(
+ _GroupAlsoByWindowDoFn(windowing))
+ self.windowing = windowing
+
+ def expand(self, pcoll):
+ self._check_pcollection(pcoll)
+ return pvalue.PCollection(pcoll.pipeline)
+
+ def to_runner_api_parameter(self, context):
+ return (
+ urns.GROUP_ALSO_BY_WINDOW_TRANSFORM,
+ wrappers_pb2.BytesValue(value=context.windowing_strategies.get_id(
+ self.windowing)))
+
+ @PTransform.register_urn(
+ urns.GROUP_ALSO_BY_WINDOW_TRANSFORM, wrappers_pb2.BytesValue)
+ def from_runner_api_parameter(payload, context):
+ return _GroupAlsoByWindow(
+ context.windowing_strategies.get_by_id(payload.value))
+
+
+class _GroupAlsoByWindowDoFn(DoFn):
+ # TODO(robertwb): Support combiner lifting.
+
+ def __init__(self, windowing):
+ super(_GroupAlsoByWindowDoFn, self).__init__()
+ self.windowing = windowing
+
+ def infer_output_type(self, input_type):
+ key_type, windowed_value_iter_type = trivial_inference.key_value_types(
+ input_type)
+ value_type = windowed_value_iter_type.inner_type.inner_type
+ return Iterable[KV[key_type, Iterable[value_type]]]
+
+ def start_bundle(self):
+ # pylint: disable=wrong-import-order, wrong-import-position
+ from apache_beam.transforms.trigger import create_trigger_driver
+ # pylint: enable=wrong-import-order, wrong-import-position
+ self.driver = create_trigger_driver(self.windowing, True)
+
+ def process(self, element):
+ k, vs = element
+ return self.driver.process_entire_key(k, vs)
+
class Partition(PTransformWithSideInputs):
"""Split a PCollection into several partitions.
@@ -1221,7 +1503,7 @@
if not windowfn.get_window_coder().is_deterministic():
raise ValueError(
'window fn (%s) does not have a determanistic coder (%s)' % (
- window_fn, windowfn.get_window_coder()))
+ windowfn, windowfn.get_window_coder()))
self.windowfn = windowfn
self.triggerfn = triggerfn
self.accumulation_mode = accumulation_mode
@@ -1256,16 +1538,16 @@
return beam_runner_api_pb2.WindowingStrategy(
window_fn=self.windowfn.to_runner_api(context),
# TODO(robertwb): Prohibit implicit multi-level merging.
- merge_status=(beam_runner_api_pb2.NEEDS_MERGE
+ merge_status=(beam_runner_api_pb2.MergeStatus.NEEDS_MERGE
if self.windowfn.is_merging()
- else beam_runner_api_pb2.NON_MERGING),
+ else beam_runner_api_pb2.MergeStatus.NON_MERGING),
window_coder_id=context.coders.get_id(
self.windowfn.get_window_coder()),
trigger=self.triggerfn.to_runner_api(context),
accumulation_mode=self.accumulation_mode,
output_time=self.timestamp_combiner,
# TODO(robertwb): Support EMIT_IF_NONEMPTY
- closing_behavior=beam_runner_api_pb2.EMIT_ALWAYS,
+ closing_behavior=beam_runner_api_pb2.ClosingBehavior.EMIT_ALWAYS,
allowed_lateness=0)
@staticmethod
@@ -1350,6 +1632,7 @@
# (Right now only WindowFn is used, but we need this to reconstitute the
# WindowInto transform, and in the future will need it at runtime to
# support meta-data driven triggers.)
+ # TODO(robertwb): Use a reference rather than embedding?
beam_runner_api_pb2.WindowingStrategy,
WindowInto.from_runner_api_parameter)
@@ -1389,7 +1672,10 @@
def expand(self, pcolls):
for pcoll in pcolls:
self._check_pcollection(pcoll)
- return pvalue.PCollection(self.pipeline)
+ result = pvalue.PCollection(self.pipeline)
+ result.element_type = typehints.Union[
+ tuple(pcoll.element_type for pcoll in pcolls)]
+ return result
def get_windowing(self, inputs):
if not inputs:
@@ -1431,15 +1717,18 @@
return Any
return Union[[trivial_inference.instance_to_type(v) for v in self.value]]
+ def get_output_type(self):
+ return (self.get_type_hints().simple_output_type(self.label) or
+ self.infer_output_type(None))
+
def expand(self, pbegin):
from apache_beam.io import iobase
assert isinstance(pbegin, pvalue.PBegin)
self.pipeline = pbegin.pipeline
- ouput_type = (self.get_type_hints().simple_output_type(self.label) or
- self.infer_output_type(None))
- coder = typecoders.registry.get_coder(ouput_type)
+ coder = typecoders.registry.get_coder(self.get_output_type())
source = self._create_source_from_iterable(self.value, coder)
- return pbegin.pipeline | iobase.Read(source).with_output_types(ouput_type)
+ return (pbegin.pipeline
+ | iobase.Read(source).with_output_types(self.get_output_type()))
def get_windowing(self, unused_inputs):
return Windowing(GlobalWindows())
diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py
index 55ad7f3..e586329 100644
--- a/sdks/python/apache_beam/transforms/create_test.py
+++ b/sdks/python/apache_beam/transforms/create_test.py
@@ -18,12 +18,12 @@
"""Unit tests for the Create and _CreateSource classes."""
import unittest
-from apache_beam.io import source_test_utils
-
from apache_beam import Create
from apache_beam.coders import FastPrimitivesCoder
+from apache_beam.io import source_test_utils
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
class CreateTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py
index 152f16e..cb7b53e 100644
--- a/sdks/python/apache_beam/transforms/display.py
+++ b/sdks/python/apache_beam/transforms/display.py
@@ -16,30 +16,33 @@
#
"""
-DisplayData, its classes, interfaces and methods.
+:class:`DisplayData`, its classes, interfaces and methods.
The classes in this module allow users and transform developers to define
-static display data to be displayed when a pipeline runs. PTransforms, DoFns
-and other pipeline components are subclasses of the HasDisplayData mixin. To
-add static display data to a component, you can override the display_data
-method of the HasDisplayData class.
+static display data to be displayed when a pipeline runs.
+:class:`~apache_beam.transforms.ptransform.PTransform` s,
+:class:`~apache_beam.transforms.core.DoFn` s
+and other pipeline components are subclasses of the :class:`HasDisplayData`
+mixin. To add static display data to a component, you can override the
+:meth:`HasDisplayData.display_data()` method.
Available classes:
-- HasDisplayData - Components that inherit from this class can have static
- display data shown in the UI.
-- DisplayDataItem - This class represents static display data elements.
-- DisplayData - Internal class that is used to create display data and
- communicate it to the API.
+* :class:`HasDisplayData` - Components that inherit from this class can have
+ static display data shown in the UI.
+* :class:`DisplayDataItem` - This class represents static display data
+ elements.
+* :class:`DisplayData` - Internal class that is used to create display data
+ and communicate it to the API.
"""
from __future__ import absolute_import
import calendar
-from datetime import datetime, timedelta
import inspect
import json
-
+from datetime import datetime
+from datetime import timedelta
__all__ = ['HasDisplayData', 'DisplayDataItem', 'DisplayData']
@@ -57,17 +60,19 @@
static display data.
Returns:
- A dictionary containing key:value pairs. The value might be an
- integer, float or string value; a DisplayDataItem for values that
- have more data (e.g. short value, label, url); or a HasDisplayData
- instance that has more display data that should be picked up. For
- example:
+ Dict[str, Any]: A dictionary containing ``key:value`` pairs.
+ The value might be an integer, float or string value; a
+ :class:`DisplayDataItem` for values that have more data
+ (e.g. short value, label, url); or a :class:`HasDisplayData` instance
+ that has more display data that should be picked up. For example::
- { 'key1': 'string_value',
- 'key2': 1234,
- 'key3': 3.14159265,
- 'key4': DisplayDataItem('apache.org', url='http://apache.org'),
- 'key5': subComponent }
+ {
+ 'key1': 'string_value',
+ 'key2': 1234,
+ 'key3': 3.14159265,
+ 'key4': DisplayDataItem('apache.org', url='http://apache.org'),
+ 'key5': subComponent
+ }
"""
return {}
@@ -111,18 +116,19 @@
@classmethod
def create_from_options(cls, pipeline_options):
- """ Creates DisplayData from a PipelineOptions instance.
+ """ Creates :class:`DisplayData` from a
+ :class:`~apache_beam.options.pipeline_options.PipelineOptions` instance.
- When creating DisplayData, this method will convert the value of any
- item of a non-supported type to its string representation.
- The normal DisplayData.create_from method rejects those items.
+ When creating :class:`DisplayData`, this method will convert the value of
+ any item of a non-supported type to its string representation.
+ The normal :meth:`.create_from()` method rejects those items.
Returns:
- A DisplayData instance with populated items.
+ DisplayData: A :class:`DisplayData` instance with populated items.
Raises:
- ValueError: If the has_display_data argument is not an instance of
- HasDisplayData.
+ ~exceptions.ValueError: If the **has_display_data** argument is
+ not an instance of :class:`HasDisplayData`.
"""
from apache_beam.options.pipeline_options import PipelineOptions
if not isinstance(pipeline_options, PipelineOptions):
@@ -138,14 +144,14 @@
@classmethod
def create_from(cls, has_display_data):
- """ Creates DisplayData from a HasDisplayData instance.
+ """ Creates :class:`DisplayData` from a :class:`HasDisplayData` instance.
Returns:
- A DisplayData instance with populated items.
+ DisplayData: A :class:`DisplayData` instance with populated items.
Raises:
- ValueError: If the has_display_data argument is not an instance of
- HasDisplayData.
+ ~exceptions.ValueError: If the **has_display_data** argument is
+ not an instance of :class:`HasDisplayData`.
"""
if not isinstance(has_display_data, HasDisplayData):
raise ValueError('Element of class {}.{} does not subclass HasDisplayData'
@@ -214,11 +220,13 @@
return False
def is_valid(self):
- """ Checks that all the necessary fields of the DisplayDataItem are
- filled in. It checks that neither key, namespace, value or type are None.
+ """ Checks that all the necessary fields of the :class:`DisplayDataItem`
+ are filled in. It checks that neither key, namespace, value or type are
+ :data:`None`.
Raises:
- ValueError: If the item does not have a key, namespace, value or type.
+ ~exceptions.ValueError: If the item does not have a key, namespace,
+ value or type.
"""
if self.key is None:
raise ValueError('Invalid DisplayDataItem. Key must not be None')
@@ -247,14 +255,15 @@
return res
def get_dict(self):
- """ Returns the internal-API dictionary representing the DisplayDataItem.
+ """ Returns the internal-API dictionary representing the
+ :class:`DisplayDataItem`.
Returns:
- A dictionary. The internal-API dictionary representing the
- DisplayDataItem
+ Dict[str, Any]: A dictionary. The internal-API dictionary representing
+ the :class:`DisplayDataItem`.
Raises:
- ValueError: if the item is not valid.
+ ~exceptions.ValueError: if the item is not valid.
"""
self.is_valid()
return self._get_dict()
diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py
index 15f1786..5c73cf3 100644
--- a/sdks/python/apache_beam/transforms/display_test.py
+++ b/sdks/python/apache_beam/transforms/display_test.py
@@ -19,17 +19,17 @@
from __future__ import absolute_import
-from datetime import datetime
import unittest
+from datetime import datetime
import hamcrest as hc
from hamcrest.core.base_matcher import BaseMatcher
import apache_beam as beam
-from apache_beam.transforms.display import HasDisplayData
+from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.transforms.display import HasDisplayData
class DisplayDataItemMatcher(BaseMatcher):
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index bd2a120..0b6d608b 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -41,6 +41,7 @@
import operator
import os
import sys
+from functools import reduce
from google.protobuf import wrappers_pb2
@@ -48,18 +49,17 @@
from apache_beam import pvalue
from apache_beam.internal import pickler
from apache_beam.internal import util
-from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms.display import HasDisplayData
from apache_beam.typehints import typehints
-from apache_beam.typehints.decorators import getcallargs_forhints
from apache_beam.typehints.decorators import TypeCheckError
from apache_beam.typehints.decorators import WithTypeHints
+from apache_beam.typehints.decorators import getcallargs_forhints
from apache_beam.typehints.trivial_inference import instance_to_type
from apache_beam.typehints.typehints import validate_composite_type_param
from apache_beam.utils import proto_utils
from apache_beam.utils import urns
-
__all__ = [
'PTransform',
'ptransform_fn',
@@ -74,27 +74,27 @@
This visits a PValueish, contstructing a (possibly mutated) copy.
"""
- def visit(self, node, *args):
- return getattr(
- self,
- 'visit_' + node.__class__.__name__,
- lambda x, *args: x)(node, *args)
-
- def visit_list(self, node, *args):
- return [self.visit(x, *args) for x in node]
-
- def visit_tuple(self, node, *args):
- return tuple(self.visit(x, *args) for x in node)
-
- def visit_dict(self, node, *args):
- return {key: self.visit(value, *args) for (key, value) in node.items()}
+ def visit_nested(self, node, *args):
+ if isinstance(node, (tuple, list)):
+ args = [self.visit(x, *args) for x in node]
+ if isinstance(node, tuple) and hasattr(node.__class__, '_make'):
+ # namedtuples require unpacked arguments in their constructor
+ return node.__class__(*args)
+ else:
+ return node.__class__(args)
+ elif isinstance(node, dict):
+ return node.__class__(
+ {key: self.visit(value, *args) for (key, value) in node.items()})
+ else:
+ return node
class _SetInputPValues(_PValueishTransform):
def visit(self, node, replacements):
if id(node) in replacements:
return replacements[id(node)]
- return super(_SetInputPValues, self).visit(node, replacements)
+ else:
+ return self.visit_nested(node, replacements)
class _MaterializedDoOutputsTuple(pvalue.DoOutputsTuple):
@@ -105,7 +105,9 @@
self._pvalue_cache = pvalue_cache
def __getitem__(self, tag):
- return self._pvalue_cache.get_unwindowed_pvalue(self._deferred[tag])
+ # Simply accessing the value should not use it up.
+ return self._pvalue_cache.get_unwindowed_pvalue(
+ self._deferred[tag], decref=False)
class _MaterializePValues(_PValueishTransform):
@@ -114,25 +116,29 @@
def visit(self, node):
if isinstance(node, pvalue.PValue):
- return self._pvalue_cache.get_unwindowed_pvalue(node)
+ # Simply accessing the value should not use it up.
+ return self._pvalue_cache.get_unwindowed_pvalue(node, decref=False)
elif isinstance(node, pvalue.DoOutputsTuple):
return _MaterializedDoOutputsTuple(node, self._pvalue_cache)
- return super(_MaterializePValues, self).visit(node)
+ else:
+ return self.visit_nested(node)
-class GetPValues(_PValueishTransform):
- def visit(self, node, pvalues=None):
- if pvalues is None:
- pvalues = []
- self.visit(node, pvalues)
- return pvalues
- elif isinstance(node, (pvalue.PValue, pvalue.DoOutputsTuple)):
+class _GetPValues(_PValueishTransform):
+ def visit(self, node, pvalues):
+ if isinstance(node, (pvalue.PValue, pvalue.DoOutputsTuple)):
pvalues.append(node)
else:
- super(GetPValues, self).visit(node, pvalues)
+ self.visit_nested(node, pvalues)
-class _ZipPValues(_PValueishTransform):
+def get_nested_pvalues(pvalueish):
+ pvalues = []
+ _GetPValues().visit(pvalueish, pvalues)
+ return pvalues
+
+
+class _ZipPValues(object):
"""Pairs each PValue in a pvalueish with a value in a parallel out sibling.
Sibling should have the same nested structure as pvalueish. Leaves in
@@ -154,10 +160,12 @@
return pairs
elif isinstance(pvalueish, (pvalue.PValue, pvalue.DoOutputsTuple)):
pairs.append((context, pvalueish, sibling))
- else:
- super(_ZipPValues, self).visit(pvalueish, sibling, pairs, context)
+ elif isinstance(pvalueish, (list, tuple)):
+ self.visit_sequence(pvalueish, sibling, pairs, context)
+ elif isinstance(pvalueish, dict):
+ self.visit_dict(pvalueish, sibling, pairs, context)
- def visit_list(self, pvalueish, sibling, pairs, context):
+ def visit_sequence(self, pvalueish, sibling, pairs, context):
if isinstance(sibling, (list, tuple)):
for ix, (p, s) in enumerate(zip(
pvalueish, list(sibling) + [None] * len(pvalueish))):
@@ -166,9 +174,6 @@
for p in pvalueish:
self.visit(p, sibling, pairs, context)
- def visit_tuple(self, pvalueish, sibling, pairs, context):
- self.visit_list(pvalueish, sibling, pairs, context)
-
def visit_dict(self, pvalueish, sibling, pairs, context):
if isinstance(sibling, dict):
for key, p in pvalueish.items():
@@ -214,38 +219,44 @@
return self.__class__.__name__
def with_input_types(self, input_type_hint):
- """Annotates the input type of a PTransform with a type-hint.
+ """Annotates the input type of a :class:`PTransform` with a type-hint.
Args:
- input_type_hint: An instance of an allowed built-in type, a custom class,
- or an instance of a typehints.TypeConstraint.
+ input_type_hint (type): An instance of an allowed built-in type, a custom
+ class, or an instance of a
+ :class:`~apache_beam.typehints.typehints.TypeConstraint`.
Raises:
- TypeError: If 'type_hint' is not a valid type-hint. See
- typehints.validate_composite_type_param for further details.
+ ~exceptions.TypeError: If **input_type_hint** is not a valid type-hint.
+ See
+ :obj:`apache_beam.typehints.typehints.validate_composite_type_param()`
+ for further details.
Returns:
- A reference to the instance of this particular PTransform object. This
- allows chaining type-hinting related methods.
+ PTransform: A reference to the instance of this particular
+ :class:`PTransform` object. This allows chaining type-hinting related
+ methods.
"""
validate_composite_type_param(input_type_hint,
'Type hints for a PTransform')
return super(PTransform, self).with_input_types(input_type_hint)
def with_output_types(self, type_hint):
- """Annotates the output type of a PTransform with a type-hint.
+ """Annotates the output type of a :class:`PTransform` with a type-hint.
Args:
- type_hint: An instance of an allowed built-in type, a custom class, or a
- typehints.TypeConstraint.
+ type_hint (type): An instance of an allowed built-in type, a custom class,
+ or a :class:`~apache_beam.typehints.typehints.TypeConstraint`.
Raises:
- TypeError: If 'type_hint' is not a valid type-hint. See
- typehints.validate_composite_type_param for further details.
+ ~exceptions.TypeError: If **type_hint** is not a valid type-hint. See
+ :obj:`~apache_beam.typehints.typehints.validate_composite_type_param()`
+ for further details.
Returns:
- A reference to the instance of this particular PTransform object. This
- allows chaining type-hinting related methods.
+ PTransform: A reference to the instance of this particular
+ :class:`PTransform` object. This allows chaining type-hinting related
+ methods.
"""
validate_composite_type_param(type_hint, 'Type hints for a PTransform')
return super(PTransform, self).with_output_types(type_hint)
@@ -426,15 +437,24 @@
_known_urns = {}
@classmethod
- def register_urn(cls, urn, parameter_type, constructor):
- cls._known_urns[urn] = parameter_type, constructor
+ def register_urn(cls, urn, parameter_type, constructor=None):
+ def register(constructor):
+ cls._known_urns[urn] = parameter_type, constructor
+ return staticmethod(constructor)
+ if constructor:
+ # Used as a statement.
+ register(constructor)
+ else:
+ # Used as a decorator.
+ return register
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.api import beam_runner_api_pb2
urn, typed_param = self.to_runner_api_parameter(context)
return beam_runner_api_pb2.FunctionSpec(
urn=urn,
- parameter=proto_utils.pack_Any(typed_param))
+ payload=typed_param.SerializeToString()
+ if typed_param is not None else None)
@classmethod
def from_runner_api(cls, proto, context):
@@ -442,7 +462,7 @@
return None
parameter_type, constructor = cls._known_urns[proto.urn]
return constructor(
- proto_utils.unpack_Any(proto.parameter, parameter_type),
+ proto_utils.parse_Bytes(proto.payload, parameter_type),
context)
def to_runner_api_parameter(self, context):
@@ -481,13 +501,16 @@
class PTransformWithSideInputs(PTransform):
- """A superclass for any PTransform (e.g. FlatMap or Combine)
+ """A superclass for any :class:`PTransform` (e.g.
+ :func:`~apache_beam.transforms.core.FlatMap` or
+ :class:`~apache_beam.transforms.core.CombineFn`)
invoking user code.
- PTransforms like FlatMap invoke user-supplied code in some kind of
- package (e.g. a DoFn) and optionally provide arguments and side inputs
- to that code. This internal-use-only class contains common functionality
- for PTransforms that fit this model.
+ :class:`PTransform` s like :func:`~apache_beam.transforms.core.FlatMap`
+ invoke user-supplied code in some kind of package (e.g. a
+ :class:`~apache_beam.transforms.core.DoFn`) and optionally provide arguments
+ and side inputs to that code. This internal-use-only class contains common
+ functionality for :class:`PTransform` s that fit this model.
"""
def __init__(self, fn, *args, **kwargs):
@@ -533,16 +556,20 @@
of an allowed built-in type, a custom class, or a
typehints.TypeConstraint.
- Example of annotating the types of side-inputs:
+ Example of annotating the types of side-inputs::
+
FlatMap().with_input_types(int, int, bool)
Raises:
- TypeError: If 'type_hint' is not a valid type-hint. See
- typehints.validate_composite_type_param for further details.
+ :class:`~exceptions.TypeError`: If **type_hint** is not a valid type-hint.
+ See
+ :func:`~apache_beam.typehints.typehints.validate_composite_type_param`
+ for further details.
Returns:
- A reference to the instance of this particular PTransform object. This
- allows chaining type-hinting related methods.
+ :class:`PTransform`: A reference to the instance of this particular
+ :class:`PTransform` object. This allows chaining type-hinting related
+ methods.
"""
super(PTransformWithSideInputs, self).with_input_types(input_type_hint)
@@ -595,32 +622,23 @@
return '%s(%s)' % (self.__class__.__name__, self.fn.default_label())
-class CallablePTransform(PTransform):
+class _PTransformFnPTransform(PTransform):
"""A class wrapper for a function-based transform."""
- def __init__(self, fn):
- # pylint: disable=super-init-not-called
- # This is a helper class for a function decorator. Only when the class
- # is called (and __call__ invoked) we will have all the information
- # needed to initialize the super class.
- self.fn = fn
- self._args = ()
- self._kwargs = {}
+ def __init__(self, fn, *args, **kwargs):
+ super(_PTransformFnPTransform, self).__init__()
+ self._fn = fn
+ self._args = args
+ self._kwargs = kwargs
def display_data(self):
- res = {'fn': (self.fn.__name__
- if hasattr(self.fn, '__name__')
- else self.fn.__class__),
+ res = {'fn': (self._fn.__name__
+ if hasattr(self._fn, '__name__')
+ else self._fn.__class__),
'args': DisplayDataItem(str(self._args)).drop_if_default('()'),
'kwargs': DisplayDataItem(str(self._kwargs)).drop_if_default('{}')}
return res
- def __call__(self, *args, **kwargs):
- super(CallablePTransform, self).__init__()
- self._args = args
- self._kwargs = kwargs
- return self
-
def expand(self, pcoll):
# Since the PTransform will be implemented entirely as a function
# (once called), we need to pass through any type-hinting information that
@@ -629,18 +647,18 @@
kwargs = dict(self._kwargs)
args = tuple(self._args)
try:
- if 'type_hints' in inspect.getargspec(self.fn).args:
+ if 'type_hints' in inspect.getargspec(self._fn).args:
args = (self.get_type_hints(),) + args
except TypeError:
# Might not be a function.
pass
- return self.fn(pcoll, *args, **kwargs)
+ return self._fn(pcoll, *args, **kwargs)
def default_label(self):
if self._args:
return '%s(%s)' % (
- label_from_callable(self.fn), label_from_callable(self._args[0]))
- return label_from_callable(self.fn)
+ label_from_callable(self._fn), label_from_callable(self._args[0]))
+ return label_from_callable(self._fn)
def ptransform_fn(fn):
@@ -684,7 +702,11 @@
operator (i.e., `|`) will inject the pcoll argument in its proper place
(first argument if no label was specified and second argument otherwise).
"""
- return CallablePTransform(fn)
+ # TODO(robertwb): Consider removing staticmethod to allow for self parameter.
+
+ def callable_ptransform_factory(*args, **kwargs):
+ return _PTransformFnPTransform(fn, *args, **kwargs)
+ return callable_ptransform_factory
def label_from_callable(fn):
@@ -693,8 +715,8 @@
elif hasattr(fn, '__name__'):
if fn.__name__ == '<lambda>':
return '<lambda at %s:%s>' % (
- os.path.basename(fn.func_code.co_filename),
- fn.func_code.co_firstlineno)
+ os.path.basename(fn.__code__.co_filename),
+ fn.__code__.co_firstlineno)
return fn.__name__
return str(fn)
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index efc5978..dac2c4f 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -18,34 +18,38 @@
"""Unit tests for the PTransform and descendants."""
from __future__ import absolute_import
+from __future__ import print_function
+import collections
import operator
import re
import unittest
+from functools import reduce
import hamcrest as hc
from nose.plugins.attrib import attr
import apache_beam as beam
+import apache_beam.pvalue as pvalue
+import apache_beam.transforms.combiners as combine
+import apache_beam.typehints as typehints
+from apache_beam.io.iobase import Read
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
-from apache_beam.io.iobase import Read
from apache_beam.options.pipeline_options import TypeOptions
-import apache_beam.pvalue as pvalue
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.transforms import window
from apache_beam.transforms.core import _GroupByKeyOnly
-import apache_beam.transforms.combiners as combine
-from apache_beam.transforms.display import DisplayData, DisplayDataItem
+from apache_beam.transforms.display import DisplayData
+from apache_beam.transforms.display import DisplayDataItem
from apache_beam.transforms.ptransform import PTransform
-import apache_beam.typehints as typehints
from apache_beam.typehints import with_input_types
from apache_beam.typehints import with_output_types
from apache_beam.typehints.typehints_test import TypeHintTestCase
from apache_beam.utils.windowed_value import WindowedValue
-
# Disable frequent lint warning due to pipe operator for chaining transforms.
# pylint: disable=expression-not-assigned
@@ -351,14 +355,16 @@
def create_accumulator(self):
return (0, 0)
- def add_input(self, (sum_, count), element):
+ def add_input(self, sum_count, element):
+ (sum_, count) = sum_count
return sum_ + element, count + 1
def merge_accumulators(self, accumulators):
sums, counts = zip(*accumulators)
return sum(sums), sum(counts)
- def extract_output(self, (sum_, count)):
+ def extract_output(self, sum_count):
+ (sum_, count) = sum_count
if not count:
return float('nan')
return sum_ / float(count)
@@ -617,7 +623,7 @@
pipeline = TestPipeline()
t = (beam.Map(lambda x: (x, 1))
| beam.GroupByKey()
- | beam.Map(lambda (x, ones): (x, sum(ones))))
+ | beam.Map(lambda x_ones: (x_ones[0], sum(x_ones[1]))))
result = pipeline | 'Start' >> beam.Create(['a', 'a', 'b']) | t
assert_that(result, equal_to([('a', 2), ('b', 1)]))
pipeline.run()
@@ -641,7 +647,7 @@
| beam.Flatten()
| beam.Map(lambda x: (x, None))
| beam.GroupByKey()
- | beam.Map(lambda (x, _): x))
+ | beam.Map(lambda kv: kv[0]))
self.assertEqual([1, 2, 3], sorted(([1, 2], [2, 3]) | DisjointUnion()))
def test_apply_to_crazy_pvaluish(self):
@@ -669,6 +675,30 @@
self.assertEqual(['x', 'x', 'y', 'y', 'z'], sorted(res['b']))
self.assertEqual([], sorted(res['c']))
+ def test_named_tuple(self):
+ MinMax = collections.namedtuple('MinMax', ['min', 'max'])
+
+ class MinMaxTransform(PTransform):
+ def expand(self, pcoll):
+ return MinMax(
+ min=pcoll | beam.CombineGlobally(min).without_defaults(),
+ max=pcoll | beam.CombineGlobally(max).without_defaults())
+ res = [1, 2, 4, 8] | MinMaxTransform()
+ self.assertIsInstance(res, MinMax)
+ self.assertEqual(res, MinMax(min=[1], max=[8]))
+
+ flat = res | beam.Flatten()
+ self.assertEqual(sorted(flat), [1, 8])
+
+ def test_tuple_twice(self):
+ class Duplicate(PTransform):
+ def expand(self, pcoll):
+ return pcoll, pcoll
+
+ res1, res2 = [1, 2, 4, 8] | Duplicate()
+ self.assertEqual(sorted(res1), [1, 2, 4, 8])
+ self.assertEqual(sorted(res2), [1, 2, 4, 8])
+
@beam.ptransform_fn
def SamplePTransform(pcoll):
@@ -694,7 +724,7 @@
pipeline = TestPipeline()
map1 = 'Map1' >> beam.Map(lambda x: (x, 1))
gbk = 'Gbk' >> beam.GroupByKey()
- map2 = 'Map2' >> beam.Map(lambda (x, ones): (x, sum(ones)))
+ map2 = 'Map2' >> beam.Map(lambda x_ones2: (x_ones2[0], sum(x_ones2[1])))
t = (map1 | gbk | map2)
result = pipeline | 'Start' >> beam.Create(['a', 'a', 'b']) | t
self.assertTrue('Map1|Gbk|Map2/Map1' in pipeline.applied_labels)
@@ -1294,7 +1324,7 @@
with self.assertRaises(typehints.TypeCheckError) as e:
(self.p
| beam.Create([(1, 3.0), (2, 4.9), (3, 9.5)])
- | ('Add' >> beam.FlatMap(lambda (x, y): [x + y])
+ | ('Add' >> beam.FlatMap(lambda x_y: [x_y[0] + x_y[1]])
.with_input_types(typehints.Tuple[int, int]).with_output_types(int))
)
self.p.run()
@@ -1302,9 +1332,11 @@
self.assertStartswith(
e.exception.message,
"Runtime type violation detected within ParDo(Add): "
- "Type-hint for argument: 'y' violated. "
- "Expected an instance of <type 'int'>, "
- "instead found 3.0, an instance of <type 'float'>.")
+ "Type-hint for argument: 'x_y' violated: "
+ "Tuple[int, int] hint type-constraint violated. "
+ "The type of element #1 in the passed tuple is incorrect. "
+ "Expected an instance of type int, instead received an instance "
+ "of type float.")
def test_pipeline_runtime_checking_violation_simple_type_output(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True
@@ -1313,9 +1345,9 @@
# The type-hinted applied via the 'returns()' method indicates the ParDo
# should output an instance of type 'int', however a 'float' will be
# generated instead.
- print "HINTS", ('ToInt' >> beam.FlatMap(
+ print("HINTS", ('ToInt' >> beam.FlatMap(
lambda x: [float(x)]).with_input_types(int).with_output_types(
- int)).get_type_hints()
+ int)).get_type_hints())
with self.assertRaises(typehints.TypeCheckError) as e:
(self.p
| beam.Create([1, 2, 3])
@@ -1342,7 +1374,7 @@
with self.assertRaises(typehints.TypeCheckError) as e:
(self.p
| beam.Create([(1, 3.0), (2, 4.9), (3, 9.5)])
- | ('Swap' >> beam.FlatMap(lambda (x, y): [x + y])
+ | ('Swap' >> beam.FlatMap(lambda x_y1: [x_y1[0] + x_y1[1]])
.with_input_types(typehints.Tuple[int, float])
.with_output_types(typehints.Tuple[float, int]))
)
@@ -1561,7 +1593,7 @@
| 'C' >> beam.Create(range(5)).with_output_types(int)
| 'Mean' >> combine.Mean.Globally())
- self.assertTrue(d.element_type is float)
+ self.assertEqual(float, d.element_type)
assert_that(d, equal_to([2.0]))
self.p.run()
@@ -1584,7 +1616,7 @@
| 'C' >> beam.Create(range(5)).with_output_types(int)
| 'Mean' >> combine.Mean.Globally())
- self.assertTrue(d.element_type is float)
+ self.assertEqual(float, d.element_type)
assert_that(d, equal_to([2.0]))
self.p.run()
@@ -1677,7 +1709,7 @@
| 'P' >> beam.Create(range(5)).with_output_types(int)
| 'CountInt' >> combine.Count.Globally())
- self.assertTrue(d.element_type is int)
+ self.assertEqual(int, d.element_type)
assert_that(d, equal_to([5]))
self.p.run()
@@ -1688,7 +1720,7 @@
| 'P' >> beam.Create(range(5)).with_output_types(int)
| 'CountInt' >> combine.Count.Globally())
- self.assertTrue(d.element_type is int)
+ self.assertEqual(int, d.element_type)
assert_that(d, equal_to([5]))
self.p.run()
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
index 6500681..1d58834 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -24,7 +24,8 @@
import apache_beam as beam
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.transforms import window
@@ -54,7 +55,7 @@
side |= beam.Map(lambda x: ('k%s' % x, 'v%s' % x))
res = main | beam.Map(lambda x, s: (x, s), side_input_type(side, **kw))
if side_input_type in (beam.pvalue.AsIter, beam.pvalue.AsList):
- res |= beam.Map(lambda (x, s): (x, sorted(s)))
+ res |= beam.Map(lambda x_s: (x_s[0], sorted(x_s[1])))
assert_that(res, equal_to(expected))
def test_global_global_windows(self):
diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py
index c0f9198..8d63d49 100644
--- a/sdks/python/apache_beam/transforms/timeutil.py
+++ b/sdks/python/apache_beam/transforms/timeutil.py
@@ -22,7 +22,6 @@
from abc import ABCMeta
from abc import abstractmethod
-
__all__ = [
'TimeDomain',
]
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index 4200995..bd99401 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -20,12 +20,14 @@
Triggers control when in processing time windows get emitted.
"""
-from abc import ABCMeta
-from abc import abstractmethod
import collections
import copy
+import itertools
+from abc import ABCMeta
+from abc import abstractmethod
from apache_beam.coders import observable
+from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.transforms import combiners
from apache_beam.transforms import core
from apache_beam.transforms.timeutil import TimeDomain
@@ -33,9 +35,9 @@
from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import WindowedValue
from apache_beam.transforms.window import WindowFn
-from apache_beam.runners.api import beam_runner_api_pb2
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import TIME_GRANULARITY
# AfterCount is experimental. No backwards compatibility guarantees.
@@ -56,8 +58,8 @@
class AccumulationMode(object):
"""Controls what to do with data when a trigger fires multiple times.
"""
- DISCARDING = beam_runner_api_pb2.DISCARDING
- ACCUMULATING = beam_runner_api_pb2.ACCUMULATING
+ DISCARDING = beam_runner_api_pb2.AccumulationMode.DISCARDING
+ ACCUMULATING = beam_runner_api_pb2.AccumulationMode.ACCUMULATING
# TODO(robertwb): Provide retractions of previous outputs.
# RETRACTING = 3
@@ -857,6 +859,19 @@
def process_timer(self, window_id, name, time_domain, timestamp, state):
pass
+ def process_entire_key(
+ self, key, windowed_values, output_watermark=MIN_TIMESTAMP):
+ state = InMemoryUnmergedState()
+ for wvalue in self.process_elements(
+ state, windowed_values, output_watermark):
+ yield wvalue.with_value((key, wvalue.value))
+ while state.timers:
+ fired = state.get_and_clear_timers()
+ for timer_window, (name, time_domain, fire_time) in fired:
+ for wvalue in self.process_timer(
+ timer_window, name, time_domain, fire_time, state):
+ yield wvalue.with_value((key, wvalue.value))
+
class _UnwindowedValues(observable.ObservableMixin):
"""Exposes iterable of windowed values as iterable of unwindowed values."""
@@ -877,6 +892,17 @@
def __reduce__(self):
return list, (list(self),)
+ def __eq__(self, other):
+ if isinstance(other, collections.Iterable):
+ return all(
+ a == b
+ for a, b in itertools.izip_longest(self, other, fillvalue=object()))
+ else:
+ return NotImplemented
+
+ def __ne__(self, other):
+ return not self == other
+
class DefaultGlobalBatchTriggerDriver(TriggerDriver):
"""Breaks a bundles into window (pane)s according to the default triggering.
@@ -887,11 +913,10 @@
pass
def process_elements(self, state, windowed_values, unused_output_watermark):
- if isinstance(windowed_values, list):
- unwindowed = [wv.value for wv in windowed_values]
- else:
- unwindowed = _UnwindowedValues(windowed_values)
- yield WindowedValue(unwindowed, MIN_TIMESTAMP, self.GLOBAL_WINDOW_TUPLE)
+ yield WindowedValue(
+ _UnwindowedValues(windowed_values),
+ MIN_TIMESTAMP,
+ self.GLOBAL_WINDOW_TUPLE)
def process_timer(self, window_id, name, time_domain, timestamp, state):
raise TypeError('Triggers never set or called for batch default windowing.')
@@ -1052,6 +1077,15 @@
self.global_state = {}
self.defensive_copy = defensive_copy
+ def copy(self):
+ cloned_object = InMemoryUnmergedState(defensive_copy=self.defensive_copy)
+ cloned_object.timers = copy.deepcopy(self.timers)
+ cloned_object.global_state = copy.deepcopy(self.global_state)
+ for window in self.state:
+ for tag in self.state[window]:
+ cloned_object.state[window][tag] = copy.copy(self.state[window][tag])
+ return cloned_object
+
def set_global_state(self, tag, value):
assert isinstance(tag, _ValueStateTag)
if self.defensive_copy:
@@ -1066,6 +1100,8 @@
def clear_timer(self, window, name, time_domain):
self.timers[window].pop((name, time_domain), None)
+ if not self.timers[window]:
+ del self.timers[window]
def get_window(self, window_id):
return window_id
@@ -1102,17 +1138,34 @@
if not self.state[window]:
self.state.pop(window, None)
- def get_and_clear_timers(self, watermark=MAX_TIMESTAMP):
+ def get_timers(self, clear=False, watermark=MAX_TIMESTAMP):
expired = []
for window, timers in list(self.timers.items()):
for (name, time_domain), timestamp in list(timers.items()):
if timestamp <= watermark:
expired.append((window, (name, time_domain, timestamp)))
- del timers[(name, time_domain)]
- if not timers:
+ if clear:
+ del timers[(name, time_domain)]
+ if not timers and clear:
del self.timers[window]
return expired
+ def get_and_clear_timers(self, watermark=MAX_TIMESTAMP):
+ return self.get_timers(clear=True, watermark=watermark)
+
+ def get_earliest_hold(self):
+ earliest_hold = MAX_TIMESTAMP
+ for unused_window, tagged_states in self.state.iteritems():
+ # TODO(BEAM-2519): currently, this assumes that the watermark hold tag is
+ # named "watermark". This is currently only true because the only place
+ # watermark holds are set is in the GeneralTriggerDriver, where we use
+ # this name. We should fix this by allowing enumeration of the tag types
+ # used in adding state.
+ if 'watermark' in tagged_states and tagged_states['watermark']:
+ hold = min(tagged_states['watermark']) - TIME_GRANULARITY
+ earliest_hold = min(earliest_hold, hold)
+ return earliest_hold
+
def __repr__(self):
state_str = '\n'.join('%s: %s' % (key, dict(state))
for key, state in self.state.items())
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 1ae1f02..3afabaf 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -27,25 +27,26 @@
import apache_beam as beam
from apache_beam.runners import pipeline_context
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.transforms import trigger
from apache_beam.transforms.core import Windowing
from apache_beam.transforms.trigger import AccumulationMode
from apache_beam.transforms.trigger import AfterAll
+from apache_beam.transforms.trigger import AfterAny
from apache_beam.transforms.trigger import AfterCount
from apache_beam.transforms.trigger import AfterEach
-from apache_beam.transforms.trigger import AfterAny
from apache_beam.transforms.trigger import AfterWatermark
from apache_beam.transforms.trigger import DefaultTrigger
from apache_beam.transforms.trigger import GeneralTriggerDriver
from apache_beam.transforms.trigger import InMemoryUnmergedState
from apache_beam.transforms.trigger import Repeatedly
from apache_beam.transforms.trigger import TriggerFn
+from apache_beam.transforms.window import MIN_TIMESTAMP
from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.window import IntervalWindow
-from apache_beam.transforms.window import MIN_TIMESTAMP
-from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import Sessions
+from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import TimestampedValue
from apache_beam.transforms.window import WindowedValue
from apache_beam.transforms.window import WindowFn
@@ -403,14 +404,20 @@
def test_after_count(self):
with TestPipeline() as p:
+ def construct_timestamped(k_t):
+ return TimestampedValue((k_t[0], k_t[1]), k_t[1])
+
+ def format_result(k_v):
+ return ('%s-%s' % (k_v[0], len(k_v[1])), set(k_v[1]))
+
result = (p
| beam.Create([1, 2, 3, 4, 5, 10, 11])
| beam.FlatMap(lambda t: [('A', t), ('B', t + 5)])
- | beam.Map(lambda (k, t): TimestampedValue((k, t), t))
+ | beam.Map(construct_timestamped)
| beam.WindowInto(FixedWindows(10), trigger=AfterCount(3),
accumulation_mode=AccumulationMode.DISCARDING)
| beam.GroupByKey()
- | beam.Map(lambda (k, v): ('%s-%s' % (k, len(v)), set(v))))
+ | beam.Map(format_result))
assert_that(result, equal_to(
{
'A-5': {1, 2, 3, 4, 5},
diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py
index a7484ac..85d4975 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -20,15 +20,25 @@
from __future__ import absolute_import
+import collections
+import contextlib
+import time
+
+from apache_beam import typehints
+from apache_beam.metrics import Metrics
+from apache_beam.transforms import window
from apache_beam.transforms.core import CombinePerKey
+from apache_beam.transforms.core import DoFn
from apache_beam.transforms.core import Flatten
from apache_beam.transforms.core import GroupByKey
from apache_beam.transforms.core import Map
+from apache_beam.transforms.core import ParDo
from apache_beam.transforms.ptransform import PTransform
from apache_beam.transforms.ptransform import ptransform_fn
-
+from apache_beam.utils import windowed_value
__all__ = [
+ 'BatchElements',
'CoGroupByKey',
'Keys',
'KvSwap',
@@ -37,6 +47,9 @@
]
+T = typehints.TypeVariable('T')
+
+
class CoGroupByKey(PTransform):
"""Groups results across several PCollections by key.
@@ -99,14 +112,16 @@
def expand(self, pcolls):
"""Performs CoGroupByKey on argument pcolls; see class docstring."""
# For associating values in K-V pairs with the PCollections they came from.
- def _pair_tag_with_value((key, value), tag):
+ def _pair_tag_with_value(key_value, tag):
+ (key, value) = key_value
return (key, (tag, value))
# Creates the key, value pairs for the output PCollection. Values are either
# lists or dicts (per the class docstring), initialized by the result of
# result_ctor(result_ctor_arg).
- def _merge_tagged_vals_under_key((key, grouped), result_ctor,
+ def _merge_tagged_vals_under_key(key_grouped, result_ctor,
result_ctor_arg):
+ (key, grouped) = key_grouped
result_value = result_ctor(result_ctor_arg)
for tag, value in grouped:
result_value[tag].append(value)
@@ -142,17 +157,17 @@
def Keys(label='Keys'): # pylint: disable=invalid-name
"""Produces a PCollection of first elements of 2-tuples in a PCollection."""
- return label >> Map(lambda (k, v): k)
+ return label >> Map(lambda k_v: k_v[0])
def Values(label='Values'): # pylint: disable=invalid-name
"""Produces a PCollection of second elements of 2-tuples in a PCollection."""
- return label >> Map(lambda (k, v): v)
+ return label >> Map(lambda k_v1: k_v1[1])
def KvSwap(label='KvSwap'): # pylint: disable=invalid-name
"""Produces a PCollection reversing 2-tuples in a PCollection."""
- return label >> Map(lambda (k, v): (v, k))
+ return label >> Map(lambda k_v2: (k_v2[1], k_v2[0]))
@ptransform_fn
@@ -162,3 +177,249 @@
| 'ToPairs' >> Map(lambda v: (v, None))
| 'Group' >> CombinePerKey(lambda vs: None)
| 'RemoveDuplicates' >> Keys())
+
+
+class _BatchSizeEstimator(object):
+ """Estimates the best size for batches given historical timing.
+ """
+
+ _MAX_DATA_POINTS = 100
+ _MAX_GROWTH_FACTOR = 2
+
+ def __init__(self,
+ min_batch_size=1,
+ max_batch_size=1000,
+ target_batch_overhead=.1,
+ target_batch_duration_secs=1,
+ clock=time.time):
+ if min_batch_size > max_batch_size:
+ raise ValueError("Minimum (%s) must not be greater than maximum (%s)" % (
+ min_batch_size, max_batch_size))
+ if target_batch_overhead and not 0 < target_batch_overhead <= 1:
+ raise ValueError("target_batch_overhead (%s) must be between 0 and 1" % (
+ target_batch_overhead))
+ if target_batch_duration_secs and target_batch_duration_secs <= 0:
+ raise ValueError("target_batch_duration_secs (%s) must be positive" % (
+ target_batch_duration_secs))
+ if max(0, target_batch_overhead, target_batch_duration_secs) == 0:
+ raise ValueError("At least one of target_batch_overhead or "
+ "target_batch_duration_secs must be positive.")
+ self._min_batch_size = min_batch_size
+ self._max_batch_size = max_batch_size
+ self._target_batch_overhead = target_batch_overhead
+ self._target_batch_duration_secs = target_batch_duration_secs
+ self._clock = clock
+ self._data = []
+ self._ignore_next_timing = False
+ self._size_distribution = Metrics.distribution(
+ 'BatchElements', 'batch_size')
+ self._time_distribution = Metrics.distribution(
+ 'BatchElements', 'msec_per_batch')
+ # Beam distributions only accept integer values, so we use this to
+ # accumulate under-reported values until they add up to whole milliseconds.
+ # (Milliseconds are chosen because that's conventionally used elsewhere in
+ # profiling-style counters.)
+ self._remainder_msecs = 0
+
+ def ignore_next_timing(self):
+ """Call to indicate the next timing should be ignored.
+
+ For example, the first emit of a ParDo operation is known to be anomalous
+ due to setup that may occur.
+ """
+ self._ignore_next_timing = False
+
+ @contextlib.contextmanager
+ def record_time(self, batch_size):
+ start = self._clock()
+ yield
+ elapsed = self._clock() - start
+ elapsed_msec = 1e3 * elapsed + self._remainder_msecs
+ self._size_distribution.update(batch_size)
+ self._time_distribution.update(int(elapsed_msec))
+ self._remainder_msecs = elapsed_msec - int(elapsed_msec)
+ if self._ignore_next_timing:
+ self._ignore_next_timing = False
+ else:
+ self._data.append((batch_size, elapsed))
+ if len(self._data) >= self._MAX_DATA_POINTS:
+ self._thin_data()
+
+ def _thin_data(self):
+ sorted_data = sorted(self._data)
+ odd_one_out = [sorted_data[-1]] if len(sorted_data) % 2 == 1 else []
+ # Sort the pairs by how different they are.
+ pairs = sorted(zip(sorted_data[::2], sorted_data[1::2]),
+ key=lambda ((x1, _1), (x2, _2)): x2 / x1)
+ # Keep the top 1/3 most different pairs, average the top 2/3 most similar.
+ threshold = 2 * len(pairs) / 3
+ self._data = (
+ list(sum(pairs[threshold:], ()))
+ + [((x1 + x2) / 2.0, (t1 + t2) / 2.0)
+ for (x1, t1), (x2, t2) in pairs[:threshold]]
+ + odd_one_out)
+
+ def next_batch_size(self):
+ if self._min_batch_size == self._max_batch_size:
+ return self._min_batch_size
+ elif len(self._data) < 1:
+ return self._min_batch_size
+ elif len(self._data) < 2:
+ # Force some variety so we have distinct batch sizes on which to do
+ # linear regression below.
+ return int(max(
+ min(self._max_batch_size,
+ self._min_batch_size * self._MAX_GROWTH_FACTOR),
+ self._min_batch_size + 1))
+
+ # Linear regression for y = a + bx, where x is batch size and y is time.
+ xs, ys = zip(*self._data)
+ n = float(len(self._data))
+ xbar = sum(xs) / n
+ ybar = sum(ys) / n
+ b = (sum([(x - xbar) * (y - ybar) for x, y in self._data])
+ / sum([(x - xbar)**2 for x in xs]))
+ a = ybar - b * xbar
+
+ # Avoid nonsensical or division-by-zero errors below due to noise.
+ a = max(a, 1e-10)
+ b = max(b, 1e-20)
+
+ last_batch_size = self._data[-1][0]
+ cap = min(last_batch_size * self._MAX_GROWTH_FACTOR, self._max_batch_size)
+
+ if self._target_batch_duration_secs:
+ # Solution to a + b*x = self._target_batch_duration_secs.
+ cap = min(cap, (self._target_batch_duration_secs - a) / b)
+
+ if self._target_batch_overhead:
+ # Solution to a / (a + b*x) = self._target_batch_overhead.
+ cap = min(cap, (a / b) * (1 / self._target_batch_overhead - 1))
+
+ # Avoid getting stuck at min_batch_size.
+ jitter = len(self._data) % 2
+ return int(max(self._min_batch_size + jitter, cap))
+
+
+class _GlobalWindowsBatchingDoFn(DoFn):
+ def __init__(self, batch_size_estimator):
+ self._batch_size_estimator = batch_size_estimator
+
+ def start_bundle(self):
+ self._batch = []
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+ # The first emit often involves non-trivial setup.
+ self._batch_size_estimator.ignore_next_timing()
+
+ def process(self, element):
+ self._batch.append(element)
+ if len(self._batch) >= self._batch_size:
+ with self._batch_size_estimator.record_time(self._batch_size):
+ yield self._batch
+ self._batch = []
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+
+ def finish_bundle(self):
+ if self._batch:
+ with self._batch_size_estimator.record_time(self._batch_size):
+ yield window.GlobalWindows.windowed_value(self._batch)
+ self._batch = None
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+
+
+class _WindowAwareBatchingDoFn(DoFn):
+
+ _MAX_LIVE_WINDOWS = 10
+
+ def __init__(self, batch_size_estimator):
+ self._batch_size_estimator = batch_size_estimator
+
+ def start_bundle(self):
+ self._batches = collections.defaultdict(list)
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+ # The first emit often involves non-trivial setup.
+ self._batch_size_estimator.ignore_next_timing()
+
+ def process(self, element, window=DoFn.WindowParam):
+ self._batches[window].append(element)
+ if len(self._batches[window]) >= self._batch_size:
+ with self._batch_size_estimator.record_time(self._batch_size):
+ yield windowed_value.WindowedValue(
+ self._batches[window], window.max_timestamp(), (window,))
+ del self._batches[window]
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+ elif len(self._batches) > self._MAX_LIVE_WINDOWS:
+ window, _ = sorted(
+ self._batches.items(),
+ key=lambda window_batch: len(window_batch[1]),
+ reverse=True)[0]
+ with self._batch_size_estimator.record_time(self._batch_size):
+ yield windowed_value.WindowedValue(
+ self._batches[window], window.max_timestamp(), (window,))
+ del self._batches[window]
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+
+ def finish_bundle(self):
+ for window, batch in self._batches.items():
+ if batch:
+ with self._batch_size_estimator.record_time(self._batch_size):
+ yield windowed_value.WindowedValue(
+ batch, window.max_timestamp(), (window,))
+ self._batches = None
+ self._batch_size = self._batch_size_estimator.next_batch_size()
+
+
+@typehints.with_input_types(T)
+@typehints.with_output_types(typehints.List[T])
+class BatchElements(PTransform):
+ """A Transform that batches elements for amortized processing.
+
+ This transform is designed to precede operations whose processing cost
+ is of the form
+
+ time = fixed_cost + num_elements * per_element_cost
+
+ where the per element cost is (often significantly) smaller than the fixed
+ cost and could be amortized over multiple elements. It consumes a PCollection
+ of element type T and produces a PCollection of element type List[T].
+
+ This transform attempts to find the best batch size between the minimim
+ and maximum parameters by profiling the time taken by (fused) downstream
+ operations. For a fixed batch size, set the min and max to be equal.
+
+ Elements are batched per-window and batches emitted in the window
+ corresponding to its contents.
+
+ Args:
+ min_batch_size: (optional) the smallest number of elements per batch
+ max_batch_size: (optional) the largest number of elements per batch
+ target_batch_overhead: (optional) a target for fixed_cost / time,
+ as used in the formula above
+ target_batch_duration_secs: (optional) a target for total time per bundle,
+ in seconds
+ clock: (optional) an alternative to time.time for measuring the cost of
+ donwstream operations (mostly for testing)
+ """
+ def __init__(self,
+ min_batch_size=1,
+ max_batch_size=1000,
+ target_batch_overhead=.05,
+ target_batch_duration_secs=1,
+ clock=time.time):
+ self._batch_size_estimator = _BatchSizeEstimator(
+ min_batch_size=min_batch_size,
+ max_batch_size=max_batch_size,
+ target_batch_overhead=target_batch_overhead,
+ target_batch_duration_secs=target_batch_duration_secs,
+ clock=clock)
+
+ def expand(self, pcoll):
+ if getattr(pcoll.pipeline.runner, 'is_streaming', False):
+ raise NotImplementedError("Requires stateful processing (BEAM-2687)")
+ elif pcoll.windowing.is_default():
+ # This is the same logic as _GlobalWindowsBatchingDoFn, but optimized
+ # for that simpler case.
+ return pcoll | ParDo(_GlobalWindowsBatchingDoFn(
+ self._batch_size_estimator))
+ else:
+ return pcoll | ParDo(_WindowAwareBatchingDoFn(self._batch_size_estimator))
diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py
new file mode 100644
index 0000000..6064e2c
--- /dev/null
+++ b/sdks/python/apache_beam/transforms/util_test.py
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for the transform.util classes."""
+
+import time
+import unittest
+
+import apache_beam as beam
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+from apache_beam.transforms import util
+from apache_beam.transforms import window
+
+
+class FakeClock(object):
+
+ def __init__(self):
+ self._now = time.time()
+
+ def __call__(self):
+ return self._now
+
+ def sleep(self, duration):
+ self._now += duration
+
+
+class BatchElementsTest(unittest.TestCase):
+
+ def test_constant_batch(self):
+ # Assumes a single bundle...
+ with TestPipeline() as p:
+ res = (
+ p
+ | beam.Create(range(35))
+ | util.BatchElements(min_batch_size=10, max_batch_size=10)
+ | beam.Map(len))
+ assert_that(res, equal_to([10, 10, 10, 5]))
+
+ def test_grows_to_max_batch(self):
+ # Assumes a single bundle...
+ with TestPipeline() as p:
+ res = (
+ p
+ | beam.Create(range(164))
+ | util.BatchElements(
+ min_batch_size=1, max_batch_size=50, clock=FakeClock())
+ | beam.Map(len))
+ assert_that(res, equal_to([1, 1, 2, 4, 8, 16, 32, 50, 50]))
+
+ def test_windowed_batches(self):
+ # Assumes a single bundle, in order...
+ with TestPipeline() as p:
+ res = (
+ p
+ | beam.Create(range(47))
+ | beam.Map(lambda t: window.TimestampedValue(t, t))
+ | beam.WindowInto(window.FixedWindows(30))
+ | util.BatchElements(
+ min_batch_size=5, max_batch_size=10, clock=FakeClock())
+ | beam.Map(len))
+ assert_that(res, equal_to([
+ 5, 5, 10, 10, # elements in [0, 30)
+ 10, 7, # elements in [30, 47)
+ ]))
+
+ def test_target_duration(self):
+ clock = FakeClock()
+ batch_estimator = util._BatchSizeEstimator(
+ target_batch_overhead=None, target_batch_duration_secs=10, clock=clock)
+ batch_duration = lambda batch_size: 1 + .7 * batch_size
+ # 1 + 12 * .7 is as close as we can get to 10 as possible.
+ expected_sizes = [1, 2, 4, 8, 12, 12, 12]
+ actual_sizes = []
+ for _ in range(len(expected_sizes)):
+ actual_sizes.append(batch_estimator.next_batch_size())
+ with batch_estimator.record_time(actual_sizes[-1]):
+ clock.sleep(batch_duration(actual_sizes[-1]))
+ self.assertEqual(expected_sizes, actual_sizes)
+
+ def test_target_overhead(self):
+ clock = FakeClock()
+ batch_estimator = util._BatchSizeEstimator(
+ target_batch_overhead=.05, target_batch_duration_secs=None, clock=clock)
+ batch_duration = lambda batch_size: 1 + .7 * batch_size
+ # At 27 items, a batch takes ~20 seconds with 5% (~1 second) overhead.
+ expected_sizes = [1, 2, 4, 8, 16, 27, 27, 27]
+ actual_sizes = []
+ for _ in range(len(expected_sizes)):
+ actual_sizes.append(batch_estimator.next_batch_size())
+ with batch_estimator.record_time(actual_sizes[-1]):
+ clock.sleep(batch_duration(actual_sizes[-1]))
+ self.assertEqual(expected_sizes, actual_sizes)
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index 94187e0..8c8bf33 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -51,20 +51,21 @@
import abc
-from google.protobuf import struct_pb2
+from google.protobuf import duration_pb2
+from google.protobuf import timestamp_pb2
from apache_beam.coders import coders
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.portability.api import beam_runner_api_pb2
+from apache_beam.portability.api import standard_window_fns_pb2
from apache_beam.transforms import timeutil
from apache_beam.utils import proto_utils
from apache_beam.utils import urns
-from apache_beam.utils.timestamp import Duration
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import Duration
from apache_beam.utils.timestamp import Timestamp
from apache_beam.utils.windowed_value import WindowedValue
-
__all__ = [
'TimestampCombiner',
'WindowFn',
@@ -85,9 +86,9 @@
class TimestampCombiner(object):
"""Determines how output timestamps of grouping operations are assigned."""
- OUTPUT_AT_EOW = beam_runner_api_pb2.END_OF_WINDOW
- OUTPUT_AT_EARLIEST = beam_runner_api_pb2.EARLIEST_IN_PANE
- OUTPUT_AT_LATEST = beam_runner_api_pb2.LATEST_IN_PANE
+ OUTPUT_AT_EOW = beam_runner_api_pb2.OutputTime.END_OF_WINDOW
+ OUTPUT_AT_EARLIEST = beam_runner_api_pb2.OutputTime.EARLIEST_IN_PANE
+ OUTPUT_AT_LATEST = beam_runner_api_pb2.OutputTime.LATEST_IN_PANE
# TODO(robertwb): Add this to the runner API or remove it.
OUTPUT_AT_EARLIEST_TRANSFORMED = 'OUTPUT_AT_EARLIEST_TRANSFORMED'
@@ -341,14 +342,18 @@
def to_runner_api_parameter(self, context):
return (urns.FIXED_WINDOWS_FN,
- proto_utils.pack_Struct(size=self.size.micros,
- offset=self.offset.micros))
+ standard_window_fns_pb2.FixedWindowsPayload(
+ size=proto_utils.from_micros(
+ duration_pb2.Duration, self.size.micros),
+ offset=proto_utils.from_micros(
+ timestamp_pb2.Timestamp, self.offset.micros)))
- @urns.RunnerApiFn.register_urn(urns.FIXED_WINDOWS_FN, struct_pb2.Struct)
+ @urns.RunnerApiFn.register_urn(
+ urns.FIXED_WINDOWS_FN, standard_window_fns_pb2.FixedWindowsPayload)
def from_runner_api_parameter(fn_parameter, unused_context):
return FixedWindows(
- size=Duration(micros=fn_parameter['size']),
- offset=Timestamp(micros=fn_parameter['offset']))
+ size=Duration(micros=fn_parameter.size.ToMicroseconds()),
+ offset=Timestamp(micros=fn_parameter.offset.ToMicroseconds()))
class SlidingWindows(NonMergingWindowFn):
@@ -392,17 +397,22 @@
def to_runner_api_parameter(self, context):
return (urns.SLIDING_WINDOWS_FN,
- proto_utils.pack_Struct(
- size=self.size.micros,
- offset=self.offset.micros,
- period=self.period.micros))
+ standard_window_fns_pb2.SlidingWindowsPayload(
+ size=proto_utils.from_micros(
+ duration_pb2.Duration, self.size.micros),
+ offset=proto_utils.from_micros(
+ timestamp_pb2.Timestamp, self.offset.micros),
+ period=proto_utils.from_micros(
+ duration_pb2.Duration, self.period.micros)))
- @urns.RunnerApiFn.register_urn(urns.SLIDING_WINDOWS_FN, struct_pb2.Struct)
+ @urns.RunnerApiFn.register_urn(
+ urns.SLIDING_WINDOWS_FN,
+ standard_window_fns_pb2.SlidingWindowsPayload)
def from_runner_api_parameter(fn_parameter, unused_context):
return SlidingWindows(
- size=Duration(micros=fn_parameter['size']),
- offset=Timestamp(micros=fn_parameter['offset']),
- period=Duration(micros=fn_parameter['period']))
+ size=Duration(micros=fn_parameter.size.ToMicroseconds()),
+ offset=Timestamp(micros=fn_parameter.offset.ToMicroseconds()),
+ period=Duration(micros=fn_parameter.period.ToMicroseconds()))
class Sessions(WindowFn):
@@ -452,10 +462,14 @@
if type(self) == type(other) == Sessions:
return self.gap_size == other.gap_size
- @urns.RunnerApiFn.register_urn(urns.SESSION_WINDOWS_FN, struct_pb2.Struct)
- def from_runner_api_parameter(fn_parameter, unused_context):
- return Sessions(gap_size=Duration(micros=fn_parameter['gap_size']))
-
def to_runner_api_parameter(self, context):
return (urns.SESSION_WINDOWS_FN,
- proto_utils.pack_Struct(gap_size=self.gap_size.micros))
+ standard_window_fns_pb2.SessionsPayload(
+ gap_size=proto_utils.from_micros(
+ duration_pb2.Duration, self.gap_size.micros)))
+
+ @urns.RunnerApiFn.register_urn(
+ urns.SESSION_WINDOWS_FN, standard_window_fns_pb2.SessionsPayload)
+ def from_runner_api_parameter(fn_parameter, unused_context):
+ return Sessions(
+ gap_size=Duration(micros=fn_parameter.gap_size.ToMicroseconds()))
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index fd1bb9d..7c1d4e9 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -21,14 +21,15 @@
from apache_beam.runners import pipeline_context
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.transforms import CombinePerKey
-from apache_beam.transforms import combiners
-from apache_beam.transforms import core
from apache_beam.transforms import Create
from apache_beam.transforms import GroupByKey
from apache_beam.transforms import Map
from apache_beam.transforms import WindowInto
+from apache_beam.transforms import combiners
+from apache_beam.transforms import core
from apache_beam.transforms.core import Windowing
from apache_beam.transforms.trigger import AccumulationMode
from apache_beam.transforms.trigger import AfterCount
@@ -36,9 +37,9 @@
from apache_beam.transforms.window import GlobalWindow
from apache_beam.transforms.window import GlobalWindows
from apache_beam.transforms.window import IntervalWindow
-from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import Sessions
from apache_beam.transforms.window import SlidingWindows
+from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import TimestampedValue
from apache_beam.transforms.window import WindowedValue
from apache_beam.transforms.window import WindowFn
@@ -50,7 +51,7 @@
return WindowFn.AssignContext(timestamp, element)
-sort_values = Map(lambda (k, vs): (k, sorted(vs)))
+sort_values = Map(lambda k_vs: (k_vs[0], sorted(k_vs[1])))
class ReifyWindowsFn(core.DoFn):
@@ -167,90 +168,85 @@
| Map(lambda x: WindowedValue((key, x), x, [GlobalWindow()])))
def test_sliding_windows(self):
- p = TestPipeline()
- pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3)
- result = (pcoll
- | 'w' >> WindowInto(SlidingWindows(period=2, size=4))
- | GroupByKey()
- | reify_windows)
- expected = [('key @ [-2.0, 2.0)', [1]),
- ('key @ [0.0, 4.0)', [1, 2, 3]),
- ('key @ [2.0, 6.0)', [2, 3])]
- assert_that(result, equal_to(expected))
- p.run()
+ with TestPipeline() as p:
+ pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3)
+ result = (pcoll
+ | 'w' >> WindowInto(SlidingWindows(period=2, size=4))
+ | GroupByKey()
+ | reify_windows)
+ expected = [('key @ [-2.0, 2.0)', [1]),
+ ('key @ [0.0, 4.0)', [1, 2, 3]),
+ ('key @ [2.0, 6.0)', [2, 3])]
+ assert_that(result, equal_to(expected))
def test_sessions(self):
- p = TestPipeline()
- pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3, 20, 35, 27)
- result = (pcoll
- | 'w' >> WindowInto(Sessions(10))
- | GroupByKey()
- | sort_values
- | reify_windows)
- expected = [('key @ [1.0, 13.0)', [1, 2, 3]),
- ('key @ [20.0, 45.0)', [20, 27, 35])]
- assert_that(result, equal_to(expected))
- p.run()
+ with TestPipeline() as p:
+ pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3, 20, 35, 27)
+ result = (pcoll
+ | 'w' >> WindowInto(Sessions(10))
+ | GroupByKey()
+ | sort_values
+ | reify_windows)
+ expected = [('key @ [1.0, 13.0)', [1, 2, 3]),
+ ('key @ [20.0, 45.0)', [20, 27, 35])]
+ assert_that(result, equal_to(expected))
def test_timestamped_value(self):
- p = TestPipeline()
- result = (p
- | 'start' >> Create([(k, k) for k in range(10)])
- | Map(lambda (x, t): TimestampedValue(x, t))
- | 'w' >> WindowInto(FixedWindows(5))
- | Map(lambda v: ('key', v))
- | GroupByKey())
- assert_that(result, equal_to([('key', [0, 1, 2, 3, 4]),
- ('key', [5, 6, 7, 8, 9])]))
- p.run()
+ with TestPipeline() as p:
+ result = (p
+ | 'start' >> Create([(k, k) for k in range(10)])
+ | Map(lambda x_t: TimestampedValue(x_t[0], x_t[1]))
+ | 'w' >> WindowInto(FixedWindows(5))
+ | Map(lambda v: ('key', v))
+ | GroupByKey())
+ assert_that(result, equal_to([('key', [0, 1, 2, 3, 4]),
+ ('key', [5, 6, 7, 8, 9])]))
def test_rewindow(self):
- p = TestPipeline()
- result = (p
- | Create([(k, k) for k in range(10)])
- | Map(lambda (x, t): TimestampedValue(x, t))
- | 'window' >> WindowInto(SlidingWindows(period=2, size=6))
- # Per the model, each element is now duplicated across
- # three windows. Rewindowing must preserve this duplication.
- | 'rewindow' >> WindowInto(FixedWindows(5))
- | 'rewindow2' >> WindowInto(FixedWindows(5))
- | Map(lambda v: ('key', v))
- | GroupByKey())
- assert_that(result, equal_to([('key', sorted([0, 1, 2, 3, 4] * 3)),
- ('key', sorted([5, 6, 7, 8, 9] * 3))]))
- p.run()
+ with TestPipeline() as p:
+ result = (p
+ | Create([(k, k) for k in range(10)])
+ | Map(lambda x_t1: TimestampedValue(x_t1[0], x_t1[1]))
+ | 'window' >> WindowInto(SlidingWindows(period=2, size=6))
+ # Per the model, each element is now duplicated across
+ # three windows. Rewindowing must preserve this duplication.
+ | 'rewindow' >> WindowInto(FixedWindows(5))
+ | 'rewindow2' >> WindowInto(FixedWindows(5))
+ | Map(lambda v: ('key', v))
+ | GroupByKey())
+ assert_that(result, equal_to([('key', sorted([0, 1, 2, 3, 4] * 3)),
+ ('key', sorted([5, 6, 7, 8, 9] * 3))]))
def test_timestamped_with_combiners(self):
- p = TestPipeline()
- result = (p
- # Create some initial test values.
- | 'start' >> Create([(k, k) for k in range(10)])
- # The purpose of the WindowInto transform is to establish a
- # FixedWindows windowing function for the PCollection.
- # It does not bucket elements into windows since the timestamps
- # from Create are not spaced 5 ms apart and very likely they all
- # fall into the same window.
- | 'w' >> WindowInto(FixedWindows(5))
- # Generate timestamped values using the values as timestamps.
- # Now there are values 5 ms apart and since Map propagates the
- # windowing function from input to output the output PCollection
- # will have elements falling into different 5ms windows.
- | Map(lambda (x, t): TimestampedValue(x, t))
- # We add a 'key' to each value representing the index of the
- # window. This is important since there is no guarantee of
- # order for the elements of a PCollection.
- | Map(lambda v: (v / 5, v)))
- # Sum all elements associated with a key and window. Although it
- # is called CombinePerKey it is really CombinePerKeyAndWindow the
- # same way GroupByKey is really GroupByKeyAndWindow.
- sum_per_window = result | CombinePerKey(sum)
- # Compute mean per key and window.
- mean_per_window = result | combiners.Mean.PerKey()
- assert_that(sum_per_window, equal_to([(0, 10), (1, 35)]),
- label='assert:sum')
- assert_that(mean_per_window, equal_to([(0, 2.0), (1, 7.0)]),
- label='assert:mean')
- p.run()
+ with TestPipeline() as p:
+ result = (p
+ # Create some initial test values.
+ | 'start' >> Create([(k, k) for k in range(10)])
+ # The purpose of the WindowInto transform is to establish a
+ # FixedWindows windowing function for the PCollection.
+ # It does not bucket elements into windows since the timestamps
+ # from Create are not spaced 5 ms apart and very likely they all
+ # fall into the same window.
+ | 'w' >> WindowInto(FixedWindows(5))
+ # Generate timestamped values using the values as timestamps.
+ # Now there are values 5 ms apart and since Map propagates the
+ # windowing function from input to output the output PCollection
+ # will have elements falling into different 5ms windows.
+ | Map(lambda x_t2: TimestampedValue(x_t2[0], x_t2[1]))
+ # We add a 'key' to each value representing the index of the
+ # window. This is important since there is no guarantee of
+ # order for the elements of a PCollection.
+ | Map(lambda v: (v / 5, v)))
+ # Sum all elements associated with a key and window. Although it
+ # is called CombinePerKey it is really CombinePerKeyAndWindow the
+ # same way GroupByKey is really GroupByKeyAndWindow.
+ sum_per_window = result | CombinePerKey(sum)
+ # Compute mean per key and window.
+ mean_per_window = result | combiners.Mean.PerKey()
+ assert_that(sum_per_window, equal_to([(0, 10), (1, 35)]),
+ label='assert:sum')
+ assert_that(mean_per_window, equal_to([(0, 2.0), (1, 7.0)]),
+ label='assert:mean')
class RunnerApiTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py
index e31b9cc..c2a2005 100644
--- a/sdks/python/apache_beam/transforms/write_ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py
@@ -20,10 +20,10 @@
import unittest
import apache_beam as beam
-
from apache_beam.io import iobase
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, is_empty
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import is_empty
from apache_beam.transforms.ptransform import PTransform
@@ -98,11 +98,10 @@
return_write_results=True):
write_to_test_sink = WriteToTestSink(return_init_result,
return_write_results)
- p = TestPipeline()
- result = p | beam.Create(data) | write_to_test_sink | beam.Map(list)
+ with TestPipeline() as p:
+ result = p | beam.Create(data) | write_to_test_sink | beam.Map(list)
- assert_that(result, is_empty())
- p.run()
+ assert_that(result, is_empty())
sink = write_to_test_sink.last_sink
self.assertIsNotNone(sink)
diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py
index 6ed388a..89dc6af 100644
--- a/sdks/python/apache_beam/typehints/decorators.py
+++ b/sdks/python/apache_beam/typehints/decorators.py
@@ -86,13 +86,13 @@
import inspect
import types
+from apache_beam.typehints import native_type_compatibility
from apache_beam.typehints import typehints
-from apache_beam.typehints.typehints import check_constraint
from apache_beam.typehints.typehints import CompositeTypeHintError
from apache_beam.typehints.typehints import SimpleTypeHintError
+from apache_beam.typehints.typehints import check_constraint
from apache_beam.typehints.typehints import validate_composite_type_param
-
__all__ = [
'with_input_types',
'with_output_types',
@@ -117,7 +117,7 @@
try:
return _original_getargspec(func)
except TypeError:
- if isinstance(func, (type, types.ClassType)):
+ if isinstance(func, type):
argspec = getargspec(func.__init__)
del argspec.args[0]
return argspec
@@ -261,7 +261,7 @@
packed_typeargs += list(typeargs[len(packed_typeargs):])
try:
callargs = inspect.getcallargs(func, *packed_typeargs, **typekwargs)
- except TypeError, e:
+ except TypeError as e:
raise TypeCheckError(e)
if argspec.defaults:
# Declare any default arguments to be Any.
@@ -309,51 +309,76 @@
be type-hinted in totality if even one parameter is type-hinted.
Once fully decorated, if the arguments passed to the resulting function
- violate the type-hint constraints defined, a TypeCheckError detailing the
- error will be raised.
+ violate the type-hint constraints defined, a :class:`TypeCheckError`
+ detailing the error will be raised.
- To be used as::
+ To be used as:
- * @with_input_types(s=str) # just @with_input_types(str) will work too.
- def upper(s):
- return s.upper()
+ .. testcode::
- Or::
+ from apache_beam.typehints import with_input_types
- * @with_input_types(ls=List[Tuple[int, int])
- def increment(ls):
- [(i + 1, j + 1) for (i,j) in ls]
+ @with_input_types(str)
+ def upper(s):
+ return s.upper()
+
+ Or:
+
+ .. testcode::
+
+ from apache_beam.typehints import with_input_types
+ from apache_beam.typehints import List
+ from apache_beam.typehints import Tuple
+
+ @with_input_types(ls=List[Tuple[int, int]])
+ def increment(ls):
+ [(i + 1, j + 1) for (i,j) in ls]
Args:
*positional_hints: Positional type-hints having identical order as the
function's formal arguments. Values for this argument must either be a
- built-in Python type or an instance of a TypeContraint created by
- 'indexing' a CompositeTypeHint instance with a type parameter.
+ built-in Python type or an instance of a
+ :class:`~apache_beam.typehints.typehints.TypeConstraint` created by
+ 'indexing' a
+ :class:`~apache_beam.typehints.typehints.CompositeTypeHint` instance
+ with a type parameter.
**keyword_hints: Keyword arguments mirroring the names of the parameters to
the decorated functions. The value of each keyword argument must either
be one of the allowed built-in Python types, a custom class, or an
- instance of a TypeContraint created by 'indexing' a CompositeTypeHint
- instance with a type parameter.
+ instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint`
+ created by 'indexing' a
+ :class:`~apache_beam.typehints.typehints.CompositeTypeHint` instance
+ with a type parameter.
Raises:
- ValueError: If not all function arguments have corresponding type-hints
- specified. Or if the inner wrapper function isn't passed a function
- object.
- TypeCheckError: If the any of the passed type-hint constraints are not a
- type or TypeContraint instance.
+ :class:`~exceptions.ValueError`: If not all function arguments have
+ corresponding type-hints specified. Or if the inner wrapper function isn't
+ passed a function object.
+ :class:`TypeCheckError`: If the any of the passed type-hint
+ constraints are not a type or
+ :class:`~apache_beam.typehints.typehints.TypeConstraint` instance.
Returns:
The original function decorated such that it enforces type-hint constraints
for all received function arguments.
"""
+ converted_positional_hints = (
+ native_type_compatibility.convert_to_beam_types(positional_hints))
+ converted_keyword_hints = (
+ native_type_compatibility.convert_to_beam_types(keyword_hints))
+ del positional_hints
+ del keyword_hints
+
def annotate(f):
if isinstance(f, types.FunctionType):
- for t in list(positional_hints) + list(keyword_hints.values()):
+ for t in (list(converted_positional_hints) +
+ list(converted_keyword_hints.values())):
validate_composite_type_param(
t, error_msg_prefix='All type hint arguments')
- get_type_hints(f).set_input_types(*positional_hints, **keyword_hints)
+ get_type_hints(f).set_input_types(*converted_positional_hints,
+ **converted_keyword_hints)
return f
return annotate
@@ -365,37 +390,53 @@
Only a single type-hint is accepted to specify the return type of the return
value. If the function to be decorated has multiple return values, then one
- should use: 'Tuple[type_1, type_2]' to annotate the types of the return
+ should use: ``Tuple[type_1, type_2]`` to annotate the types of the return
values.
If the ultimate return value for the function violates the specified type-hint
- a TypeCheckError will be raised detailing the type-constraint violation.
+ a :class:`TypeCheckError` will be raised detailing the type-constraint
+ violation.
- This decorator is intended to be used like::
+ This decorator is intended to be used like:
- * @with_output_types(Set[Coordinate])
- def parse_ints(ints):
- ....
- return [Coordinate.from_int(i) for i in ints]
+ .. testcode::
- Or with a simple type-hint::
+ from apache_beam.typehints import with_output_types
+ from apache_beam.typehints import Set
- * @with_output_types(bool)
- def negate(p):
- return not p if p else p
+ class Coordinate:
+ def __init__(self, x, y):
+ self.x = x
+ self.y = y
+
+ @with_output_types(Set[Coordinate])
+ def parse_ints(ints):
+ return {Coordinate(i, i) for i in ints}
+
+ Or with a simple type-hint:
+
+ .. testcode::
+
+ from apache_beam.typehints import with_output_types
+
+ @with_output_types(bool)
+ def negate(p):
+ return not p if p else p
Args:
*return_type_hint: A type-hint specifying the proper return type of the
function. This argument should either be a built-in Python type or an
- instance of a 'TypeConstraint' created by 'indexing' a
- 'CompositeTypeHint'.
+ instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint`
+ created by 'indexing' a
+ :class:`~apache_beam.typehints.typehints.CompositeTypeHint`.
**kwargs: Not used.
Raises:
- ValueError: If any kwarg parameters are passed in, or the length of
- 'return_type_hint' is greater than 1. Or if the inner wrapper function
- isn't passed a function object.
- TypeCheckError: If the 'return_type_hint' object is in invalid type-hint.
+ :class:`~exceptions.ValueError`: If any kwarg parameters are passed in,
+ or the length of **return_type_hint** is greater than ``1``. Or if the
+ inner wrapper function isn't passed a function object.
+ :class:`TypeCheckError`: If the **return_type_hint** object is
+ in invalid type-hint.
Returns:
The original function decorated such that it enforces type-hint constraints
@@ -410,7 +451,8 @@
"order to specify multiple return types, use the 'Tuple' "
"type-hint.")
- return_type_hint = return_type_hint[0]
+ return_type_hint = native_type_compatibility.convert_to_beam_type(
+ return_type_hint[0])
validate_composite_type_param(
return_type_hint,
@@ -420,6 +462,7 @@
def annotate(f):
get_type_hints(f).set_output_types(return_type_hint)
return f
+
return annotate
diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility.py b/sdks/python/apache_beam/typehints/native_type_compatibility.py
new file mode 100644
index 0000000..8a8e07e
--- /dev/null
+++ b/sdks/python/apache_beam/typehints/native_type_compatibility.py
@@ -0,0 +1,166 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Module to convert Python's native typing types to Beam types."""
+
+import collections
+import typing
+
+from apache_beam.typehints import typehints
+
+# Describes an entry in the type map in convert_to_beam_type.
+# match is a function that takes a user type and returns whether the conversion
+# should trigger.
+# arity is the expected arity of the user type. -1 means it's variadic.
+# beam_type is the Beam type the user type should map to.
+_TypeMapEntry = collections.namedtuple(
+ '_TypeMapEntry', ['match', 'arity', 'beam_type'])
+
+
+def _get_arg(typ, index):
+ """Returns the index-th argument to the given type."""
+ return typ.__args__[index]
+
+
+def _len_arg(typ):
+ """Returns the length of the arguments to the given type."""
+ try:
+ return len(typ.__args__)
+ except AttributeError:
+ # For Any type, which takes no arguments.
+ return 0
+
+
+def _safe_issubclass(derived, parent):
+ """Like issubclass, but swallows TypeErrors.
+
+ This is useful for when either parameter might not actually be a class,
+ e.g. typing.Union isn't actually a class.
+
+ Args:
+ derived: As in issubclass.
+ parent: As in issubclass.
+
+ Returns:
+ issubclass(derived, parent), or False if a TypeError was raised.
+ """
+ try:
+ return issubclass(derived, parent)
+ except TypeError:
+ return False
+
+
+def _match_issubclass(match_against):
+ return lambda user_type: _safe_issubclass(user_type, match_against)
+
+
+def _match_same_type(match_against):
+ # For Union types. They can't be compared with isinstance either, so we
+ # have to compare their types directly.
+ return lambda user_type: type(user_type) == type(match_against)
+
+
+def _match_is_named_tuple(user_type):
+ return (_safe_issubclass(user_type, typing.Tuple) and
+ hasattr(user_type, '_field_types'))
+
+
+def convert_to_beam_type(typ):
+ """Convert a given typing type to a Beam type.
+
+ Args:
+ typ (type): typing type.
+
+ Returns:
+ type: The given type converted to a Beam type as far as we can do the
+ conversion.
+
+ Raises:
+ ~exceptions.ValueError: The type was malformed.
+ """
+
+ type_map = [
+ _TypeMapEntry(
+ match=_match_same_type(typing.Any),
+ arity=0,
+ beam_type=typehints.Any),
+ _TypeMapEntry(
+ match=_match_issubclass(typing.Dict),
+ arity=2,
+ beam_type=typehints.Dict),
+ _TypeMapEntry(
+ match=_match_issubclass(typing.List),
+ arity=1,
+ beam_type=typehints.List),
+ _TypeMapEntry(
+ match=_match_issubclass(typing.Set),
+ arity=1,
+ beam_type=typehints.Set),
+ # NamedTuple is a subclass of Tuple, but it needs special handling.
+ # We just convert it to Any for now.
+ # This MUST appear before the entry for the normal Tuple.
+ _TypeMapEntry(
+ match=_match_is_named_tuple, arity=0, beam_type=typehints.Any),
+ _TypeMapEntry(
+ match=_match_issubclass(typing.Tuple),
+ arity=-1,
+ beam_type=typehints.Tuple),
+ _TypeMapEntry(
+ match=_match_same_type(typing.Union),
+ arity=-1,
+ beam_type=typehints.Union)
+ ]
+
+ # Find the first matching entry.
+ matched_entry = next((entry for entry in type_map if entry.match(typ)), None)
+ if not matched_entry:
+ # No match: return original type.
+ return typ
+
+ if matched_entry.arity == -1:
+ arity = _len_arg(typ)
+ else:
+ arity = matched_entry.arity
+ if _len_arg(typ) != arity:
+ raise ValueError('expecting type %s to have arity %d, had arity %d '
+ 'instead' % (str(typ), arity, _len_arg(typ)))
+ typs = [convert_to_beam_type(_get_arg(typ, i)) for i in xrange(arity)]
+ if arity == 0:
+ # Nullary types (e.g. Any) don't accept empty tuples as arguments.
+ return matched_entry.beam_type
+ elif arity == 1:
+ # Unary types (e.g. Set) don't accept 1-tuples as arguments
+ return matched_entry.beam_type[typs[0]]
+ else:
+ return matched_entry.beam_type[tuple(typs)]
+
+
+def convert_to_beam_types(args):
+ """Convert the given list or dictionary of args to Beam types.
+
+ Args:
+ args: Either an iterable of types, or a dictionary where the values are
+ types.
+
+ Returns:
+ If given an iterable, a list of converted types. If given a dictionary,
+ a dictionary with the same keys, and values which have been converted.
+ """
+ if isinstance(args, dict):
+ return {k: convert_to_beam_type(v) for k, v in args.iteritems()}
+ else:
+ return [convert_to_beam_type(v) for v in args]
diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
new file mode 100644
index 0000000..4171507
--- /dev/null
+++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Test for Beam type compatibility library."""
+
+import typing
+import unittest
+
+from apache_beam.typehints import native_type_compatibility
+from apache_beam.typehints import typehints
+
+_TestNamedTuple = typing.NamedTuple('_TestNamedTuple',
+ [('age', int), ('name', bytes)])
+_TestFlatAlias = typing.Tuple[bytes, float]
+_TestNestedAlias = typing.List[_TestFlatAlias]
+
+
+class _TestClass(object):
+ pass
+
+
+class NativeTypeCompatibilityTest(unittest.TestCase):
+
+ def test_convert_to_beam_type(self):
+ test_cases = [
+ ('raw bytes', bytes, bytes),
+ ('raw int', int, int),
+ ('raw float', float, float),
+ ('any', typing.Any, typehints.Any),
+ ('simple dict', typing.Dict[bytes, int],
+ typehints.Dict[bytes, int]),
+ ('simple list', typing.List[int], typehints.List[int]),
+ ('simple optional', typing.Optional[int], typehints.Optional[int]),
+ ('simple set', typing.Set[float], typehints.Set[float]),
+ ('simple unary tuple', typing.Tuple[bytes],
+ typehints.Tuple[bytes]),
+ ('simple union', typing.Union[int, bytes, float],
+ typehints.Union[int, bytes, float]),
+ ('namedtuple', _TestNamedTuple, typehints.Any),
+ ('test class', _TestClass, _TestClass),
+ ('test class in list', typing.List[_TestClass],
+ typehints.List[_TestClass]),
+ ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[
+ bytes, typing.Union[int, bytes, float]]]],
+ typehints.Tuple[bytes, typehints.List[typehints.Tuple[
+ bytes, typehints.Union[int, bytes, float]]]]),
+ ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]),
+ ('nested alias', _TestNestedAlias,
+ typehints.List[typehints.Tuple[bytes, float]]),
+ ('complex dict',
+ typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
+ typehints.Dict[bytes, typehints.List[typehints.Tuple[
+ bytes, _TestClass]]])
+ ]
+
+ for test_case in test_cases:
+ # Unlike typing types, Beam types are guaranteed to compare equal.
+ description = test_case[0]
+ typing_type = test_case[1]
+ beam_type = test_case[2]
+ self.assertEqual(
+ native_type_compatibility.convert_to_beam_type(typing_type),
+ beam_type, description)
+
+ def test_convert_to_beam_types(self):
+ typing_types = [bytes, typing.List[bytes],
+ typing.List[typing.Tuple[bytes, int]],
+ typing.Union[int, typing.List[int]]]
+ beam_types = [bytes, typehints.List[bytes],
+ typehints.List[typehints.Tuple[bytes, int]],
+ typehints.Union[int, typehints.List[int]]]
+ self.assertEqual(
+ native_type_compatibility.convert_to_beam_types(typing_types),
+ beam_types)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py
index 83f444c..dcca6d0 100644
--- a/sdks/python/apache_beam/typehints/opcodes.py
+++ b/sdks/python/apache_beam/typehints/opcodes.py
@@ -26,11 +26,22 @@
For internal use only; no backwards-compatibility guarantees.
"""
-import types
+from __future__ import absolute_import
-from trivial_inference import union, element_type, Const, BoundMethod
-import typehints
-from typehints import Any, Dict, Iterable, List, Tuple, Union
+import types
+from functools import reduce
+
+from . import typehints
+from .trivial_inference import BoundMethod
+from .trivial_inference import Const
+from .trivial_inference import element_type
+from .trivial_inference import union
+from .typehints import Any
+from .typehints import Dict
+from .typehints import Iterable
+from .typehints import List
+from .typehints import Tuple
+from .typehints import Union
def pop_one(state, unused_arg):
@@ -254,7 +265,7 @@
name = state.get_name(arg)
if isinstance(o, Const) and hasattr(o.value, name):
state.stack.append(Const(getattr(o.value, name)))
- elif (isinstance(o, (type, types.ClassType))
+ elif (isinstance(o, type)
and isinstance(getattr(o, name, None), types.MethodType)):
state.stack.append(Const(BoundMethod(getattr(o, name))))
else:
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index 977ea06..a68bd18 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -19,12 +19,16 @@
For internal use only; no backwards-compatibility guarantees.
"""
+from __future__ import absolute_import
+from __future__ import print_function
+
import __builtin__
import collections
import dis
import pprint
import sys
import types
+from functools import reduce
from apache_beam.typehints import Any
from apache_beam.typehints import typehints
@@ -40,8 +44,7 @@
"""
t = type(o)
if o is None:
- # TODO(robertwb): Eliminate inconsistent use of None vs. NoneType.
- return None
+ return type(None)
elif t not in typehints.DISALLOWED_PRIMITIVE_TYPES:
if t == types.InstanceType:
return o.__class__
@@ -104,7 +107,10 @@
def __init__(self, f, local_vars=None, stack=()):
self.f = f
- self.co = f.func_code
+ if sys.version_info[0] >= 3:
+ self.co = f.__code__
+ else:
+ self.co = f.func_code
self.vars = list(local_vars)
self.stack = list(stack)
@@ -121,12 +127,12 @@
ncellvars = len(self.co.co_cellvars)
if i < ncellvars:
return Any
- return Const(self.f.func_closure[i - ncellvars].cell_contents)
+ return Const(self.f.__closure__[i - ncellvars].cell_contents)
def get_global(self, i):
name = self.get_name(i)
- if name in self.f.func_globals:
- return Const(self.f.func_globals[name])
+ if name in self.f.__globals__:
+ return Const(self.f.__globals__[name])
if name in __builtin__.__dict__:
return Const(__builtin__.__dict__[name])
return Any
@@ -228,14 +234,14 @@
elif isinstance(c, types.FunctionType):
return infer_return_type_func(c, input_types, debug, depth)
elif isinstance(c, types.MethodType):
- if c.im_self is not None:
- input_types = [Const(c.im_self)] + input_types
- return infer_return_type_func(c.im_func, input_types, debug, depth)
+ if c.__self__ is not None:
+ input_types = [Const(c.__self__)] + input_types
+ return infer_return_type_func(c.__func__, input_types, debug, depth)
elif isinstance(c, BoundMethod):
- input_types = [c.unbound.im_class] + input_types
+ input_types = [c.unbound.__self__.__class__] + input_types
return infer_return_type_func(
- c.unbound.im_func, input_types, debug, depth)
- elif isinstance(c, (type, types.ClassType)):
+ c.unbound.__func__, input_types, debug, depth)
+ elif isinstance(c, type):
if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
return {
list: typehints.List[Any],
@@ -273,12 +279,12 @@
TypeInferenceError: if no type can be inferred.
"""
if debug:
- print
- print f, id(f), input_types
- import opcodes
+ print()
+ print(f, id(f), input_types)
+ from . import opcodes
simple_ops = dict((k.upper(), v) for k, v in opcodes.__dict__.items())
- co = f.func_code
+ co = f.__code__
code = co.co_code
end = len(code)
pc = 0
@@ -300,38 +306,38 @@
op = ord(code[pc])
if debug:
- print '-->' if pc == last_pc else ' ',
- print repr(pc).rjust(4),
- print dis.opname[op].ljust(20),
+ print('-->' if pc == last_pc else ' ', end=' ')
+ print(repr(pc).rjust(4), end=' ')
+ print(dis.opname[op].ljust(20), end=' ')
pc += 1
if op >= dis.HAVE_ARGUMENT:
arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
extended_arg = 0
pc += 2
if op == dis.EXTENDED_ARG:
- extended_arg = arg * 65536L
+ extended_arg = arg * 65536
if debug:
- print str(arg).rjust(5),
+ print(str(arg).rjust(5), end=' ')
if op in dis.hasconst:
- print '(' + repr(co.co_consts[arg]) + ')',
+ print('(' + repr(co.co_consts[arg]) + ')', end=' ')
elif op in dis.hasname:
- print '(' + co.co_names[arg] + ')',
+ print('(' + co.co_names[arg] + ')', end=' ')
elif op in dis.hasjrel:
- print '(to ' + repr(pc + arg) + ')',
+ print('(to ' + repr(pc + arg) + ')', end=' ')
elif op in dis.haslocal:
- print '(' + co.co_varnames[arg] + ')',
+ print('(' + co.co_varnames[arg] + ')', end=' ')
elif op in dis.hascompare:
- print '(' + dis.cmp_op[arg] + ')',
+ print('(' + dis.cmp_op[arg] + ')', end=' ')
elif op in dis.hasfree:
if free is None:
free = co.co_cellvars + co.co_freevars
- print '(' + free[arg] + ')',
+ print('(' + free[arg] + ')', end=' ')
# Acutally emulate the op.
if state is None and states[start] is None:
# No control reaches here (yet).
if debug:
- print
+ print()
continue
state |= states[start]
@@ -359,7 +365,22 @@
else:
return_type = Any
state.stack[-pop_count:] = [return_type]
+ elif (opname == 'BINARY_SUBSCR'
+ and isinstance(state.stack[1], Const)
+ and isinstance(state.stack[0], typehints.IndexableTypeConstraint)):
+ if debug:
+ print("Executing special case binary subscript")
+ idx = state.stack.pop()
+ src = state.stack.pop()
+ try:
+ state.stack.append(src._constraint_for_index(idx.value))
+ except Exception as e:
+ if debug:
+ print("Exception {0} during special case indexing".format(e))
+ state.stack.append(Any)
elif opname in simple_ops:
+ if debug:
+ print("Executing simple op " + opname)
simple_ops[opname](state, arg)
elif opname == 'RETURN_VALUE':
returns.add(state.stack[-1])
@@ -399,8 +420,8 @@
states[jmp] = new_state
if debug:
- print
- print state
+ print()
+ print(state)
pprint.pprint(dict(item for item in states.items() if item[1]))
if yields:
@@ -409,5 +430,5 @@
result = reduce(union, Const.unwrap_all(returns))
if debug:
- print f, id(f), input_types, '->', result
+ print(f, id(f), input_types, '->', result)
return result
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index ac00baa..37b2258 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -18,7 +18,6 @@
"""Tests for apache_beam.typehints.trivial_inference."""
import unittest
-
from apache_beam.typehints import trivial_inference
from apache_beam.typehints import typehints
@@ -33,6 +32,11 @@
def testIdentity(self):
self.assertReturnType(int, lambda x: x, [int])
+ def testIndexing(self):
+ self.assertReturnType(int, lambda x: x[0], [typehints.Tuple[int, str]])
+ self.assertReturnType(str, lambda x: x[1], [typehints.Tuple[int, str]])
+ self.assertReturnType(str, lambda x: x[1], [typehints.List[str]])
+
def testTuples(self):
self.assertReturnType(
typehints.Tuple[typehints.Tuple[()], int], lambda x: ((), x), [int])
@@ -40,7 +44,8 @@
typehints.Tuple[str, int, float], lambda x: (x, 0, 1.0), [str])
def testUnpack(self):
- def reverse((a, b)):
+ def reverse(a_b):
+ (a, b) = a_b
return b, a
any_tuple = typehints.Tuple[typehints.Any, typehints.Any]
self.assertReturnType(
@@ -60,6 +65,13 @@
self.assertReturnType(any_tuple,
reverse, [trivial_inference.Const((1, 2, 3))])
+ def testNoneReturn(self):
+ def func(a):
+ if a == 5:
+ return a
+ return None
+ self.assertReturnType(typehints.Union[int, type(None)], func, [int])
+
def testListComprehension(self):
self.assertReturnType(
typehints.List[int],
diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py
index 89a5f5c..c47e9ba 100644
--- a/sdks/python/apache_beam/typehints/typecheck.py
+++ b/sdks/python/apache_beam/typehints/typecheck.py
@@ -28,13 +28,13 @@
from apache_beam.pvalue import TaggedOutput
from apache_beam.transforms.core import DoFn
from apache_beam.transforms.window import WindowedValue
-from apache_beam.typehints.decorators import _check_instance_type
-from apache_beam.typehints.decorators import getcallargs_forhints
from apache_beam.typehints.decorators import GeneratorWrapper
from apache_beam.typehints.decorators import TypeCheckError
-from apache_beam.typehints.typehints import check_constraint
+from apache_beam.typehints.decorators import _check_instance_type
+from apache_beam.typehints.decorators import getcallargs_forhints
from apache_beam.typehints.typehints import CompositeTypeHintError
from apache_beam.typehints.typehints import SimpleTypeHintError
+from apache_beam.typehints.typehints import check_constraint
class AbstractDoFnWrapper(DoFn):
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 589dc0e..2581457 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -17,15 +17,16 @@
"""Unit tests for the type-hint objects and decorators."""
import inspect
+import typing
import unittest
-
import apache_beam as beam
from apache_beam import pvalue
from apache_beam import typehints
from apache_beam.options.pipeline_options import OptionsContext
from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.testing.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
from apache_beam.typehints import WithTypeHints
# These test often construct a pipeline as value | PTransform to test side
@@ -98,6 +99,34 @@
[1, 2, 3] | (beam.ParDo(my_do_fn) | 'again' >> beam.ParDo(my_do_fn))
+class NativeTypesTest(unittest.TestCase):
+
+ def test_good_main_input(self):
+ @typehints.with_input_types(typing.Tuple[str, int])
+ def munge(s_i):
+ (s, i) = s_i
+ return (s + 's', i * 2)
+ result = [('apple', 5), ('pear', 3)] | beam.Map(munge)
+ self.assertEqual([('apples', 10), ('pears', 6)], sorted(result))
+
+ def test_bad_main_input(self):
+ @typehints.with_input_types(typing.Tuple[str, str])
+ def munge(s_i):
+ (s, i) = s_i
+ return (s + 's', i * 2)
+ with self.assertRaises(typehints.TypeCheckError):
+ [('apple', 5), ('pear', 3)] | beam.Map(munge)
+
+ def test_bad_main_output(self):
+ @typehints.with_input_types(typing.Tuple[int, int])
+ @typehints.with_output_types(typing.Tuple[str, str])
+ def munge(a_b):
+ (a, b) = a_b
+ return (str(a), str(b))
+ with self.assertRaises(typehints.TypeCheckError):
+ [(5, 4), (3, 2)] | beam.Map(munge) | 'Again' >> beam.Map(munge)
+
+
class SideInputTest(unittest.TestCase):
def _run_repeat_test(self, repeat):
@@ -168,12 +197,11 @@
@typehints.with_input_types(str, int)
def repeat(s, times):
return s * times
- p = TestPipeline()
- main_input = p | beam.Create(['a', 'bb', 'c'])
- side_input = p | 'side' >> beam.Create([3])
- result = main_input | beam.Map(repeat, pvalue.AsSingleton(side_input))
- assert_that(result, equal_to(['aaa', 'bbbbbb', 'ccc']))
- p.run()
+ with TestPipeline() as p:
+ main_input = p | beam.Create(['a', 'bb', 'c'])
+ side_input = p | 'side' >> beam.Create([3])
+ result = main_input | beam.Map(repeat, pvalue.AsSingleton(side_input))
+ assert_that(result, equal_to(['aaa', 'bbbbbb', 'ccc']))
bad_side_input = p | 'bad_side' >> beam.Create(['z'])
with self.assertRaises(typehints.TypeCheckError):
@@ -183,12 +211,11 @@
@typehints.with_input_types(str, typehints.Iterable[str])
def concat(glue, items):
return glue.join(sorted(items))
- p = TestPipeline()
- main_input = p | beam.Create(['a', 'bb', 'c'])
- side_input = p | 'side' >> beam.Create(['x', 'y', 'z'])
- result = main_input | beam.Map(concat, pvalue.AsIter(side_input))
- assert_that(result, equal_to(['xayaz', 'xbbybbz', 'xcycz']))
- p.run()
+ with TestPipeline() as p:
+ main_input = p | beam.Create(['a', 'bb', 'c'])
+ side_input = p | 'side' >> beam.Create(['x', 'y', 'z'])
+ result = main_input | beam.Map(concat, pvalue.AsIter(side_input))
+ assert_that(result, equal_to(['xayaz', 'xbbybbz', 'xcycz']))
bad_side_input = p | 'bad_side' >> beam.Create([1, 2, 3])
with self.assertRaises(typehints.TypeCheckError):
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index cc430be..6e1d8b7 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -65,15 +65,14 @@
import collections
import copy
+import sys
import types
-
__all__ = [
'Any',
'Union',
'Optional',
'Tuple',
- 'Tuple',
'List',
'KV',
'Dict',
@@ -109,9 +108,10 @@
"""The base-class for all created type-constraints defined below.
- A TypeConstraint is the result of parameterizing a CompositeTypeHint with
- with one of the allowed Python types or another CompositeTypeHint. It
- binds and enforces a specific version of a generalized TypeHint.
+ A :class:`TypeConstraint` is the result of parameterizing a
+ :class:`CompositeTypeHint` with with one of the allowed Python types or
+ another :class:`CompositeTypeHint`. It binds and enforces a specific
+ version of a generalized TypeHint.
"""
def _consistent_with_check_(self, sub):
@@ -135,12 +135,14 @@
instance: An instance of a Python object.
Raises:
- TypeError: The passed 'instance' doesn't satisfy this TypeConstraint.
- Subclasses of TypeConstraint are free to raise any of the subclasses of
- TypeError defined above, depending on the manner of the type hint error.
+ :class:`~exceptions.TypeError`: The passed **instance** doesn't satisfy
+ this :class:`TypeConstraint`. Subclasses of
+ :class:`TypeConstraint` are free to raise any of the subclasses of
+ :class:`~exceptions.TypeError` defined above, depending on
+ the manner of the type hint error.
- All TypeConstraint sub-classes must define this method in other for the
- class object to be created.
+ All :class:`TypeConstraint` sub-classes must define this method in other
+ for the class object to be created.
"""
raise NotImplementedError
@@ -183,7 +185,21 @@
return type_constraint
-class SequenceTypeConstraint(TypeConstraint):
+class IndexableTypeConstraint(TypeConstraint):
+ """An internal common base-class for all type constraints with indexing.
+ E.G. SequenceTypeConstraint + Tuple's of fixed size.
+ """
+
+ def _constraint_for_index(self, idx):
+ """Returns the type at the given index. This is used to allow type inference
+ to determine the correct type for a specific index. On lists this will also
+ be the same, however for tuples the value will depend on the position. This
+ was added as part of the futurize changes since more of the expressions now
+ index into tuples."""
+ raise NotImplementedError
+
+
+class SequenceTypeConstraint(IndexableTypeConstraint):
"""A common base-class for all sequence related type-constraint classes.
A sequence is defined as an arbitrary length homogeneous container type. Type
@@ -213,6 +229,10 @@
def _inner_types(self):
yield self.inner_type
+ def _constraint_for_index(self, idx):
+ """Returns the type at the given index."""
+ return self.inner_type
+
def _consistent_with_check_(self, sub):
return (isinstance(sub, self.__class__)
and is_consistent_with(sub.inner_type, self.inner_type))
@@ -296,23 +316,28 @@
def validate_composite_type_param(type_param, error_msg_prefix):
- """Determines if an object is a valid type parameter to a CompositeTypeHint.
+ """Determines if an object is a valid type parameter to a
+ :class:`CompositeTypeHint`.
- Implements sanity checking to disallow things like:
- * List[1, 2, 3] or Dict[5].
+ Implements sanity checking to disallow things like::
+
+ List[1, 2, 3] or Dict[5].
Args:
type_param: An object instance.
- error_msg_prefix: A string prefix used to format an error message in the
- case of an exception.
+ error_msg_prefix (:class:`str`): A string prefix used to format an error
+ message in the case of an exception.
Raises:
- TypeError: If the passed 'type_param' is not a valid type parameter for a
- CompositeTypeHint.
+ ~exceptions.TypeError: If the passed **type_param** is not a valid type
+ parameter for a :class:`CompositeTypeHint`.
"""
# Must either be a TypeConstraint instance or a basic Python type.
+ possible_classes = [type, TypeConstraint]
+ if sys.version_info[0] == 2:
+ possible_classes.append(types.ClassType)
is_not_type_constraint = (
- not isinstance(type_param, (type, types.ClassType, TypeConstraint))
+ not isinstance(type_param, tuple(possible_classes))
and type_param is not None)
is_forbidden_type = (isinstance(type_param, type) and
type_param in DISALLOWED_PRIMITIVE_TYPES)
@@ -337,7 +362,7 @@
A qualified name for the passed Python object fit for string formatting.
"""
return repr(o) if isinstance(
- o, (TypeConstraint, types.NoneType)) else o.__name__
+ o, (TypeConstraint, type(None))) else o.__name__
def check_constraint(type_constraint, object_instance):
@@ -488,7 +513,7 @@
if Any in params:
return Any
elif len(params) == 1:
- return iter(params).next()
+ return next(iter(params))
return self.UnionConstraint(params)
@@ -498,7 +523,7 @@
class OptionalHint(UnionHint):
"""An Option type-hint. Optional[X] accepts instances of X or None.
- The Optional[X] factory function proxies to Union[X, None]
+ The Optional[X] factory function proxies to Union[X, type(None)]
"""
def __getitem__(self, py_type):
@@ -507,7 +532,7 @@
raise TypeError('An Option type-hint only accepts a single type '
'parameter.')
- return Union[py_type, None]
+ return Union[py_type, type(None)]
class TupleHint(CompositeTypeHint):
@@ -543,7 +568,7 @@
for elem in sub.tuple_types)
return super(TupleSequenceConstraint, self)._consistent_with_check_(sub)
- class TupleConstraint(TypeConstraint):
+ class TupleConstraint(IndexableTypeConstraint):
def __init__(self, type_params):
self.tuple_types = tuple(type_params)
@@ -563,6 +588,10 @@
for t in self.tuple_types:
yield t
+ def _constraint_for_index(self, idx):
+ """Returns the type at the given index."""
+ return self.tuple_types[idx]
+
def _consistent_with_check_(self, sub):
return (isinstance(sub, self.__class__)
and len(sub.tuple_types) == len(self.tuple_types)
diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py
index f1b92e0..af575f4 100644
--- a/sdks/python/apache_beam/typehints/typehints_test.py
+++ b/sdks/python/apache_beam/typehints/typehints_test.py
@@ -20,7 +20,6 @@
import inspect
import unittest
-
import apache_beam.typehints.typehints as typehints
from apache_beam.typehints import Any
from apache_beam.typehints import Tuple
@@ -28,12 +27,12 @@
from apache_beam.typehints import Union
from apache_beam.typehints import with_input_types
from apache_beam.typehints import with_output_types
+from apache_beam.typehints.decorators import GeneratorWrapper
from apache_beam.typehints.decorators import _check_instance_type
from apache_beam.typehints.decorators import _interleave_type_check
from apache_beam.typehints.decorators import _positional_arg_hints
from apache_beam.typehints.decorators import get_type_hints
from apache_beam.typehints.decorators import getcallargs_forhints
-from apache_beam.typehints.decorators import GeneratorWrapper
from apache_beam.typehints.typehints import is_consistent_with
diff --git a/sdks/python/apache_beam/utils/annotations_test.py b/sdks/python/apache_beam/utils/annotations_test.py
index 32af8a9..ddd1b9f 100644
--- a/sdks/python/apache_beam/utils/annotations_test.py
+++ b/sdks/python/apache_beam/utils/annotations_test.py
@@ -17,6 +17,7 @@
import unittest
import warnings
+
from apache_beam.utils.annotations import deprecated
from apache_beam.utils.annotations import experimental
diff --git a/sdks/python/apache_beam/utils/counters.py b/sdks/python/apache_beam/utils/counters.py
index b379461..ae97434 100644
--- a/sdks/python/apache_beam/utils/counters.py
+++ b/sdks/python/apache_beam/utils/counters.py
@@ -24,8 +24,63 @@
"""
import threading
+from collections import namedtuple
+
from apache_beam.transforms import cy_combiners
+# Information identifying the IO being measured by a counter.
+IOTargetName = namedtuple('IOTargetName', ['side_input_step_name',
+ 'side_input_index',
+ 'original_shuffle_step_name'])
+
+
+def side_input_id(step_name, input_index):
+ """Create an IOTargetName that identifies the reading of a side input."""
+ return IOTargetName(step_name, input_index, None)
+
+
+def shuffle_id(step_name):
+ """Create an IOTargetName that identifies a GBK step."""
+ return IOTargetName(None, None, step_name)
+
+
+_CounterName = namedtuple('_CounterName', ['name',
+ 'stage_name',
+ 'step_name',
+ 'system_name',
+ 'namespace',
+ 'origin',
+ 'output_index',
+ 'io_target'])
+
+
+class CounterName(_CounterName):
+ """Naming information for a counter."""
+ SYSTEM = object()
+ USER = object()
+
+ def __new__(cls, name, stage_name=None, step_name=None,
+ system_name=None, namespace=None,
+ origin=None, output_index=None, io_target=None):
+ origin = origin or CounterName.SYSTEM
+ return super(CounterName, cls).__new__(cls, name, stage_name, step_name,
+ system_name, namespace,
+ origin, output_index, io_target)
+
+ def __str__(self):
+ return '%s' % self._str_internal()
+
+ def __repr__(self):
+ return '<CounterName<%s> at %s>' % (self._str_internal(), hex(id(self)))
+
+ def _str_internal(self):
+ if self.origin == CounterName.USER:
+ return 'user-%s-%s' % (self.step_name, self.name)
+ elif self.origin == CounterName.SYSTEM and self.output_index:
+ return '%s-out%s-%s' % (self.step_name, self.output_index, self.name)
+ else:
+ return '%s-%s-%s' % (self.stage_name, self.step_name, self.name)
+
class Counter(object):
"""A counter aggregates a series of values.
@@ -52,8 +107,8 @@
"""Creates a Counter object.
Args:
- name: the name of this counter. Typically has three parts:
- "step-output-counter".
+ name: the name of this counter. It may be a string,
+ or a CounterName object.
combine_fn: the CombineFn to use for aggregation
"""
self.name = name
@@ -90,10 +145,6 @@
self._fast_add_input(value)
-# Counters that represent Accumulators have names starting with this
-USER_COUNTER_PREFIX = 'user-'
-
-
class CounterFactory(object):
"""Keeps track of unique counters."""
@@ -128,21 +179,6 @@
self.counters[name] = counter
return counter
- def get_aggregator_counter(self, step_name, aggregator):
- """Returns an AggregationCounter for this step's aggregator.
-
- Passing in the same values will return the same counter.
-
- Args:
- step_name: the name of this step.
- aggregator: an Aggregator object.
- Returns:
- A new or existing counter.
- """
- return self.get_counter(
- '%s%s-%s' % (USER_COUNTER_PREFIX, step_name, aggregator.name),
- aggregator.combine_fn)
-
def get_counters(self):
"""Returns the current set of counters.
@@ -154,32 +190,3 @@
"""
with self._lock:
return self.counters.values()
-
- def get_aggregator_values(self, aggregator_or_name):
- """Returns dict of step names to values of the aggregator."""
- with self._lock:
- return get_aggregator_values(
- aggregator_or_name, self.counters, lambda counter: counter.value())
-
-
-def get_aggregator_values(aggregator_or_name, counter_dict,
- value_extractor=None):
- """Extracts the named aggregator value from a set of counters.
-
- Args:
- aggregator_or_name: an Aggregator object or the name of one.
- counter_dict: a dict object of {name: value_wrapper}
- value_extractor: a function to convert the value_wrapper into a value.
- If None, no extraction is done and the value is return unchanged.
-
- Returns:
- dict of step names to values of the aggregator.
- """
- name = aggregator_or_name
- if value_extractor is None:
- value_extractor = lambda x: x
- if not isinstance(aggregator_or_name, basestring):
- name = aggregator_or_name.name
- return {n: value_extractor(c) for n, c in counter_dict.iteritems()
- if n.startswith(USER_COUNTER_PREFIX)
- and n.endswith('-%s' % name)}
diff --git a/sdks/python/apache_beam/utils/counters_test.py b/sdks/python/apache_beam/utils/counters_test.py
new file mode 100644
index 0000000..37cab88
--- /dev/null
+++ b/sdks/python/apache_beam/utils/counters_test.py
@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for counters and counter names."""
+
+from __future__ import absolute_import
+
+import unittest
+
+from apache_beam.utils import counters
+from apache_beam.utils.counters import CounterName
+
+
+class CounterNameTest(unittest.TestCase):
+
+ def test_equal_objects(self):
+ self.assertEqual(CounterName('counter_name',
+ 'stage_name',
+ 'step_name'),
+ CounterName('counter_name',
+ 'stage_name',
+ 'step_name'))
+ self.assertNotEqual(CounterName('counter_name',
+ 'stage_name',
+ 'step_name'),
+ CounterName('counter_name',
+ 'stage_name',
+ 'step_nam'))
+
+ # Testing objects with an IOTarget.
+ self.assertEqual(CounterName('counter_name',
+ 'stage_name',
+ 'step_name',
+ io_target=counters.side_input_id(1, 's9')),
+ CounterName('counter_name',
+ 'stage_name',
+ 'step_name',
+ io_target=counters.side_input_id(1, 's9')))
+ self.assertNotEqual(CounterName('counter_name',
+ 'stage_name',
+ 'step_name',
+ io_target=counters.side_input_id(1, 's')),
+ CounterName('counter_name',
+ 'stage_name',
+ 'step_name',
+ io_target=counters.side_input_id(1, 's9')))
+
+ def test_hash_two_objects(self):
+ self.assertEqual(hash(CounterName('counter_name',
+ 'stage_name',
+ 'step_name')),
+ hash(CounterName('counter_name',
+ 'stage_name',
+ 'step_name')))
+ self.assertNotEqual(hash(CounterName('counter_name',
+ 'stage_name',
+ 'step_name')),
+ hash(CounterName('counter_name',
+ 'stage_name',
+ 'step_nam')))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/apache_beam/utils/plugin.py b/sdks/python/apache_beam/utils/plugin.py
new file mode 100644
index 0000000..563b93c
--- /dev/null
+++ b/sdks/python/apache_beam/utils/plugin.py
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A BeamPlugin base class.
+
+For experimental usage only; no backwards-compatibility guarantees.
+"""
+
+
+class BeamPlugin(object):
+ """Plugin base class to be extended by dependent users such as FileSystem.
+ Any instantiated subclass will be imported at worker startup time."""
+
+ @classmethod
+ def get_all_subclasses(cls):
+ """Get all the subclasses of the BeamPlugin class."""
+ all_subclasses = []
+ for subclass in cls.__subclasses__():
+ all_subclasses.append(subclass)
+ all_subclasses.extend(subclass.get_all_subclasses())
+ return all_subclasses
+
+ @classmethod
+ def get_all_plugin_paths(cls):
+ """Get full import paths of the BeamPlugin subclass."""
+ def fullname(o):
+ return o.__module__ + "." + o.__name__
+ return [fullname(o) for o in cls.get_all_subclasses()]
diff --git a/sdks/python/apache_beam/utils/processes_test.py b/sdks/python/apache_beam/utils/processes_test.py
index 920b621..2dd45f4 100644
--- a/sdks/python/apache_beam/utils/processes_test.py
+++ b/sdks/python/apache_beam/utils/processes_test.py
@@ -18,7 +18,6 @@
import unittest
-
import mock
from apache_beam.utils import processes
diff --git a/sdks/python/apache_beam/utils/proto_utils.py b/sdks/python/apache_beam/utils/proto_utils.py
index 090a821..d7693f3 100644
--- a/sdks/python/apache_beam/utils/proto_utils.py
+++ b/sdks/python/apache_beam/utils/proto_utils.py
@@ -46,6 +46,17 @@
return msg
+def parse_Bytes(bytes, msg_class):
+ """Parses the String of bytes into msg_class.
+
+ Returns the input bytes if msg_class is None."""
+ if msg_class is None:
+ return bytes
+ msg = msg_class()
+ msg.ParseFromString(bytes)
+ return msg
+
+
def pack_Struct(**kwargs):
"""Returns a struct containing the values indicated by kwargs.
"""
@@ -53,3 +64,9 @@
for key, value in kwargs.items():
msg[key] = value # pylint: disable=unsubscriptable-object, unsupported-assignment-operation
return msg
+
+
+def from_micros(cls, micros):
+ result = cls()
+ result.FromMicroseconds(micros)
+ return result
diff --git a/sdks/python/apache_beam/utils/retry.py b/sdks/python/apache_beam/utils/retry.py
index 1a8b907..927da14 100644
--- a/sdks/python/apache_beam/utils/retry.py
+++ b/sdks/python/apache_beam/utils/retry.py
@@ -31,6 +31,8 @@
import time
import traceback
+from apache_beam.io.filesystem import BeamIOError
+
# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
# TODO(sourabhbajaj): Remove the GCP specific error code to a submodule
@@ -99,6 +101,11 @@
return retry_on_server_errors_filter(exception)
+def retry_on_beam_io_error_filter(exception):
+ """Filter allowing retries on Beam IO errors."""
+ return isinstance(exception, BeamIOError)
+
+
SERVER_ERROR_OR_TIMEOUT_CODES = [408, 500, 502, 503, 504, 598, 599]
@@ -175,7 +182,7 @@
exn_traceback = sys.exc_info()[2]
try:
try:
- sleep_interval = retry_intervals.next()
+ sleep_interval = next(retry_intervals)
except StopIteration:
# Re-raise the original exception since we finished the retries.
raise exn, None, exn_traceback # pylint: disable=raising-bad-type
diff --git a/sdks/python/apache_beam/utils/retry_test.py b/sdks/python/apache_beam/utils/retry_test.py
index 1b03c83..e5f07e8 100644
--- a/sdks/python/apache_beam/utils/retry_test.py
+++ b/sdks/python/apache_beam/utils/retry_test.py
@@ -19,6 +19,8 @@
import unittest
+from apache_beam.utils import retry
+
# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
# TODO(sourabhbajaj): Remove the GCP specific error code to a submodule
@@ -29,9 +31,6 @@
# pylint: enable=wrong-import-order, wrong-import-position
-from apache_beam.utils import retry
-
-
class FakeClock(object):
"""A fake clock object implementing sleep() and recording calls."""
diff --git a/sdks/python/apache_beam/utils/timestamp.py b/sdks/python/apache_beam/utils/timestamp.py
index 5d1b48c..b3e840e 100644
--- a/sdks/python/apache_beam/utils/timestamp.py
+++ b/sdks/python/apache_beam/utils/timestamp.py
@@ -208,3 +208,8 @@
def __mod__(self, other):
other = Duration.of(other)
return Duration(micros=self.micros % other.micros)
+
+
+# The minimum granularity / interval expressible in a Timestamp / Duration
+# object.
+TIME_GRANULARITY = Duration(micros=1)
diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py
index 379b5ff..c6135ba 100644
--- a/sdks/python/apache_beam/utils/urns.py
+++ b/sdks/python/apache_beam/utils/urns.py
@@ -25,19 +25,44 @@
from apache_beam.internal import pickler
from apache_beam.utils import proto_utils
+PICKLED_WINDOW_FN = "beam:windowfn:pickled_python:v0.1"
+GLOBAL_WINDOWS_FN = "beam:windowfn:global_windows:v0.1"
+FIXED_WINDOWS_FN = "beam:windowfn:fixed_windows:v0.1"
+SLIDING_WINDOWS_FN = "beam:windowfn:sliding_windows:v0.1"
+SESSION_WINDOWS_FN = "beam:windowfn:session_windows:v0.1"
-PICKLED_WINDOW_FN = "beam:window_fn:pickled_python:v0.1"
-GLOBAL_WINDOWS_FN = "beam:window_fn:global_windows:v0.1"
-FIXED_WINDOWS_FN = "beam:window_fn:fixed_windows:v0.1"
-SLIDING_WINDOWS_FN = "beam:window_fn:sliding_windows:v0.1"
-SESSION_WINDOWS_FN = "beam:window_fn:session_windows:v0.1"
-
-PICKLED_CODER = "beam:coder:pickled_python:v0.1"
+PICKLED_DO_FN = "beam:dofn:pickled_python:v0.1"
+PICKLED_DO_FN_INFO = "beam:dofn:pickled_python_info:v0.1"
+PICKLED_COMBINE_FN = "beam:combinefn:pickled_python:v0.1"
PICKLED_TRANSFORM = "beam:ptransform:pickled_python:v0.1"
+PARDO_TRANSFORM = "beam:ptransform:pardo:v0.1"
+GROUP_BY_KEY_TRANSFORM = "beam:ptransform:group_by_key:v0.1"
+GROUP_BY_KEY_ONLY_TRANSFORM = "beam:ptransform:group_by_key_only:v0.1"
+GROUP_ALSO_BY_WINDOW_TRANSFORM = "beam:ptransform:group_also_by_window:v0.1"
+COMBINE_PER_KEY_TRANSFORM = "beam:ptransform:combine_per_key:v0.1"
+COMBINE_GROUPED_VALUES_TRANSFORM = "beam:ptransform:combine_grouped_values:v0.1"
FLATTEN_TRANSFORM = "beam:ptransform:flatten:v0.1"
+READ_TRANSFORM = "beam:ptransform:read:v0.1"
WINDOW_INTO_TRANSFORM = "beam:ptransform:window_into:v0.1"
+PICKLED_SOURCE = "beam:source:pickled_python:v0.1"
+
+PICKLED_CODER = "beam:coder:pickled_python:v0.1"
+BYTES_CODER = "urn:beam:coders:bytes:0.1"
+VAR_INT_CODER = "urn:beam:coders:varint:0.1"
+INTERVAL_WINDOW_CODER = "urn:beam:coders:interval_window:0.1"
+ITERABLE_CODER = "urn:beam:coders:stream:0.1"
+KV_CODER = "urn:beam:coders:kv:0.1"
+LENGTH_PREFIX_CODER = "urn:beam:coders:length_prefix:0.1"
+GLOBAL_WINDOW_CODER = "urn:beam:coders:urn:beam:coders:global_window:0.1"
+WINDOWED_VALUE_CODER = "urn:beam:coders:windowed_value:0.1"
+
+ITERABLE_ACCESS = "urn:beam:sideinput:iterable"
+MULTIMAP_ACCESS = "urn:beam:sideinput:multimap"
+PICKLED_PYTHON_VIEWFN = "beam:view_fn:pickled_python_data:v0.1"
+PICKLED_WINDOW_MAPPING_FN = "beam:window_mapping_fn:pickled_python:v0.1"
+
class RunnerApiFn(object):
"""Abstract base class that provides urn registration utilities.
@@ -50,7 +75,8 @@
to register serialization via pickling.
"""
- __metaclass__ = abc.ABCMeta
+ # TODO(BEAM-2685): Issue with dill + local classes + abc metaclass
+ # __metaclass__ = abc.ABCMeta
_known_urns = {}
@@ -102,12 +128,13 @@
Prefer overriding self.to_runner_api_parameter.
"""
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.api import beam_runner_api_pb2
urn, typed_param = self.to_runner_api_parameter(context)
return beam_runner_api_pb2.SdkFunctionSpec(
spec=beam_runner_api_pb2.FunctionSpec(
urn=urn,
- parameter=proto_utils.pack_Any(typed_param)))
+ payload=typed_param.SerializeToString()
+ if typed_param is not None else None))
@classmethod
def from_runner_api(cls, fn_proto, context):
@@ -117,5 +144,5 @@
"""
parameter_type, constructor = cls._known_urns[fn_proto.spec.urn]
return constructor(
- proto_utils.unpack_Any(fn_proto.spec.parameter, parameter_type),
+ proto_utils.parse_Bytes(fn_proto.spec.payload, parameter_type),
context)
diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py
index ae92a23..b956661 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -18,4 +18,4 @@
"""Apache Beam SDK version information and utilities."""
-__version__ = '2.1.0.dev'
+__version__ = '2.3.0.dev'
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
new file mode 100644
index 0000000..826e36c
--- /dev/null
+++ b/sdks/python/container/Dockerfile
@@ -0,0 +1,27 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM python:2
+MAINTAINER "Apache Beam <dev@beam.apache.org>"
+
+# TODO(herohde): preinstall various packages for better startup
+# performance and reliability.
+
+ADD target/linux_amd64/boot /opt/apache/beam/
+
+ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
new file mode 100644
index 0000000..fea0935
--- /dev/null
+++ b/sdks/python/container/boot.go
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// boot is the boot code for the Python SDK harness container. It is responsible
+// for retrieving and install staged files and invoking python correctly.
+package main
+
+import (
+ "context"
+ "flag"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/artifact"
+ pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
+ "github.com/apache/beam/sdks/go/pkg/beam/provision"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/execx"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+ "github.com/golang/protobuf/proto"
+)
+
+var (
+ // Contract: https://s.apache.org/beam-fn-api-container-contract.
+
+ id = flag.String("id", "", "Local identifier (required).")
+ loggingEndpoint = flag.String("logging_endpoint", "", "Logging endpoint (required).")
+ artifactEndpoint = flag.String("artifact_endpoint", "", "Artifact endpoint (required).")
+ provisionEndpoint = flag.String("provision_endpoint", "", "Provision endpoint (required).")
+ controlEndpoint = flag.String("control_endpoint", "", "Control endpoint (required).")
+ semiPersistDir = flag.String("semi_persist_dir", "/tmp", "Local semi-persistent directory (optional).")
+)
+
+func main() {
+ flag.Parse()
+ if *id == "" {
+ log.Fatal("No id provided.")
+ }
+ if *loggingEndpoint == "" {
+ log.Fatal("No logging endpoint provided.")
+ }
+ if *artifactEndpoint == "" {
+ log.Fatal("No artifact endpoint provided.")
+ }
+ if *provisionEndpoint == "" {
+ log.Fatal("No provision endpoint provided.")
+ }
+ if *controlEndpoint == "" {
+ log.Fatal("No control endpoint provided.")
+ }
+
+ log.Printf("Initializing python harness: %v", strings.Join(os.Args, " "))
+
+ ctx := grpcx.WriteWorkerID(context.Background(), *id)
+
+ // (1) Obtain the pipeline options
+
+ info, err := provision.Info(ctx, *provisionEndpoint)
+ if err != nil {
+ log.Fatalf("Failed to obtain provisioning information: %v", err)
+ }
+ options, err := provision.ProtoToJSON(info.GetPipelineOptions())
+ if err != nil {
+ log.Fatalf("Failed to convert pipeline options: %v", err)
+ }
+
+ // (2) Retrieve and install the staged packages.
+
+ dir := filepath.Join(*semiPersistDir, "staged")
+
+ _, err = artifact.Materialize(ctx, *artifactEndpoint, dir)
+ if err != nil {
+ log.Fatalf("Failed to retrieve staged files: %v", err)
+ }
+
+ // TODO(herohde): the packages to install should be specified explicitly. It
+ // would also be possible to install the SDK in the Dockerfile.
+ if err := pipInstall(joinPaths(dir, "dataflow_python_sdk.tar[gcp]")); err != nil {
+ log.Fatalf("Failed to install SDK: %v", err)
+ }
+
+ // (3) Invoke python
+
+ os.Setenv("PIPELINE_OPTIONS", options)
+ os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pb.ApiServiceDescriptor{Url: *loggingEndpoint}))
+ os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pb.ApiServiceDescriptor{Url: *controlEndpoint}))
+
+ args := []string{
+ "-m",
+ "apache_beam.runners.worker.sdk_worker_main",
+ }
+ log.Printf("Executing: python %v", strings.Join(args, " "))
+
+ log.Fatalf("Python exited: %v", execx.Execute("python", args...))
+}
+
+// pipInstall runs pip install with the given args.
+func pipInstall(args []string) error {
+ return execx.Execute("pip", append([]string{"install"}, args...)...)
+}
+
+// joinPaths joins the dir to every artifact path. Each / in the path is
+// interpreted as a directory separator.
+func joinPaths(dir string, paths ...string) []string {
+ var ret []string
+ for _, p := range paths {
+ ret = append(ret, filepath.Join(dir, filepath.FromSlash(p)))
+ }
+ return ret
+}
diff --git a/sdks/python/container/pom.xml b/sdks/python/container/pom.xml
new file mode 100644
index 0000000..45b8cbf
--- /dev/null
+++ b/sdks/python/container/pom.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-python</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-python-container</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: SDKs :: Python :: Container</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+ <go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${go.source.base}/github.com/apache/beam/cmd/boot</outputDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>*.go</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- CAVEAT: for latest shared files, run mvn install in sdks/go -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ <classifier>pkg-sources</classifier>
+ <overWrite>true</overWrite>
+ <outputDirectory>${go.source.dir}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>google.golang.org/api/storage/v1</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>boot</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>linux_amd64/boot</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <configuration>
+ <repository>${docker-repository-root}/python</repository>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
new file mode 100644
index 0000000..c7bf55f
--- /dev/null
+++ b/sdks/python/gen_protos.py
@@ -0,0 +1,141 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Generates Python proto modules and grpc stubs for Beam protos."""
+
+import glob
+import logging
+import multiprocessing
+import os
+import pkg_resources
+import platform
+import shutil
+import subprocess
+import sys
+import time
+import warnings
+
+
+GRPC_TOOLS = 'grpcio-tools>=1.3.5'
+
+BEAM_PROTO_PATHS = [
+ os.path.join('..', '..', 'model', 'pipeline', 'src', 'main', 'proto'),
+ os.path.join('..', '..', 'model', 'job-management', 'src', 'main', 'proto'),
+ os.path.join('..', '..', 'model', 'fn-execution', 'src', 'main', 'proto'),
+]
+
+PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'api')
+
+
+def generate_proto_files(force=False):
+
+ try:
+ import grpc_tools
+ except ImportError:
+ warnings.warn('Installing grpcio-tools is recommended for development.')
+
+ py_sdk_root = os.path.dirname(os.path.abspath(__file__))
+ common = os.path.join(py_sdk_root, '..', 'common')
+ proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
+ proto_files = sum(
+ [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
+ out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
+ out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
+
+ if out_files and not proto_files and not force:
+ # We have out_files but no protos; assume they're up to date.
+ # This is actually the common case (e.g. installation from an sdist).
+ logging.info('No proto files; using existing generated files.')
+ return
+
+ elif not out_files and not proto_files:
+ if not os.path.exists(common):
+ raise RuntimeError(
+ 'Not in apache git tree; unable to find proto definitions.')
+ else:
+ raise RuntimeError(
+ 'No proto files found in %s.' % proto_dirs)
+
+ # Regenerate iff the proto files are newer.
+ elif force or not out_files or len(out_files) < len(proto_files) or (
+ min(os.path.getmtime(path) for path in out_files)
+ <= max(os.path.getmtime(path) for path in proto_files)):
+ try:
+ from grpc_tools import protoc
+ except ImportError:
+ if platform.system() == 'Windows':
+ # For Windows, grpcio-tools has to be installed manually.
+ raise RuntimeError(
+ 'Cannot generate protos for Windows since grpcio-tools package is '
+ 'not installed. Please install this package manually '
+ 'using \'pip install grpcio-tools\'.')
+
+ # Use a subprocess to avoid messing with this process' path and imports.
+ # Note that this requires a separate module from setup.py for Windows:
+ # https://docs.python.org/2/library/multiprocessing.html#windows
+ p = multiprocessing.Process(
+ target=_install_grpcio_tools_and_generate_proto_files)
+ p.start()
+ p.join()
+ if p.exitcode:
+ raise ValueError("Proto generation failed (see log for details).")
+ else:
+ logging.info('Regenerating out-of-date Python proto definitions.')
+ builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto')
+ args = (
+ [sys.executable] + # expecting to be called from command line
+ ['--proto_path=%s' % builtin_protos] +
+ ['--proto_path=%s' % d for d in proto_dirs] +
+ ['--python_out=%s' % out_dir] +
+ # TODO(robertwb): Remove the prefix once it's the default.
+ ['--grpc_python_out=grpc_2_0:%s' % out_dir] +
+ proto_files)
+ ret_code = protoc.main(args)
+ if ret_code:
+ raise RuntimeError(
+ 'Protoc returned non-zero status (see logs for details): '
+ '%s' % ret_code)
+
+
+# Though wheels are available for grpcio-tools, setup_requires uses
+# easy_install which doesn't understand them. This means that it is
+# compiled from scratch (which is expensive as it compiles the full
+# protoc compiler). Instead, we attempt to install a wheel in a temporary
+# directory and add it to the path as needed.
+# See https://github.com/pypa/setuptools/issues/377
+def _install_grpcio_tools_and_generate_proto_files():
+ install_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), '.eggs', 'grpcio-wheels')
+ build_path = install_path + '-build'
+ if os.path.exists(build_path):
+ shutil.rmtree(build_path)
+ logging.warning('Installing grpcio-tools into %s' % install_path)
+ try:
+ start = time.time()
+ subprocess.check_call(
+ ['pip', 'install', '--target', install_path, '--build', build_path,
+ '--upgrade', GRPC_TOOLS])
+ logging.warning(
+ 'Installing grpcio-tools took %0.2f seconds.' % (time.time() - start))
+ finally:
+ shutil.rmtree(build_path)
+ sys.path.append(install_path)
+ generate_proto_files()
+
+
+if __name__ == '__main__':
+ generate_proto_files(force=True)
diff --git a/sdks/python/generate_pydoc.sh b/sdks/python/generate_pydoc.sh
index 1fea6f1..9ae019c 100755
--- a/sdks/python/generate_pydoc.sh
+++ b/sdks/python/generate_pydoc.sh
@@ -31,43 +31,138 @@
mkdir -p target/docs/source
-# Exclude internal/experimental files from the documentation.
-excluded_internal_code=(
+# Sphinx apidoc autodoc options
+export SPHINX_APIDOC_OPTIONS=\
+members,\
+undoc-members,\
+show-inheritance
+
+# Exclude internal, test, and Cython paths/patterns from the documentation.
+excluded_patterns=(
+ apache_beam/coders/stream.*
+ apache_beam/coders/coder_impl.*
apache_beam/examples/
apache_beam/internal/clients/
- apache_beam/io/gcp/internal/clients/
+ apache_beam/io/gcp/internal/
+ apache_beam/io/gcp/tests/
+ apache_beam/metrics/execution.*
+ apache_beam/runners/common.*
apache_beam/runners/api/
apache_beam/runners/test/
+ apache_beam/runners/dataflow/internal/
apache_beam/runners/portability/
apache_beam/runners/worker/
- apache_beam/runners/dataflow/internal/clients/
- apache_beam/testing/data/)
+ apache_beam/transforms/cy_combiners.*
+ apache_beam/utils/counters.*
+ apache_beam/utils/windowed_value.*
+ *_pb2.py
+ *_test.py
+ *_test_common.py
+)
-python $(type -p sphinx-apidoc) -f -o target/docs/source apache_beam \
- "${excluded_internal_code[@]}" "*_test.py"
-
-# Remove Cython modules from doc template; they won't load
-sed -i -e '/.. automodule:: apache_beam.coders.stream/d' \
- target/docs/source/apache_beam.coders.rst
+python $(type -p sphinx-apidoc) -fMeT -o target/docs/source apache_beam \
+ "${excluded_patterns[@]}"
# Create the configuration and index files
+#=== conf.py ===#
cat > target/docs/source/conf.py <<'EOF'
import os
import sys
+import sphinx_rtd_theme
+
sys.path.insert(0, os.path.abspath('../../..'))
+exclude_patterns = [
+ '_build',
+ 'target/docs/source/apache_beam.rst',
+]
+
extensions = [
'sphinx.ext.autodoc',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.intersphinx',
'sphinx.ext.napoleon',
'sphinx.ext.viewcode',
]
master_doc = 'index'
-html_theme = 'sphinxdoc'
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
project = 'Apache Beam'
+
+autoclass_content = 'both'
+autodoc_member_order = 'bysource'
+
+doctest_global_setup = '''
+import apache_beam as beam
+'''
+
+intersphinx_mapping = {
+ 'python': ('https://docs.python.org/2', None),
+ 'hamcrest': ('https://pyhamcrest.readthedocs.io/en/latest/', None),
+}
+
+# Since private classes are skipped by sphinx, if there is any cross reference
+# to them, it will be broken. This can happen if a class inherits from a
+# private class.
+ignore_identifiers = [
+ # Ignore "custom" builtin types
+ '',
+ 'Any',
+ 'Dict',
+ 'Iterable',
+ 'List',
+ 'Set',
+ 'Tuple',
+
+ # Ignore broken built-in type references
+ 'tuple',
+
+ # Ignore private classes
+ 'apache_beam.coders.coders._PickleCoderBase',
+ 'apache_beam.coders.coders.FastCoder',
+ 'apache_beam.io._AvroSource',
+ 'apache_beam.io.gcp.bigquery.RowAsDictJsonCoder',
+ 'apache_beam.io.gcp.datastore.v1.datastoreio._Mutate',
+ 'apache_beam.io.gcp.internal.clients.bigquery.'
+ 'bigquery_v2_messages.TableSchema',
+ 'apache_beam.io.iobase.SourceBase',
+ 'apache_beam.io.source_test_utils.ExpectedSplitOutcome',
+ 'apache_beam.metrics.metric.MetricResults',
+ 'apache_beam.pipeline.PipelineVisitor',
+ 'apache_beam.pipeline.PTransformOverride',
+ 'apache_beam.pvalue.AsSideInput',
+ 'apache_beam.pvalue.DoOutputsTuple',
+ 'apache_beam.pvalue.PValue',
+ 'apache_beam.runners.direct.executor.CallableTask',
+ 'apache_beam.transforms.core.CallableWrapperCombineFn',
+ 'apache_beam.transforms.ptransform.PTransformWithSideInputs',
+ 'apache_beam.transforms.trigger._ParallelTriggerFn',
+ 'apache_beam.transforms.trigger.InMemoryUnmergedState',
+ 'apache_beam.typehints.typehints.AnyTypeConstraint',
+ 'apache_beam.typehints.typehints.CompositeTypeHint',
+ 'apache_beam.typehints.typehints.TypeConstraint',
+ 'apache_beam.typehints.typehints.validate_composite_type_param()',
+
+ # Private classes which are used within the same module
+ 'WindowedTypeConstraint', # apache_beam.typehints.typehints
+
+ # stdlib classes without documentation
+ 'unittest.case.TestCase'
+]
+
+# When inferring a base class it will use ':py:class'; if inferring a function
+# argument type or return type, it will use ':py:obj'. We'll generate both.
+nitpicky = True
+nitpick_ignore = []
+nitpick_ignore += [('py:class', iden) for iden in ignore_identifiers]
+nitpick_ignore += [('py:obj', iden) for iden in ignore_identifiers]
EOF
+
+#=== index.rst ===#
cat > target/docs/source/index.rst <<'EOF'
-.. include:: ./modules.rst
+.. include:: ./apache_beam.rst
+ :start-line: 2
EOF
# Build the documentation using sphinx
@@ -76,10 +171,21 @@
target/docs/_build -c target/docs/source \
-w "target/docs/sphinx-build.warnings.log"
+# Fail if there are errors or warnings in docs
+! grep -q "ERROR:" target/docs/sphinx-build.warnings.log || exit 1
+! grep -q "WARNING:" target/docs/sphinx-build.warnings.log || exit 1
+
+# Run tests for code samples, these can be:
+# - Code blocks using '.. testsetup::', '.. testcode::' and '.. testoutput::'
+# - Interactive code starting with '>>>'
+python -msphinx -M doctest target/docs/source \
+ target/docs/_build -c target/docs/source \
+ -w "target/docs/sphinx-doctest.warnings.log"
+
+# Fail if there are errors or warnings in docs
+! grep -q "ERROR:" target/docs/sphinx-doctest.warnings.log || exit 1
+! grep -q "WARNING:" target/docs/sphinx-doctest.warnings.log || exit 1
+
# Message is useful only when this script is run locally. In a remote
# test environment, this path will be removed when the test completes.
echo "Browse to file://$PWD/target/docs/_build/index.html"
-
-# Fail if there are errors or warnings in docs
-! grep -q "ERROR:" target/docs/sphinx-build.warnings.log
-! grep -q "WARNING:" target/docs/sphinx-build.warnings.log
diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml
index 1295654..3e67e3b 100644
--- a/sdks/python/pom.xml
+++ b/sdks/python/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,6 +32,10 @@
<name>Apache Beam :: SDKs :: Python</name>
+ <modules>
+ <module>container</module>
+ </modules>
+
<properties>
<!-- python.interpreter.bin & python.pip.bin
is set dynamically by findSupportedPython.groovy -->
@@ -59,6 +63,7 @@
<groupId>org.codehaus.gmaven</groupId>
<artifactId>groovy-maven-plugin</artifactId>
<version>${groovy-maven-plugin.version}</version>
+ <inherited>false</inherited>
<executions>
<execution>
<id>find-supported-python-for-clean</id>
@@ -85,6 +90,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
+ <inherited>false</inherited>
<executions>
<execution>
<id>setuptools-clean</id>
@@ -189,6 +195,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
+ <inherited>false</inherited>
<executions>
<execution>
<id>setuptools-test</id>
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index ddc3dc7..5e1c6b2 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -66,26 +66,6 @@
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
-# Install test dependencies for ValidatesRunner tests.
-echo "pyhamcrest" > postcommit_requirements.txt
-echo "mock" >> postcommit_requirements.txt
-
-# Run ValidatesRunner tests on Google Cloud Dataflow service
-echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
-python setup.py nosetests \
- --attr ValidatesRunner \
- --nocapture \
- --processes=4 \
- --process-timeout=900 \
- --test-pipeline-options=" \
- --runner=TestDataflowRunner \
- --project=$PROJECT \
- --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
- --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
- --sdk_location=$SDK_LOCATION \
- --requirements_file=postcommit_requirements.txt \
- --num_workers=1"
-
# Run integration tests on the Google Cloud Dataflow service
# and validate that jobs finish successfully.
echo ">>> RUNNING TEST DATAFLOW RUNNER it tests"
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
index a5e3fa1..4c57e75 100755
--- a/sdks/python/run_pylint.sh
+++ b/sdks/python/run_pylint.sh
@@ -33,7 +33,7 @@
if test $# -gt 0; then
case "$@" in
--help) usage; exit 1;;
- *) MODULE="$@";;
+ *) MODULE="$*";;
esac
fi
@@ -46,9 +46,7 @@
"apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py"
"apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py"
"apache_beam/coders/proto2_coder_test_messages_pb2.py"
-"apache_beam/runners/api/beam_fn_api_pb2.py"
-"apache_beam/runners/api/beam_fn_api_pb2_grpc.py"
-"apache_beam/runners/api/beam_runner_api_pb2.py"
+apache_beam/portability/api/*pb2*.py
)
FILES_TO_IGNORE=""
@@ -61,6 +59,45 @@
echo "Skipping lint for generated files: $FILES_TO_IGNORE"
echo "Running pylint for module $MODULE:"
-pylint $MODULE --ignore-patterns="$FILES_TO_IGNORE"
+pylint -j8 "$MODULE" --ignore-patterns="$FILES_TO_IGNORE"
echo "Running pycodestyle for module $MODULE:"
-pycodestyle $MODULE --exclude="$FILES_TO_IGNORE"
+pycodestyle "$MODULE" --exclude="$FILES_TO_IGNORE"
+echo "Running isort for module $MODULE:"
+# Skip files where isort is behaving weirdly
+ISORT_EXCLUDED=(
+ "apiclient.py"
+ "avroio_test.py"
+ "datastore_wordcount.py"
+ "datastoreio_test.py"
+ "iobase_test.py"
+ "fast_coders_test.py"
+ "slow_coders_test.py"
+)
+SKIP_PARAM=""
+for file in "${ISORT_EXCLUDED[@]}"; do
+ SKIP_PARAM="$SKIP_PARAM --skip $file"
+done
+for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
+ SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)"
+done
+pushd "$MODULE"
+isort -p apache_beam -w 120 -y -c -ot -cs -sl ${SKIP_PARAM}
+popd
+FUTURIZE_EXCLUDED=(
+ "typehints.py"
+ "pb2"
+ "trivial_infernce.py"
+)
+FUTURIZE_GREP_PARAM=$( IFS='|'; echo "${ids[*]}" )
+echo "Checking for files requiring stage 1 refactoring from futurize"
+futurize_results=$(futurize -j 8 --stage1 apache_beam 2>&1 |grep Refactored)
+futurize_filtered=$(echo "$futurize_results" |grep -v "$FUTURIZE_GREP_PARAM" || echo "")
+count=${#futurize_filtered}
+if [ "$count" != "0" ]; then
+ echo "Some of the changes require futurize stage 1 changes."
+ echo "The files with required changes:"
+ echo "$futurize_filtered"
+ echo "You can run futurize apache_beam to see the proposed changes."
+ exit 1
+fi
+echo "No future changes needed"
diff --git a/sdks/python/run_validatesrunner.sh b/sdks/python/run_validatesrunner.sh
new file mode 100755
index 0000000..7d20a75
--- /dev/null
+++ b/sdks/python/run_validatesrunner.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script will be run by Jenkins as a post commit test. In order to run
+# locally make the following changes:
+#
+# LOCAL_PATH -> Path of tox and virtualenv if you have them already installed.
+# GCS_LOCATION -> Temporary location to use for service tests.
+# PROJECT -> Project name to use for service jobs.
+#
+# Execute from the root of the repository: sdks/python/run_postcommit.sh
+
+set -e
+set -v
+
+# pip install --user installation location.
+LOCAL_PATH=$HOME/.local/bin/
+
+# INFRA does not install virtualenv
+pip install virtualenv --user
+
+# Virtualenv for the rest of the script to run setup & e2e tests
+${LOCAL_PATH}/virtualenv sdks/python
+. sdks/python/bin/activate
+cd sdks/python
+pip install -e .[gcp,test]
+
+# Where to store integration test outputs.
+GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
+
+PROJECT=apache-beam-testing
+
+# Create a tarball
+python setup.py sdist
+
+SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+
+# Install test dependencies for ValidatesRunner tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+echo "mock" >> postcommit_requirements.txt
+
+# Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
+python setup.py nosetests \
+ --attr ValidatesRunner \
+ --nocapture \
+ --processes=4 \
+ --process-timeout=900 \
+ --test-pipeline-options=" \
+ --runner=TestDataflowRunner \
+ --project=$PROJECT \
+ --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
+ --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
+ --sdk_location=$SDK_LOCATION \
+ --requirements_file=postcommit_requirements.txt \
+ --num_workers=1"
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 9bf3cf4..46f4f8a 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -19,13 +19,21 @@
from distutils.version import StrictVersion
+import glob
import os
+import pkg_resources
import platform
import shutil
+import subprocess
+import sys
import warnings
import setuptools
+from setuptools.command.build_py import build_py
+from setuptools.command.sdist import sdist
+from setuptools.command.test import test
+
from pkg_resources import get_distribution, DistributionNotFound
@@ -94,23 +102,54 @@
'httplib2>=0.8,<0.10',
'mock>=1.0.1,<3.0.0',
'oauth2client>=2.0.1,<4.0.0',
- 'protobuf==3.2.0',
+ 'protobuf>=3.2.0,<=3.3.0',
'pyyaml>=3.12,<4.0.0',
+ 'pyvcf>=0.6.8,<0.7.0',
+ # Six 1.11.0 incompatible with apitools.
+ # TODO(BEAM-2964): Remove the upper bound.
+ 'six>=1.9,<1.11',
+ 'typing>=3.6.0,<3.7.0',
+ 'futures>=3.1.1,<4.0.0',
+ ]
+
+REQUIRED_SETUP_PACKAGES = [
+ 'nose>=1.0',
]
REQUIRED_TEST_PACKAGES = [
- 'pyhamcrest>=1.9,<2.0'
+ 'pyhamcrest>=1.9,<2.0',
+ # Six required by nose plugins management.
+ # Six 1.11.0 incompatible with apitools.
+ # TODO(BEAM-2964): Remove the upper bound.
+ 'six>=1.9,<1.11',
]
GCP_REQUIREMENTS = [
- 'google-apitools==0.5.10',
- 'proto-google-cloud-datastore-v1==0.90.0',
+ 'google-apitools>=0.5.10,<=0.5.11',
+ 'proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4',
'googledatastore==7.0.1',
+ 'google-cloud-pubsub==0.26.0',
# GCP packages required by tests
- 'google-cloud-bigquery>=0.23.0,<0.24.0',
+ 'google-cloud-bigquery==0.25.0',
]
+# We must generate protos after setup_requires are installed.
+def generate_protos_first(original_cmd):
+ try:
+ # See https://issues.apache.org/jira/browse/BEAM-2366
+ # pylint: disable=wrong-import-position
+ import gen_protos
+ class cmd(original_cmd, object):
+ def run(self):
+ gen_protos.generate_proto_files()
+ super(cmd, self).run()
+ return cmd
+ except ImportError:
+ warnings.warn("Could not import gen_protos, skipping proto generation.")
+ return original_cmd
+
+
setuptools.setup(
name=PACKAGE_NAME,
version=PACKAGE_VERSION,
@@ -122,7 +161,7 @@
author_email=PACKAGE_EMAIL,
packages=setuptools.find_packages(),
package_data={'apache_beam': [
- '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'testing/data/*']},
+ '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'testing/data/*.yaml']},
ext_modules=cythonize([
'apache_beam/**/*.pyx',
'apache_beam/coders/coder_impl.py',
@@ -135,8 +174,9 @@
'apache_beam/utils/counters.py',
'apache_beam/utils/windowed_value.py',
]),
- setup_requires=['nose>=1.0'],
+ setup_requires=REQUIRED_SETUP_PACKAGES,
install_requires=REQUIRED_PACKAGES,
+ python_requires='>=2.7,<3.0',
test_suite='nose.collector',
tests_require=REQUIRED_TEST_PACKAGES,
extras_require={
@@ -153,11 +193,16 @@
'Programming Language :: Python :: 2.7',
'Topic :: Software Development :: Libraries',
'Topic :: Software Development :: Libraries :: Python Modules',
- ],
+ ],
license='Apache License, Version 2.0',
keywords=PACKAGE_KEYWORDS,
entry_points={
'nose.plugins.0.10': [
'beam_test_plugin = test_config:BeamTestPlugin'
- ]}
- )
+ ]},
+ cmdclass={
+ 'build_py': generate_protos_first(build_py),
+ 'sdist': generate_protos_first(sdist),
+ 'test': generate_protos_first(test),
+ },
+)
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2592b17..039b0e8 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -28,8 +28,12 @@
# autocomplete_test depends on nose when invoked directly.
deps =
nose==1.3.7
+ grpcio-tools==1.3.5
+whitelist_externals=find
commands =
python --version
+ # Clean up all previous python generated files.
+ - find apache_beam -type f -name '*.pyc' -delete
pip install -e .[test]
python apache_beam/examples/complete/autocomplete_test.py
python setup.py test
@@ -44,16 +48,21 @@
# autocomplete_test depends on nose when invoked directly.
deps =
nose==1.3.7
+ grpcio-tools==1.3.5
cython==0.25.2
-whitelist_externals=find
+whitelist_externals=
+ find
+ time
commands =
python --version
+ # Clean up all previous python generated files.
+ - find apache_beam -type f -name '*.pyc' -delete
# Clean up all previous cython generated files.
- find apache_beam -type f -name '*.c' -delete
- find apache_beam -type f -name '*.so' -delete
- find target/build -type f -name '*.c' -delete
- find target/build -type f -name '*.so' -delete
- pip install -e .[test]
+ time pip install -e .[test]
python apache_beam/examples/complete/autocomplete_test.py
python setup.py test
# Clean up all cython generated files. Ignore if deletion fails.
@@ -67,9 +76,12 @@
# autocomplete_test depends on nose when invoked directly.
deps =
nose==1.3.7
+whitelist_externals=find
commands =
pip install -e .[test,gcp]
python --version
+ # Clean up all previous python generated files.
+ - find apache_beam -type f -name '*.pyc' -delete
python apache_beam/examples/complete/autocomplete_test.py
python setup.py test
passenv = TRAVIS*
@@ -78,17 +90,23 @@
deps=
nose==1.3.7
pycodestyle==2.3.1
- pylint==1.7.1
+ pylint==1.7.2
+ future==0.16.0
+ isort==4.2.15
+whitelist_externals=time
commands =
- pip install -e .[test]
- {toxinidir}/run_pylint.sh
+ time pip install -e .[test]
+ time {toxinidir}/run_pylint.sh
passenv = TRAVIS*
[testenv:docs]
deps=
nose==1.3.7
+ grpcio-tools==1.3.5
Sphinx==1.5.5
+ sphinx_rtd_theme==0.2.4
+whitelist_externals=time
commands =
- pip install -e .[test,gcp,docs]
- {toxinidir}/generate_pydoc.sh
+ time pip install -e .[test,gcp,docs]
+ time {toxinidir}/generate_pydoc.sh
passenv = TRAVIS*
